@haathie/pgmb 0.2.6 → 0.2.8

package/README.md

@@ -32,6 +32,9 @@ Install PGMB by running the following command:
 npm install @haathie/pgmb
 ```
 
+Note: PGMB directly exports typescript files, so if you're using a bundler -- ensure it can handle typescript files. NodeJs (v22+), Deno, Bun can all run typescript files natively -- so about time we utilised this!
+For commonjs compatibility, the compiled JS files are also exported.
+
 Before using PGMB, you'll need to run the setup script to create the required tables, functions & triggers in your database. You can do this by running:
 
 ```sh
@@ -494,6 +497,30 @@ const pgmb = new PgmbClient({
 })
 ```
 
+## Configuring Knobs
+
+PGMB provides a number of configuration options to tune its behaviour (eg. how often to poll for events, read events, expire subscriptions, etc.). These can be configured via relevant env vars too, the names for which can be found [here](src/client.ts#L105)
+``` ts
+// poll every 500ms
+process.env.PGMB_READ_EVENTS_INTERVAL_MS = '500' // default: 1000
+const pgmb = new PgmbClient(opts)
+```
+
+## Production Considerations
+
+PGMB relies on 2 functions that need to be run periodically & only once globally to ensure smooth operation, i.e.
+1. `poll_for_events()` -- finds unread events & assigns them to relevant subscriptions.
+	It's okay if this runs simultaneously in multiple processes, but that can create unnecessary contention on the `unread_events` table, which can bubble up to other tables.
+2. `maintain_events_table()` -- removes old partitions & creates new partitions for the events table. It's also okay if this runs simultaneously in multiple processes, as it has advisory locks to ensure only a single process is maintaining the events table at any time, but running this too frequently can cause unnecessary overhead.
+
+If you have the `pg_cron` extension installed, `pgmb` will automatically setup these functions to run periodically via `pg_cron` on initialization. The default intervals are:
+- `poll_for_events()` -- every `1 second`
+- `maintain_events_table()` -- on every `30th minute`
+
+These can be easily configured by changing the values in the `config` table in the `pgmb` schema. Changes automatically get applied to the underlying cron jobs via triggers.
+
+If you do not have `pg_cron` installed, PGMB will run these functions in the same process as the client itself, at the above intervals. This is okay for development & small deployments.
+
 ## General Notes
 
 - **Does the client automatically reconnect on errors & temporary network issues?**

package/lib/client.js

@@ -11,12 +11,14 @@ const abortable_async_iterator_ts_1 = require("./abortable-async-iterator.js");
 const batcher_ts_1 = require("./batcher.js");
 const queries_ts_1 = require("./queries.js");
 const retry_handler_ts_1 = require("./retry-handler.js");
+const utils_ts_1 = require("./utils.js");
 const webhook_handler_ts_1 = require("./webhook-handler.js");
 class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
     client;
     logger;
     groupId;
-    sleepDurationMs;
+    readEventsIntervalMs;
+    pollEventsIntervalMs;
     readChunkSize;
     subscriptionMaintenanceMs;
     tableMaintenanceMs;
@@ -29,14 +31,13 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
     #webhookHandlerOpts;
     #readClient;
     #endAc = new AbortController();
-    #shouldPoll;
     #readTask;
     #pollTask;
     #subMaintainTask;
     #tableMaintainTask;
     #inMemoryCursor = null;
     #activeCheckpoints = [];
-    constructor({ client, groupId, logger = (0, pino_1.pino)(), sleepDurationMs = 750, readChunkSize = 1000, maxActiveCheckpoints = 10, poll, subscriptionMaintenanceMs = 60 * 1000, webhookHandlerOpts: { splitBy: whSplitBy, ...whHandlerOpts } = {}, getWebhookInfo = () => ({}), tableMaintainanceMs = 5 * 60 * 1000, readNextEvents = queries_ts_1.readNextEvents.run.bind(queries_ts_1.readNextEvents), findEvents, ...batcherOpts }) {
+    constructor({ client, groupId, logger = (0, pino_1.pino)(), readEventsIntervalMs = (0, utils_ts_1.getEnvNumber)('PGMB_READ_EVENTS_INTERVAL_MS', 1000), readChunkSize = (0, utils_ts_1.getEnvNumber)('PGMB_READ_CHUNK_SIZE', 1000), maxActiveCheckpoints = (0, utils_ts_1.getEnvNumber)('PGMB_MAX_ACTIVE_CHECKPOINTS', 10), pollEventsIntervalMs = (0, utils_ts_1.getEnvNumber)('PGMB_POLL_EVENTS_INTERVAL_MS', 1000), subscriptionMaintenanceMs = (0, utils_ts_1.getEnvNumber)('PGMB_SUBSCRIPTION_MAINTENANCE_S', 60) * 1000, tableMaintainanceMs = (0, utils_ts_1.getEnvNumber)('PGMB_TABLE_MAINTENANCE_M', 15) * 60 * 1000, webhookHandlerOpts: { splitBy: whSplitBy, ...whHandlerOpts } = {}, getWebhookInfo = () => ({}), readNextEvents = queries_ts_1.readNextEvents.run.bind(queries_ts_1.readNextEvents), findEvents, ...batcherOpts }) {
         super({
             ...batcherOpts,
             logger,
@@ -45,9 +46,9 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
         this.client = client;
         this.logger = logger;
         this.groupId = groupId;
-        this.sleepDurationMs = sleepDurationMs;
+        this.readEventsIntervalMs = readEventsIntervalMs;
         this.readChunkSize = readChunkSize;
-        this.#shouldPoll = !!poll;
+        this.pollEventsIntervalMs = pollEventsIntervalMs;
         this.subscriptionMaintenanceMs = subscriptionMaintenanceMs;
         this.maxActiveCheckpoints = maxActiveCheckpoints;
         this.webhookHandler = (0, webhook_handler_ts_1.createWebhookHandler)(whHandlerOpts);
@@ -62,24 +63,31 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
         if ('connect' in this.client) {
             this.client.on('remove', this.#onPoolClientRemoved);
         }
-        // maintain event table
-        await queries_ts_1.maintainEventsTable.run(undefined, this.client);
-        this.logger.debug('maintained events table');
+        const [pgCronRslt] = await queries_ts_1.getConfigValue
+            .run({ key: 'use_pg_cron' }, this.client);
+        const isPgCronEnabled = pgCronRslt?.value === 'true';
+        if (!isPgCronEnabled) {
+            // maintain event table
+            await queries_ts_1.maintainEventsTable.run(undefined, this.client);
+            this.logger.debug('maintained events table');
+            if (this.pollEventsIntervalMs) {
+                this.#pollTask = this.#startLoop(queries_ts_1.pollForEvents.run.bind(queries_ts_1.pollForEvents, undefined, this.client), this.pollEventsIntervalMs);
+            }
+            if (this.tableMaintenanceMs) {
+                this.#tableMaintainTask = this.#startLoop(queries_ts_1.maintainEventsTable.run
+                    .bind(queries_ts_1.maintainEventsTable, undefined, this.client), this.tableMaintenanceMs);
+            }
+        }
         await queries_ts_1.assertGroup.run({ id: this.groupId }, this.client);
         this.logger.debug({ groupId: this.groupId }, 'asserted group exists');
         // clean up expired subscriptions on start
         const [{ deleted }] = await queries_ts_1.removeExpiredSubscriptions.run({ groupId: this.groupId, activeIds: [] }, this.client);
         this.logger.debug({ deleted }, 'removed expired subscriptions');
-        this.#readTask = this.#startLoop(this.readChanges.bind(this), this.sleepDurationMs);
-        if (this.#shouldPoll) {
-            this.#pollTask = this.#startLoop(queries_ts_1.pollForEvents.run.bind(queries_ts_1.pollForEvents, undefined, this.client), this.sleepDurationMs);
-        }
+        this.#readTask = this.#startLoop(this.readChanges.bind(this), this.readEventsIntervalMs);
         if (this.subscriptionMaintenanceMs) {
             this.#subMaintainTask = this.#startLoop(this.#maintainSubscriptions, this.subscriptionMaintenanceMs);
         }
-        if (this.tableMaintenanceMs) {
-            this.#tableMaintainTask = this.#startLoop(queries_ts_1.maintainEventsTable.run.bind(queries_ts_1.maintainEventsTable, undefined, this.client), this.tableMaintenanceMs);
-        }
+        this.logger.info({ isPgCronEnabled }, 'pgmb client initialised');
     }
     async end() {
         await super.end();

package/lib/queries.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.maintainEventsTable = exports.removeExpiredSubscriptions = exports.findEvents = exports.scheduleEventRetry = exports.writeScheduledEvents = exports.writeEvents = exports.releaseGroupLock = exports.setGroupCursor = exports.replayEvents = exports.readNextEventsText = exports.readNextEvents = exports.pollForEvents = exports.markSubscriptionsActive = exports.deleteSubscriptions = exports.assertSubscription = exports.assertGroup = void 0;
+exports.maintainEventsTable = exports.updateConfigValue = exports.getConfigValue = exports.removeExpiredSubscriptions = exports.findEvents = exports.scheduleEventRetry = exports.writeScheduledEvents = exports.writeEvents = exports.releaseGroupLock = exports.setGroupCursor = exports.replayEvents = exports.readNextEventsText = exports.readNextEvents = exports.pollForEvents = exports.markSubscriptionsActive = exports.deleteSubscriptions = exports.assertSubscription = exports.assertGroup = void 0;
 /** Types generated for queries found in "sql/queries.sql" */
 const runtime_1 = require("@pgtyped/runtime");
 const assertGroupIR = { "usedParamSet": { "id": true }, "params": [{ "name": "id", "required": true, "transform": { "type": "scalar" }, "locs": [{ "a": 50, "b": 53 }] }], "statement": "INSERT INTO pgmb.subscription_groups (id)\nVALUES (:id!)\nON CONFLICT DO NOTHING" };
@@ -225,6 +225,25 @@ const removeExpiredSubscriptionsIR = { "usedParamSet": { "groupId": true, "activ
  * ```
  */
 exports.removeExpiredSubscriptions = new runtime_1.PreparedQuery(removeExpiredSubscriptionsIR);
+const getConfigValueIR = { "usedParamSet": { "key": true }, "params": [{ "name": "key", "required": true, "transform": { "type": "scalar" }, "locs": [{ "a": 29, "b": 33 }] }], "statement": "SELECT pgmb.get_config_value(:key!::pgmb.config_type) AS \"value\"" };
+/**
+ * Query generated from SQL:
+ * ```
+ * SELECT pgmb.get_config_value(:key!::pgmb.config_type) AS "value"
+ * ```
+ */
+exports.getConfigValue = new runtime_1.PreparedQuery(getConfigValueIR);
+const updateConfigValueIR = { "usedParamSet": { "value": true, "key": true }, "params": [{ "name": "value", "required": true, "transform": { "type": "scalar" }, "locs": [{ "a": 31, "b": 37 }] }, { "name": "key", "required": true, "transform": { "type": "scalar" }, "locs": [{ "a": 56, "b": 60 }] }], "statement": "UPDATE pgmb.config\nSET value = :value!::TEXT\nWHERE id = :key!::pgmb.config_type\nRETURNING 1 AS \"updated!\"" };
+/**
+ * Query generated from SQL:
+ * ```
+ * UPDATE pgmb.config
+ * SET value = :value!::TEXT
+ * WHERE id = :key!::pgmb.config_type
+ * RETURNING 1 AS "updated!"
+ * ```
+ */
+exports.updateConfigValue = new runtime_1.PreparedQuery(updateConfigValueIR);
 const maintainEventsTableIR = { "usedParamSet": {}, "params": [], "statement": "SELECT pgmb.maintain_events_table()" };
 /**
  * Query generated from SQL:

package/lib/utils.js

@@ -6,6 +6,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.getDateFromMessageId = getDateFromMessageId;
 exports.getCreateDateFromSubscriptionId = getCreateDateFromSubscriptionId;
 exports.createTopicalSubscriptionParams = createTopicalSubscriptionParams;
+exports.getEnvNumber = getEnvNumber;
 const node_assert_1 = __importDefault(require("node:assert"));
 /**
  * Extract the date from a message ID, same as the PG function
@@ -50,3 +51,13 @@ function createTopicalSubscriptionParams({ topics, partition, additionalFilters
         ...rest
     };
 }
+/**
+ * Get an environment variable as a number
+ */
+function getEnvNumber(key, defaultValue = 0) {
+    const num = +(process.env[key] || defaultValue);
+    if (isNaN(num) || !isFinite(num)) {
+        return defaultValue;
+    }
+    return num;
+}

package/package.json

@@ -1,21 +1,27 @@
 {
   "name": "@haathie/pgmb",
-  "version": "0.2.6",
+  "version": "0.2.8",
   "description": "PG message broker, with a type-safe typescript client with built-in webhook & SSE support.",
-  "main": "lib/index.js",
   "publishConfig": {
     "registry": "https://registry.npmjs.org",
     "access": "public"
   },
+  "exports": {
+    ".": {
+      "import": "./src/index.ts",
+      "types": "./src/index.ts",
+      "require": "./lib/index.js"
+    }
+  },
   "repository": "https://github.com/haathie/pgmb",
   "scripts": {
     "test": "TZ=UTC NODE_ENV=test node --env-file ./.env.test --test tests/*.test.ts",
-    "prepare": "tsc",
-    "build": "tsc",
+    "prepare": "npm run build",
+    "build": "tsc -p tsconfig.build.json",
     "lint": "eslint ./ --ext .js,.ts,.jsx,.tsx",
     "lint:fix": "eslint ./ --fix --ext .js,.ts,.jsx,.tsx",
     "benchmark": "TZ=utc node --env-file ./.env.test src/benchmark/run.ts",
-    "gen:pgtyped": "pgtyped --config ./pgtyped.config.json"
+    "pg:typegen": "pgtyped --config ./pgtyped.config.json"
   },
   "devDependencies": {
     "@adiwajshing/eslint-config": "git+https://github.com/adiwajshing/eslint-config",
@@ -27,13 +33,14 @@
     "@types/pg": "^8.11.14",
     "amqplib": "^0.10.7",
     "chance": "^1.1.12",
-    "eslint": "^8.19.0",
+    "eslint": "^9.0.0",
     "eventsource": "^4.1.0",
     "pg": "^8.16.3",
     "typescript": "^5.0.0"
   },
   "files": [
     "lib",
+    "src",
     "sql"
   ],
   "keywords": [

package/sql/pgmb-0.2.0-0.2.8.sql

@@ -0,0 +1,81 @@
+SET search_path TO pgmb;
+
+ALTER TYPE config_type ADD VALUE 'use_pg_cron';
+ALTER TYPE config_type ADD VALUE 'pg_cron_poll_for_events_cron';
+ALTER TYPE config_type ADD VALUE 'pg_cron_partition_maintenance_cron';
+
+INSERT INTO config(id, value) VALUES
+('poll_chunk_size', '10000'),
+('pg_cron_poll_for_events_cron', '1 second'),
+-- every 30 minutes
+('pg_cron_partition_maintenance_cron', '*/30 * * * *');
+
+CREATE OR REPLACE FUNCTION manage_cron_jobs_trigger_fn()
+RETURNS TRIGGER AS $$
+DECLARE
+	poll_job_name CONSTANT TEXT := 'pgmb_poll';
+	maintain_job_name CONSTANT TEXT := 'pgmb_maintain_table_partitions';
+BEGIN
+	IF get_config_value('use_pg_cron') = 'true' THEN
+		-- Schedule/update event polling job
+		PERFORM cron.schedule(
+			poll_job_name,
+			get_config_value('pg_cron_poll_for_events_cron'),
+			$CMD$
+				-- ensure we don't accidentally run for too long
+				SET SESSION statement_timeout = '10s';
+				SELECT pgmb.poll_for_events();
+			$CMD$
+		);
+		RAISE LOG 'Scheduled pgmb polling job: %', poll_job_name;
+
+		-- Schedule/update partition maintenance job
+		PERFORM cron.schedule(
+			'pgmb_maintain_table_partitions',
+			get_config_value('pg_cron_partition_maintenance_cron'),
+			$CMD$ SELECT pgmb.maintain_events_table(); $CMD$
+		);
+
+		RAISE LOG 'Scheduled pgmb partition maintenance job: %',
+			maintain_job_name;
+	ELSIF (SELECT 1 FROM pg_namespace WHERE nspname = 'cron') THEN
+		RAISE LOG 'Unscheduling pgmb cron jobs.';
+		-- Unschedule jobs. cron.unschedule does not fail if job does not exist.
+		PERFORM cron.unschedule(poll_job_name);
+		PERFORM cron.unschedule(maintain_job_name);
+	END IF;
+
+	RETURN NULL;
+END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER
+SET search_path TO pgmb;
+
+CREATE TRIGGER manage_cron_jobs_trigger
+AFTER INSERT OR UPDATE OF value ON config
+FOR EACH ROW
+WHEN (
+	NEW.id IN (
+		'use_pg_cron',
+		'pg_cron_poll_for_events_cron',
+		'pg_cron_partition_maintenance_cron'
+	)
+)
+EXECUTE FUNCTION manage_cron_jobs_trigger_fn();
+
+DO $$
+BEGIN
+	IF (
+		SELECT EXISTS (
+			SELECT 1
+			FROM pg_available_extensions
+			WHERE name = 'pg_cron'
+		)
+	) THEN
+		CREATE EXTENSION IF NOT EXISTS pg_cron;
+		INSERT INTO config(id, value) VALUES ('use_pg_cron', 'true');
+	ELSE
+		RAISE LOG 'pg_cron extension not available. Skipping pg_cron setup.';
+		INSERT INTO config(id, value) VALUES ('use_pg_cron', 'false');
+	END IF;
+END
+$$;

package/sql/pgmb.sql

@@ -22,7 +22,10 @@ CREATE TYPE config_type AS ENUM(
 	-- how far into the future to create partitions
 	'future_intervals_to_create',
 	'partition_interval',
-	'poll_chunk_size'
+	'poll_chunk_size',
+	'use_pg_cron',
+	'pg_cron_poll_for_events_cron',
+	'pg_cron_partition_maintenance_cron'
 );
 
 CREATE TABLE IF NOT EXISTS config(
@@ -37,13 +40,15 @@ CREATE OR REPLACE FUNCTION get_config_value(
 	SELECT value FROM config WHERE id = config_id
 $$ LANGUAGE sql STRICT STABLE PARALLEL SAFE SET SEARCH_PATH TO pgmb;
 
-INSERT INTO config(id, value)
-	VALUES
-		('plugin_version', '0.2.0'),
-		('partition_retention_period', '60 minutes'),
-		('future_intervals_to_create', '120 minutes'),
-		('partition_interval', '30 minutes'),
-		('poll_chunk_size', '10000');
+INSERT INTO config(id, value) VALUES
+('plugin_version', '0.2.0'),
+('partition_retention_period', '60 minutes'),
+('future_intervals_to_create', '120 minutes'),
+('partition_interval', '30 minutes'),
+('poll_chunk_size', '10000'),
+('pg_cron_poll_for_events_cron', '1 second'),
+-- every 30 minutes
+('pg_cron_partition_maintenance_cron', '*/30 * * * *');
 
 -- we'll create the events table next & its functions ---------------
 
@@ -365,33 +370,11 @@ CREATE INDEX ON subscriptions(
 	add_interval_imm(last_active_at, expiry_interval)
 ) WHERE expiry_interval IS NOT NULL;
 
-DO $$
-DECLARE
-	has_btree_gin BOOLEAN;
-BEGIN
-	has_btree_gin := (
-		SELECT EXISTS (
-			SELECT 1
-			FROM pg_available_extensions
-			WHERE name = 'btree_gin'
-		)
-	);
-	-- create btree_gin extension if not exists, if the extension
-	-- is not available, we create a simpler regular GIN index instead.
-	IF has_btree_gin THEN
-		CREATE EXTENSION IF NOT EXISTS btree_gin;
-		-- fastupdate=false, slows down subscription creation, but ensures the costlier
-		-- "poll_for_events" function is executed faster.
-		CREATE INDEX "sub_gin" ON subscriptions USING GIN(conditions_sql, params)
-			WITH (fastupdate = false);
-	ELSE
-		RAISE NOTICE 'btree_gin extension is not available, using
-			regular GIN index for subscriptions.params';
-		CREATE INDEX "sub_gin" ON subscriptions USING GIN(params)
-			WITH (fastupdate = false);
-	END IF;
-END
-$$;
+-- fastupdate=false, slows down subscription creation, but ensures the costlier
+-- "poll_for_events" function is executed faster.
+CREATE EXTENSION IF NOT EXISTS btree_gin;
+CREATE INDEX "sub_gin" ON subscriptions
+USING GIN(conditions_sql, params) WITH (fastupdate = false);
 
 -- materialized view to hold distinct conditions_sql statements.
 -- We utilise changes in this view to determine when to prepare the
@@ -863,6 +846,78 @@ SET search_path TO pgmb;
 -- create the initial partitions
 SELECT maintain_events_table();
 
+-- setup pg_cron if it's available ----------------
+
+CREATE OR REPLACE FUNCTION manage_cron_jobs_trigger_fn()
+RETURNS TRIGGER AS $$
+DECLARE
+	poll_job_name CONSTANT TEXT := 'pgmb_poll';
+	maintain_job_name CONSTANT TEXT := 'pgmb_maintain_table_partitions';
+BEGIN
+	IF get_config_value('use_pg_cron') = 'true' THEN
+		-- Schedule/update event polling job
+		PERFORM cron.schedule(
+			poll_job_name,
+			get_config_value('pg_cron_poll_for_events_cron'),
+			$CMD$
+				-- ensure we don't accidentally run for too long
+				SET SESSION statement_timeout = '10s';
+				SELECT pgmb.poll_for_events();
+			$CMD$
+		);
+		RAISE LOG 'Scheduled pgmb polling job: %', poll_job_name;
+
+		-- Schedule/update partition maintenance job
+		PERFORM cron.schedule(
+			'pgmb_maintain_table_partitions',
+			get_config_value('pg_cron_partition_maintenance_cron'),
+			$CMD$ SELECT pgmb.maintain_events_table(); $CMD$
+		);
+
+		RAISE LOG 'Scheduled pgmb partition maintenance job: %',
+			maintain_job_name;
+	ELSIF (SELECT 1 FROM pg_namespace WHERE nspname = 'cron') THEN
+		RAISE LOG 'Unscheduling pgmb cron jobs.';
+		-- Unschedule jobs. cron.unschedule does not fail if job does not exist.
+		PERFORM cron.unschedule(poll_job_name);
+		PERFORM cron.unschedule(maintain_job_name);
+	END IF;
+
+	RETURN NULL;
+END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER
+SET search_path TO pgmb;
+
+CREATE TRIGGER manage_cron_jobs_trigger
+AFTER INSERT OR UPDATE OF value ON config
+FOR EACH ROW
+WHEN (
+	NEW.id IN (
+		'use_pg_cron',
+		'pg_cron_poll_for_events_cron',
+		'pg_cron_partition_maintenance_cron'
+	)
+)
+EXECUTE FUNCTION manage_cron_jobs_trigger_fn();
+
+DO $$
+BEGIN
+	IF (
+		SELECT EXISTS (
+			SELECT 1
+			FROM pg_available_extensions
+			WHERE name = 'pg_cron'
+		)
+	) THEN
+		CREATE EXTENSION IF NOT EXISTS pg_cron;
+		INSERT INTO config(id, value) VALUES ('use_pg_cron', 'true');
+	ELSE
+		RAISE LOG 'pg_cron extension not available. Skipping pg_cron setup.';
+		INSERT INTO config(id, value) VALUES ('use_pg_cron', 'false');
+	END IF;
+END
+$$;
+
 -- triggers to add events for specific tables ---------------------------
 
 -- Function to create a topic string for subscriptions.

package/sql/queries.sql

@@ -150,5 +150,14 @@ WITH deleted AS (
 )
 SELECT COUNT(*) AS "deleted!" FROM deleted;
 
+/* @name getConfigValue */
+SELECT pgmb.get_config_value(:key!::pgmb.config_type) AS "value";
+
+/* @name updateConfigValue */
+UPDATE pgmb.config
+SET value = :value!::TEXT
+WHERE id = :key!::pgmb.config_type
+RETURNING 1 AS "updated!";
+
 /* @name maintainEventsTable */
 SELECT pgmb.maintain_events_table();

package/src/abortable-async-iterator.ts

@@ -0,0 +1,98 @@
+import assert from 'assert'
+
+type AAResult<T> = IteratorResult<T>
+
+export class AbortableAsyncIterator<T> implements AsyncIterableIterator<T> {
+	readonly signal: AbortSignal
+	readonly onEnd: () => void
+
+	ended = false
+
+	#resolve: (() => void) | undefined
+	#reject: ((reason?: unknown) => void) | undefined
+	#queue: T[] = []
+	#locked = false
+
+	constructor(signal: AbortSignal, onEnd: () => void = () => {}) {
+		this.signal = signal
+		this.onEnd = onEnd
+		signal.addEventListener('abort', this.#onAbort)
+	}
+
+	async next(): Promise<AAResult<T>> {
+		assert(!this.ended, 'Iterator has already been completed')
+		assert(!this.#locked, 'Concurrent calls to next() are not allowed')
+
+		let nextItem = this.#queue.shift()
+		if(nextItem) {
+			return { value: nextItem, done: false }
+		}
+
+		this.#locked = true
+		try {
+			await this.#setupNextPromise()
+		} finally {
+			this.#locked = false
+		}
+
+		nextItem = this.#queue.shift()
+		if(nextItem) {
+			return { value: nextItem, done: false }
+		}
+
+		return { value: undefined, done: true }
+	}
+
+	enqueue(value: T) {
+		assert(!this.ended, 'Iterator has already been completed')
+		this.#queue.push(value)
+		this.#resolve?.()
+	}
+
+	throw(reason?: unknown): Promise<AAResult<T>> {
+		this.signal.throwIfAborted()
+		this.#reject?.(reason)
+		this.#end()
+		return Promise.resolve({ done: true, value: undefined })
+	}
+
+	return(value?: any): Promise<AAResult<T>> {
+		this.#resolve?.()
+		this.#end()
+		return Promise.resolve({ done: true, value })
+	}
+
+	#setupNextPromise() {
+		return new Promise<void>((resolve, reject) => {
+			this.#resolve = () => {
+				resolve()
+				this.#cleanupTask()
+			}
+
+			this.#reject = err => {
+				reject(err)
+				this.#cleanupTask()
+			}
+		})
+	}
+
+	#cleanupTask() {
+		this.#resolve = undefined
+		this.#reject = undefined
+	}
+
+	#onAbort = (reason: any) => {
+		this.#reject?.(reason)
+		this.#end()
+		this.ended = true
+	}
+
+	#end() {
+		this.signal.removeEventListener('abort', this.#onAbort)
+		this.onEnd()
+	}
+
+	[Symbol.asyncIterator]() {
+		return this
+	}
+}

package/src/batcher.ts

@@ -0,0 +1,90 @@
+import type { IEventData, PGMBEventBatcherOpts } from './types.ts'
+
+type Batch<T> = {
+	messages: T[]
+}
+
+export class PGMBEventBatcher<T extends IEventData> {
+
+	#publish: PGMBEventBatcherOpts<T>['publish']
+	#flushIntervalMs: number | undefined
+	#maxBatchSize: number
+	#currentBatch: Batch<T> = { messages: [] }
+	#flushTimeout: NodeJS.Timeout | undefined
+	#flushTask: Promise<void> | undefined
+	#logger: PGMBEventBatcherOpts<T>['logger']
+	#shouldLog?: PGMBEventBatcherOpts<T>['shouldLog']
+	#batch = 0
+
+	constructor({
+		shouldLog,
+		publish,
+		flushIntervalMs,
+		maxBatchSize = 2500,
+		logger
+	}: PGMBEventBatcherOpts<T>) {
+		this.#publish = publish
+		this.#flushIntervalMs = flushIntervalMs
+		this.#maxBatchSize = maxBatchSize
+		this.#logger = logger
+		this.#shouldLog = shouldLog
+	}
+
+	async end() {
+		clearTimeout(this.#flushTimeout)
+		await this.#flushTask
+		await this.flush()
+	}
+
+	/**
+	 * Enqueue a message to be published, will be flushed to the database
+	 * when flush() is called (either manually or via interval)
+	 */
+	enqueue(msg: T) {
+		this.#currentBatch.messages.push(msg)
+		if(this.#currentBatch.messages.length >= this.#maxBatchSize) {
+			this.flush()
+			return
+		}
+
+		if(this.#flushTimeout || !this.#flushIntervalMs) {
+			return
+		}
+
+		this.#flushTimeout = setTimeout(() => this.flush(), this.#flushIntervalMs)
+	}
+
+	async flush() {
+		if(!this.#currentBatch.messages.length) {
+			return
+		}
+
+		const batch = this.#currentBatch
+		this.#currentBatch = { messages: [] }
+		clearTimeout(this.#flushTimeout)
+		this.#flushTimeout = undefined
+
+		await this.#flushTask
+
+		this.#flushTask = this.#publishBatch(batch)
+		return this.#flushTask
+	}
+
+	async #publishBatch({ messages }: Batch<T>) {
+		const batch = ++this.#batch
+		try {
+			const ids = await this.#publish(...messages)
+			for(const [i, { id }] of ids.entries()) {
+				if(this.#shouldLog && !this.#shouldLog(messages[i])) {
+					continue
+				}
+
+				this.#logger
+					?.info({ batch, id, message: messages[i] }, 'published message')
+			}
+		} catch(err) {
+			this.#logger
+				?.error({ batch, err, msgs: messages }, 'failed to publish messages')
+		}
+	}
+}