@haathie/pgmb 0.2.7 → 0.2.9

package/README.md

@@ -513,17 +513,13 @@ PGMB relies on 2 functions that need to be run periodically & only once globally
 	It's okay if this runs simultaneously in multiple processes, but that can create unnecessary contention on the `unread_events` table, which can bubble up to other tables.
 2. `maintain_events_table()` -- removes old partitions & creates new partitions for the events table. It's also okay if this runs simultaneously in multiple processes, as it has advisory locks to ensure only a single process is maintaining the events table at any time, but running this too frequently can cause unnecessary overhead.
 
-If you're only running a single instance of your service, you can simply run these functions in the same process as your PGMB client (default behaviour).
-However, for larger deployments with multiple instances of your service, it's recommended to run these functions in a separate process to avoid contention, and disable polling & table maintainence:
-``` ts
-const pgmb = new PgmbClient({
-	pollEventsIntervalMs: 0,
-	tableMaintainanceMs: 0,
-	...otherOpts
-})
-```
+If you have the `pg_cron` extension installed, `pgmb` will automatically setup these functions to run periodically via `pg_cron` on initialization. The default intervals are:
+- `poll_for_events()` -- every `1 second`
+- `maintain_events_table()` -- on every `30th minute`
+
+These can be easily configured by changing the values in the `config` table in the `pgmb` schema. Changes automatically get applied to the underlying cron jobs via triggers.
 
-Something like [pg_cron](https://github.com/citusdata/pg_cron) is a good option.
+If you do not have `pg_cron` installed, PGMB will run these functions in the same process as the client itself, at the above intervals. This is okay for development & small deployments.
 
 ## General Notes
 

package/lib/client.js

@@ -63,24 +63,31 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
         if ('connect' in this.client) {
             this.client.on('remove', this.#onPoolClientRemoved);
         }
-        // maintain event table
-        await queries_ts_1.maintainEventsTable.run(undefined, this.client);
-        this.logger.debug('maintained events table');
+        const [pgCronRslt] = await queries_ts_1.getConfigValue
+            .run({ key: 'use_pg_cron' }, this.client);
+        const isPgCronEnabled = pgCronRslt?.value === 'true';
+        if (!isPgCronEnabled) {
+            // maintain event table
+            await queries_ts_1.maintainEventsTable.run(undefined, this.client);
+            this.logger.debug('maintained events table');
+            if (this.pollEventsIntervalMs) {
+                this.#pollTask = this.#startLoop(queries_ts_1.pollForEvents.run.bind(queries_ts_1.pollForEvents, undefined, this.client), this.pollEventsIntervalMs);
+            }
+            if (this.tableMaintenanceMs) {
+                this.#tableMaintainTask = this.#startLoop(queries_ts_1.maintainEventsTable.run
+                    .bind(queries_ts_1.maintainEventsTable, undefined, this.client), this.tableMaintenanceMs);
+            }
+        }
         await queries_ts_1.assertGroup.run({ id: this.groupId }, this.client);
         this.logger.debug({ groupId: this.groupId }, 'asserted group exists');
         // clean up expired subscriptions on start
         const [{ deleted }] = await queries_ts_1.removeExpiredSubscriptions.run({ groupId: this.groupId, activeIds: [] }, this.client);
         this.logger.debug({ deleted }, 'removed expired subscriptions');
         this.#readTask = this.#startLoop(this.readChanges.bind(this), this.readEventsIntervalMs);
-        if (this.pollEventsIntervalMs) {
-            this.#pollTask = this.#startLoop(queries_ts_1.pollForEvents.run.bind(queries_ts_1.pollForEvents, undefined, this.client), this.pollEventsIntervalMs);
-        }
         if (this.subscriptionMaintenanceMs) {
             this.#subMaintainTask = this.#startLoop(this.#maintainSubscriptions, this.subscriptionMaintenanceMs);
         }
-        if (this.tableMaintenanceMs) {
-            this.#tableMaintainTask = this.#startLoop(queries_ts_1.maintainEventsTable.run.bind(queries_ts_1.maintainEventsTable, undefined, this.client), this.tableMaintenanceMs);
-        }
+        this.logger.info({ isPgCronEnabled }, 'pgmb client initialised');
     }
     async end() {
         await super.end();

package/lib/queries.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.maintainEventsTable = exports.removeExpiredSubscriptions = exports.findEvents = exports.scheduleEventRetry = exports.writeScheduledEvents = exports.writeEvents = exports.releaseGroupLock = exports.setGroupCursor = exports.replayEvents = exports.readNextEventsText = exports.readNextEvents = exports.pollForEvents = exports.markSubscriptionsActive = exports.deleteSubscriptions = exports.assertSubscription = exports.assertGroup = void 0;
+exports.maintainEventsTable = exports.updateConfigValue = exports.getConfigValue = exports.removeExpiredSubscriptions = exports.findEvents = exports.scheduleEventRetry = exports.writeScheduledEvents = exports.writeEvents = exports.releaseGroupLock = exports.setGroupCursor = exports.replayEvents = exports.readNextEventsText = exports.readNextEvents = exports.pollForEvents = exports.markSubscriptionsActive = exports.deleteSubscriptions = exports.assertSubscription = exports.assertGroup = void 0;
 /** Types generated for queries found in "sql/queries.sql" */
 const runtime_1 = require("@pgtyped/runtime");
 const assertGroupIR = { "usedParamSet": { "id": true }, "params": [{ "name": "id", "required": true, "transform": { "type": "scalar" }, "locs": [{ "a": 50, "b": 53 }] }], "statement": "INSERT INTO pgmb.subscription_groups (id)\nVALUES (:id!)\nON CONFLICT DO NOTHING" };
@@ -225,6 +225,25 @@ const removeExpiredSubscriptionsIR = { "usedParamSet": { "groupId": true, "activ
  * ```
  */
 exports.removeExpiredSubscriptions = new runtime_1.PreparedQuery(removeExpiredSubscriptionsIR);
+const getConfigValueIR = { "usedParamSet": { "key": true }, "params": [{ "name": "key", "required": true, "transform": { "type": "scalar" }, "locs": [{ "a": 29, "b": 33 }] }], "statement": "SELECT pgmb.get_config_value(:key!::pgmb.config_type) AS \"value\"" };
+/**
+ * Query generated from SQL:
+ * ```
+ * SELECT pgmb.get_config_value(:key!::pgmb.config_type) AS "value"
+ * ```
+ */
+exports.getConfigValue = new runtime_1.PreparedQuery(getConfigValueIR);
+const updateConfigValueIR = { "usedParamSet": { "value": true, "key": true }, "params": [{ "name": "value", "required": true, "transform": { "type": "scalar" }, "locs": [{ "a": 31, "b": 37 }] }, { "name": "key", "required": true, "transform": { "type": "scalar" }, "locs": [{ "a": 56, "b": 60 }] }], "statement": "UPDATE pgmb.config\nSET value = :value!::TEXT\nWHERE id = :key!::pgmb.config_type\nRETURNING 1 AS \"updated!\"" };
+/**
+ * Query generated from SQL:
+ * ```
+ * UPDATE pgmb.config
+ * SET value = :value!::TEXT
+ * WHERE id = :key!::pgmb.config_type
+ * RETURNING 1 AS "updated!"
+ * ```
+ */
+exports.updateConfigValue = new runtime_1.PreparedQuery(updateConfigValueIR);
 const maintainEventsTableIR = { "usedParamSet": {}, "params": [], "statement": "SELECT pgmb.maintain_events_table()" };
 /**
  * Query generated from SQL:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@haathie/pgmb",
-  "version": "0.2.7",
+  "version": "0.2.9",
   "description": "PG message broker, with a type-safe typescript client with built-in webhook & SSE support.",
   "publishConfig": {
     "registry": "https://registry.npmjs.org",
@@ -21,7 +21,7 @@
     "lint": "eslint ./ --ext .js,.ts,.jsx,.tsx",
     "lint:fix": "eslint ./ --fix --ext .js,.ts,.jsx,.tsx",
     "benchmark": "TZ=utc node --env-file ./.env.test src/benchmark/run.ts",
-    "gen:pgtyped": "pgtyped --config ./pgtyped.config.json"
+    "pg:typegen": "pgtyped --config ./pgtyped.config.json"
   },
   "devDependencies": {
     "@adiwajshing/eslint-config": "git+https://github.com/adiwajshing/eslint-config",

package/sql/pgmb-0.2.0-0.2.8.sql

@@ -0,0 +1,80 @@
+SET search_path TO pgmb;
+
+ALTER TYPE config_type ADD VALUE 'use_pg_cron';
+ALTER TYPE config_type ADD VALUE 'pg_cron_poll_for_events_cron';
+ALTER TYPE config_type ADD VALUE 'pg_cron_partition_maintenance_cron';
+
+INSERT INTO config(id, value) VALUES
+('pg_cron_poll_for_events_cron', '1 second'),
+-- every 30 minutes
+('pg_cron_partition_maintenance_cron', '*/30 * * * *');
+
+CREATE OR REPLACE FUNCTION manage_cron_jobs_trigger_fn()
+RETURNS TRIGGER AS $$
+DECLARE
+	poll_job_name CONSTANT TEXT := 'pgmb_poll';
+	maintain_job_name CONSTANT TEXT := 'pgmb_maintain_table_partitions';
+BEGIN
+	IF get_config_value('use_pg_cron') = 'true' THEN
+		-- Schedule/update event polling job
+		PERFORM cron.schedule(
+			poll_job_name,
+			get_config_value('pg_cron_poll_for_events_cron'),
+			$CMD$
+				-- ensure we don't accidentally run for too long
+				SET SESSION statement_timeout = '10s';
+				SELECT pgmb.poll_for_events();
+			$CMD$
+		);
+		RAISE LOG 'Scheduled pgmb polling job: %', poll_job_name;
+
+		-- Schedule/update partition maintenance job
+		PERFORM cron.schedule(
+			'pgmb_maintain_table_partitions',
+			get_config_value('pg_cron_partition_maintenance_cron'),
+			$CMD$ SELECT pgmb.maintain_events_table(); $CMD$
+		);
+
+		RAISE LOG 'Scheduled pgmb partition maintenance job: %',
+			maintain_job_name;
+	ELSIF (SELECT 1 FROM pg_namespace WHERE nspname = 'cron') THEN
+		RAISE LOG 'Unscheduling pgmb cron jobs.';
+		-- Unschedule jobs. cron.unschedule does not fail if job does not exist.
+		PERFORM cron.unschedule(poll_job_name);
+		PERFORM cron.unschedule(maintain_job_name);
+	END IF;
+
+	RETURN NULL;
+END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER
+SET search_path TO pgmb;
+
+CREATE TRIGGER manage_cron_jobs_trigger
+AFTER INSERT OR UPDATE OF value ON config
+FOR EACH ROW
+WHEN (
+	NEW.id IN (
+		'use_pg_cron',
+		'pg_cron_poll_for_events_cron',
+		'pg_cron_partition_maintenance_cron'
+	)
+)
+EXECUTE FUNCTION manage_cron_jobs_trigger_fn();
+
+DO $$
+BEGIN
+	IF (
+		SELECT EXISTS (
+			SELECT 1
+			FROM pg_available_extensions
+			WHERE name = 'pg_cron'
+		)
+	) THEN
+		CREATE EXTENSION IF NOT EXISTS pg_cron;
+		INSERT INTO config(id, value) VALUES ('use_pg_cron', 'true');
+	ELSE
+		RAISE LOG 'pg_cron extension not available. Skipping pg_cron setup.';
+		INSERT INTO config(id, value) VALUES ('use_pg_cron', 'false');
+	END IF;
+END
+$$;

package/sql/pgmb.sql

@@ -22,7 +22,10 @@ CREATE TYPE config_type AS ENUM(
 	-- how far into the future to create partitions
 	'future_intervals_to_create',
 	'partition_interval',
-	'poll_chunk_size'
+	'poll_chunk_size',
+	'use_pg_cron',
+	'pg_cron_poll_for_events_cron',
+	'pg_cron_partition_maintenance_cron'
 );
 
 CREATE TABLE IF NOT EXISTS config(
@@ -37,13 +40,15 @@ CREATE OR REPLACE FUNCTION get_config_value(
 	SELECT value FROM config WHERE id = config_id
 $$ LANGUAGE sql STRICT STABLE PARALLEL SAFE SET SEARCH_PATH TO pgmb;
 
-INSERT INTO config(id, value)
-	VALUES
-		('plugin_version', '0.2.0'),
-		('partition_retention_period', '60 minutes'),
-		('future_intervals_to_create', '120 minutes'),
-		('partition_interval', '30 minutes'),
-		('poll_chunk_size', '10000');
+INSERT INTO config(id, value) VALUES
+('plugin_version', '0.2.0'),
+('partition_retention_period', '60 minutes'),
+('future_intervals_to_create', '120 minutes'),
+('partition_interval', '30 minutes'),
+('poll_chunk_size', '10000'),
+('pg_cron_poll_for_events_cron', '1 second'),
+-- every 30 minutes
+('pg_cron_partition_maintenance_cron', '*/30 * * * *');
 
 -- we'll create the events table next & its functions ---------------
 
@@ -365,33 +370,11 @@ CREATE INDEX ON subscriptions(
 	add_interval_imm(last_active_at, expiry_interval)
 ) WHERE expiry_interval IS NOT NULL;
 
-DO $$
-DECLARE
-	has_btree_gin BOOLEAN;
-BEGIN
-	has_btree_gin := (
-		SELECT EXISTS (
-			SELECT 1
-			FROM pg_available_extensions
-			WHERE name = 'btree_gin'
-		)
-	);
-	-- create btree_gin extension if not exists, if the extension
-	-- is not available, we create a simpler regular GIN index instead.
-	IF has_btree_gin THEN
-		CREATE EXTENSION IF NOT EXISTS btree_gin;
-		-- fastupdate=false, slows down subscription creation, but ensures the costlier
-		-- "poll_for_events" function is executed faster.
-		CREATE INDEX "sub_gin" ON subscriptions USING GIN(conditions_sql, params)
-			WITH (fastupdate = false);
-	ELSE
-		RAISE NOTICE 'btree_gin extension is not available, using
-			regular GIN index for subscriptions.params';
-		CREATE INDEX "sub_gin" ON subscriptions USING GIN(params)
-			WITH (fastupdate = false);
-	END IF;
-END
-$$;
+-- fastupdate=false, slows down subscription creation, but ensures the costlier
+-- "poll_for_events" function is executed faster.
+CREATE EXTENSION IF NOT EXISTS btree_gin;
+CREATE INDEX "sub_gin" ON subscriptions
+USING GIN(conditions_sql, params) WITH (fastupdate = false);
 
 -- materialized view to hold distinct conditions_sql statements.
 -- We utilise changes in this view to determine when to prepare the
@@ -863,6 +846,78 @@ SET search_path TO pgmb;
 -- create the initial partitions
 SELECT maintain_events_table();
 
+-- setup pg_cron if it's available ----------------
+
+CREATE OR REPLACE FUNCTION manage_cron_jobs_trigger_fn()
+RETURNS TRIGGER AS $$
+DECLARE
+	poll_job_name CONSTANT TEXT := 'pgmb_poll';
+	maintain_job_name CONSTANT TEXT := 'pgmb_maintain_table_partitions';
+BEGIN
+	IF get_config_value('use_pg_cron') = 'true' THEN
+		-- Schedule/update event polling job
+		PERFORM cron.schedule(
+			poll_job_name,
+			get_config_value('pg_cron_poll_for_events_cron'),
+			$CMD$
+				-- ensure we don't accidentally run for too long
+				SET SESSION statement_timeout = '10s';
+				SELECT pgmb.poll_for_events();
+			$CMD$
+		);
+		RAISE LOG 'Scheduled pgmb polling job: %', poll_job_name;
+
+		-- Schedule/update partition maintenance job
+		PERFORM cron.schedule(
+			'pgmb_maintain_table_partitions',
+			get_config_value('pg_cron_partition_maintenance_cron'),
+			$CMD$ SELECT pgmb.maintain_events_table(); $CMD$
+		);
+
+		RAISE LOG 'Scheduled pgmb partition maintenance job: %',
+			maintain_job_name;
+	ELSIF (SELECT 1 FROM pg_namespace WHERE nspname = 'cron') THEN
+		RAISE LOG 'Unscheduling pgmb cron jobs.';
+		-- Unschedule jobs. cron.unschedule does not fail if job does not exist.
+		PERFORM cron.unschedule(poll_job_name);
+		PERFORM cron.unschedule(maintain_job_name);
+	END IF;
+
+	RETURN NULL;
+END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER
+SET search_path TO pgmb;
+
+CREATE TRIGGER manage_cron_jobs_trigger
+AFTER INSERT OR UPDATE OF value ON config
+FOR EACH ROW
+WHEN (
+	NEW.id IN (
+		'use_pg_cron',
+		'pg_cron_poll_for_events_cron',
+		'pg_cron_partition_maintenance_cron'
+	)
+)
+EXECUTE FUNCTION manage_cron_jobs_trigger_fn();
+
+DO $$
+BEGIN
+	IF (
+		SELECT EXISTS (
+			SELECT 1
+			FROM pg_available_extensions
+			WHERE name = 'pg_cron'
+		)
+	) THEN
+		CREATE EXTENSION IF NOT EXISTS pg_cron;
+		INSERT INTO config(id, value) VALUES ('use_pg_cron', 'true');
+	ELSE
+		RAISE LOG 'pg_cron extension not available. Skipping pg_cron setup.';
+		INSERT INTO config(id, value) VALUES ('use_pg_cron', 'false');
+	END IF;
+END
+$$;
+
 -- triggers to add events for specific tables ---------------------------
 
 -- Function to create a topic string for subscriptions.

package/sql/queries.sql

@@ -150,5 +150,14 @@ WITH deleted AS (
 )
 SELECT COUNT(*) AS "deleted!" FROM deleted;
 
+/* @name getConfigValue */
+SELECT pgmb.get_config_value(:key!::pgmb.config_type) AS "value";
+
+/* @name updateConfigValue */
+UPDATE pgmb.config
+SET value = :value!::TEXT
+WHERE id = :key!::pgmb.config_type
+RETURNING 1 AS "updated!";
+
 /* @name maintainEventsTable */
 SELECT pgmb.maintain_events_table();

package/src/client.ts

@@ -7,6 +7,7 @@ import {
 	assertGroup,
 	assertSubscription,
 	deleteSubscriptions,
+	getConfigValue,
 	maintainEventsTable,
 	markSubscriptionsActive,
 	pollForEvents,
@@ -151,9 +152,29 @@ export class PgmbClient<
 			this.client.on('remove', this.#onPoolClientRemoved)
 		}
 
-		// maintain event table
-		await maintainEventsTable.run(undefined, this.client)
-		this.logger.debug('maintained events table')
+		const [pgCronRslt] = await getConfigValue
+			.run({ key: 'use_pg_cron' }, this.client)
+		const isPgCronEnabled = pgCronRslt?.value === 'true'
+		if(!isPgCronEnabled) {
+			// maintain event table
+			await maintainEventsTable.run(undefined, this.client)
+			this.logger.debug('maintained events table')
+
+			if(this.pollEventsIntervalMs) {
+				this.#pollTask = this.#startLoop(
+					pollForEvents.run.bind(pollForEvents, undefined, this.client),
+					this.pollEventsIntervalMs,
+				)
+			}
+
+			if(this.tableMaintenanceMs) {
+				this.#tableMaintainTask = this.#startLoop(
+					maintainEventsTable.run
+						.bind(maintainEventsTable, undefined, this.client),
+					this.tableMaintenanceMs,
+				)
+			}
+		}
 
 		await assertGroup.run({ id: this.groupId }, this.client)
 		this.logger.debug({ groupId: this.groupId }, 'asserted group exists')
@@ -169,13 +190,6 @@ export class PgmbClient<
 			this.readEventsIntervalMs,
 		)
 
-		if(this.pollEventsIntervalMs) {
-			this.#pollTask = this.#startLoop(
-				pollForEvents.run.bind(pollForEvents, undefined, this.client),
-				this.pollEventsIntervalMs,
-			)
-		}
-
 		if(this.subscriptionMaintenanceMs) {
 			this.#subMaintainTask = this.#startLoop(
 				this.#maintainSubscriptions,
@@ -183,16 +197,7 @@ export class PgmbClient<
 			)
 		}
 
-		if(this.tableMaintenanceMs) {
-			this.#tableMaintainTask = this.#startLoop(
-				maintainEventsTable.run.bind(
-					maintainEventsTable,
-					undefined,
-					this.client,
-				),
-				this.tableMaintenanceMs,
-			)
-		}
+		this.logger.info({ isPgCronEnabled }, 'pgmb client initialised')
 	}
 
 	async end() {

package/src/queries.ts

@@ -1,6 +1,8 @@
 /** Types generated for queries found in "sql/queries.sql" */
 import { PreparedQuery } from '@pgtyped/runtime';
 
+export type config_type = 'future_intervals_to_create' | 'partition_interval' | 'partition_retention_period' | 'pg_cron_partition_maintenance_cron' | 'pg_cron_poll_for_events_cron' | 'plugin_version' | 'poll_chunk_size' | 'use_pg_cron';
+
 export type DateOrString = Date | string;
 
 export type DateOrStringArray = (DateOrString)[];
@@ -543,6 +545,64 @@ const removeExpiredSubscriptionsIR: any = {"usedParamSet":{"groupId":true,"activ
 export const removeExpiredSubscriptions = new PreparedQuery<IRemoveExpiredSubscriptionsParams,IRemoveExpiredSubscriptionsResult>(removeExpiredSubscriptionsIR);
 
 
+/** 'GetConfigValue' parameters type */
+export interface IGetConfigValueParams {
+  key: config_type;
+}
+
+/** 'GetConfigValue' return type */
+export interface IGetConfigValueResult {
+  value: string | null;
+}
+
+/** 'GetConfigValue' query type */
+export interface IGetConfigValueQuery {
+  params: IGetConfigValueParams;
+  result: IGetConfigValueResult;
+}
+
+const getConfigValueIR: any = {"usedParamSet":{"key":true},"params":[{"name":"key","required":true,"transform":{"type":"scalar"},"locs":[{"a":29,"b":33}]}],"statement":"SELECT pgmb.get_config_value(:key!::pgmb.config_type) AS \"value\""};
+
+/**
+ * Query generated from SQL:
+ * ```
+ * SELECT pgmb.get_config_value(:key!::pgmb.config_type) AS "value"
+ * ```
+ */
+export const getConfigValue = new PreparedQuery<IGetConfigValueParams,IGetConfigValueResult>(getConfigValueIR);
+
+
+/** 'UpdateConfigValue' parameters type */
+export interface IUpdateConfigValueParams {
+  key: config_type;
+  value: string;
+}
+
+/** 'UpdateConfigValue' return type */
+export interface IUpdateConfigValueResult {
+  updated: number;
+}
+
+/** 'UpdateConfigValue' query type */
+export interface IUpdateConfigValueQuery {
+  params: IUpdateConfigValueParams;
+  result: IUpdateConfigValueResult;
+}
+
+const updateConfigValueIR: any = {"usedParamSet":{"value":true,"key":true},"params":[{"name":"value","required":true,"transform":{"type":"scalar"},"locs":[{"a":31,"b":37}]},{"name":"key","required":true,"transform":{"type":"scalar"},"locs":[{"a":56,"b":60}]}],"statement":"UPDATE pgmb.config\nSET value = :value!::TEXT\nWHERE id = :key!::pgmb.config_type\nRETURNING 1 AS \"updated!\""};
+
+/**
+ * Query generated from SQL:
+ * ```
+ * UPDATE pgmb.config
+ * SET value = :value!::TEXT
+ * WHERE id = :key!::pgmb.config_type
+ * RETURNING 1 AS "updated!"
+ * ```
+ */
+export const updateConfigValue = new PreparedQuery<IUpdateConfigValueParams,IUpdateConfigValueResult>(updateConfigValueIR);
+
+
 /** 'MaintainEventsTable' parameters type */
 export type IMaintainEventsTableParams = void;