@haathie/pgmb 0.2.12 → 0.2.14

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@haathie/pgmb",
-  "version": "0.2.12",
+  "version": "0.2.14",
   "description": "PG message broker, with a type-safe typescript client with built-in webhook & SSE support.",
   "publishConfig": {
     "registry": "https://registry.npmjs.org",
@@ -15,7 +15,7 @@
     "build": "tsc -p tsconfig.build.json",
     "lint": "eslint ./ --ext .js,.ts,.jsx,.tsx",
     "lint:fix": "eslint ./ --fix --ext .js,.ts,.jsx,.tsx",
-    "benchmark": "TZ=utc node --env-file ./.env.test src/benchmark/run.ts",
+    "benchmark": "TZ=utc node --env-file ./.env.test benchmark/run.ts",
     "pg:typegen": "pgtyped --config ./pgtyped.config.json"
   },
   "devDependencies": {

package/sql/pgmb-0.2.8-0.2.14.sql

@@ -0,0 +1,113 @@
+SET search_path TO pgmb;
+
+-- Partition maintenance function for events table. Creates partitions for
+-- the current and next interval. Deletes partitions that are older than the
+-- configured time interval.
+-- Exact partition size and oldest partition interval can be configured
+-- using the "subscriptions_config" table.
+CREATE OR REPLACE FUNCTION maintain_time_partitions_using_event_id(
+	table_id regclass,
+	partition_interval INTERVAL,
+	future_interval INTERVAL,
+	retention_period INTERVAL,
+	additional_sql TEXT DEFAULT NULL,
+	current_ts timestamptz DEFAULT NOW()
+)
+RETURNS void AS $$
+DECLARE
+	ts_trunc timestamptz := date_bin(partition_interval, current_ts, '2000-1-1');
+	oldest_pt_to_keep text := pgmb
+		.get_time_partition_name(table_id, ts_trunc - retention_period);
+	p_info RECORD;
+	lock_key CONSTANT BIGINT :=
+		hashtext('pgmb.maintain_tp.' || table_id::text);
+	ranges_to_create tstzrange[];
+	cur_range tstzrange;
+	max_retries constant int = 50;
+BEGIN
+	ASSERT partition_interval >= interval '1 minute',
+		'partition_interval must be at least 1 minute';
+	ASSERT future_interval >= partition_interval,
+		'future_interval must be at least as large as partition_interval';
+
+	IF NOT pg_try_advisory_xact_lock(lock_key) THEN
+		-- another process is already maintaining partitions for this table
+		RETURN;
+	END IF;
+
+	-- find all intervals we need to create partitions for
+	WITH existing_part_ranges AS (
+		SELECT
+			tstzrange(
+	  		extract_date_from_event_id(lower_bound),
+	     	extract_date_from_event_id(upper_bound),
+	      '[]'
+	    ) as range
+    FROM pgmb.get_partitions_and_bounds(table_id)
+	),
+	future_tzs AS (
+		SELECT
+			tstzrange(dt, dt + partition_interval, '[]') AS range
+		FROM generate_series(
+			ts_trunc,
+			ts_trunc + future_interval,
+			partition_interval
+		) AS gs(dt)
+	),
+	diffs AS (
+		SELECT
+			CASE WHEN epr.range IS NOT NULL
+			THEN (ftz.range::tstzmultirange - epr.range::tstzmultirange)
+			ELSE ftz.range::tstzmultirange
+			END AS ranges
+		FROM future_tzs ftz
+		LEFT JOIN existing_part_ranges epr ON ftz.range && epr.range
+	)
+	select ARRAY_AGG(u.range) FROM diffs
+	CROSS JOIN LATERAL unnest(diffs.ranges) AS u(range)
+	INTO ranges_to_create;
+
+	ranges_to_create := COALESCE(ranges_to_create, ARRAY[]::tstzrange[]);
+
+	FOR i IN 1..max_retries LOOP
+		BEGIN
+			-- go from now to future_interval
+			FOREACH cur_range IN ARRAY ranges_to_create LOOP
+				DECLARE
+					start_ev_id event_id := pgmb.create_event_id(lower(cur_range), 0);
+					end_ev_id event_id := pgmb.create_event_id(upper(cur_range), 0);
+					pt_name TEXT := pgmb.get_time_partition_name(table_id, lower(cur_range));
+				BEGIN
+					RAISE NOTICE 'creating partition "%". start: %, end: %',
+						pt_name, lower(cur_range), upper(cur_range);
+
+					EXECUTE FORMAT(
+						'CREATE TABLE %I PARTITION OF %I FOR VALUES FROM (%L) TO (%L)',
+						pt_name, table_id, start_ev_id, end_ev_id
+					);
+
+					IF additional_sql IS NOT NULL THEN
+						EXECUTE REPLACE(additional_sql, '$1', pt_name);
+					END IF;
+				END;
+			END LOOP;
+
+			-- Drop old partitions
+			FOR p_info IN (
+				SELECT inhrelid::regclass AS child
+				FROM pg_catalog.pg_inherits
+				WHERE inhparent = table_id
+					AND inhrelid::regclass::text < oldest_pt_to_keep
+			) LOOP
+				EXECUTE format('DROP TABLE %I', p_info.child);
+			END LOOP;
+			EXIT;
+		EXCEPTION WHEN lock_not_available OR deadlock_detected THEN
+			IF i = max_retries THEN
+				RAISE;
+			END IF;
+			PERFORM pg_sleep(1);
+		END;
+	END LOOP;
+END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;

package/sql/pgmb.sql

@@ -43,12 +43,12 @@ $$ LANGUAGE sql STRICT STABLE PARALLEL SAFE SET SEARCH_PATH TO pgmb;
 INSERT INTO config(id, value) VALUES
 ('plugin_version', '0.2.0'),
 ('partition_retention_period', '60 minutes'),
-('future_intervals_to_create', '120 minutes'),
+('future_intervals_to_create', '3 hours'),
 ('partition_interval', '30 minutes'),
 ('poll_chunk_size', '10000'),
 ('pg_cron_poll_for_events_cron', '1 second'),
 -- every 30 minutes
-('pg_cron_partition_maintenance_cron', '*/30 * * * *');
+('pg_cron_partition_maintenance_cron', '0 * * * *');
 
 -- we'll create the events table next & its functions ---------------
 
@@ -211,6 +211,7 @@ DECLARE
 		hashtext('pgmb.maintain_tp.' || table_id::text);
 	ranges_to_create tstzrange[];
 	cur_range tstzrange;
+	max_retries constant int = 50;
 BEGIN
 	ASSERT partition_interval >= interval '1 minute',
 		'partition_interval must be at least 1 minute';
@@ -256,36 +257,46 @@ BEGIN
 
 	ranges_to_create := COALESCE(ranges_to_create, ARRAY[]::tstzrange[]);
 
-	-- go from now to future_interval
-	FOREACH cur_range IN ARRAY ranges_to_create LOOP
-		DECLARE
-			start_ev_id event_id := pgmb.create_event_id(lower(cur_range), 0);
-			end_ev_id event_id := pgmb.create_event_id(upper(cur_range), 0);
-			pt_name TEXT := pgmb.get_time_partition_name(table_id, lower(cur_range));
+	FOR i IN 1..max_retries LOOP
 		BEGIN
-			RAISE NOTICE 'creating partition "%". start: %, end: %',
-				pt_name, lower(cur_range), upper(cur_range);
-
-			EXECUTE FORMAT(
-				'CREATE TABLE %I PARTITION OF %I FOR VALUES FROM (%L) TO (%L)',
-				pt_name, table_id, start_ev_id, end_ev_id
-			);
-
-			IF additional_sql IS NOT NULL THEN
-				EXECUTE REPLACE(additional_sql, '$1', pt_name);
+			-- go from now to future_interval
+			FOREACH cur_range IN ARRAY ranges_to_create LOOP
+				DECLARE
+					start_ev_id event_id := pgmb.create_event_id(lower(cur_range), 0);
+					end_ev_id event_id := pgmb.create_event_id(upper(cur_range), 0);
+					pt_name TEXT := pgmb.get_time_partition_name(table_id, lower(cur_range));
+				BEGIN
+					RAISE NOTICE 'creating partition "%". start: %, end: %',
+						pt_name, lower(cur_range), upper(cur_range);
+
+					EXECUTE FORMAT(
+						'CREATE TABLE %I PARTITION OF %I FOR VALUES FROM (%L) TO (%L)',
+						pt_name, table_id, start_ev_id, end_ev_id
+					);
+
+					IF additional_sql IS NOT NULL THEN
+						EXECUTE REPLACE(additional_sql, '$1', pt_name);
+					END IF;
+				END;
+			END LOOP;
+
+			-- Drop old partitions
+			FOR p_info IN (
+				SELECT inhrelid::regclass AS child
+				FROM pg_catalog.pg_inherits
+				WHERE inhparent = table_id
+					AND inhrelid::regclass::text < oldest_pt_to_keep
+			) LOOP
+				EXECUTE format('DROP TABLE %I', p_info.child);
+			END LOOP;
+			EXIT;
+		EXCEPTION WHEN lock_not_available OR deadlock_detected THEN
+			IF i = max_retries THEN
+				RAISE;
 			END IF;
+			PERFORM pg_sleep(1);
 		END;
 	END LOOP;
-
-	-- Drop old partitions
-	FOR p_info IN (
-		SELECT inhrelid::regclass AS child
-		FROM pg_catalog.pg_inherits
-		WHERE inhparent = table_id
-			AND inhrelid::regclass::text < oldest_pt_to_keep
-	) LOOP
-		EXECUTE format('DROP TABLE %I', p_info.child);
-	END LOOP;
 END;
 $$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;