@haathie/pgmb 0.2.14 → 0.2.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.js +2 -2
- package/lib/queries.d.ts +8 -8
- package/lib/queries.js +6 -6
- package/package.json +3 -2
- package/sql/pgmb-0.2.14-0.2.15.sql +165 -0
- package/sql/pgmb.sql +68 -68
- package/sql/queries.sql +3 -3
package/lib/client.js
CHANGED
|
@@ -62,14 +62,14 @@ export class PgmbClient extends PGMBEventBatcher {
|
|
|
62
62
|
const isPgCronEnabled = pgCronRslt?.value === 'true';
|
|
63
63
|
if (!isPgCronEnabled) {
|
|
64
64
|
// maintain event table
|
|
65
|
-
await maintainEventsTable.run(
|
|
65
|
+
await maintainEventsTable.run({}, this.client);
|
|
66
66
|
this.logger.debug('maintained events table');
|
|
67
67
|
if (this.pollEventsIntervalMs) {
|
|
68
68
|
this.#pollTask = this.#startLoop(pollForEvents.run.bind(pollForEvents, undefined, this.client), this.pollEventsIntervalMs);
|
|
69
69
|
}
|
|
70
70
|
if (this.tableMaintenanceMs) {
|
|
71
71
|
this.#tableMaintainTask = this.#startLoop(maintainEventsTable.run
|
|
72
|
-
.bind(maintainEventsTable,
|
|
72
|
+
.bind(maintainEventsTable, {}, this.client), this.tableMaintenanceMs);
|
|
73
73
|
}
|
|
74
74
|
}
|
|
75
75
|
await assertGroup.run({ id: this.groupId }, this.client);
|
package/lib/queries.d.ts
CHANGED
|
@@ -103,7 +103,7 @@ export interface IMarkSubscriptionsActiveQuery {
|
|
|
103
103
|
* UPDATE pgmb.subscriptions
|
|
104
104
|
* SET
|
|
105
105
|
* last_active_at = NOW()
|
|
106
|
-
* WHERE id IN (SELECT * FROM unnest(:ids!::
|
|
106
|
+
* WHERE id IN (SELECT * FROM unnest(:ids!::text[]))
|
|
107
107
|
* ```
|
|
108
108
|
*/
|
|
109
109
|
export declare const markSubscriptionsActive: PreparedQuery<IMarkSubscriptionsActiveParams, void>;
|
|
@@ -427,7 +427,7 @@ export interface IRemoveExpiredSubscriptionsQuery {
|
|
|
427
427
|
* WHERE group_id = :groupId!
|
|
428
428
|
* AND expiry_interval IS NOT NULL
|
|
429
429
|
* AND pgmb.add_interval_imm(last_active_at, expiry_interval) < NOW()
|
|
430
|
-
* AND id NOT IN (select * from unnest(:activeIds!::
|
|
430
|
+
* AND id NOT IN (select * from unnest(:activeIds!::text[]))
|
|
431
431
|
* RETURNING id
|
|
432
432
|
* )
|
|
433
433
|
* SELECT COUNT(*) AS "deleted!" FROM deleted
|
|
@@ -479,11 +479,11 @@ export interface IUpdateConfigValueQuery {
|
|
|
479
479
|
*/
|
|
480
480
|
export declare const updateConfigValue: PreparedQuery<IUpdateConfigValueParams, IUpdateConfigValueResult>;
|
|
481
481
|
/** 'MaintainEventsTable' parameters type */
|
|
482
|
-
export
|
|
483
|
-
|
|
484
|
-
export interface IMaintainEventsTableResult {
|
|
485
|
-
maintainEventsTable: undefined | null;
|
|
482
|
+
export interface IMaintainEventsTableParams {
|
|
483
|
+
ts?: DateOrString | null | void;
|
|
486
484
|
}
|
|
485
|
+
/** 'MaintainEventsTable' return type */
|
|
486
|
+
export type IMaintainEventsTableResult = void;
|
|
487
487
|
/** 'MaintainEventsTable' query type */
|
|
488
488
|
export interface IMaintainEventsTableQuery {
|
|
489
489
|
params: IMaintainEventsTableParams;
|
|
@@ -492,7 +492,7 @@ export interface IMaintainEventsTableQuery {
|
|
|
492
492
|
/**
|
|
493
493
|
* Query generated from SQL:
|
|
494
494
|
* ```
|
|
495
|
-
*
|
|
495
|
+
* CALL pgmb.maintain_events_table(COALESCE(:ts, NOW()))
|
|
496
496
|
* ```
|
|
497
497
|
*/
|
|
498
|
-
export declare const maintainEventsTable: PreparedQuery<
|
|
498
|
+
export declare const maintainEventsTable: PreparedQuery<IMaintainEventsTableParams, void>;
|
package/lib/queries.js
CHANGED
|
@@ -46,14 +46,14 @@ const deleteSubscriptionsIR = { "usedParamSet": { "ids": true }, "params": [{ "n
|
|
|
46
46
|
* ```
|
|
47
47
|
*/
|
|
48
48
|
export const deleteSubscriptions = new PreparedQuery(deleteSubscriptionsIR);
|
|
49
|
-
const markSubscriptionsActiveIR = { "usedParamSet": { "ids": true }, "params": [{ "name": "ids", "required": true, "transform": { "type": "scalar" }, "locs": [{ "a": 88, "b": 92 }] }], "statement": "UPDATE pgmb.subscriptions\nSET\n\tlast_active_at = NOW()\nWHERE id IN (SELECT * FROM unnest(:ids!::
|
|
49
|
+
const markSubscriptionsActiveIR = { "usedParamSet": { "ids": true }, "params": [{ "name": "ids", "required": true, "transform": { "type": "scalar" }, "locs": [{ "a": 88, "b": 92 }] }], "statement": "UPDATE pgmb.subscriptions\nSET\n\tlast_active_at = NOW()\nWHERE id IN (SELECT * FROM unnest(:ids!::text[]))" };
|
|
50
50
|
/**
|
|
51
51
|
* Query generated from SQL:
|
|
52
52
|
* ```
|
|
53
53
|
* UPDATE pgmb.subscriptions
|
|
54
54
|
* SET
|
|
55
55
|
* last_active_at = NOW()
|
|
56
|
-
* WHERE id IN (SELECT * FROM unnest(:ids!::
|
|
56
|
+
* WHERE id IN (SELECT * FROM unnest(:ids!::text[]))
|
|
57
57
|
* ```
|
|
58
58
|
*/
|
|
59
59
|
export const markSubscriptionsActive = new PreparedQuery(markSubscriptionsActiveIR);
|
|
@@ -206,7 +206,7 @@ const findEventsIR = { "usedParamSet": { "ids": true }, "params": [{ "name": "id
|
|
|
206
206
|
* ```
|
|
207
207
|
*/
|
|
208
208
|
export const findEvents = new PreparedQuery(findEventsIR);
|
|
209
|
-
const removeExpiredSubscriptionsIR = { "usedParamSet": { "groupId": true, "activeIds": true }, "params": [{ "name": "groupId", "required": true, "transform": { "type": "scalar" }, "locs": [{ "a": 68, "b": 76 }] }, { "name": "activeIds", "required": true, "transform": { "type": "scalar" }, "locs": [{ "a": 219, "b": 229 }] }], "statement": "WITH deleted AS (\n\tDELETE FROM pgmb.subscriptions\n\tWHERE group_id = :groupId!\n\t\tAND expiry_interval IS NOT NULL\n\t\tAND pgmb.add_interval_imm(last_active_at, expiry_interval) < NOW()\n\t\tAND id NOT IN (select * from unnest(:activeIds!::
|
|
209
|
+
const removeExpiredSubscriptionsIR = { "usedParamSet": { "groupId": true, "activeIds": true }, "params": [{ "name": "groupId", "required": true, "transform": { "type": "scalar" }, "locs": [{ "a": 68, "b": 76 }] }, { "name": "activeIds", "required": true, "transform": { "type": "scalar" }, "locs": [{ "a": 219, "b": 229 }] }], "statement": "WITH deleted AS (\n\tDELETE FROM pgmb.subscriptions\n\tWHERE group_id = :groupId!\n\t\tAND expiry_interval IS NOT NULL\n\t\tAND pgmb.add_interval_imm(last_active_at, expiry_interval) < NOW()\n\t\tAND id NOT IN (select * from unnest(:activeIds!::text[]))\n\tRETURNING id\n)\nSELECT COUNT(*) AS \"deleted!\" FROM deleted" };
|
|
210
210
|
/**
|
|
211
211
|
* Query generated from SQL:
|
|
212
212
|
* ```
|
|
@@ -215,7 +215,7 @@ const removeExpiredSubscriptionsIR = { "usedParamSet": { "groupId": true, "activ
|
|
|
215
215
|
* WHERE group_id = :groupId!
|
|
216
216
|
* AND expiry_interval IS NOT NULL
|
|
217
217
|
* AND pgmb.add_interval_imm(last_active_at, expiry_interval) < NOW()
|
|
218
|
-
* AND id NOT IN (select * from unnest(:activeIds!::
|
|
218
|
+
* AND id NOT IN (select * from unnest(:activeIds!::text[]))
|
|
219
219
|
* RETURNING id
|
|
220
220
|
* )
|
|
221
221
|
* SELECT COUNT(*) AS "deleted!" FROM deleted
|
|
@@ -241,11 +241,11 @@ const updateConfigValueIR = { "usedParamSet": { "value": true, "key": true }, "p
|
|
|
241
241
|
* ```
|
|
242
242
|
*/
|
|
243
243
|
export const updateConfigValue = new PreparedQuery(updateConfigValueIR);
|
|
244
|
-
const maintainEventsTableIR = { "usedParamSet": {}, "params": [], "statement": "
|
|
244
|
+
const maintainEventsTableIR = { "usedParamSet": { "ts": true }, "params": [{ "name": "ts", "required": false, "transform": { "type": "scalar" }, "locs": [{ "a": 41, "b": 43 }] }], "statement": "CALL pgmb.maintain_events_table(COALESCE(:ts, NOW()))" };
|
|
245
245
|
/**
|
|
246
246
|
* Query generated from SQL:
|
|
247
247
|
* ```
|
|
248
|
-
*
|
|
248
|
+
* CALL pgmb.maintain_events_table(COALESCE(:ts, NOW()))
|
|
249
249
|
* ```
|
|
250
250
|
*/
|
|
251
251
|
export const maintainEventsTable = new PreparedQuery(maintainEventsTableIR);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@haathie/pgmb",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.15",
|
|
4
4
|
"description": "PG message broker, with a type-safe typescript client with built-in webhook & SSE support.",
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"registry": "https://registry.npmjs.org",
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"main": "lib/index.js",
|
|
11
11
|
"repository": "https://github.com/haathie/pgmb",
|
|
12
12
|
"scripts": {
|
|
13
|
-
"test": "
|
|
13
|
+
"test": "node --env-file ./.test.env --test tests/*.test.ts",
|
|
14
14
|
"prepare": "npm run build",
|
|
15
15
|
"build": "tsc -p tsconfig.build.json",
|
|
16
16
|
"lint": "eslint ./ --ext .js,.ts,.jsx,.tsx",
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
},
|
|
21
21
|
"devDependencies": {
|
|
22
22
|
"@adiwajshing/eslint-config": "git+https://github.com/adiwajshing/eslint-config",
|
|
23
|
+
"@electric-sql/pglite": "^0.4.1",
|
|
23
24
|
"@pgtyped/cli": "^2.4.3",
|
|
24
25
|
"@types/amqplib": "^0.10.0",
|
|
25
26
|
"@types/chance": "^1.1.6",
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
SET search_path TO pgmb;
|
|
2
|
+
|
|
3
|
+
CREATE OR REPLACE FUNCTION maintain_time_partitions_using_event_id(
|
|
4
|
+
table_id regclass,
|
|
5
|
+
partition_interval INTERVAL,
|
|
6
|
+
future_interval INTERVAL,
|
|
7
|
+
retention_period INTERVAL,
|
|
8
|
+
additional_sql TEXT DEFAULT NULL,
|
|
9
|
+
current_ts timestamptz DEFAULT NOW()
|
|
10
|
+
)
|
|
11
|
+
RETURNS void AS $$
|
|
12
|
+
DECLARE
|
|
13
|
+
ts_trunc timestamptz := date_bin(partition_interval, current_ts, '2000-1-1');
|
|
14
|
+
oldest_pt_to_keep text := pgmb
|
|
15
|
+
.get_time_partition_name(table_id, ts_trunc - retention_period);
|
|
16
|
+
lock_key CONSTANT BIGINT :=
|
|
17
|
+
hashtext('pgmb.maintain_tp.' || table_id::text);
|
|
18
|
+
ranges_to_create tstzrange[];
|
|
19
|
+
partitions_to_drop regclass[];
|
|
20
|
+
p_to_drop regclass;
|
|
21
|
+
cur_range tstzrange;
|
|
22
|
+
max_retries constant int = 50;
|
|
23
|
+
BEGIN
|
|
24
|
+
ASSERT partition_interval >= interval '1 minute',
|
|
25
|
+
'partition_interval must be at least 1 minute';
|
|
26
|
+
ASSERT future_interval >= partition_interval,
|
|
27
|
+
'future_interval must be at least as large as partition_interval';
|
|
28
|
+
|
|
29
|
+
IF NOT pg_try_advisory_xact_lock(lock_key) THEN
|
|
30
|
+
-- another process is already maintaining partitions for this table
|
|
31
|
+
RETURN;
|
|
32
|
+
END IF;
|
|
33
|
+
|
|
34
|
+
-- find all intervals we need to create partitions for
|
|
35
|
+
WITH existing_part_ranges AS (
|
|
36
|
+
SELECT
|
|
37
|
+
tstzrange(
|
|
38
|
+
extract_date_from_event_id(lower_bound),
|
|
39
|
+
extract_date_from_event_id(upper_bound),
|
|
40
|
+
'[]'
|
|
41
|
+
) as range
|
|
42
|
+
FROM pgmb.get_partitions_and_bounds(table_id)
|
|
43
|
+
),
|
|
44
|
+
future_tzs AS (
|
|
45
|
+
SELECT
|
|
46
|
+
tstzrange(dt, dt + partition_interval, '[]') AS range
|
|
47
|
+
FROM generate_series(
|
|
48
|
+
ts_trunc,
|
|
49
|
+
ts_trunc + future_interval,
|
|
50
|
+
partition_interval
|
|
51
|
+
) AS gs(dt)
|
|
52
|
+
),
|
|
53
|
+
diffs AS (
|
|
54
|
+
SELECT
|
|
55
|
+
CASE WHEN epr.range IS NOT NULL
|
|
56
|
+
THEN (ftz.range::tstzmultirange - epr.range::tstzmultirange)
|
|
57
|
+
ELSE ftz.range::tstzmultirange
|
|
58
|
+
END AS ranges
|
|
59
|
+
FROM future_tzs ftz
|
|
60
|
+
LEFT JOIN existing_part_ranges epr ON ftz.range && epr.range
|
|
61
|
+
)
|
|
62
|
+
select ARRAY_AGG(u.range) FROM diffs
|
|
63
|
+
CROSS JOIN LATERAL unnest(diffs.ranges) AS u(range)
|
|
64
|
+
INTO ranges_to_create;
|
|
65
|
+
|
|
66
|
+
ranges_to_create := COALESCE(ranges_to_create, ARRAY[]::tstzrange[]);
|
|
67
|
+
|
|
68
|
+
SELECT ARRAY_AGG(inhrelid::regclass) INTO partitions_to_drop
|
|
69
|
+
FROM pg_catalog.pg_inherits
|
|
70
|
+
WHERE inhparent = table_id
|
|
71
|
+
AND inhrelid::regclass::text < oldest_pt_to_keep;
|
|
72
|
+
partitions_to_drop := COALESCE(partitions_to_drop, ARRAY[]::regclass[]);
|
|
73
|
+
|
|
74
|
+
-- check if nothing to do
|
|
75
|
+
IF
|
|
76
|
+
array_length(partitions_to_drop, 1) = 0
|
|
77
|
+
AND array_length(ranges_to_create, 1) = 0
|
|
78
|
+
THEN
|
|
79
|
+
RETURN;
|
|
80
|
+
END IF;
|
|
81
|
+
|
|
82
|
+
-- go from now to future_interval
|
|
83
|
+
FOREACH cur_range IN ARRAY ranges_to_create LOOP
|
|
84
|
+
DECLARE
|
|
85
|
+
start_ev_id event_id := pgmb.create_event_id(lower(cur_range), 0);
|
|
86
|
+
end_ev_id event_id := pgmb.create_event_id(upper(cur_range), 0);
|
|
87
|
+
pt_name TEXT := pgmb.get_time_partition_name(table_id, lower(cur_range));
|
|
88
|
+
BEGIN
|
|
89
|
+
RAISE NOTICE 'creating partition "%". start: %, end: %',
|
|
90
|
+
pt_name, lower(cur_range), upper(cur_range);
|
|
91
|
+
|
|
92
|
+
EXECUTE FORMAT(
|
|
93
|
+
'CREATE TABLE %I PARTITION OF %I FOR VALUES FROM (%L) TO (%L)',
|
|
94
|
+
pt_name, table_id, start_ev_id, end_ev_id
|
|
95
|
+
);
|
|
96
|
+
|
|
97
|
+
IF additional_sql IS NOT NULL THEN
|
|
98
|
+
EXECUTE REPLACE(additional_sql, '$1', pt_name);
|
|
99
|
+
END IF;
|
|
100
|
+
END;
|
|
101
|
+
END LOOP;
|
|
102
|
+
|
|
103
|
+
-- Drop old partitions
|
|
104
|
+
FOREACH p_to_drop IN ARRAY partitions_to_drop LOOP
|
|
105
|
+
EXECUTE format('DROP TABLE %I', p_to_drop);
|
|
106
|
+
END LOOP;
|
|
107
|
+
END;
|
|
108
|
+
$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
|
|
109
|
+
|
|
110
|
+
CREATE FUNCTION maintain_append_only_table(
|
|
111
|
+
tbl regclass,
|
|
112
|
+
current_ts timestamptz DEFAULT NOW()
|
|
113
|
+
)
|
|
114
|
+
RETURNS VOID AS $$
|
|
115
|
+
SELECT maintain_time_partitions_using_event_id(
|
|
116
|
+
tbl,
|
|
117
|
+
partition_interval := get_config_value('partition_interval')::interval,
|
|
118
|
+
future_interval := get_config_value('future_intervals_to_create')::interval,
|
|
119
|
+
retention_period := get_config_value('partition_retention_period')::interval,
|
|
120
|
+
-- turn off autovacuum on the events table, since we're not
|
|
121
|
+
-- going to be updating/deleting rows from it.
|
|
122
|
+
-- Also set fillfactor to 100 since we're only inserting.
|
|
123
|
+
additional_sql := 'ALTER TABLE $1 SET(
|
|
124
|
+
fillfactor = 100,
|
|
125
|
+
autovacuum_enabled = false,
|
|
126
|
+
toast.autovacuum_enabled = false
|
|
127
|
+
);',
|
|
128
|
+
current_ts := current_ts
|
|
129
|
+
);
|
|
130
|
+
$$ LANGUAGE sql VOLATILE PARALLEL UNSAFE SECURITY DEFINER
|
|
131
|
+
SET search_path TO pgmb;
|
|
132
|
+
|
|
133
|
+
CREATE OR REPLACE PROCEDURE maintain_events_table(
|
|
134
|
+
current_ts timestamptz DEFAULT NOW()
|
|
135
|
+
) AS $$
|
|
136
|
+
DECLARE
|
|
137
|
+
pi INTERVAL := pgmb.get_config_value('partition_interval');
|
|
138
|
+
fic INTERVAL := pgmb.get_config_value('future_intervals_to_create');
|
|
139
|
+
rp INTERVAL := pgmb.get_config_value('partition_retention_period');
|
|
140
|
+
BEGIN
|
|
141
|
+
SET search_path TO pgmb;
|
|
142
|
+
|
|
143
|
+
PERFORM maintain_append_only_table('events'::regclass, current_ts);
|
|
144
|
+
COMMIT;
|
|
145
|
+
|
|
146
|
+
PERFORM maintain_append_only_table('subscription_events'::regclass, current_ts);
|
|
147
|
+
COMMIT;
|
|
148
|
+
END;
|
|
149
|
+
$$ LANGUAGE plpgsql;
|
|
150
|
+
|
|
151
|
+
DO $$
|
|
152
|
+
BEGIN
|
|
153
|
+
IF get_config_value('use_pg_cron') <> 'true' THEN
|
|
154
|
+
RETURN;
|
|
155
|
+
END IF;
|
|
156
|
+
|
|
157
|
+
SELECT cron.schedule(
|
|
158
|
+
'pgmb_maintain_table_partitions',
|
|
159
|
+
get_config_value('pg_cron_partition_maintenance_cron'),
|
|
160
|
+
$CMD$ CALL pgmb.maintain_events_table(); $CMD$
|
|
161
|
+
);
|
|
162
|
+
END
|
|
163
|
+
$$;
|
|
164
|
+
|
|
165
|
+
DROP FUNCTION IF EXISTS maintain_events_table(timestamptz);
|
package/sql/pgmb.sql
CHANGED
|
@@ -206,10 +206,11 @@ DECLARE
|
|
|
206
206
|
ts_trunc timestamptz := date_bin(partition_interval, current_ts, '2000-1-1');
|
|
207
207
|
oldest_pt_to_keep text := pgmb
|
|
208
208
|
.get_time_partition_name(table_id, ts_trunc - retention_period);
|
|
209
|
-
p_info RECORD;
|
|
210
209
|
lock_key CONSTANT BIGINT :=
|
|
211
210
|
hashtext('pgmb.maintain_tp.' || table_id::text);
|
|
212
211
|
ranges_to_create tstzrange[];
|
|
212
|
+
partitions_to_drop regclass[];
|
|
213
|
+
p_to_drop regclass;
|
|
213
214
|
cur_range tstzrange;
|
|
214
215
|
max_retries constant int = 50;
|
|
215
216
|
BEGIN
|
|
@@ -257,46 +258,45 @@ BEGIN
|
|
|
257
258
|
|
|
258
259
|
ranges_to_create := COALESCE(ranges_to_create, ARRAY[]::tstzrange[]);
|
|
259
260
|
|
|
260
|
-
|
|
261
|
+
SELECT ARRAY_AGG(inhrelid::regclass) INTO partitions_to_drop
|
|
262
|
+
FROM pg_catalog.pg_inherits
|
|
263
|
+
WHERE inhparent = table_id
|
|
264
|
+
AND inhrelid::regclass::text < oldest_pt_to_keep;
|
|
265
|
+
partitions_to_drop := COALESCE(partitions_to_drop, ARRAY[]::regclass[]);
|
|
266
|
+
|
|
267
|
+
-- check if nothing to do
|
|
268
|
+
IF
|
|
269
|
+
array_length(partitions_to_drop, 1) = 0
|
|
270
|
+
AND array_length(ranges_to_create, 1) = 0
|
|
271
|
+
THEN
|
|
272
|
+
RETURN;
|
|
273
|
+
END IF;
|
|
274
|
+
|
|
275
|
+
-- go from now to future_interval
|
|
276
|
+
FOREACH cur_range IN ARRAY ranges_to_create LOOP
|
|
277
|
+
DECLARE
|
|
278
|
+
start_ev_id event_id := pgmb.create_event_id(lower(cur_range), 0);
|
|
279
|
+
end_ev_id event_id := pgmb.create_event_id(upper(cur_range), 0);
|
|
280
|
+
pt_name TEXT := pgmb.get_time_partition_name(table_id, lower(cur_range));
|
|
261
281
|
BEGIN
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
EXECUTE FORMAT(
|
|
273
|
-
'CREATE TABLE %I PARTITION OF %I FOR VALUES FROM (%L) TO (%L)',
|
|
274
|
-
pt_name, table_id, start_ev_id, end_ev_id
|
|
275
|
-
);
|
|
276
|
-
|
|
277
|
-
IF additional_sql IS NOT NULL THEN
|
|
278
|
-
EXECUTE REPLACE(additional_sql, '$1', pt_name);
|
|
279
|
-
END IF;
|
|
280
|
-
END;
|
|
281
|
-
END LOOP;
|
|
282
|
-
|
|
283
|
-
-- Drop old partitions
|
|
284
|
-
FOR p_info IN (
|
|
285
|
-
SELECT inhrelid::regclass AS child
|
|
286
|
-
FROM pg_catalog.pg_inherits
|
|
287
|
-
WHERE inhparent = table_id
|
|
288
|
-
AND inhrelid::regclass::text < oldest_pt_to_keep
|
|
289
|
-
) LOOP
|
|
290
|
-
EXECUTE format('DROP TABLE %I', p_info.child);
|
|
291
|
-
END LOOP;
|
|
292
|
-
EXIT;
|
|
293
|
-
EXCEPTION WHEN lock_not_available OR deadlock_detected THEN
|
|
294
|
-
IF i = max_retries THEN
|
|
295
|
-
RAISE;
|
|
282
|
+
RAISE NOTICE 'creating partition "%". start: %, end: %',
|
|
283
|
+
pt_name, lower(cur_range), upper(cur_range);
|
|
284
|
+
|
|
285
|
+
EXECUTE FORMAT(
|
|
286
|
+
'CREATE TABLE %I PARTITION OF %I FOR VALUES FROM (%L) TO (%L)',
|
|
287
|
+
pt_name, table_id, start_ev_id, end_ev_id
|
|
288
|
+
);
|
|
289
|
+
|
|
290
|
+
IF additional_sql IS NOT NULL THEN
|
|
291
|
+
EXECUTE REPLACE(additional_sql, '$1', pt_name);
|
|
296
292
|
END IF;
|
|
297
|
-
PERFORM pg_sleep(1);
|
|
298
293
|
END;
|
|
299
294
|
END LOOP;
|
|
295
|
+
|
|
296
|
+
-- Drop old partitions
|
|
297
|
+
FOREACH p_to_drop IN ARRAY partitions_to_drop LOOP
|
|
298
|
+
EXECUTE format('DROP TABLE %I', p_to_drop);
|
|
299
|
+
END LOOP;
|
|
300
300
|
END;
|
|
301
301
|
$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
|
|
302
302
|
|
|
@@ -810,20 +810,20 @@ END
|
|
|
810
810
|
$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE
|
|
811
811
|
SET search_path TO pgmb;
|
|
812
812
|
|
|
813
|
-
|
|
813
|
+
-- contains fn to maintain partitions for an append-only table, this
|
|
814
|
+
-- can be used for both "events" and "subscription_events" tables.
|
|
815
|
+
-- It trims old partitions that are outside the retention period, and creates new
|
|
816
|
+
-- ones. Also ensures partitions aren't autovacuumed.
|
|
817
|
+
CREATE FUNCTION maintain_append_only_table(
|
|
818
|
+
tbl regclass,
|
|
814
819
|
current_ts timestamptz DEFAULT NOW()
|
|
815
820
|
)
|
|
816
821
|
RETURNS VOID AS $$
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
PERFORM maintain_time_partitions_using_event_id(
|
|
823
|
-
'pgmb.events'::regclass,
|
|
824
|
-
partition_interval := pi,
|
|
825
|
-
future_interval := fic,
|
|
826
|
-
retention_period := rp,
|
|
822
|
+
SELECT maintain_time_partitions_using_event_id(
|
|
823
|
+
tbl,
|
|
824
|
+
partition_interval := get_config_value('partition_interval')::interval,
|
|
825
|
+
future_interval := get_config_value('future_intervals_to_create')::interval,
|
|
826
|
+
retention_period := get_config_value('partition_retention_period')::interval,
|
|
827
827
|
-- turn off autovacuum on the events table, since we're not
|
|
828
828
|
-- going to be updating/deleting rows from it.
|
|
829
829
|
-- Also set fillfactor to 100 since we're only inserting.
|
|
@@ -834,28 +834,28 @@ BEGIN
|
|
|
834
834
|
);',
|
|
835
835
|
current_ts := current_ts
|
|
836
836
|
);
|
|
837
|
+
$$ LANGUAGE sql VOLATILE PARALLEL UNSAFE SECURITY DEFINER
|
|
838
|
+
SET search_path TO pgmb;
|
|
837
839
|
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
);
|
|
840
|
+
CREATE OR REPLACE PROCEDURE maintain_events_table(
|
|
841
|
+
current_ts timestamptz DEFAULT NOW()
|
|
842
|
+
) AS $$
|
|
843
|
+
BEGIN
|
|
844
|
+
SET search_path TO pgmb;
|
|
845
|
+
-- we commit after each maintainance function to release locks on the
|
|
846
|
+
-- partitions as soon as possible. This avoids blocking "poll_for_events",
|
|
847
|
+
-- & "read_next_events" functions, which when executing all concurrently,
|
|
848
|
+
-- may cause deadlocks due to lock contention on the partitions.
|
|
849
|
+
PERFORM maintain_append_only_table('events'::regclass, current_ts);
|
|
850
|
+
COMMIT;
|
|
851
|
+
|
|
852
|
+
PERFORM maintain_append_only_table('subscription_events'::regclass, current_ts);
|
|
853
|
+
COMMIT;
|
|
853
854
|
END;
|
|
854
|
-
$$ LANGUAGE plpgsql
|
|
855
|
-
SET search_path TO pgmb;
|
|
855
|
+
$$ LANGUAGE plpgsql;
|
|
856
856
|
|
|
857
|
-
|
|
858
|
-
SELECT
|
|
857
|
+
SELECT maintain_append_only_table('events'::regclass);
|
|
858
|
+
SELECT maintain_append_only_table('subscription_events'::regclass);
|
|
859
859
|
|
|
860
860
|
-- setup pg_cron if it's available ----------------
|
|
861
861
|
|
|
@@ -882,7 +882,7 @@ BEGIN
|
|
|
882
882
|
PERFORM cron.schedule(
|
|
883
883
|
'pgmb_maintain_table_partitions',
|
|
884
884
|
get_config_value('pg_cron_partition_maintenance_cron'),
|
|
885
|
-
$CMD$
|
|
885
|
+
$CMD$ CALL pgmb.maintain_events_table(); $CMD$
|
|
886
886
|
);
|
|
887
887
|
|
|
888
888
|
RAISE LOG 'Scheduled pgmb partition maintenance job: %',
|
package/sql/queries.sql
CHANGED
|
@@ -38,7 +38,7 @@ WHERE id IN :ids!;
|
|
|
38
38
|
UPDATE pgmb.subscriptions
|
|
39
39
|
SET
|
|
40
40
|
last_active_at = NOW()
|
|
41
|
-
WHERE id IN (SELECT * FROM unnest(:ids!::
|
|
41
|
+
WHERE id IN (SELECT * FROM unnest(:ids!::text[]));
|
|
42
42
|
|
|
43
43
|
/* @name pollForEvents */
|
|
44
44
|
SELECT count AS "count!" FROM pgmb.poll_for_events() AS count;
|
|
@@ -145,7 +145,7 @@ WITH deleted AS (
|
|
|
145
145
|
WHERE group_id = :groupId!
|
|
146
146
|
AND expiry_interval IS NOT NULL
|
|
147
147
|
AND pgmb.add_interval_imm(last_active_at, expiry_interval) < NOW()
|
|
148
|
-
AND id NOT IN (select * from unnest(:activeIds!::
|
|
148
|
+
AND id NOT IN (select * from unnest(:activeIds!::text[]))
|
|
149
149
|
RETURNING id
|
|
150
150
|
)
|
|
151
151
|
SELECT COUNT(*) AS "deleted!" FROM deleted;
|
|
@@ -160,4 +160,4 @@ WHERE id = :key!::pgmb.config_type
|
|
|
160
160
|
RETURNING 1 AS "updated!";
|
|
161
161
|
|
|
162
162
|
/* @name maintainEventsTable */
|
|
163
|
-
|
|
163
|
+
CALL pgmb.maintain_events_table(COALESCE(:ts, NOW()));
|