@event-driven-io/emmett-postgresql 0.43.0-beta.14 → 0.43.0-beta.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +824 -1449
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +7 -43
- package/dist/index.d.ts +7 -43
- package/dist/index.js +827 -1444
- package/dist/index.js.map +1 -1
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AssertionError, EmmettError, ExpectedVersionConflictError, NO_CONCURRENCY_CHECK, STREAM_DOES_NOT_EXIST, STREAM_EXISTS, assertDeepEqual, assertEqual, assertExpectedVersionMatchesCurrent, assertFails, assertIsNotNull, assertIsNull, assertThatArray, assertTrue, asyncAwaiter, asyncRetry, bigInt, bigIntProcessorCheckpoint, defaultProcessorPartition, defaultProcessorVersion, downcastRecordedMessages, getCheckpoint, getProcessorInstanceId, getProjectorId, getWorkflowId, hashText, isBigint, isErrorConstructor, parseBigIntProcessorCheckpoint, projection, projector, reactor, reduceAsync, unknownTag
|
|
1
|
+
import { AssertionError, EmmettError, ExpectedVersionConflictError, NO_CONCURRENCY_CHECK, STREAM_DOES_NOT_EXIST, STREAM_EXISTS, assertDeepEqual, assertEqual, assertExpectedVersionMatchesCurrent, assertFails, assertIsNotNull, assertIsNull, assertThatArray, assertTrue, asyncAwaiter, asyncRetry, bigInt, bigIntProcessorCheckpoint, defaultProcessorPartition, defaultProcessorVersion, downcastRecordedMessages, getCheckpoint, getProcessorInstanceId, getProjectorId, getWorkflowId, hashText, isBigint, isErrorConstructor, parseBigIntProcessorCheckpoint, projection, projector, reactor, reduceAsync, unknownTag, upcastRecordedMessage, workflowProcessor } from "@event-driven-io/emmett";
|
|
2
2
|
import { DumboError, JSONSerializer, SQL, UniqueConstraintError, dumbo, fromDatabaseDriverType, getFormatter, mapRows, runSQLMigrations, single, singleOrNull, sqlMigration } from "@event-driven-io/dumbo";
|
|
3
3
|
import { v4, v7 } from "uuid";
|
|
4
4
|
import { pongoClient } from "@event-driven-io/pongo";
|
|
@@ -8,7 +8,7 @@ import { pgDriver } from "@event-driven-io/pongo/pg";
|
|
|
8
8
|
const emmettPrefix = "emt";
|
|
9
9
|
const globalTag = "global";
|
|
10
10
|
const defaultTag = `${"emt"}:default`;
|
|
11
|
-
const unknownTag = `${"emt"}:unknown`;
|
|
11
|
+
const unknownTag$1 = `${"emt"}:unknown`;
|
|
12
12
|
const globalNames = {
|
|
13
13
|
module: `${"emt"}:module:${globalTag}`,
|
|
14
14
|
tenant: `${"emt"}:tenant:${globalTag}`
|
|
@@ -98,36 +98,43 @@ const postgreSQLEventStoreMessageBatchPuller = ({ executor, batchSize, eachBatch
|
|
|
98
98
|
let isRunning = false;
|
|
99
99
|
let start;
|
|
100
100
|
const pullMessages = async (options) => {
|
|
101
|
-
let after;
|
|
102
101
|
try {
|
|
103
|
-
after
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
102
|
+
let after;
|
|
103
|
+
try {
|
|
104
|
+
after = options.startFrom === "BEGINNING" ? 0n : options.startFrom === "END" ? (await readLastMessageGlobalPosition(executor)).currentGlobalPosition ?? 0n : parseBigIntProcessorCheckpoint(options.startFrom.lastCheckpoint);
|
|
105
|
+
} catch (error) {
|
|
106
|
+
console.log("Error occurred during reading last processor checkpoint:", error);
|
|
107
|
+
options.started?.reject(error);
|
|
108
|
+
throw error;
|
|
109
|
+
}
|
|
110
|
+
options.started?.resolve();
|
|
111
|
+
const readMessagesOptions = {
|
|
112
|
+
after,
|
|
113
|
+
batchSize
|
|
114
|
+
};
|
|
115
|
+
let waitTime = 100;
|
|
116
|
+
while (isRunning && !signal?.aborted) {
|
|
117
|
+
const { messages, currentGlobalPosition, areMessagesLeft } = await readMessagesBatch(executor, readMessagesOptions);
|
|
118
|
+
if (messages.length > 0) {
|
|
119
|
+
const result = await eachBatch(messages);
|
|
120
|
+
if (result && result.type === "STOP") {
|
|
121
|
+
isRunning = false;
|
|
122
|
+
break;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
readMessagesOptions.after = currentGlobalPosition;
|
|
126
|
+
await new Promise((resolve) => setTimeout(resolve, waitTime));
|
|
127
|
+
if (stopWhen?.noMessagesLeft === true && !areMessagesLeft) {
|
|
128
|
+
console.log(`No messages left to process after reaching global position ${currentGlobalPosition}. Stopping the puller.`);
|
|
119
129
|
isRunning = false;
|
|
120
130
|
break;
|
|
121
131
|
}
|
|
132
|
+
if (!areMessagesLeft) waitTime = Math.min(waitTime * 2, 1e3);
|
|
133
|
+
else waitTime = pullingFrequencyInMs;
|
|
122
134
|
}
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
isRunning = false;
|
|
127
|
-
break;
|
|
128
|
-
}
|
|
129
|
-
if (!areMessagesLeft) waitTime = Math.min(waitTime * 2, 1e3);
|
|
130
|
-
else waitTime = pullingFrequencyInMs;
|
|
135
|
+
} catch (error) {
|
|
136
|
+
console.log("Error occurred during message pulling:", error);
|
|
137
|
+
throw error;
|
|
131
138
|
}
|
|
132
139
|
};
|
|
133
140
|
return {
|
|
@@ -239,7 +246,7 @@ CREATE OR REPLACE FUNCTION emt_try_acquire_processor_lock(
|
|
|
239
246
|
p_processor_id TEXT,
|
|
240
247
|
p_version INT,
|
|
241
248
|
p_partition TEXT DEFAULT '${SQL.plain(defaultTag)}',
|
|
242
|
-
p_processor_instance_id TEXT DEFAULT '${SQL.plain(unknownTag)}',
|
|
249
|
+
p_processor_instance_id TEXT DEFAULT '${SQL.plain(unknownTag$1)}',
|
|
243
250
|
p_projection_name TEXT DEFAULT NULL,
|
|
244
251
|
p_projection_type VARCHAR(1) DEFAULT NULL,
|
|
245
252
|
p_projection_kind TEXT DEFAULT NULL,
|
|
@@ -248,6 +255,9 @@ CREATE OR REPLACE FUNCTION emt_try_acquire_processor_lock(
|
|
|
248
255
|
RETURNS TABLE (acquired BOOLEAN, checkpoint TEXT)
|
|
249
256
|
LANGUAGE plpgsql
|
|
250
257
|
AS $emt_try_acquire_processor_lock$
|
|
258
|
+
DECLARE
|
|
259
|
+
current_position TEXT;
|
|
260
|
+
v_current_time TIMESTAMPTZ := clock_timestamp();
|
|
251
261
|
BEGIN
|
|
252
262
|
RETURN QUERY
|
|
253
263
|
WITH lock_check AS (
|
|
@@ -265,16 +275,16 @@ BEGIN
|
|
|
265
275
|
created_at,
|
|
266
276
|
last_updated
|
|
267
277
|
)
|
|
268
|
-
SELECT p_processor_id, p_partition, p_version, p_processor_instance_id, 'running', '${SQL.plain(bigInt.toNormalizedString(0n))}', '0'::xid8,
|
|
278
|
+
SELECT p_processor_id, p_partition, p_version, p_processor_instance_id, 'running', '${SQL.plain(bigInt.toNormalizedString(0n))}', '0'::xid8, v_current_time, v_current_time
|
|
269
279
|
WHERE (SELECT lock_acquired FROM lock_check) = true
|
|
270
280
|
ON CONFLICT (processor_id, partition, version) DO UPDATE
|
|
271
281
|
SET processor_instance_id = p_processor_instance_id,
|
|
272
282
|
status = 'running',
|
|
273
|
-
last_updated =
|
|
283
|
+
last_updated = v_current_time
|
|
274
284
|
WHERE ${SQL.plain(processorsTable.name)}.processor_instance_id = p_processor_instance_id
|
|
275
|
-
OR ${SQL.plain(processorsTable.name)}.processor_instance_id = '${SQL.plain(unknownTag)}'
|
|
285
|
+
OR ${SQL.plain(processorsTable.name)}.processor_instance_id = '${SQL.plain(unknownTag$1)}'
|
|
276
286
|
OR ${SQL.plain(processorsTable.name)}.status = 'stopped'
|
|
277
|
-
OR ${SQL.plain(processorsTable.name)}.last_updated <
|
|
287
|
+
OR ${SQL.plain(processorsTable.name)}.last_updated < v_current_time - (p_lock_timeout_seconds || ' seconds')::interval
|
|
278
288
|
RETURNING last_processed_checkpoint
|
|
279
289
|
),
|
|
280
290
|
projection_status AS (
|
|
@@ -306,7 +316,7 @@ CREATE OR REPLACE FUNCTION emt_release_processor_lock(
|
|
|
306
316
|
p_processor_id TEXT,
|
|
307
317
|
p_partition TEXT,
|
|
308
318
|
p_version INT,
|
|
309
|
-
p_processor_instance_id TEXT DEFAULT '${SQL.plain(unknownTag)}',
|
|
319
|
+
p_processor_instance_id TEXT DEFAULT '${SQL.plain(unknownTag$1)}',
|
|
310
320
|
p_projection_name TEXT DEFAULT NULL
|
|
311
321
|
)
|
|
312
322
|
RETURNS BOOLEAN
|
|
@@ -326,7 +336,7 @@ BEGIN
|
|
|
326
336
|
|
|
327
337
|
UPDATE ${SQL.plain(processorsTable.name)}
|
|
328
338
|
SET status = 'stopped',
|
|
329
|
-
processor_instance_id = '${SQL.plain(unknownTag)}',
|
|
339
|
+
processor_instance_id = '${SQL.plain(unknownTag$1)}',
|
|
330
340
|
last_updated = now()
|
|
331
341
|
WHERE processor_id = p_processor_id
|
|
332
342
|
AND partition = p_partition
|
|
@@ -379,6 +389,7 @@ const tryAcquireProcessorLock = async (execute, options) => {
|
|
|
379
389
|
projectionKind: options.projection?.kind ?? null,
|
|
380
390
|
lockTimeoutSeconds: options.lockTimeoutSeconds ?? 300
|
|
381
391
|
})));
|
|
392
|
+
console.log(`Lock acquisition attempt for processor '${options.processorId}' with processor instance '${options.processorInstanceId}' and lock '${options.lockKey}' resulted in: ${acquired}`);
|
|
382
393
|
return acquired ? {
|
|
383
394
|
acquired: true,
|
|
384
395
|
checkpoint
|
|
@@ -404,6 +415,7 @@ const releaseProcessorLock = async (execute, options) => {
|
|
|
404
415
|
processorInstanceId: options.processorInstanceId,
|
|
405
416
|
projectionName: options.projectionName ?? null
|
|
406
417
|
})));
|
|
418
|
+
console.log(`Lock for processor '${options.processorId}' with processor instance '${options.processorInstanceId}' and lock '${options.lockKey}' released: ${result}`);
|
|
407
419
|
return result;
|
|
408
420
|
};
|
|
409
421
|
|
|
@@ -415,17 +427,26 @@ const postgreSQLProcessorLock = (options) => {
|
|
|
415
427
|
const lockKey = options.lockKey ?? toProcessorLockKey(options);
|
|
416
428
|
return {
|
|
417
429
|
tryAcquire: async (context) => {
|
|
418
|
-
if (acquired)
|
|
430
|
+
if (acquired) {
|
|
431
|
+
console.log(`Lock for processor '${options.processorId}' is already acquired by this instance. Reusing the lock.`);
|
|
432
|
+
return true;
|
|
433
|
+
}
|
|
419
434
|
const result = await tryAcquireProcessorLockWithRetry(context.execute, {
|
|
420
435
|
...options,
|
|
421
436
|
lockKey
|
|
422
437
|
});
|
|
423
|
-
if (!result.acquired && options.lockAcquisitionPolicy?.type !== "skip")
|
|
438
|
+
if (!result.acquired && options.lockAcquisitionPolicy?.type !== "skip") {
|
|
439
|
+
console.log(`Failed to acquire lock for processor '${options.processorId}' with policy '${options.lockAcquisitionPolicy?.type}'.`);
|
|
440
|
+
throw new EmmettError(`Failed to acquire lock for processor '${options.processorId}'`);
|
|
441
|
+
}
|
|
424
442
|
acquired = result.acquired;
|
|
425
443
|
return acquired;
|
|
426
444
|
},
|
|
427
445
|
release: async (context) => {
|
|
428
|
-
if (!acquired)
|
|
446
|
+
if (!acquired) {
|
|
447
|
+
console.log(`Lock for processor '${options.processorId}' is not acquired by this instance. Skipping release.`);
|
|
448
|
+
return;
|
|
449
|
+
}
|
|
429
450
|
const { projection, ...releaseOptions } = options;
|
|
430
451
|
await releaseProcessorLock(context.execute, {
|
|
431
452
|
...releaseOptions,
|
|
@@ -1216,156 +1237,118 @@ const appendEventsRaw = (execute, streamId, streamType, messages, options) => si
|
|
|
1216
1237
|
})));
|
|
1217
1238
|
|
|
1218
1239
|
//#endregion
|
|
1219
|
-
//#region src/eventStore/schema/
|
|
1220
|
-
const
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
END IF;
|
|
1231
|
-
|
|
1232
|
-
IF EXISTS (SELECT 1 FROM pg_proc WHERE proname = 'add_module_for_all_tenants') THEN
|
|
1233
|
-
DROP FUNCTION add_module_for_all_tenants(TEXT);
|
|
1234
|
-
END IF;
|
|
1235
|
-
|
|
1236
|
-
IF EXISTS (SELECT 1 FROM pg_proc WHERE proname = 'add_tenant_for_all_modules') THEN
|
|
1237
|
-
DROP FUNCTION add_tenant_for_all_modules(TEXT);
|
|
1238
|
-
END IF;
|
|
1239
|
-
END $$;
|
|
1240
|
-
`;
|
|
1241
|
-
const dropOldAppendToSQLWithoutGlobalPositions = SQL`
|
|
1242
|
-
DO $$
|
|
1243
|
-
DECLARE
|
|
1244
|
-
v_current_return_type text;
|
|
1245
|
-
BEGIN
|
|
1246
|
-
-- Get the current return type definition as text
|
|
1247
|
-
SELECT pg_get_function_result(p.oid)
|
|
1248
|
-
INTO v_current_return_type
|
|
1249
|
-
FROM pg_proc p
|
|
1250
|
-
JOIN pg_namespace n ON p.pronamespace = n.oid
|
|
1251
|
-
WHERE n.nspname = current_schema() -- or specify your schema
|
|
1252
|
-
AND p.proname = 'emt_append_to_stream'
|
|
1253
|
-
AND p.pronargs = 10; -- number of arguments
|
|
1254
|
-
|
|
1255
|
-
-- Check if it contains the old column name
|
|
1256
|
-
IF v_current_return_type IS NOT NULL AND
|
|
1257
|
-
v_current_return_type LIKE '%last_global_position%' AND
|
|
1258
|
-
v_current_return_type NOT LIKE '%global_positions%' THEN
|
|
1259
|
-
DROP FUNCTION emt_append_to_stream(text[], jsonb[], jsonb[], text[], text[], text[], text, text, bigint, text);
|
|
1260
|
-
RAISE NOTICE 'Old version of function dropped. Return type was: %', v_current_return_type;
|
|
1261
|
-
END IF;
|
|
1262
|
-
END $$;
|
|
1263
|
-
`;
|
|
1264
|
-
const migrationFromEventsToMessagesSQL = SQL`
|
|
1265
|
-
DO $$
|
|
1240
|
+
//#region src/eventStore/schema/storeProcessorCheckpoint.ts
|
|
1241
|
+
const storeSubscriptionCheckpointSQL = createFunctionIfDoesNotExistSQL("store_processor_checkpoint", SQL`
|
|
1242
|
+
CREATE OR REPLACE FUNCTION store_processor_checkpoint(
|
|
1243
|
+
p_processor_id TEXT,
|
|
1244
|
+
p_version BIGINT,
|
|
1245
|
+
p_position TEXT,
|
|
1246
|
+
p_check_position TEXT,
|
|
1247
|
+
p_transaction_id xid8,
|
|
1248
|
+
p_partition TEXT DEFAULT '${SQL.plain(defaultTag)}',
|
|
1249
|
+
p_processor_instance_id TEXT DEFAULT '${SQL.plain(unknownTag$1)}'
|
|
1250
|
+
) RETURNS INT AS $spc$
|
|
1266
1251
|
DECLARE
|
|
1267
|
-
|
|
1252
|
+
current_position TEXT;
|
|
1268
1253
|
BEGIN
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
END LOOP;
|
|
1254
|
+
-- Handle the case when p_check_position is provided
|
|
1255
|
+
IF p_check_position IS NOT NULL THEN
|
|
1256
|
+
-- Try to update if the position matches p_check_position
|
|
1257
|
+
UPDATE "${SQL.plain(processorsTable.name)}"
|
|
1258
|
+
SET
|
|
1259
|
+
"last_processed_checkpoint" = p_position,
|
|
1260
|
+
"last_processed_transaction_id" = p_transaction_id,
|
|
1261
|
+
"last_updated" = now()
|
|
1262
|
+
WHERE "processor_id" = p_processor_id
|
|
1263
|
+
AND "last_processed_checkpoint" = p_check_position
|
|
1264
|
+
AND "partition" = p_partition
|
|
1265
|
+
AND "version" = p_version;
|
|
1282
1266
|
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
-- Rename columns
|
|
1287
|
-
ALTER TABLE emt_messages
|
|
1288
|
-
RENAME COLUMN event_data TO message_data;
|
|
1289
|
-
ALTER TABLE emt_messages
|
|
1290
|
-
RENAME COLUMN event_metadata TO message_metadata;
|
|
1291
|
-
ALTER TABLE emt_messages
|
|
1292
|
-
RENAME COLUMN event_schema_version TO message_schema_version;
|
|
1293
|
-
ALTER TABLE emt_messages
|
|
1294
|
-
RENAME COLUMN event_type TO message_type;
|
|
1295
|
-
ALTER TABLE emt_messages
|
|
1296
|
-
RENAME COLUMN event_id TO message_id;
|
|
1297
|
-
ALTER TABLE emt_messages
|
|
1298
|
-
ADD COLUMN message_kind CHAR(1) NOT NULL DEFAULT 'E';
|
|
1267
|
+
IF FOUND THEN
|
|
1268
|
+
RETURN 1; -- Successfully updated
|
|
1269
|
+
END IF;
|
|
1299
1270
|
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
ALTER COLUMN global_position
|
|
1307
|
-
SET DEFAULT nextval('emt_global_message_position');
|
|
1308
|
-
END IF;
|
|
1309
|
-
END IF;
|
|
1310
|
-
END $$;`;
|
|
1311
|
-
const migration_0_38_7_and_older = sqlMigration("emt:postgresql:eventstore:0.38.7:migrate-events-to-messages", [
|
|
1312
|
-
dropFutureConceptModuleAndTenantFunctions,
|
|
1313
|
-
dropOldAppendToSQLWithoutGlobalPositions,
|
|
1314
|
-
migrationFromEventsToMessagesSQL
|
|
1315
|
-
]);
|
|
1271
|
+
-- Retrieve the current position
|
|
1272
|
+
SELECT "last_processed_checkpoint" INTO current_position
|
|
1273
|
+
FROM "${SQL.plain(processorsTable.name)}"
|
|
1274
|
+
WHERE "processor_id" = p_processor_id
|
|
1275
|
+
AND "partition" = p_partition
|
|
1276
|
+
AND "version" = p_version;
|
|
1316
1277
|
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
-- Rename all partitions first
|
|
1327
|
-
FOR partition_record IN
|
|
1328
|
-
SELECT tablename
|
|
1329
|
-
FROM pg_tables
|
|
1330
|
-
WHERE tablename LIKE 'emt_events_%'
|
|
1331
|
-
ORDER BY tablename DESC -- to handle child partitions first
|
|
1332
|
-
LOOP
|
|
1333
|
-
EXECUTE format('ALTER TABLE %I RENAME TO %I',
|
|
1334
|
-
partition_record.tablename,
|
|
1335
|
-
REPLACE(partition_record.tablename, 'events', 'messages'));
|
|
1336
|
-
END LOOP;
|
|
1278
|
+
-- Return appropriate codes based on current position
|
|
1279
|
+
IF current_position = p_position THEN
|
|
1280
|
+
RETURN 0; -- Idempotent check: position already set
|
|
1281
|
+
ELSIF current_position > p_position THEN
|
|
1282
|
+
RETURN 3; -- Current ahead: another process has progressed further
|
|
1283
|
+
ELSE
|
|
1284
|
+
RETURN 2; -- Mismatch: check position doesn't match current
|
|
1285
|
+
END IF;
|
|
1286
|
+
END IF;
|
|
1337
1287
|
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
ALTER TABLE emt_messages
|
|
1351
|
-
RENAME COLUMN event_id TO message_id;
|
|
1352
|
-
ALTER TABLE emt_messages
|
|
1353
|
-
ADD COLUMN message_kind CHAR(1) NOT NULL DEFAULT 'E';
|
|
1288
|
+
-- Handle the case when p_check_position is NULL: Insert if not exists
|
|
1289
|
+
BEGIN
|
|
1290
|
+
INSERT INTO "${SQL.plain(processorsTable.name)}"("processor_id", "version", "last_processed_checkpoint", "partition", "last_processed_transaction_id", "created_at", "last_updated")
|
|
1291
|
+
VALUES (p_processor_id, p_version, p_position, p_partition, p_transaction_id, now(), now());
|
|
1292
|
+
RETURN 1; -- Successfully inserted
|
|
1293
|
+
EXCEPTION WHEN unique_violation THEN
|
|
1294
|
+
-- If insertion failed, it means the row already exists
|
|
1295
|
+
SELECT "last_processed_checkpoint" INTO current_position
|
|
1296
|
+
FROM "${SQL.plain(processorsTable.name)}"
|
|
1297
|
+
WHERE "processor_id" = p_processor_id
|
|
1298
|
+
AND "partition" = p_partition
|
|
1299
|
+
AND "version" = p_version;
|
|
1354
1300
|
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1301
|
+
IF current_position = p_position THEN
|
|
1302
|
+
RETURN 0; -- Idempotent check: position already set
|
|
1303
|
+
ELSIF current_position > p_position THEN
|
|
1304
|
+
RETURN 3; -- Current ahead: another process has progressed further
|
|
1305
|
+
ELSE
|
|
1306
|
+
RETURN 2; -- Insertion failed, row already exists with different position
|
|
1307
|
+
END IF;
|
|
1308
|
+
END;
|
|
1309
|
+
END;
|
|
1310
|
+
$spc$ LANGUAGE plpgsql;
|
|
1311
|
+
`);
|
|
1312
|
+
const callStoreProcessorCheckpoint = (params) => SQL`
|
|
1313
|
+
SELECT store_processor_checkpoint(
|
|
1314
|
+
${params.processorId},
|
|
1315
|
+
${params.version},
|
|
1316
|
+
${params.position},
|
|
1317
|
+
${params.checkPosition},
|
|
1318
|
+
pg_current_xact_id(),
|
|
1319
|
+
${params.partition},
|
|
1320
|
+
${params.processorInstanceId}
|
|
1321
|
+
) as result;`;
|
|
1322
|
+
const storeProcessorCheckpoint = async (execute, options) => {
|
|
1323
|
+
try {
|
|
1324
|
+
const { result } = await single(execute.command(callStoreProcessorCheckpoint({
|
|
1325
|
+
processorId: options.processorId,
|
|
1326
|
+
version: options.version ?? 1,
|
|
1327
|
+
position: options.newCheckpoint !== null ? options.newCheckpoint : null,
|
|
1328
|
+
checkPosition: options.lastProcessedCheckpoint !== null ? options.lastProcessedCheckpoint : null,
|
|
1329
|
+
partition: options.partition ?? defaultTag,
|
|
1330
|
+
processorInstanceId: options.processorInstanceId ?? unknownTag$1
|
|
1331
|
+
})));
|
|
1332
|
+
return result === 1 ? {
|
|
1333
|
+
success: true,
|
|
1334
|
+
newCheckpoint: options.newCheckpoint
|
|
1335
|
+
} : {
|
|
1336
|
+
success: false,
|
|
1337
|
+
reason: result === 0 ? "IGNORED" : result === 3 ? "CURRENT_AHEAD" : "MISMATCH"
|
|
1338
|
+
};
|
|
1339
|
+
} catch (error) {
|
|
1340
|
+
console.log(error);
|
|
1341
|
+
throw error;
|
|
1342
|
+
}
|
|
1343
|
+
};
|
|
1344
|
+
|
|
1345
|
+
//#endregion
|
|
1346
|
+
//#region src/eventStore/schema/tables.ts
|
|
1347
|
+
const streamsTableSQL = SQL`
|
|
1348
|
+
CREATE TABLE IF NOT EXISTS ${SQL.identifier(streamsTable.name)}(
|
|
1366
1349
|
stream_id TEXT NOT NULL,
|
|
1367
1350
|
stream_position BIGINT NOT NULL,
|
|
1368
|
-
partition TEXT NOT NULL DEFAULT '
|
|
1351
|
+
partition TEXT NOT NULL DEFAULT '${SQL.plain(defaultTag)}',
|
|
1369
1352
|
stream_type TEXT NOT NULL,
|
|
1370
1353
|
stream_metadata JSONB NOT NULL,
|
|
1371
1354
|
is_archived BOOLEAN NOT NULL DEFAULT FALSE,
|
|
@@ -1373,40 +1356,62 @@ END $$;CREATE TABLE IF NOT EXISTS emt_streams(
|
|
|
1373
1356
|
) PARTITION BY LIST (partition);
|
|
1374
1357
|
|
|
1375
1358
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_streams_unique
|
|
1376
|
-
ON
|
|
1377
|
-
INCLUDE (stream_position)
|
|
1359
|
+
ON ${SQL.identifier(streamsTable.name)}(stream_id, partition, is_archived)
|
|
1360
|
+
INCLUDE (stream_position);`;
|
|
1361
|
+
const messagesTableSQL = SQL`
|
|
1378
1362
|
CREATE SEQUENCE IF NOT EXISTS emt_global_message_position;
|
|
1379
1363
|
|
|
1380
|
-
CREATE TABLE IF NOT EXISTS
|
|
1381
|
-
stream_id TEXT NOT NULL,
|
|
1364
|
+
CREATE TABLE IF NOT EXISTS ${SQL.identifier(messagesTable.name)}(
|
|
1382
1365
|
stream_position BIGINT NOT NULL,
|
|
1383
|
-
partition TEXT NOT NULL DEFAULT 'global',
|
|
1384
|
-
message_kind CHAR(1) NOT NULL DEFAULT 'E',
|
|
1385
|
-
message_data JSONB NOT NULL,
|
|
1386
|
-
message_metadata JSONB NOT NULL,
|
|
1387
|
-
message_schema_version TEXT NOT NULL,
|
|
1388
|
-
message_type TEXT NOT NULL,
|
|
1389
|
-
message_id TEXT NOT NULL,
|
|
1390
|
-
is_archived BOOLEAN NOT NULL DEFAULT FALSE,
|
|
1391
1366
|
global_position BIGINT DEFAULT nextval('emt_global_message_position'),
|
|
1392
1367
|
transaction_id XID8 NOT NULL,
|
|
1393
1368
|
created TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
1369
|
+
is_archived BOOLEAN NOT NULL DEFAULT FALSE,
|
|
1370
|
+
message_kind VARCHAR(1) NOT NULL DEFAULT 'E',
|
|
1371
|
+
stream_id TEXT NOT NULL,
|
|
1372
|
+
partition TEXT NOT NULL DEFAULT '${SQL.plain(defaultTag)}',
|
|
1373
|
+
message_schema_version TEXT NOT NULL,
|
|
1374
|
+
message_id TEXT NOT NULL,
|
|
1375
|
+
message_type TEXT NOT NULL,
|
|
1376
|
+
message_data JSONB NOT NULL,
|
|
1377
|
+
message_metadata JSONB NOT NULL,
|
|
1394
1378
|
PRIMARY KEY (stream_id, stream_position, partition, is_archived)
|
|
1379
|
+
) PARTITION BY LIST (partition);`;
|
|
1380
|
+
const processorsTableSQL = SQL`
|
|
1381
|
+
CREATE TABLE IF NOT EXISTS ${SQL.identifier(processorsTable.name)}(
|
|
1382
|
+
last_processed_transaction_id XID8 NOT NULL,
|
|
1383
|
+
version INT NOT NULL DEFAULT 1,
|
|
1384
|
+
processor_id TEXT NOT NULL,
|
|
1385
|
+
partition TEXT NOT NULL DEFAULT '${SQL.plain(defaultTag)}',
|
|
1386
|
+
status TEXT NOT NULL DEFAULT 'stopped',
|
|
1387
|
+
last_processed_checkpoint TEXT NOT NULL,
|
|
1388
|
+
processor_instance_id TEXT DEFAULT '${SQL.plain(unknownTag$1)}',
|
|
1389
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
1390
|
+
last_updated TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
1391
|
+
PRIMARY KEY (processor_id, partition, version)
|
|
1395
1392
|
) PARTITION BY LIST (partition);
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1393
|
+
`;
|
|
1394
|
+
const projectionsTableSQL = SQL`
|
|
1395
|
+
CREATE TABLE IF NOT EXISTS ${SQL.identifier(projectionsTable.name)}(
|
|
1396
|
+
version INT NOT NULL DEFAULT 1,
|
|
1397
|
+
type VARCHAR(1) NOT NULL,
|
|
1398
|
+
name TEXT NOT NULL,
|
|
1399
|
+
partition TEXT NOT NULL DEFAULT '${SQL.plain(defaultTag)}',
|
|
1400
|
+
kind TEXT NOT NULL,
|
|
1401
|
+
status TEXT NOT NULL,
|
|
1402
|
+
definition JSONB NOT NULL DEFAULT '{}'::jsonb,
|
|
1403
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
1404
|
+
last_updated TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
1405
|
+
PRIMARY KEY (name, partition, version)
|
|
1403
1406
|
) PARTITION BY LIST (partition);
|
|
1404
|
-
|
|
1407
|
+
`;
|
|
1408
|
+
const sanitizeNameSQL = createFunctionIfDoesNotExistSQL("emt_sanitize_name", SQL`CREATE OR REPLACE FUNCTION emt_sanitize_name(input_name TEXT) RETURNS TEXT AS $emt_sanitize_name$
|
|
1405
1409
|
BEGIN
|
|
1406
1410
|
RETURN REGEXP_REPLACE(input_name, '[^a-zA-Z0-9_]', '_', 'g');
|
|
1407
1411
|
END;
|
|
1408
|
-
|
|
1409
|
-
|
|
1412
|
+
$emt_sanitize_name$ LANGUAGE plpgsql;`);
|
|
1413
|
+
const addTablePartitions = createFunctionIfDoesNotExistSQL("emt_add_table_partition", SQL`
|
|
1414
|
+
CREATE OR REPLACE FUNCTION emt_add_table_partition(tableName TEXT, partition_name TEXT) RETURNS void AS $emt_add_table_partition$
|
|
1410
1415
|
DECLARE
|
|
1411
1416
|
v_main_partiton_name TEXT;
|
|
1412
1417
|
v_active_partiton_name TEXT;
|
|
@@ -1436,20 +1441,215 @@ CREATE OR REPLACE FUNCTION emt_sanitize_name(input_name TEXT) RETURNS TEXT AS $$
|
|
|
1436
1441
|
v_archived_partiton_name, v_main_partiton_name
|
|
1437
1442
|
);
|
|
1438
1443
|
END;
|
|
1439
|
-
|
|
1440
|
-
|
|
1444
|
+
$emt_add_table_partition$ LANGUAGE plpgsql;`);
|
|
1445
|
+
const addPartitionSQL = createFunctionIfDoesNotExistSQL("emt_add_partition", SQL`
|
|
1446
|
+
CREATE OR REPLACE FUNCTION emt_add_partition(partition_name TEXT) RETURNS void AS $emt_add_partition$
|
|
1441
1447
|
BEGIN
|
|
1442
|
-
PERFORM emt_add_table_partition('
|
|
1443
|
-
PERFORM emt_add_table_partition('
|
|
1448
|
+
PERFORM emt_add_table_partition('${SQL.plain(messagesTable.name)}', partition_name);
|
|
1449
|
+
PERFORM emt_add_table_partition('${SQL.plain(streamsTable.name)}', partition_name);
|
|
1450
|
+
|
|
1451
|
+
EXECUTE format('
|
|
1452
|
+
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
1453
|
+
FOR VALUES IN (%L);',
|
|
1454
|
+
emt_sanitize_name('${SQL.plain(processorsTable.name)}' || '_' || partition_name), '${SQL.plain(processorsTable.name)}', partition_name
|
|
1455
|
+
);
|
|
1444
1456
|
|
|
1445
1457
|
EXECUTE format('
|
|
1446
1458
|
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
1447
1459
|
FOR VALUES IN (%L);',
|
|
1448
|
-
emt_sanitize_name('
|
|
1460
|
+
emt_sanitize_name('${SQL.plain(projectionsTable.name)}' || '_' || partition_name), '${SQL.plain(projectionsTable.name)}', partition_name
|
|
1449
1461
|
);
|
|
1450
1462
|
END;
|
|
1451
|
-
|
|
1463
|
+
$emt_add_partition$ LANGUAGE plpgsql;`);
|
|
1464
|
+
const addModuleSQL = SQL`
|
|
1465
|
+
CREATE OR REPLACE FUNCTION add_module(new_module TEXT) RETURNS void AS $$
|
|
1466
|
+
BEGIN
|
|
1467
|
+
-- For ${SQL.plain(messagesTable.name)} table
|
|
1468
|
+
EXECUTE format('
|
|
1469
|
+
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
1470
|
+
FOR VALUES IN (emt_sanitize_name(%L || ''__'' || %L)) PARTITION BY LIST (is_archived);',
|
|
1471
|
+
emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}'), '${SQL.plain(messagesTable.name)}', new_module, '${SQL.plain(globalTag)}'
|
|
1472
|
+
);
|
|
1473
|
+
|
|
1474
|
+
EXECUTE format('
|
|
1475
|
+
CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
|
|
1476
|
+
FOR VALUES IN (FALSE);',
|
|
1477
|
+
emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}' || '_active'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}')
|
|
1478
|
+
);
|
|
1479
|
+
|
|
1480
|
+
EXECUTE format('
|
|
1481
|
+
CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
|
|
1482
|
+
FOR VALUES IN (TRUE);',
|
|
1483
|
+
emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}' || '_archived'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}')
|
|
1484
|
+
);
|
|
1485
|
+
|
|
1486
|
+
-- For ${SQL.plain(streamsTable.name)} table
|
|
1487
|
+
EXECUTE format('
|
|
1488
|
+
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
1489
|
+
FOR VALUES IN (emt_sanitize_name(%L || ''__'' || %L)) PARTITION BY LIST (is_archived);',
|
|
1490
|
+
emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}'), '${SQL.plain(streamsTable.name)}', new_module, '${SQL.plain(globalTag)}'
|
|
1491
|
+
);
|
|
1492
|
+
|
|
1493
|
+
EXECUTE format('
|
|
1494
|
+
CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
|
|
1495
|
+
FOR VALUES IN (FALSE);',
|
|
1496
|
+
emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}' || '_active'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}')
|
|
1497
|
+
);
|
|
1498
|
+
|
|
1499
|
+
EXECUTE format('
|
|
1500
|
+
CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
|
|
1501
|
+
FOR VALUES IN (TRUE);',
|
|
1502
|
+
emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}' || '_archived'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}')
|
|
1503
|
+
);
|
|
1504
|
+
END;
|
|
1505
|
+
$$ LANGUAGE plpgsql;
|
|
1506
|
+
`;
|
|
1507
|
+
const addTenantSQL = SQL`
|
|
1508
|
+
CREATE OR REPLACE FUNCTION add_tenant(new_module TEXT, new_tenant TEXT) RETURNS void AS $$
|
|
1509
|
+
BEGIN
|
|
1510
|
+
-- For ${SQL.plain(messagesTable.name)} table
|
|
1511
|
+
EXECUTE format('
|
|
1512
|
+
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
1513
|
+
FOR VALUES IN (emt_sanitize_name(''%s__%s'')) PARTITION BY LIST (is_archived);',
|
|
1514
|
+
emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || new_tenant), '${SQL.plain(messagesTable.name)}', new_module, new_tenant
|
|
1515
|
+
);
|
|
1516
|
+
|
|
1517
|
+
EXECUTE format('
|
|
1518
|
+
CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
|
|
1519
|
+
FOR VALUES IN (FALSE);',
|
|
1520
|
+
emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || new_tenant || '_active'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || new_tenant)
|
|
1521
|
+
);
|
|
1522
|
+
|
|
1523
|
+
EXECUTE format('
|
|
1524
|
+
CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
|
|
1525
|
+
FOR VALUES IN (TRUE);',
|
|
1526
|
+
emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || new_tenant || '_archived'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || new_tenant)
|
|
1527
|
+
);
|
|
1528
|
+
|
|
1529
|
+
-- For ${SQL.plain(streamsTable.name)} table
|
|
1530
|
+
EXECUTE format('
|
|
1531
|
+
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
1532
|
+
FOR VALUES IN (emt_sanitize_name(''%s__%s'')) PARTITION BY LIST (is_archived);',
|
|
1533
|
+
emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || new_tenant), '${SQL.plain(streamsTable.name)}', new_module, new_tenant
|
|
1534
|
+
);
|
|
1535
|
+
|
|
1536
|
+
EXECUTE format('
|
|
1537
|
+
CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
|
|
1538
|
+
FOR VALUES IN (FALSE);',
|
|
1539
|
+
emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || new_tenant || '_active'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || new_tenant)
|
|
1540
|
+
);
|
|
1541
|
+
|
|
1542
|
+
EXECUTE format('
|
|
1543
|
+
CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
|
|
1544
|
+
FOR VALUES IN (TRUE);',
|
|
1545
|
+
emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || new_tenant || '_archived'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || new_tenant)
|
|
1546
|
+
);
|
|
1547
|
+
END;
|
|
1548
|
+
$$ LANGUAGE plpgsql;
|
|
1549
|
+
`;
|
|
1550
|
+
const addModuleForAllTenantsSQL = SQL`
|
|
1551
|
+
CREATE OR REPLACE FUNCTION add_module_for_all_tenants(new_module TEXT) RETURNS void AS $$
|
|
1552
|
+
DECLARE
|
|
1553
|
+
tenant_record RECORD;
|
|
1554
|
+
BEGIN
|
|
1555
|
+
PERFORM add_module(new_module);
|
|
1556
|
+
|
|
1557
|
+
FOR tenant_record IN SELECT DISTINCT tenant FROM ${SQL.plain(messagesTable.name)}
|
|
1558
|
+
LOOP
|
|
1559
|
+
-- For ${SQL.plain(messagesTable.name)} table
|
|
1560
|
+
EXECUTE format('
|
|
1561
|
+
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
1562
|
+
FOR VALUES IN (emt_sanitize_name(''%s__%s'')) PARTITION BY LIST (is_archived);',
|
|
1563
|
+
emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || tenant_record.tenant), '${SQL.plain(messagesTable.name)}', new_module, tenant_record.tenant
|
|
1564
|
+
);
|
|
1565
|
+
|
|
1566
|
+
EXECUTE format('
|
|
1567
|
+
CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
|
|
1568
|
+
FOR VALUES IN (FALSE);',
|
|
1569
|
+
emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || tenant_record.tenant || '_active'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || tenant_record.tenant)
|
|
1570
|
+
);
|
|
1571
|
+
|
|
1572
|
+
EXECUTE format('
|
|
1573
|
+
CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
|
|
1574
|
+
FOR VALUES IN (TRUE);',
|
|
1575
|
+
emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || tenant_record.tenant || '_archived'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || tenant_record.tenant)
|
|
1576
|
+
);
|
|
1577
|
+
|
|
1578
|
+
-- For ${SQL.plain(streamsTable.name)} table
|
|
1579
|
+
EXECUTE format('
|
|
1580
|
+
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
1581
|
+
FOR VALUES IN (emt_sanitize_name(''%s__%s'')) PARTITION BY LIST (is_archived);',
|
|
1582
|
+
emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || tenant_record.tenant), '${SQL.plain(streamsTable.name)}', new_module, tenant_record.tenant
|
|
1583
|
+
);
|
|
1584
|
+
|
|
1585
|
+
EXECUTE format('
|
|
1586
|
+
CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
|
|
1587
|
+
FOR VALUES IN (FALSE);',
|
|
1588
|
+
emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || tenant_record.tenant || '_active'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || tenant_record.tenant)
|
|
1589
|
+
);
|
|
1590
|
+
|
|
1591
|
+
EXECUTE format('
|
|
1592
|
+
CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
|
|
1593
|
+
FOR VALUES IN (TRUE);',
|
|
1594
|
+
emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || tenant_record.tenant || '_archived'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || tenant_record.tenant)
|
|
1595
|
+
);
|
|
1596
|
+
END LOOP;
|
|
1597
|
+
END;
|
|
1598
|
+
$$ LANGUAGE plpgsql;
|
|
1599
|
+
`;
|
|
1600
|
+
const addTenantForAllModulesSQL = SQL`
|
|
1601
|
+
CREATE OR REPLACE FUNCTION add_tenant_for_all_modules(new_tenant TEXT) RETURNS void AS $$
|
|
1602
|
+
DECLARE
|
|
1603
|
+
module_record RECORD;
|
|
1604
|
+
BEGIN
|
|
1605
|
+
FOR module_record IN SELECT DISTINCT partitionname FROM pg_partman.part_config WHERE parent_table = '${SQL.plain(messagesTable.name)}'
|
|
1606
|
+
LOOP
|
|
1607
|
+
-- For ${SQL.plain(messagesTable.name)} table
|
|
1608
|
+
EXECUTE format('
|
|
1609
|
+
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
1610
|
+
FOR VALUES IN (emt_sanitize_name(''%s__%s'')) PARTITION BY LIST (is_archived);',
|
|
1611
|
+
emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || module_record.partitionname || '__' || new_tenant), '${SQL.plain(messagesTable.name)}', module_record.partitionname, new_tenant
|
|
1612
|
+
);
|
|
1613
|
+
|
|
1614
|
+
EXECUTE format('
|
|
1615
|
+
CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
|
|
1616
|
+
FOR VALUES IN (FALSE);',
|
|
1617
|
+
emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || module_record.partitionname || '__' || new_tenant || '_active'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || module_record.partitionname || '__' || new_tenant)
|
|
1618
|
+
);
|
|
1619
|
+
|
|
1620
|
+
EXECUTE format('
|
|
1621
|
+
CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
|
|
1622
|
+
FOR VALUES IN (TRUE);',
|
|
1623
|
+
emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || module_record.partitionname || '__' || new_tenant || '_archived'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || module_record.partitionname || '__' || new_tenant)
|
|
1624
|
+
);
|
|
1625
|
+
|
|
1626
|
+
-- For ${SQL.plain(streamsTable.name)} table
|
|
1627
|
+
EXECUTE format('
|
|
1628
|
+
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
1629
|
+
FOR VALUES IN (emt_sanitize_name(''%s__%s'')) PARTITION BY LIST (is_archived);',
|
|
1630
|
+
emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || module_record.partitionname || '__' || new_tenant), '${SQL.plain(streamsTable.name)}', module_record.partitionname, new_tenant
|
|
1631
|
+
);
|
|
1632
|
+
|
|
1633
|
+
EXECUTE format('
|
|
1634
|
+
CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
|
|
1635
|
+
FOR VALUES IN (FALSE);',
|
|
1636
|
+
emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || module_record.partitionname || '__' || new_tenant || '_active'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || module_record.partitionname || '__' || new_tenant)
|
|
1637
|
+
);
|
|
1638
|
+
|
|
1639
|
+
EXECUTE format('
|
|
1640
|
+
CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
|
|
1641
|
+
FOR VALUES IN (TRUE);',
|
|
1642
|
+
emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || module_record.partitionname || '__' || new_tenant || '_archived'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || module_record.partitionname || '__' || new_tenant)
|
|
1643
|
+
);
|
|
1644
|
+
END LOOP;
|
|
1645
|
+
END;
|
|
1646
|
+
$$ LANGUAGE plpgsql;
|
|
1647
|
+
`;
|
|
1648
|
+
const addDefaultPartitionSQL = SQL`SELECT emt_add_partition('${SQL.plain(defaultTag)}');`;
|
|
1452
1649
|
|
|
1650
|
+
//#endregion
|
|
1651
|
+
//#region src/eventStore/schema/migrations/0_38_7/0_38_7.migration.ts
|
|
1652
|
+
const dropFutureConceptModuleAndTenantFunctions = SQL`
|
|
1453
1653
|
DO $$
|
|
1454
1654
|
BEGIN
|
|
1455
1655
|
-- Check and drop functions related to future concept of modules and tenants
|
|
@@ -1469,9 +1669,9 @@ CREATE OR REPLACE FUNCTION emt_sanitize_name(input_name TEXT) RETURNS TEXT AS $$
|
|
|
1469
1669
|
DROP FUNCTION add_tenant_for_all_modules(TEXT);
|
|
1470
1670
|
END IF;
|
|
1471
1671
|
END $$;
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
DO $$
|
|
1672
|
+
`;
|
|
1673
|
+
const dropOldAppendToSQLWithoutGlobalPositions = SQL`
|
|
1674
|
+
DO $$
|
|
1475
1675
|
DECLARE
|
|
1476
1676
|
v_current_return_type text;
|
|
1477
1677
|
BEGIN
|
|
@@ -1490,158 +1690,156 @@ CREATE OR REPLACE FUNCTION emt_sanitize_name(input_name TEXT) RETURNS TEXT AS $$
|
|
|
1490
1690
|
v_current_return_type NOT LIKE '%global_positions%' THEN
|
|
1491
1691
|
DROP FUNCTION emt_append_to_stream(text[], jsonb[], jsonb[], text[], text[], text[], text, text, bigint, text);
|
|
1492
1692
|
RAISE NOTICE 'Old version of function dropped. Return type was: %', v_current_return_type;
|
|
1493
|
-
END IF;
|
|
1494
|
-
END $$;
|
|
1495
|
-
CREATE OR REPLACE FUNCTION emt_append_to_stream(
|
|
1496
|
-
v_message_ids text[],
|
|
1497
|
-
v_messages_data jsonb[],
|
|
1498
|
-
v_messages_metadata jsonb[],
|
|
1499
|
-
v_message_schema_versions text[],
|
|
1500
|
-
v_message_types text[],
|
|
1501
|
-
v_message_kinds text[],
|
|
1502
|
-
v_stream_id text,
|
|
1503
|
-
v_stream_type text,
|
|
1504
|
-
v_expected_stream_position bigint DEFAULT NULL,
|
|
1505
|
-
v_partition text DEFAULT emt_sanitize_name('default_partition')
|
|
1506
|
-
) RETURNS TABLE (
|
|
1507
|
-
success boolean,
|
|
1508
|
-
next_stream_position bigint,
|
|
1509
|
-
global_positions bigint[],
|
|
1510
|
-
transaction_id xid8
|
|
1511
|
-
) LANGUAGE plpgsql
|
|
1512
|
-
AS $$
|
|
1513
|
-
DECLARE
|
|
1514
|
-
v_next_stream_position bigint;
|
|
1515
|
-
v_position bigint;
|
|
1516
|
-
v_updated_rows int;
|
|
1517
|
-
v_transaction_id xid8;
|
|
1518
|
-
v_global_positions bigint[];
|
|
1519
|
-
BEGIN
|
|
1520
|
-
v_transaction_id := pg_current_xact_id();
|
|
1521
|
-
|
|
1522
|
-
IF v_expected_stream_position IS NULL THEN
|
|
1523
|
-
SELECT COALESCE(
|
|
1524
|
-
(SELECT stream_position
|
|
1525
|
-
FROM emt_streams
|
|
1526
|
-
WHERE stream_id = v_stream_id
|
|
1527
|
-
AND partition = v_partition
|
|
1528
|
-
AND is_archived = FALSE
|
|
1529
|
-
LIMIT 1),
|
|
1530
|
-
0
|
|
1531
|
-
) INTO v_expected_stream_position;
|
|
1532
|
-
END IF;
|
|
1533
|
-
|
|
1534
|
-
v_next_stream_position := v_expected_stream_position + array_upper(v_messages_data, 1);
|
|
1535
|
-
|
|
1536
|
-
IF v_expected_stream_position = 0 THEN
|
|
1537
|
-
INSERT INTO emt_streams
|
|
1538
|
-
(stream_id, stream_position, partition, stream_type, stream_metadata, is_archived)
|
|
1539
|
-
VALUES
|
|
1540
|
-
(v_stream_id, v_next_stream_position, v_partition, v_stream_type, '{}', FALSE);
|
|
1541
|
-
ELSE
|
|
1542
|
-
UPDATE emt_streams as s
|
|
1543
|
-
SET stream_position = v_next_stream_position
|
|
1544
|
-
WHERE stream_id = v_stream_id AND stream_position = v_expected_stream_position AND partition = v_partition AND is_archived = FALSE;
|
|
1545
|
-
|
|
1546
|
-
get diagnostics v_updated_rows = row_count;
|
|
1547
1693
|
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1694
|
+
CREATE OR REPLACE FUNCTION emt_append_to_stream(
|
|
1695
|
+
v_message_ids text[],
|
|
1696
|
+
v_messages_data jsonb[],
|
|
1697
|
+
v_messages_metadata jsonb[],
|
|
1698
|
+
v_message_schema_versions text[],
|
|
1699
|
+
v_message_types text[],
|
|
1700
|
+
v_message_kinds text[],
|
|
1701
|
+
v_stream_id text,
|
|
1702
|
+
v_stream_type text,
|
|
1703
|
+
v_expected_stream_position bigint DEFAULT NULL,
|
|
1704
|
+
v_partition text DEFAULT emt_sanitize_name('default_partition')
|
|
1705
|
+
) RETURNS TABLE (
|
|
1706
|
+
success boolean,
|
|
1707
|
+
next_stream_position bigint,
|
|
1708
|
+
global_positions bigint[],
|
|
1709
|
+
transaction_id xid8
|
|
1710
|
+
) LANGUAGE plpgsql
|
|
1711
|
+
AS $emt_append_to_stream$
|
|
1712
|
+
DECLARE
|
|
1713
|
+
v_next_stream_position bigint;
|
|
1714
|
+
v_position bigint;
|
|
1715
|
+
v_updated_rows int;
|
|
1716
|
+
v_transaction_id xid8;
|
|
1717
|
+
v_global_positions bigint[];
|
|
1718
|
+
BEGIN
|
|
1719
|
+
v_transaction_id := pg_current_xact_id();
|
|
1720
|
+
|
|
1721
|
+
IF v_expected_stream_position IS NULL THEN
|
|
1722
|
+
SELECT COALESCE(
|
|
1723
|
+
(SELECT stream_position
|
|
1724
|
+
FROM emt_streams
|
|
1725
|
+
WHERE stream_id = v_stream_id
|
|
1726
|
+
AND partition = v_partition
|
|
1727
|
+
AND is_archived = FALSE
|
|
1728
|
+
LIMIT 1),
|
|
1729
|
+
0
|
|
1730
|
+
) INTO v_expected_stream_position;
|
|
1731
|
+
END IF;
|
|
1732
|
+
|
|
1733
|
+
v_next_stream_position := v_expected_stream_position + array_upper(v_messages_data, 1);
|
|
1734
|
+
|
|
1735
|
+
IF v_expected_stream_position = 0 THEN
|
|
1736
|
+
INSERT INTO emt_streams
|
|
1737
|
+
(stream_id, stream_position, partition, stream_type, stream_metadata, is_archived)
|
|
1738
|
+
VALUES
|
|
1739
|
+
(v_stream_id, v_next_stream_position, v_partition, v_stream_type, '{}', FALSE);
|
|
1740
|
+
ELSE
|
|
1741
|
+
UPDATE emt_streams as s
|
|
1742
|
+
SET stream_position = v_next_stream_position
|
|
1743
|
+
WHERE stream_id = v_stream_id AND stream_position = v_expected_stream_position AND partition = v_partition AND is_archived = FALSE;
|
|
1744
|
+
|
|
1745
|
+
get diagnostics v_updated_rows = row_count;
|
|
1746
|
+
|
|
1747
|
+
IF v_updated_rows = 0 THEN
|
|
1748
|
+
RETURN QUERY SELECT FALSE, NULL::bigint, NULL::bigint[], NULL::xid8;
|
|
1749
|
+
RETURN;
|
|
1750
|
+
END IF;
|
|
1751
|
+
END IF;
|
|
1752
|
+
|
|
1753
|
+
WITH ev AS (
|
|
1754
|
+
SELECT row_number() OVER () + v_expected_stream_position AS stream_position,
|
|
1755
|
+
message_data,
|
|
1756
|
+
message_metadata,
|
|
1757
|
+
schema_version,
|
|
1758
|
+
message_id,
|
|
1759
|
+
message_type,
|
|
1760
|
+
message_kind
|
|
1761
|
+
FROM (
|
|
1762
|
+
SELECT *
|
|
1763
|
+
FROM
|
|
1764
|
+
unnest(v_message_ids, v_messages_data, v_messages_metadata, v_message_schema_versions, v_message_types, v_message_kinds)
|
|
1765
|
+
AS message(message_id, message_data, message_metadata, schema_version, message_type, message_kind)
|
|
1766
|
+
) AS message
|
|
1767
|
+
),
|
|
1768
|
+
all_messages_insert AS (
|
|
1769
|
+
INSERT INTO emt_messages
|
|
1770
|
+
(stream_id, stream_position, partition, message_data, message_metadata, message_schema_version, message_type, message_kind, message_id, transaction_id)
|
|
1771
|
+
SELECT
|
|
1772
|
+
v_stream_id, ev.stream_position, v_partition, ev.message_data, ev.message_metadata, ev.schema_version, ev.message_type, ev.message_kind, ev.message_id, v_transaction_id
|
|
1773
|
+
FROM ev
|
|
1774
|
+
RETURNING global_position
|
|
1775
|
+
)
|
|
1776
|
+
SELECT
|
|
1777
|
+
array_agg(global_position ORDER BY global_position) INTO v_global_positions
|
|
1778
|
+
FROM
|
|
1779
|
+
all_messages_insert;
|
|
1780
|
+
|
|
1781
|
+
RETURN QUERY SELECT TRUE, v_next_stream_position, v_global_positions, v_transaction_id;
|
|
1782
|
+
END;
|
|
1783
|
+
$emt_append_to_stream$;
|
|
1552
1784
|
END IF;
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
message_metadata,
|
|
1558
|
-
schema_version,
|
|
1559
|
-
message_id,
|
|
1560
|
-
message_type,
|
|
1561
|
-
message_kind
|
|
1562
|
-
FROM (
|
|
1563
|
-
SELECT *
|
|
1564
|
-
FROM
|
|
1565
|
-
unnest(v_message_ids, v_messages_data, v_messages_metadata, v_message_schema_versions, v_message_types, v_message_kinds)
|
|
1566
|
-
AS message(message_id, message_data, message_metadata, schema_version, message_type, message_kind)
|
|
1567
|
-
) AS message
|
|
1568
|
-
),
|
|
1569
|
-
all_messages_insert AS (
|
|
1570
|
-
INSERT INTO emt_messages
|
|
1571
|
-
(stream_id, stream_position, partition, message_data, message_metadata, message_schema_version, message_type, message_kind, message_id, transaction_id)
|
|
1572
|
-
SELECT
|
|
1573
|
-
v_stream_id, ev.stream_position, v_partition, ev.message_data, ev.message_metadata, ev.schema_version, ev.message_type, ev.message_kind, ev.message_id, v_transaction_id
|
|
1574
|
-
FROM ev
|
|
1575
|
-
RETURNING global_position
|
|
1576
|
-
)
|
|
1577
|
-
SELECT
|
|
1578
|
-
array_agg(global_position ORDER BY global_position) INTO v_global_positions
|
|
1579
|
-
FROM
|
|
1580
|
-
all_messages_insert;
|
|
1581
|
-
|
|
1582
|
-
RETURN QUERY SELECT TRUE, v_next_stream_position, v_global_positions, v_transaction_id;
|
|
1583
|
-
END;
|
|
1584
|
-
$$;
|
|
1585
|
-
SELECT emt_add_partition('emt:default');
|
|
1586
|
-
CREATE OR REPLACE FUNCTION store_subscription_checkpoint(
|
|
1587
|
-
p_subscription_id VARCHAR(100),
|
|
1588
|
-
p_version BIGINT,
|
|
1589
|
-
p_position BIGINT,
|
|
1590
|
-
p_check_position BIGINT,
|
|
1591
|
-
p_transaction_id xid8,
|
|
1592
|
-
p_partition TEXT DEFAULT 'emt:default'
|
|
1593
|
-
) RETURNS INT AS $$
|
|
1785
|
+
END $$;
|
|
1786
|
+
`;
|
|
1787
|
+
const migrationFromEventsToMessagesSQL = SQL`
|
|
1788
|
+
DO $$
|
|
1594
1789
|
DECLARE
|
|
1595
|
-
|
|
1790
|
+
partition_record RECORD;
|
|
1596
1791
|
BEGIN
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
-- Retrieve the current position
|
|
1611
|
-
SELECT "last_processed_position" INTO current_position
|
|
1612
|
-
FROM "emt_subscriptions"
|
|
1613
|
-
WHERE "subscription_id" = p_subscription_id AND "partition" = p_partition;
|
|
1792
|
+
-- Rename the main table and its columns if it exists
|
|
1793
|
+
IF EXISTS (SELECT 1 FROM pg_tables WHERE tablename = 'emt_events') THEN
|
|
1794
|
+
-- Rename all partitions first
|
|
1795
|
+
FOR partition_record IN
|
|
1796
|
+
SELECT tablename
|
|
1797
|
+
FROM pg_tables
|
|
1798
|
+
WHERE tablename LIKE 'emt_events_%'
|
|
1799
|
+
ORDER BY tablename DESC -- to handle child partitions first
|
|
1800
|
+
LOOP
|
|
1801
|
+
EXECUTE format('ALTER TABLE %I RENAME TO %I',
|
|
1802
|
+
partition_record.tablename,
|
|
1803
|
+
REPLACE(partition_record.tablename, 'events', 'messages'));
|
|
1804
|
+
END LOOP;
|
|
1614
1805
|
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1806
|
+
-- Rename the main table
|
|
1807
|
+
ALTER TABLE emt_events RENAME TO emt_messages;
|
|
1808
|
+
|
|
1809
|
+
-- Rename columns
|
|
1810
|
+
ALTER TABLE emt_messages
|
|
1811
|
+
RENAME COLUMN event_data TO message_data;
|
|
1812
|
+
ALTER TABLE emt_messages
|
|
1813
|
+
RENAME COLUMN event_metadata TO message_metadata;
|
|
1814
|
+
ALTER TABLE emt_messages
|
|
1815
|
+
RENAME COLUMN event_schema_version TO message_schema_version;
|
|
1816
|
+
ALTER TABLE emt_messages
|
|
1817
|
+
RENAME COLUMN event_type TO message_type;
|
|
1818
|
+
ALTER TABLE emt_messages
|
|
1819
|
+
RENAME COLUMN event_id TO message_id;
|
|
1820
|
+
ALTER TABLE emt_messages
|
|
1821
|
+
ADD COLUMN message_kind CHAR(1) NOT NULL DEFAULT 'E';
|
|
1624
1822
|
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1823
|
+
-- Rename sequence if it exists
|
|
1824
|
+
IF EXISTS (SELECT 1 FROM pg_sequences WHERE sequencename = 'emt_global_event_position') THEN
|
|
1825
|
+
ALTER SEQUENCE emt_global_event_position
|
|
1826
|
+
RENAME TO emt_global_message_position;
|
|
1827
|
+
|
|
1828
|
+
ALTER TABLE emt_messages
|
|
1829
|
+
ALTER COLUMN global_position
|
|
1830
|
+
SET DEFAULT nextval('emt_global_message_position');
|
|
1831
|
+
END IF;
|
|
1832
|
+
END IF;
|
|
1833
|
+
END $$;`;
|
|
1834
|
+
const migration_0_38_7_and_older = sqlMigration("emt:postgresql:eventstore:0.38.7:migrate-events-to-messages:with_append_to_stream_update", [
|
|
1835
|
+
dropFutureConceptModuleAndTenantFunctions,
|
|
1836
|
+
dropOldAppendToSQLWithoutGlobalPositions,
|
|
1837
|
+
migrationFromEventsToMessagesSQL
|
|
1838
|
+
]);
|
|
1635
1839
|
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
RETURN 2; -- Insertion failed, row already exists with different position
|
|
1640
|
-
END IF;
|
|
1641
|
-
END;
|
|
1642
|
-
END;
|
|
1643
|
-
$$ LANGUAGE plpgsql;
|
|
1644
|
-
`;
|
|
1840
|
+
//#endregion
|
|
1841
|
+
//#region src/eventStore/schema/migrations/0_38_7/index.ts
|
|
1842
|
+
const migrations_0_38_7 = [migration_0_38_7_and_older];
|
|
1645
1843
|
|
|
1646
1844
|
//#endregion
|
|
1647
1845
|
//#region src/eventStore/schema/migrations/0_42_0/0_42_0.migration.ts
|
|
@@ -2176,1064 +2374,216 @@ END;
|
|
|
2176
2374
|
$emt_deactivate_projection$;
|
|
2177
2375
|
`;
|
|
2178
2376
|
const migration_0_42_0_2_AddProcessorProjectionFunctions = sqlMigration("emt:postgresql:eventstore:0.42.0-2:add-processor-projection-functions", [migration_0_42_0_2_AddProcessorProjectionFunctionsSQL]);
|
|
2377
|
+
const migration_0_42_0_3_FixProcessorLockTimeoutSQL = SQL`
|
|
2378
|
+
CREATE OR REPLACE FUNCTION emt_try_acquire_processor_lock(
|
|
2379
|
+
p_lock_key BIGINT,
|
|
2380
|
+
p_processor_id TEXT,
|
|
2381
|
+
p_version INT,
|
|
2382
|
+
p_partition TEXT DEFAULT '${SQL.plain(defaultTag)}',
|
|
2383
|
+
p_processor_instance_id TEXT DEFAULT '${SQL.plain(unknownTag$1)}',
|
|
2384
|
+
p_projection_name TEXT DEFAULT NULL,
|
|
2385
|
+
p_projection_type VARCHAR(1) DEFAULT NULL,
|
|
2386
|
+
p_projection_kind TEXT DEFAULT NULL,
|
|
2387
|
+
p_lock_timeout_seconds INT DEFAULT 300
|
|
2388
|
+
)
|
|
2389
|
+
RETURNS TABLE (acquired BOOLEAN, checkpoint TEXT)
|
|
2390
|
+
LANGUAGE plpgsql
|
|
2391
|
+
AS $emt_try_acquire_processor_lock$
|
|
2392
|
+
DECLARE
|
|
2393
|
+
current_position TEXT;
|
|
2394
|
+
v_current_time TIMESTAMPTZ := clock_timestamp();
|
|
2395
|
+
BEGIN
|
|
2396
|
+
RETURN QUERY
|
|
2397
|
+
WITH lock_check AS (
|
|
2398
|
+
SELECT pg_try_advisory_xact_lock(p_lock_key) AS lock_acquired
|
|
2399
|
+
),
|
|
2400
|
+
ownership_check AS (
|
|
2401
|
+
INSERT INTO emt_processors (
|
|
2402
|
+
processor_id,
|
|
2403
|
+
partition,
|
|
2404
|
+
version,
|
|
2405
|
+
processor_instance_id,
|
|
2406
|
+
status,
|
|
2407
|
+
last_processed_checkpoint,
|
|
2408
|
+
last_processed_transaction_id,
|
|
2409
|
+
created_at,
|
|
2410
|
+
last_updated
|
|
2411
|
+
)
|
|
2412
|
+
SELECT p_processor_id, p_partition, p_version, p_processor_instance_id, 'running', '0000000000000000000', '0'::xid8, v_current_time, v_current_time
|
|
2413
|
+
WHERE (SELECT lock_acquired FROM lock_check) = true
|
|
2414
|
+
ON CONFLICT (processor_id, partition, version) DO UPDATE
|
|
2415
|
+
SET processor_instance_id = p_processor_instance_id,
|
|
2416
|
+
status = 'running',
|
|
2417
|
+
last_updated = v_current_time
|
|
2418
|
+
WHERE emt_processors.processor_instance_id = p_processor_instance_id
|
|
2419
|
+
OR emt_processors.processor_instance_id = 'emt:unknown'
|
|
2420
|
+
OR emt_processors.status = 'stopped'
|
|
2421
|
+
OR emt_processors.last_updated < v_current_time - (p_lock_timeout_seconds || ' seconds')::interval
|
|
2422
|
+
RETURNING last_processed_checkpoint
|
|
2423
|
+
),
|
|
2424
|
+
projection_status AS (
|
|
2425
|
+
INSERT INTO emt_projections (
|
|
2426
|
+
name,
|
|
2427
|
+
partition,
|
|
2428
|
+
version,
|
|
2429
|
+
type,
|
|
2430
|
+
kind,
|
|
2431
|
+
status,
|
|
2432
|
+
definition
|
|
2433
|
+
)
|
|
2434
|
+
SELECT p_projection_name, p_partition, p_version, p_projection_type, p_projection_kind, 'async_processing', '{}'::jsonb
|
|
2435
|
+
WHERE p_projection_name IS NOT NULL
|
|
2436
|
+
AND (SELECT last_processed_checkpoint FROM ownership_check) IS NOT NULL
|
|
2437
|
+
ON CONFLICT (name, partition, version) DO UPDATE
|
|
2438
|
+
SET status = 'async_processing'
|
|
2439
|
+
RETURNING name
|
|
2440
|
+
)
|
|
2441
|
+
SELECT
|
|
2442
|
+
(SELECT COUNT(*) > 0 FROM ownership_check),
|
|
2443
|
+
(SELECT oc.last_processed_checkpoint FROM ownership_check oc);
|
|
2444
|
+
END;
|
|
2445
|
+
$emt_try_acquire_processor_lock$;
|
|
2446
|
+
`;
|
|
2447
|
+
const migration_0_42_0_3_FixProcessorLockTimeout = sqlMigration("emt:postgresql:eventstore:0.42.0-3:fix-processor-lock-timeout", [migration_0_42_0_3_FixProcessorLockTimeoutSQL]);
|
|
2179
2448
|
|
|
2180
2449
|
//#endregion
|
|
2181
|
-
//#region src/eventStore/schema/migrations/0_42_0/
|
|
2182
|
-
const
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
stream_type TEXT NOT NULL,
|
|
2188
|
-
stream_metadata JSONB NOT NULL,
|
|
2189
|
-
is_archived BOOLEAN NOT NULL DEFAULT FALSE,
|
|
2190
|
-
PRIMARY KEY (stream_id, partition, is_archived)
|
|
2191
|
-
) PARTITION BY LIST (partition);
|
|
2192
|
-
|
|
2193
|
-
CREATE UNIQUE INDEX IF NOT EXISTS idx_streams_unique
|
|
2194
|
-
ON emt_streams(stream_id, partition, is_archived)
|
|
2195
|
-
INCLUDE (stream_position);
|
|
2196
|
-
CREATE SEQUENCE IF NOT EXISTS emt_global_message_position;
|
|
2450
|
+
//#region src/eventStore/schema/migrations/0_42_0/index.ts
|
|
2451
|
+
const migrations_0_42_0 = [
|
|
2452
|
+
migration_0_42_0_FromSubscriptionsToProcessors,
|
|
2453
|
+
migration_0_42_0_2_AddProcessorProjectionFunctions,
|
|
2454
|
+
migration_0_42_0_3_FixProcessorLockTimeout
|
|
2455
|
+
];
|
|
2197
2456
|
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
message_data JSONB NOT NULL,
|
|
2211
|
-
message_metadata JSONB NOT NULL,
|
|
2212
|
-
PRIMARY KEY (stream_id, stream_position, partition, is_archived)
|
|
2213
|
-
) PARTITION BY LIST (partition);
|
|
2214
|
-
CREATE TABLE IF NOT EXISTS emt_projections(
|
|
2215
|
-
version INT NOT NULL DEFAULT 1,
|
|
2216
|
-
type VARCHAR(1) NOT NULL,
|
|
2217
|
-
name TEXT NOT NULL,
|
|
2218
|
-
partition TEXT NOT NULL DEFAULT 'emt:default',
|
|
2219
|
-
kind TEXT NOT NULL,
|
|
2220
|
-
status TEXT NOT NULL,
|
|
2221
|
-
definition JSONB NOT NULL DEFAULT '{}'::jsonb,
|
|
2222
|
-
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
2223
|
-
last_updated TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
2224
|
-
PRIMARY KEY (name, partition, version)
|
|
2225
|
-
) PARTITION BY LIST (partition);
|
|
2226
|
-
|
|
2227
|
-
CREATE TABLE IF NOT EXISTS emt_processors(
|
|
2228
|
-
last_processed_transaction_id XID8 NOT NULL,
|
|
2229
|
-
version INT NOT NULL DEFAULT 1,
|
|
2230
|
-
processor_id TEXT NOT NULL,
|
|
2231
|
-
partition TEXT NOT NULL DEFAULT 'emt:default',
|
|
2232
|
-
status TEXT NOT NULL DEFAULT 'stopped',
|
|
2233
|
-
last_processed_checkpoint TEXT NOT NULL,
|
|
2234
|
-
processor_instance_id TEXT DEFAULT 'emt:unknown',
|
|
2235
|
-
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
2236
|
-
last_updated TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
2237
|
-
PRIMARY KEY (processor_id, partition, version)
|
|
2238
|
-
) PARTITION BY LIST (partition);
|
|
2239
|
-
|
|
2240
|
-
DO $$
|
|
2241
|
-
BEGIN
|
|
2242
|
-
IF NOT EXISTS (SELECT 1 FROM pg_proc WHERE proname = 'emt_sanitize_name') THEN
|
|
2243
|
-
CREATE OR REPLACE FUNCTION emt_sanitize_name(input_name TEXT) RETURNS TEXT AS $emt_sanitize_name$
|
|
2244
|
-
BEGIN
|
|
2245
|
-
RETURN REGEXP_REPLACE(input_name, '[^a-zA-Z0-9_]', '_', 'g');
|
|
2246
|
-
END;
|
|
2247
|
-
$emt_sanitize_name$ LANGUAGE plpgsql;
|
|
2248
|
-
END IF;
|
|
2249
|
-
END $$;
|
|
2250
|
-
|
|
2251
|
-
DO $$
|
|
2252
|
-
BEGIN
|
|
2253
|
-
IF NOT EXISTS (SELECT 1 FROM pg_proc WHERE proname = 'emt_add_table_partition') THEN
|
|
2254
|
-
|
|
2255
|
-
CREATE OR REPLACE FUNCTION emt_add_table_partition(tableName TEXT, partition_name TEXT) RETURNS void AS $emt_add_table_partition$
|
|
2256
|
-
DECLARE
|
|
2257
|
-
v_main_partiton_name TEXT;
|
|
2258
|
-
v_active_partiton_name TEXT;
|
|
2259
|
-
v_archived_partiton_name TEXT;
|
|
2260
|
-
BEGIN
|
|
2261
|
-
v_main_partiton_name := emt_sanitize_name(tableName || '_' || partition_name);
|
|
2262
|
-
v_active_partiton_name := emt_sanitize_name(v_main_partiton_name || '_active');
|
|
2263
|
-
v_archived_partiton_name := emt_sanitize_name(v_main_partiton_name || '_archived');
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
-- create default partition
|
|
2267
|
-
EXECUTE format('
|
|
2268
|
-
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
2269
|
-
FOR VALUES IN (%L) PARTITION BY LIST (is_archived);',
|
|
2270
|
-
v_main_partiton_name, tableName, partition_name
|
|
2271
|
-
);
|
|
2272
|
-
|
|
2273
|
-
EXECUTE format('
|
|
2274
|
-
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
2275
|
-
FOR VALUES IN (FALSE);',
|
|
2276
|
-
v_active_partiton_name, v_main_partiton_name
|
|
2277
|
-
);
|
|
2278
|
-
|
|
2279
|
-
EXECUTE format('
|
|
2280
|
-
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
2281
|
-
FOR VALUES IN (TRUE);',
|
|
2282
|
-
v_archived_partiton_name, v_main_partiton_name
|
|
2283
|
-
);
|
|
2284
|
-
END;
|
|
2285
|
-
$emt_add_table_partition$ LANGUAGE plpgsql;
|
|
2286
|
-
END IF;
|
|
2287
|
-
END $$;
|
|
2288
|
-
|
|
2289
|
-
DO $$
|
|
2290
|
-
BEGIN
|
|
2291
|
-
IF NOT EXISTS (SELECT 1 FROM pg_proc WHERE proname = 'emt_add_partition') THEN
|
|
2292
|
-
|
|
2293
|
-
CREATE OR REPLACE FUNCTION emt_add_partition(partition_name TEXT) RETURNS void AS $emt_add_partition$
|
|
2294
|
-
BEGIN
|
|
2295
|
-
PERFORM emt_add_table_partition('emt_messages', partition_name);
|
|
2296
|
-
PERFORM emt_add_table_partition('emt_streams', partition_name);
|
|
2297
|
-
|
|
2298
|
-
EXECUTE format('
|
|
2299
|
-
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
2300
|
-
FOR VALUES IN (%L);',
|
|
2301
|
-
emt_sanitize_name('emt_processors' || '_' || partition_name), 'emt_processors', partition_name
|
|
2302
|
-
);
|
|
2303
|
-
|
|
2304
|
-
EXECUTE format('
|
|
2305
|
-
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
2306
|
-
FOR VALUES IN (%L);',
|
|
2307
|
-
emt_sanitize_name('emt_projections' || '_' || partition_name), 'emt_projections', partition_name
|
|
2308
|
-
);
|
|
2309
|
-
END;
|
|
2310
|
-
$emt_add_partition$ LANGUAGE plpgsql;
|
|
2311
|
-
END IF;
|
|
2312
|
-
END $$;
|
|
2313
|
-
|
|
2314
|
-
DO $$
|
|
2315
|
-
BEGIN
|
|
2316
|
-
IF NOT EXISTS (SELECT 1 FROM pg_proc WHERE proname = 'emt_append_to_stream') THEN
|
|
2317
|
-
CREATE OR REPLACE FUNCTION emt_append_to_stream(
|
|
2318
|
-
v_message_ids text[],
|
|
2319
|
-
v_messages_data jsonb[],
|
|
2320
|
-
v_messages_metadata jsonb[],
|
|
2321
|
-
v_message_schema_versions text[],
|
|
2322
|
-
v_message_types text[],
|
|
2323
|
-
v_message_kinds text[],
|
|
2324
|
-
v_stream_id text,
|
|
2325
|
-
v_stream_type text,
|
|
2326
|
-
v_expected_stream_position bigint DEFAULT NULL,
|
|
2327
|
-
v_partition text DEFAULT emt_sanitize_name('default_partition')
|
|
2328
|
-
) RETURNS TABLE (
|
|
2329
|
-
success boolean,
|
|
2330
|
-
next_stream_position bigint,
|
|
2331
|
-
global_positions bigint[],
|
|
2332
|
-
transaction_id xid8
|
|
2333
|
-
) LANGUAGE plpgsql
|
|
2334
|
-
AS $emt_append_to_stream$
|
|
2335
|
-
DECLARE
|
|
2336
|
-
v_next_stream_position bigint;
|
|
2337
|
-
v_position bigint;
|
|
2338
|
-
v_updated_rows int;
|
|
2339
|
-
v_transaction_id xid8;
|
|
2340
|
-
v_global_positions bigint[];
|
|
2341
|
-
BEGIN
|
|
2342
|
-
v_transaction_id := pg_current_xact_id();
|
|
2343
|
-
|
|
2344
|
-
IF v_expected_stream_position IS NULL THEN
|
|
2345
|
-
SELECT COALESCE(
|
|
2346
|
-
(SELECT stream_position
|
|
2347
|
-
FROM emt_streams
|
|
2348
|
-
WHERE stream_id = v_stream_id
|
|
2349
|
-
AND partition = v_partition
|
|
2350
|
-
AND is_archived = FALSE
|
|
2351
|
-
LIMIT 1),
|
|
2352
|
-
0
|
|
2353
|
-
) INTO v_expected_stream_position;
|
|
2354
|
-
END IF;
|
|
2355
|
-
|
|
2356
|
-
v_next_stream_position := v_expected_stream_position + array_upper(v_messages_data, 1);
|
|
2357
|
-
|
|
2358
|
-
IF v_expected_stream_position = 0 THEN
|
|
2359
|
-
INSERT INTO emt_streams
|
|
2360
|
-
(stream_id, stream_position, partition, stream_type, stream_metadata, is_archived)
|
|
2361
|
-
VALUES
|
|
2362
|
-
(v_stream_id, v_next_stream_position, v_partition, v_stream_type, '{}', FALSE);
|
|
2363
|
-
ELSE
|
|
2364
|
-
UPDATE emt_streams as s
|
|
2365
|
-
SET stream_position = v_next_stream_position
|
|
2366
|
-
WHERE stream_id = v_stream_id AND stream_position = v_expected_stream_position AND partition = v_partition AND is_archived = FALSE;
|
|
2367
|
-
|
|
2368
|
-
get diagnostics v_updated_rows = row_count;
|
|
2369
|
-
|
|
2370
|
-
IF v_updated_rows = 0 THEN
|
|
2371
|
-
RETURN QUERY SELECT FALSE, NULL::bigint, NULL::bigint[], NULL::xid8;
|
|
2372
|
-
RETURN;
|
|
2373
|
-
END IF;
|
|
2374
|
-
END IF;
|
|
2375
|
-
|
|
2376
|
-
WITH ev AS (
|
|
2377
|
-
SELECT row_number() OVER () + v_expected_stream_position AS stream_position,
|
|
2378
|
-
message_data,
|
|
2379
|
-
message_metadata,
|
|
2380
|
-
schema_version,
|
|
2381
|
-
message_id,
|
|
2382
|
-
message_type,
|
|
2383
|
-
message_kind
|
|
2384
|
-
FROM (
|
|
2385
|
-
SELECT *
|
|
2386
|
-
FROM
|
|
2387
|
-
unnest(v_message_ids, v_messages_data, v_messages_metadata, v_message_schema_versions, v_message_types, v_message_kinds)
|
|
2388
|
-
AS message(message_id, message_data, message_metadata, schema_version, message_type, message_kind)
|
|
2389
|
-
) AS message
|
|
2390
|
-
),
|
|
2391
|
-
all_messages_insert AS (
|
|
2392
|
-
INSERT INTO emt_messages
|
|
2393
|
-
(stream_id, stream_position, partition, message_data, message_metadata, message_schema_version, message_type, message_kind, message_id, transaction_id)
|
|
2394
|
-
SELECT
|
|
2395
|
-
v_stream_id, ev.stream_position, v_partition, ev.message_data, ev.message_metadata, ev.schema_version, ev.message_type, ev.message_kind, ev.message_id, v_transaction_id
|
|
2396
|
-
FROM ev
|
|
2397
|
-
RETURNING global_position
|
|
2398
|
-
)
|
|
2399
|
-
SELECT
|
|
2400
|
-
array_agg(global_position ORDER BY global_position) INTO v_global_positions
|
|
2401
|
-
FROM
|
|
2402
|
-
all_messages_insert;
|
|
2403
|
-
|
|
2404
|
-
RETURN QUERY SELECT TRUE, v_next_stream_position, v_global_positions, v_transaction_id;
|
|
2405
|
-
END;
|
|
2406
|
-
$emt_append_to_stream$;
|
|
2407
|
-
|
|
2408
|
-
END IF;
|
|
2409
|
-
END $$;
|
|
2410
|
-
SELECT emt_add_partition('emt:default');
|
|
2411
|
-
DO $$
|
|
2412
|
-
BEGIN
|
|
2413
|
-
IF NOT EXISTS (SELECT 1 FROM pg_proc WHERE proname = 'store_processor_checkpoint') THEN
|
|
2414
|
-
|
|
2415
|
-
CREATE OR REPLACE FUNCTION store_processor_checkpoint(
|
|
2416
|
-
p_processor_id TEXT,
|
|
2417
|
-
p_version BIGINT,
|
|
2418
|
-
p_position TEXT,
|
|
2419
|
-
p_check_position TEXT,
|
|
2420
|
-
p_transaction_id xid8,
|
|
2421
|
-
p_partition TEXT DEFAULT 'emt:default',
|
|
2422
|
-
p_processor_instance_id TEXT DEFAULT 'emt:unknown'
|
|
2423
|
-
) RETURNS INT AS $spc$
|
|
2424
|
-
DECLARE
|
|
2425
|
-
current_position TEXT;
|
|
2426
|
-
BEGIN
|
|
2427
|
-
-- Handle the case when p_check_position is provided
|
|
2428
|
-
IF p_check_position IS NOT NULL THEN
|
|
2429
|
-
-- Try to update if the position matches p_check_position
|
|
2430
|
-
UPDATE "emt_processors"
|
|
2431
|
-
SET
|
|
2432
|
-
"last_processed_checkpoint" = p_position,
|
|
2433
|
-
"last_processed_transaction_id" = p_transaction_id,
|
|
2434
|
-
"last_updated" = now()
|
|
2435
|
-
WHERE "processor_id" = p_processor_id
|
|
2436
|
-
AND "last_processed_checkpoint" = p_check_position
|
|
2437
|
-
AND "partition" = p_partition
|
|
2438
|
-
AND "version" = p_version;
|
|
2439
|
-
|
|
2440
|
-
IF FOUND THEN
|
|
2441
|
-
RETURN 1; -- Successfully updated
|
|
2442
|
-
END IF;
|
|
2443
|
-
|
|
2444
|
-
-- Retrieve the current position
|
|
2445
|
-
SELECT "last_processed_checkpoint" INTO current_position
|
|
2446
|
-
FROM "emt_processors"
|
|
2447
|
-
WHERE "processor_id" = p_processor_id
|
|
2448
|
-
AND "partition" = p_partition
|
|
2449
|
-
AND "version" = p_version;
|
|
2450
|
-
|
|
2451
|
-
-- Return appropriate codes based on current position
|
|
2452
|
-
IF current_position = p_position THEN
|
|
2453
|
-
RETURN 0; -- Idempotent check: position already set
|
|
2454
|
-
ELSIF current_position > p_position THEN
|
|
2455
|
-
RETURN 3; -- Current ahead: another process has progressed further
|
|
2456
|
-
ELSE
|
|
2457
|
-
RETURN 2; -- Mismatch: check position doesn't match current
|
|
2458
|
-
END IF;
|
|
2459
|
-
END IF;
|
|
2460
|
-
|
|
2461
|
-
-- Handle the case when p_check_position is NULL: Insert if not exists
|
|
2462
|
-
BEGIN
|
|
2463
|
-
INSERT INTO "emt_processors"("processor_id", "version", "last_processed_checkpoint", "partition", "last_processed_transaction_id", "created_at", "last_updated")
|
|
2464
|
-
VALUES (p_processor_id, p_version, p_position, p_partition, p_transaction_id, now(), now());
|
|
2465
|
-
RETURN 1; -- Successfully inserted
|
|
2466
|
-
EXCEPTION WHEN unique_violation THEN
|
|
2467
|
-
-- If insertion failed, it means the row already exists
|
|
2468
|
-
SELECT "last_processed_checkpoint" INTO current_position
|
|
2469
|
-
FROM "emt_processors"
|
|
2470
|
-
WHERE "processor_id" = p_processor_id
|
|
2471
|
-
AND "partition" = p_partition
|
|
2472
|
-
AND "version" = p_version;
|
|
2473
|
-
|
|
2474
|
-
IF current_position = p_position THEN
|
|
2475
|
-
RETURN 0; -- Idempotent check: position already set
|
|
2476
|
-
ELSIF current_position > p_position THEN
|
|
2477
|
-
RETURN 3; -- Current ahead: another process has progressed further
|
|
2478
|
-
ELSE
|
|
2479
|
-
RETURN 2; -- Insertion failed, row already exists with different position
|
|
2480
|
-
END IF;
|
|
2481
|
-
END;
|
|
2482
|
-
END;
|
|
2483
|
-
$spc$ LANGUAGE plpgsql;
|
|
2484
|
-
|
|
2485
|
-
END IF;
|
|
2486
|
-
END $$;
|
|
2487
|
-
|
|
2488
|
-
CREATE OR REPLACE FUNCTION emt_try_acquire_processor_lock(
|
|
2489
|
-
p_lock_key BIGINT,
|
|
2490
|
-
p_processor_id TEXT,
|
|
2491
|
-
p_version INT,
|
|
2492
|
-
p_partition TEXT DEFAULT 'emt:default',
|
|
2493
|
-
p_processor_instance_id TEXT DEFAULT 'emt:unknown',
|
|
2494
|
-
p_projection_name TEXT DEFAULT NULL,
|
|
2495
|
-
p_projection_type VARCHAR(1) DEFAULT NULL,
|
|
2496
|
-
p_projection_kind TEXT DEFAULT NULL,
|
|
2497
|
-
p_lock_timeout_seconds INT DEFAULT 300
|
|
2498
|
-
)
|
|
2499
|
-
RETURNS TABLE (acquired BOOLEAN, checkpoint TEXT)
|
|
2500
|
-
LANGUAGE plpgsql
|
|
2501
|
-
AS $emt_try_acquire_processor_lock$
|
|
2502
|
-
BEGIN
|
|
2503
|
-
RETURN QUERY
|
|
2504
|
-
WITH lock_check AS (
|
|
2505
|
-
SELECT pg_try_advisory_xact_lock(p_lock_key) AS lock_acquired
|
|
2506
|
-
),
|
|
2507
|
-
ownership_check AS (
|
|
2508
|
-
INSERT INTO emt_processors (
|
|
2509
|
-
processor_id,
|
|
2510
|
-
partition,
|
|
2511
|
-
version,
|
|
2512
|
-
processor_instance_id,
|
|
2513
|
-
status,
|
|
2514
|
-
last_processed_checkpoint,
|
|
2515
|
-
last_processed_transaction_id,
|
|
2516
|
-
created_at,
|
|
2517
|
-
last_updated
|
|
2518
|
-
)
|
|
2519
|
-
SELECT p_processor_id, p_partition, p_version, p_processor_instance_id, 'running', '0000000000000000000', '0'::xid8, now(), now()
|
|
2520
|
-
WHERE (SELECT lock_acquired FROM lock_check) = true
|
|
2521
|
-
ON CONFLICT (processor_id, partition, version) DO UPDATE
|
|
2522
|
-
SET processor_instance_id = p_processor_instance_id,
|
|
2523
|
-
status = 'running',
|
|
2524
|
-
last_updated = now()
|
|
2525
|
-
WHERE emt_processors.processor_instance_id = p_processor_instance_id
|
|
2526
|
-
OR emt_processors.processor_instance_id = 'emt:unknown'
|
|
2527
|
-
OR emt_processors.status = 'stopped'
|
|
2528
|
-
OR emt_processors.last_updated < now() - (p_lock_timeout_seconds || ' seconds')::interval
|
|
2529
|
-
RETURNING last_processed_checkpoint
|
|
2530
|
-
),
|
|
2531
|
-
projection_status AS (
|
|
2532
|
-
INSERT INTO emt_projections (
|
|
2533
|
-
name,
|
|
2534
|
-
partition,
|
|
2535
|
-
version,
|
|
2536
|
-
type,
|
|
2537
|
-
kind,
|
|
2538
|
-
status,
|
|
2539
|
-
definition
|
|
2540
|
-
)
|
|
2541
|
-
SELECT p_projection_name, p_partition, p_version, p_projection_type, p_projection_kind, 'async_processing', '{}'::jsonb
|
|
2542
|
-
WHERE p_projection_name IS NOT NULL
|
|
2543
|
-
AND (SELECT last_processed_checkpoint FROM ownership_check) IS NOT NULL
|
|
2544
|
-
ON CONFLICT (name, partition, version) DO UPDATE
|
|
2545
|
-
SET status = 'async_processing'
|
|
2546
|
-
RETURNING name
|
|
2547
|
-
)
|
|
2548
|
-
SELECT
|
|
2549
|
-
(SELECT COUNT(*) > 0 FROM ownership_check),
|
|
2550
|
-
(SELECT oc.last_processed_checkpoint FROM ownership_check oc);
|
|
2551
|
-
END;
|
|
2552
|
-
$emt_try_acquire_processor_lock$;
|
|
2553
|
-
|
|
2554
|
-
CREATE OR REPLACE FUNCTION emt_release_processor_lock(
|
|
2555
|
-
p_lock_key BIGINT,
|
|
2556
|
-
p_processor_id TEXT,
|
|
2557
|
-
p_partition TEXT,
|
|
2558
|
-
p_version INT,
|
|
2559
|
-
p_processor_instance_id TEXT DEFAULT 'emt:unknown',
|
|
2560
|
-
p_projection_name TEXT DEFAULT NULL
|
|
2561
|
-
)
|
|
2562
|
-
RETURNS BOOLEAN
|
|
2563
|
-
LANGUAGE plpgsql
|
|
2564
|
-
AS $emt_release_processor_lock$
|
|
2565
|
-
DECLARE
|
|
2566
|
-
v_rows_updated INT;
|
|
2567
|
-
BEGIN
|
|
2568
|
-
IF p_projection_name IS NOT NULL THEN
|
|
2569
|
-
UPDATE emt_projections
|
|
2570
|
-
SET status = 'active',
|
|
2571
|
-
last_updated = now()
|
|
2572
|
-
WHERE partition = p_partition
|
|
2573
|
-
AND name = p_projection_name
|
|
2574
|
-
AND version = p_version;
|
|
2575
|
-
END IF;
|
|
2576
|
-
|
|
2577
|
-
UPDATE emt_processors
|
|
2578
|
-
SET status = 'stopped',
|
|
2579
|
-
processor_instance_id = 'emt:unknown',
|
|
2580
|
-
last_updated = now()
|
|
2581
|
-
WHERE processor_id = p_processor_id
|
|
2582
|
-
AND partition = p_partition
|
|
2583
|
-
AND version = p_version
|
|
2584
|
-
AND processor_instance_id = p_processor_instance_id;
|
|
2585
|
-
|
|
2586
|
-
GET DIAGNOSTICS v_rows_updated = ROW_COUNT;
|
|
2587
|
-
|
|
2588
|
-
PERFORM pg_advisory_unlock(p_lock_key);
|
|
2589
|
-
|
|
2590
|
-
RETURN v_rows_updated > 0;
|
|
2591
|
-
END;
|
|
2592
|
-
$emt_release_processor_lock$;
|
|
2593
|
-
|
|
2594
|
-
CREATE OR REPLACE FUNCTION emt_try_acquire_projection_lock(
|
|
2595
|
-
p_lock_key BIGINT,
|
|
2596
|
-
p_partition TEXT,
|
|
2597
|
-
p_name TEXT,
|
|
2598
|
-
p_version INT
|
|
2599
|
-
)
|
|
2600
|
-
RETURNS TABLE (acquired BOOLEAN, is_active BOOLEAN)
|
|
2601
|
-
LANGUAGE plpgsql
|
|
2602
|
-
AS $emt_try_acquire_projection_lock$
|
|
2603
|
-
BEGIN
|
|
2604
|
-
RETURN QUERY
|
|
2605
|
-
WITH lock_check AS (
|
|
2606
|
-
SELECT pg_try_advisory_xact_lock_shared(p_lock_key) AS acquired
|
|
2607
|
-
),
|
|
2608
|
-
status_check AS (
|
|
2609
|
-
SELECT status = 'active' AS is_active
|
|
2610
|
-
FROM emt_projections
|
|
2611
|
-
WHERE partition = p_partition AND name = p_name AND version = p_version
|
|
2612
|
-
)
|
|
2613
|
-
SELECT
|
|
2614
|
-
COALESCE((SELECT lc.acquired FROM lock_check lc), false),
|
|
2615
|
-
COALESCE((SELECT sc.is_active FROM status_check sc), true);
|
|
2616
|
-
END;
|
|
2617
|
-
$emt_try_acquire_projection_lock$;
|
|
2618
|
-
|
|
2619
|
-
CREATE OR REPLACE FUNCTION emt_register_projection(
|
|
2620
|
-
p_lock_key BIGINT,
|
|
2621
|
-
p_name TEXT,
|
|
2622
|
-
p_partition TEXT,
|
|
2623
|
-
p_version INT,
|
|
2624
|
-
p_type VARCHAR(1),
|
|
2625
|
-
p_kind TEXT,
|
|
2626
|
-
p_status TEXT,
|
|
2627
|
-
p_definition JSONB
|
|
2628
|
-
)
|
|
2629
|
-
RETURNS BOOLEAN
|
|
2630
|
-
LANGUAGE plpgsql
|
|
2631
|
-
AS $emt_register_projection$
|
|
2632
|
-
DECLARE
|
|
2633
|
-
v_result BOOLEAN;
|
|
2634
|
-
BEGIN
|
|
2635
|
-
WITH lock_check AS (
|
|
2636
|
-
SELECT pg_try_advisory_xact_lock(p_lock_key) AS lock_acquired
|
|
2637
|
-
),
|
|
2638
|
-
upsert_result AS (
|
|
2639
|
-
INSERT INTO emt_projections (
|
|
2640
|
-
name, partition, version, type, kind, status, definition, created_at, last_updated
|
|
2641
|
-
)
|
|
2642
|
-
SELECT p_name, p_partition, p_version, p_type, p_kind, p_status, p_definition, now(), now()
|
|
2643
|
-
WHERE (SELECT lock_acquired FROM lock_check) = true
|
|
2644
|
-
ON CONFLICT (name, partition, version) DO UPDATE
|
|
2645
|
-
SET definition = EXCLUDED.definition,
|
|
2646
|
-
last_updated = now()
|
|
2647
|
-
RETURNING name
|
|
2648
|
-
)
|
|
2649
|
-
SELECT COUNT(*) > 0 INTO v_result FROM upsert_result;
|
|
2650
|
-
|
|
2651
|
-
RETURN v_result;
|
|
2652
|
-
END;
|
|
2653
|
-
$emt_register_projection$;
|
|
2654
|
-
|
|
2655
|
-
CREATE OR REPLACE FUNCTION emt_activate_projection(
|
|
2656
|
-
p_lock_key BIGINT,
|
|
2657
|
-
p_name TEXT,
|
|
2658
|
-
p_partition TEXT,
|
|
2659
|
-
p_version INT
|
|
2660
|
-
)
|
|
2661
|
-
RETURNS BOOLEAN
|
|
2662
|
-
LANGUAGE plpgsql
|
|
2663
|
-
AS $emt_activate_projection$
|
|
2664
|
-
DECLARE
|
|
2665
|
-
v_result BOOLEAN;
|
|
2666
|
-
BEGIN
|
|
2667
|
-
WITH lock_check AS (
|
|
2668
|
-
SELECT pg_try_advisory_xact_lock(p_lock_key) AS lock_acquired
|
|
2669
|
-
),
|
|
2670
|
-
update_result AS (
|
|
2671
|
-
UPDATE emt_projections
|
|
2672
|
-
SET status = 'active',
|
|
2673
|
-
last_updated = now()
|
|
2674
|
-
WHERE name = p_name
|
|
2675
|
-
AND partition = p_partition
|
|
2676
|
-
AND version = p_version
|
|
2677
|
-
AND (SELECT lock_acquired FROM lock_check) = true
|
|
2678
|
-
RETURNING name
|
|
2679
|
-
)
|
|
2680
|
-
SELECT COUNT(*) > 0 INTO v_result FROM update_result;
|
|
2681
|
-
|
|
2682
|
-
RETURN v_result;
|
|
2683
|
-
END;
|
|
2684
|
-
$emt_activate_projection$;
|
|
2685
|
-
|
|
2686
|
-
CREATE OR REPLACE FUNCTION emt_deactivate_projection(
|
|
2687
|
-
p_lock_key BIGINT,
|
|
2688
|
-
p_name TEXT,
|
|
2689
|
-
p_partition TEXT,
|
|
2690
|
-
p_version INT
|
|
2691
|
-
)
|
|
2692
|
-
RETURNS BOOLEAN
|
|
2693
|
-
LANGUAGE plpgsql
|
|
2694
|
-
AS $emt_deactivate_projection$
|
|
2695
|
-
DECLARE
|
|
2696
|
-
v_result BOOLEAN;
|
|
2697
|
-
BEGIN
|
|
2698
|
-
WITH lock_check AS (
|
|
2699
|
-
SELECT pg_try_advisory_xact_lock(p_lock_key) AS lock_acquired
|
|
2700
|
-
),
|
|
2701
|
-
update_result AS (
|
|
2702
|
-
UPDATE emt_projections
|
|
2703
|
-
SET status = 'inactive',
|
|
2704
|
-
last_updated = now()
|
|
2705
|
-
WHERE name = p_name
|
|
2706
|
-
AND partition = p_partition
|
|
2707
|
-
AND version = p_version
|
|
2708
|
-
AND (SELECT lock_acquired FROM lock_check) = true
|
|
2709
|
-
RETURNING name
|
|
2710
|
-
)
|
|
2711
|
-
SELECT COUNT(*) > 0 INTO v_result FROM update_result;
|
|
2712
|
-
|
|
2713
|
-
RETURN v_result;
|
|
2714
|
-
END;
|
|
2715
|
-
$emt_deactivate_projection$;
|
|
2716
|
-
`;
|
|
2717
|
-
|
|
2718
|
-
//#endregion
|
|
2719
|
-
//#region src/eventStore/schema/storeProcessorCheckpoint.ts
|
|
2720
|
-
const storeSubscriptionCheckpointSQL = createFunctionIfDoesNotExistSQL("store_processor_checkpoint", SQL`
|
|
2721
|
-
CREATE OR REPLACE FUNCTION store_processor_checkpoint(
|
|
2722
|
-
p_processor_id TEXT,
|
|
2723
|
-
p_version BIGINT,
|
|
2724
|
-
p_position TEXT,
|
|
2725
|
-
p_check_position TEXT,
|
|
2726
|
-
p_transaction_id xid8,
|
|
2727
|
-
p_partition TEXT DEFAULT '${SQL.plain(defaultTag)}',
|
|
2728
|
-
p_processor_instance_id TEXT DEFAULT '${SQL.plain(unknownTag)}'
|
|
2729
|
-
) RETURNS INT AS $spc$
|
|
2730
|
-
DECLARE
|
|
2731
|
-
current_position TEXT;
|
|
2732
|
-
BEGIN
|
|
2733
|
-
-- Handle the case when p_check_position is provided
|
|
2734
|
-
IF p_check_position IS NOT NULL THEN
|
|
2735
|
-
-- Try to update if the position matches p_check_position
|
|
2736
|
-
UPDATE "${SQL.plain(processorsTable.name)}"
|
|
2737
|
-
SET
|
|
2738
|
-
"last_processed_checkpoint" = p_position,
|
|
2739
|
-
"last_processed_transaction_id" = p_transaction_id,
|
|
2740
|
-
"last_updated" = now()
|
|
2741
|
-
WHERE "processor_id" = p_processor_id
|
|
2742
|
-
AND "last_processed_checkpoint" = p_check_position
|
|
2743
|
-
AND "partition" = p_partition
|
|
2744
|
-
AND "version" = p_version;
|
|
2745
|
-
|
|
2746
|
-
IF FOUND THEN
|
|
2747
|
-
RETURN 1; -- Successfully updated
|
|
2748
|
-
END IF;
|
|
2749
|
-
|
|
2750
|
-
-- Retrieve the current position
|
|
2751
|
-
SELECT "last_processed_checkpoint" INTO current_position
|
|
2752
|
-
FROM "${SQL.plain(processorsTable.name)}"
|
|
2753
|
-
WHERE "processor_id" = p_processor_id
|
|
2754
|
-
AND "partition" = p_partition
|
|
2755
|
-
AND "version" = p_version;
|
|
2756
|
-
|
|
2757
|
-
-- Return appropriate codes based on current position
|
|
2758
|
-
IF current_position = p_position THEN
|
|
2759
|
-
RETURN 0; -- Idempotent check: position already set
|
|
2760
|
-
ELSIF current_position > p_position THEN
|
|
2761
|
-
RETURN 3; -- Current ahead: another process has progressed further
|
|
2762
|
-
ELSE
|
|
2763
|
-
RETURN 2; -- Mismatch: check position doesn't match current
|
|
2764
|
-
END IF;
|
|
2765
|
-
END IF;
|
|
2766
|
-
|
|
2767
|
-
-- Handle the case when p_check_position is NULL: Insert if not exists
|
|
2768
|
-
BEGIN
|
|
2769
|
-
INSERT INTO "${SQL.plain(processorsTable.name)}"("processor_id", "version", "last_processed_checkpoint", "partition", "last_processed_transaction_id", "created_at", "last_updated")
|
|
2770
|
-
VALUES (p_processor_id, p_version, p_position, p_partition, p_transaction_id, now(), now());
|
|
2771
|
-
RETURN 1; -- Successfully inserted
|
|
2772
|
-
EXCEPTION WHEN unique_violation THEN
|
|
2773
|
-
-- If insertion failed, it means the row already exists
|
|
2774
|
-
SELECT "last_processed_checkpoint" INTO current_position
|
|
2775
|
-
FROM "${SQL.plain(processorsTable.name)}"
|
|
2776
|
-
WHERE "processor_id" = p_processor_id
|
|
2777
|
-
AND "partition" = p_partition
|
|
2778
|
-
AND "version" = p_version;
|
|
2779
|
-
|
|
2780
|
-
IF current_position = p_position THEN
|
|
2781
|
-
RETURN 0; -- Idempotent check: position already set
|
|
2782
|
-
ELSIF current_position > p_position THEN
|
|
2783
|
-
RETURN 3; -- Current ahead: another process has progressed further
|
|
2784
|
-
ELSE
|
|
2785
|
-
RETURN 2; -- Insertion failed, row already exists with different position
|
|
2786
|
-
END IF;
|
|
2787
|
-
END;
|
|
2788
|
-
END;
|
|
2789
|
-
$spc$ LANGUAGE plpgsql;
|
|
2790
|
-
`);
|
|
2791
|
-
const callStoreProcessorCheckpoint = (params) => SQL`
|
|
2792
|
-
SELECT store_processor_checkpoint(
|
|
2793
|
-
${params.processorId},
|
|
2794
|
-
${params.version},
|
|
2795
|
-
${params.position},
|
|
2796
|
-
${params.checkPosition},
|
|
2797
|
-
pg_current_xact_id(),
|
|
2798
|
-
${params.partition},
|
|
2799
|
-
${params.processorInstanceId}
|
|
2800
|
-
) as result;`;
|
|
2801
|
-
const storeProcessorCheckpoint = async (execute, options) => {
|
|
2802
|
-
try {
|
|
2803
|
-
const { result } = await single(execute.command(callStoreProcessorCheckpoint({
|
|
2804
|
-
processorId: options.processorId,
|
|
2805
|
-
version: options.version ?? 1,
|
|
2806
|
-
position: options.newCheckpoint !== null ? options.newCheckpoint : null,
|
|
2807
|
-
checkPosition: options.lastProcessedCheckpoint !== null ? options.lastProcessedCheckpoint : null,
|
|
2808
|
-
partition: options.partition ?? defaultTag,
|
|
2809
|
-
processorInstanceId: options.processorInstanceId ?? unknownTag
|
|
2810
|
-
})));
|
|
2811
|
-
return result === 1 ? {
|
|
2812
|
-
success: true,
|
|
2813
|
-
newCheckpoint: options.newCheckpoint
|
|
2814
|
-
} : {
|
|
2815
|
-
success: false,
|
|
2816
|
-
reason: result === 0 ? "IGNORED" : result === 3 ? "CURRENT_AHEAD" : "MISMATCH"
|
|
2817
|
-
};
|
|
2818
|
-
} catch (error) {
|
|
2819
|
-
console.log(error);
|
|
2820
|
-
throw error;
|
|
2821
|
-
}
|
|
2822
|
-
};
|
|
2823
|
-
|
|
2824
|
-
//#endregion
|
|
2825
|
-
//#region src/eventStore/schema/migrations/0_43_0/index.ts
|
|
2826
|
-
const migration_0_43_0_cleanupLegacySubscriptionSQL = SQL`
|
|
2827
|
-
DO $$
|
|
2828
|
-
BEGIN
|
|
2829
|
-
IF EXISTS (SELECT 1 FROM pg_tables WHERE tablename = 'emt_subscriptions') THEN
|
|
2830
|
-
-- Restore clean emt_add_partition (remove creation of emt_subscriptions partitions)
|
|
2831
|
-
CREATE OR REPLACE FUNCTION emt_add_partition(partition_name TEXT) RETURNS void AS $fnpar$
|
|
2832
|
-
BEGIN
|
|
2833
|
-
PERFORM emt_add_table_partition('${SQL.plain(messagesTable.name)}', partition_name);
|
|
2834
|
-
PERFORM emt_add_table_partition('${SQL.plain(streamsTable.name)}', partition_name);
|
|
2835
|
-
|
|
2836
|
-
EXECUTE format('
|
|
2837
|
-
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
2838
|
-
FOR VALUES IN (%L);',
|
|
2839
|
-
emt_sanitize_name('${SQL.plain(processorsTable.name)}' || '_' || partition_name), '${SQL.plain(processorsTable.name)}', partition_name
|
|
2840
|
-
);
|
|
2841
|
-
|
|
2842
|
-
EXECUTE format('
|
|
2843
|
-
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
2844
|
-
FOR VALUES IN (%L);',
|
|
2845
|
-
emt_sanitize_name('${SQL.plain(projectionsTable.name)}' || '_' || partition_name), '${SQL.plain(projectionsTable.name)}', partition_name
|
|
2846
|
-
);
|
|
2847
|
-
END;
|
|
2848
|
-
$fnpar$ LANGUAGE plpgsql;
|
|
2849
|
-
|
|
2850
|
-
-- Drop old subscriptions table if it exists
|
|
2851
|
-
DROP TABLE IF EXISTS emt_subscriptions CASCADE;
|
|
2852
|
-
|
|
2853
|
-
-- Drop old function if it exists
|
|
2854
|
-
DROP FUNCTION IF EXISTS store_subscription_checkpoint(character varying, bigint, bigint, bigint, xid8, text);
|
|
2855
|
-
|
|
2856
|
-
-- Restore clean store_processor_checkpoint (remove dual-write logic)
|
|
2857
|
-
CREATE OR REPLACE FUNCTION store_processor_checkpoint(
|
|
2858
|
-
p_processor_id TEXT,
|
|
2859
|
-
p_version BIGINT,
|
|
2860
|
-
p_position TEXT,
|
|
2861
|
-
p_check_position TEXT,
|
|
2862
|
-
p_transaction_id xid8,
|
|
2863
|
-
p_partition TEXT DEFAULT '${SQL.plain(defaultTag)}',
|
|
2864
|
-
p_processor_instance_id TEXT DEFAULT 'emt:unknown'
|
|
2865
|
-
) RETURNS INT AS $fn$
|
|
2866
|
-
DECLARE
|
|
2867
|
-
current_position TEXT;
|
|
2868
|
-
BEGIN
|
|
2869
|
-
IF p_check_position IS NOT NULL THEN
|
|
2870
|
-
UPDATE "emt_processors"
|
|
2871
|
-
SET
|
|
2872
|
-
"last_processed_checkpoint" = p_position,
|
|
2873
|
-
"last_processed_transaction_id" = p_transaction_id,
|
|
2874
|
-
"last_updated" = now()
|
|
2875
|
-
WHERE "processor_id" = p_processor_id
|
|
2876
|
-
AND "last_processed_checkpoint" = p_check_position
|
|
2877
|
-
AND "partition" = p_partition
|
|
2878
|
-
AND "version" = p_version;
|
|
2879
|
-
|
|
2880
|
-
IF FOUND THEN
|
|
2881
|
-
RETURN 1;
|
|
2882
|
-
END IF;
|
|
2883
|
-
|
|
2884
|
-
SELECT "last_processed_checkpoint" INTO current_position
|
|
2885
|
-
FROM "emt_processors"
|
|
2886
|
-
WHERE "processor_id" = p_processor_id
|
|
2887
|
-
AND "partition" = p_partition
|
|
2888
|
-
AND "version" = p_version ;
|
|
2889
|
-
|
|
2890
|
-
IF current_position = p_position THEN
|
|
2891
|
-
RETURN 0;
|
|
2892
|
-
ELSIF current_position > p_position THEN
|
|
2893
|
-
RETURN 3;
|
|
2894
|
-
ELSE
|
|
2895
|
-
RETURN 2;
|
|
2896
|
-
END IF;
|
|
2897
|
-
END IF;
|
|
2898
|
-
|
|
2899
|
-
BEGIN
|
|
2900
|
-
INSERT INTO "emt_processors"("processor_id", "version", "last_processed_checkpoint", "partition", "last_processed_transaction_id", "created_at", "last_updated")
|
|
2901
|
-
VALUES (p_processor_id, p_version, p_position, p_partition, p_transaction_id, now(), now());
|
|
2902
|
-
RETURN 1;
|
|
2903
|
-
EXCEPTION WHEN unique_violation THEN
|
|
2904
|
-
SELECT "last_processed_checkpoint" INTO current_position
|
|
2905
|
-
FROM "emt_processors"
|
|
2906
|
-
WHERE "processor_id" = p_processor_id
|
|
2907
|
-
AND "partition" = p_partition
|
|
2908
|
-
AND "version" = p_version;
|
|
2909
|
-
|
|
2910
|
-
IF current_position = p_position THEN
|
|
2911
|
-
RETURN 0;
|
|
2912
|
-
ELSE
|
|
2913
|
-
RETURN 2;
|
|
2914
|
-
END IF;
|
|
2915
|
-
END;
|
|
2916
|
-
END;
|
|
2917
|
-
$fn$ LANGUAGE plpgsql;
|
|
2918
|
-
END IF;
|
|
2919
|
-
END $$;
|
|
2920
|
-
`;
|
|
2921
|
-
const migration_0_43_0_cleanupLegacySubscription = sqlMigration("emt:postgresql:eventstore:0.43.0:cleanup-legacy-subscription", [migration_0_43_0_cleanupLegacySubscriptionSQL]);
|
|
2922
|
-
const cleanupLegacySubscriptionTables = async (connectionString) => {
|
|
2923
|
-
const pool = dumbo({ connectionString });
|
|
2924
|
-
try {
|
|
2925
|
-
await pool.withTransaction(async ({ execute }) => {
|
|
2926
|
-
await execute.command(migration_0_43_0_cleanupLegacySubscriptionSQL);
|
|
2927
|
-
});
|
|
2928
|
-
} finally {
|
|
2929
|
-
await pool.close();
|
|
2930
|
-
}
|
|
2931
|
-
};
|
|
2932
|
-
|
|
2933
|
-
//#endregion
|
|
2934
|
-
//#region src/eventStore/schema/tables.ts
|
|
2935
|
-
const streamsTableSQL = SQL`
|
|
2936
|
-
CREATE TABLE IF NOT EXISTS ${SQL.identifier(streamsTable.name)}(
|
|
2937
|
-
stream_id TEXT NOT NULL,
|
|
2938
|
-
stream_position BIGINT NOT NULL,
|
|
2939
|
-
partition TEXT NOT NULL DEFAULT '${SQL.plain(defaultTag)}',
|
|
2940
|
-
stream_type TEXT NOT NULL,
|
|
2941
|
-
stream_metadata JSONB NOT NULL,
|
|
2942
|
-
is_archived BOOLEAN NOT NULL DEFAULT FALSE,
|
|
2943
|
-
PRIMARY KEY (stream_id, partition, is_archived)
|
|
2944
|
-
) PARTITION BY LIST (partition);
|
|
2945
|
-
|
|
2946
|
-
CREATE UNIQUE INDEX IF NOT EXISTS idx_streams_unique
|
|
2947
|
-
ON ${SQL.identifier(streamsTable.name)}(stream_id, partition, is_archived)
|
|
2948
|
-
INCLUDE (stream_position);`;
|
|
2949
|
-
const messagesTableSQL = SQL`
|
|
2950
|
-
CREATE SEQUENCE IF NOT EXISTS emt_global_message_position;
|
|
2951
|
-
|
|
2952
|
-
CREATE TABLE IF NOT EXISTS ${SQL.identifier(messagesTable.name)}(
|
|
2953
|
-
stream_position BIGINT NOT NULL,
|
|
2954
|
-
global_position BIGINT DEFAULT nextval('emt_global_message_position'),
|
|
2955
|
-
transaction_id XID8 NOT NULL,
|
|
2956
|
-
created TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
2957
|
-
is_archived BOOLEAN NOT NULL DEFAULT FALSE,
|
|
2958
|
-
message_kind VARCHAR(1) NOT NULL DEFAULT 'E',
|
|
2959
|
-
stream_id TEXT NOT NULL,
|
|
2960
|
-
partition TEXT NOT NULL DEFAULT '${SQL.plain(defaultTag)}',
|
|
2961
|
-
message_schema_version TEXT NOT NULL,
|
|
2962
|
-
message_id TEXT NOT NULL,
|
|
2963
|
-
message_type TEXT NOT NULL,
|
|
2964
|
-
message_data JSONB NOT NULL,
|
|
2965
|
-
message_metadata JSONB NOT NULL,
|
|
2966
|
-
PRIMARY KEY (stream_id, stream_position, partition, is_archived)
|
|
2967
|
-
) PARTITION BY LIST (partition);`;
|
|
2968
|
-
const processorsTableSQL = SQL`
|
|
2969
|
-
CREATE TABLE IF NOT EXISTS ${SQL.identifier(processorsTable.name)}(
|
|
2970
|
-
last_processed_transaction_id XID8 NOT NULL,
|
|
2971
|
-
version INT NOT NULL DEFAULT 1,
|
|
2972
|
-
processor_id TEXT NOT NULL,
|
|
2973
|
-
partition TEXT NOT NULL DEFAULT '${SQL.plain(defaultTag)}',
|
|
2974
|
-
status TEXT NOT NULL DEFAULT 'stopped',
|
|
2975
|
-
last_processed_checkpoint TEXT NOT NULL,
|
|
2976
|
-
processor_instance_id TEXT DEFAULT '${SQL.plain(unknownTag)}',
|
|
2977
|
-
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
2978
|
-
last_updated TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
2979
|
-
PRIMARY KEY (processor_id, partition, version)
|
|
2980
|
-
) PARTITION BY LIST (partition);
|
|
2981
|
-
`;
|
|
2982
|
-
const projectionsTableSQL = SQL`
|
|
2983
|
-
CREATE TABLE IF NOT EXISTS ${SQL.identifier(projectionsTable.name)}(
|
|
2984
|
-
version INT NOT NULL DEFAULT 1,
|
|
2985
|
-
type VARCHAR(1) NOT NULL,
|
|
2986
|
-
name TEXT NOT NULL,
|
|
2987
|
-
partition TEXT NOT NULL DEFAULT '${SQL.plain(defaultTag)}',
|
|
2988
|
-
kind TEXT NOT NULL,
|
|
2989
|
-
status TEXT NOT NULL,
|
|
2990
|
-
definition JSONB NOT NULL DEFAULT '{}'::jsonb,
|
|
2991
|
-
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
2992
|
-
last_updated TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
2993
|
-
PRIMARY KEY (name, partition, version)
|
|
2994
|
-
) PARTITION BY LIST (partition);
|
|
2995
|
-
`;
|
|
2996
|
-
const sanitizeNameSQL = createFunctionIfDoesNotExistSQL("emt_sanitize_name", SQL`CREATE OR REPLACE FUNCTION emt_sanitize_name(input_name TEXT) RETURNS TEXT AS $emt_sanitize_name$
|
|
2997
|
-
BEGIN
|
|
2998
|
-
RETURN REGEXP_REPLACE(input_name, '[^a-zA-Z0-9_]', '_', 'g');
|
|
2999
|
-
END;
|
|
3000
|
-
$emt_sanitize_name$ LANGUAGE plpgsql;`);
|
|
3001
|
-
const addTablePartitions = createFunctionIfDoesNotExistSQL("emt_add_table_partition", SQL`
|
|
3002
|
-
CREATE OR REPLACE FUNCTION emt_add_table_partition(tableName TEXT, partition_name TEXT) RETURNS void AS $emt_add_table_partition$
|
|
3003
|
-
DECLARE
|
|
3004
|
-
v_main_partiton_name TEXT;
|
|
3005
|
-
v_active_partiton_name TEXT;
|
|
3006
|
-
v_archived_partiton_name TEXT;
|
|
3007
|
-
BEGIN
|
|
3008
|
-
v_main_partiton_name := emt_sanitize_name(tableName || '_' || partition_name);
|
|
3009
|
-
v_active_partiton_name := emt_sanitize_name(v_main_partiton_name || '_active');
|
|
3010
|
-
v_archived_partiton_name := emt_sanitize_name(v_main_partiton_name || '_archived');
|
|
3011
|
-
|
|
3012
|
-
|
|
3013
|
-
-- create default partition
|
|
3014
|
-
EXECUTE format('
|
|
3015
|
-
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
3016
|
-
FOR VALUES IN (%L) PARTITION BY LIST (is_archived);',
|
|
3017
|
-
v_main_partiton_name, tableName, partition_name
|
|
3018
|
-
);
|
|
3019
|
-
|
|
3020
|
-
EXECUTE format('
|
|
3021
|
-
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
3022
|
-
FOR VALUES IN (FALSE);',
|
|
3023
|
-
v_active_partiton_name, v_main_partiton_name
|
|
3024
|
-
);
|
|
3025
|
-
|
|
3026
|
-
EXECUTE format('
|
|
3027
|
-
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
3028
|
-
FOR VALUES IN (TRUE);',
|
|
3029
|
-
v_archived_partiton_name, v_main_partiton_name
|
|
3030
|
-
);
|
|
3031
|
-
END;
|
|
3032
|
-
$emt_add_table_partition$ LANGUAGE plpgsql;`);
|
|
3033
|
-
const addPartitionSQL = createFunctionIfDoesNotExistSQL("emt_add_partition", SQL`
|
|
3034
|
-
CREATE OR REPLACE FUNCTION emt_add_partition(partition_name TEXT) RETURNS void AS $emt_add_partition$
|
|
3035
|
-
BEGIN
|
|
3036
|
-
PERFORM emt_add_table_partition('${SQL.plain(messagesTable.name)}', partition_name);
|
|
3037
|
-
PERFORM emt_add_table_partition('${SQL.plain(streamsTable.name)}', partition_name);
|
|
3038
|
-
|
|
3039
|
-
EXECUTE format('
|
|
3040
|
-
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
3041
|
-
FOR VALUES IN (%L);',
|
|
3042
|
-
emt_sanitize_name('${SQL.plain(processorsTable.name)}' || '_' || partition_name), '${SQL.plain(processorsTable.name)}', partition_name
|
|
3043
|
-
);
|
|
3044
|
-
|
|
3045
|
-
EXECUTE format('
|
|
3046
|
-
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
3047
|
-
FOR VALUES IN (%L);',
|
|
3048
|
-
emt_sanitize_name('${SQL.plain(projectionsTable.name)}' || '_' || partition_name), '${SQL.plain(projectionsTable.name)}', partition_name
|
|
3049
|
-
);
|
|
3050
|
-
END;
|
|
3051
|
-
$emt_add_partition$ LANGUAGE plpgsql;`);
|
|
3052
|
-
const addModuleSQL = SQL`
|
|
3053
|
-
CREATE OR REPLACE FUNCTION add_module(new_module TEXT) RETURNS void AS $$
|
|
3054
|
-
BEGIN
|
|
3055
|
-
-- For ${SQL.plain(messagesTable.name)} table
|
|
3056
|
-
EXECUTE format('
|
|
3057
|
-
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
3058
|
-
FOR VALUES IN (emt_sanitize_name(%L || ''__'' || %L)) PARTITION BY LIST (is_archived);',
|
|
3059
|
-
emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}'), '${SQL.plain(messagesTable.name)}', new_module, '${SQL.plain(globalTag)}'
|
|
3060
|
-
);
|
|
3061
|
-
|
|
3062
|
-
EXECUTE format('
|
|
3063
|
-
CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
|
|
3064
|
-
FOR VALUES IN (FALSE);',
|
|
3065
|
-
emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}' || '_active'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}')
|
|
3066
|
-
);
|
|
3067
|
-
|
|
3068
|
-
EXECUTE format('
|
|
3069
|
-
CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
|
|
3070
|
-
FOR VALUES IN (TRUE);',
|
|
3071
|
-
emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}' || '_archived'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}')
|
|
3072
|
-
);
|
|
3073
|
-
|
|
3074
|
-
-- For ${SQL.plain(streamsTable.name)} table
|
|
3075
|
-
EXECUTE format('
|
|
3076
|
-
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
3077
|
-
FOR VALUES IN (emt_sanitize_name(%L || ''__'' || %L)) PARTITION BY LIST (is_archived);',
|
|
3078
|
-
emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}'), '${SQL.plain(streamsTable.name)}', new_module, '${SQL.plain(globalTag)}'
|
|
3079
|
-
);
|
|
3080
|
-
|
|
3081
|
-
EXECUTE format('
|
|
3082
|
-
CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
|
|
3083
|
-
FOR VALUES IN (FALSE);',
|
|
3084
|
-
emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}' || '_active'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}')
|
|
3085
|
-
);
|
|
3086
|
-
|
|
3087
|
-
EXECUTE format('
|
|
3088
|
-
CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
|
|
3089
|
-
FOR VALUES IN (TRUE);',
|
|
3090
|
-
emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}' || '_archived'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}')
|
|
3091
|
-
);
|
|
3092
|
-
END;
|
|
3093
|
-
$$ LANGUAGE plpgsql;
|
|
3094
|
-
`;
|
|
3095
|
-
const addTenantSQL = SQL`
|
|
3096
|
-
CREATE OR REPLACE FUNCTION add_tenant(new_module TEXT, new_tenant TEXT) RETURNS void AS $$
|
|
3097
|
-
BEGIN
|
|
3098
|
-
-- For ${SQL.plain(messagesTable.name)} table
|
|
3099
|
-
EXECUTE format('
|
|
3100
|
-
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
3101
|
-
FOR VALUES IN (emt_sanitize_name(''%s__%s'')) PARTITION BY LIST (is_archived);',
|
|
3102
|
-
emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || new_tenant), '${SQL.plain(messagesTable.name)}', new_module, new_tenant
|
|
3103
|
-
);
|
|
3104
|
-
|
|
3105
|
-
EXECUTE format('
|
|
3106
|
-
CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
|
|
3107
|
-
FOR VALUES IN (FALSE);',
|
|
3108
|
-
emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || new_tenant || '_active'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || new_tenant)
|
|
3109
|
-
);
|
|
3110
|
-
|
|
3111
|
-
EXECUTE format('
|
|
3112
|
-
CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
|
|
3113
|
-
FOR VALUES IN (TRUE);',
|
|
3114
|
-
emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || new_tenant || '_archived'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || new_tenant)
|
|
3115
|
-
);
|
|
3116
|
-
|
|
3117
|
-
-- For ${SQL.plain(streamsTable.name)} table
|
|
2457
|
+
//#endregion
|
|
2458
|
+
//#region src/eventStore/schema/migrations/0_43_0/0_43_0.migration.ts
|
|
2459
|
+
const migration_0_43_0_cleanupLegacySubscriptionSQL = SQL`
|
|
2460
|
+
DO $$
|
|
2461
|
+
BEGIN
|
|
2462
|
+
IF EXISTS (SELECT 1 FROM pg_tables WHERE tablename = 'emt_subscriptions') THEN
|
|
2463
|
+
-- Restore clean emt_add_partition (remove creation of emt_subscriptions partitions)
|
|
2464
|
+
CREATE OR REPLACE FUNCTION emt_add_partition(partition_name TEXT) RETURNS void AS $fnpar$
|
|
2465
|
+
BEGIN
|
|
2466
|
+
PERFORM emt_add_table_partition('${SQL.plain(messagesTable.name)}', partition_name);
|
|
2467
|
+
PERFORM emt_add_table_partition('${SQL.plain(streamsTable.name)}', partition_name);
|
|
2468
|
+
|
|
3118
2469
|
EXECUTE format('
|
|
3119
2470
|
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
3120
|
-
FOR VALUES IN (
|
|
3121
|
-
emt_sanitize_name('${SQL.plain(
|
|
3122
|
-
);
|
|
3123
|
-
|
|
3124
|
-
EXECUTE format('
|
|
3125
|
-
CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
|
|
3126
|
-
FOR VALUES IN (FALSE);',
|
|
3127
|
-
emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || new_tenant || '_active'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || new_tenant)
|
|
2471
|
+
FOR VALUES IN (%L);',
|
|
2472
|
+
emt_sanitize_name('${SQL.plain(processorsTable.name)}' || '_' || partition_name), '${SQL.plain(processorsTable.name)}', partition_name
|
|
3128
2473
|
);
|
|
3129
|
-
|
|
2474
|
+
|
|
3130
2475
|
EXECUTE format('
|
|
3131
|
-
CREATE TABLE IF NOT EXISTS %
|
|
3132
|
-
FOR VALUES IN (
|
|
3133
|
-
emt_sanitize_name('${SQL.plain(
|
|
2476
|
+
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
2477
|
+
FOR VALUES IN (%L);',
|
|
2478
|
+
emt_sanitize_name('${SQL.plain(projectionsTable.name)}' || '_' || partition_name), '${SQL.plain(projectionsTable.name)}', partition_name
|
|
3134
2479
|
);
|
|
3135
2480
|
END;
|
|
3136
|
-
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
|
|
3140
|
-
|
|
3141
|
-
|
|
3142
|
-
|
|
3143
|
-
|
|
3144
|
-
|
|
3145
|
-
|
|
3146
|
-
|
|
3147
|
-
|
|
3148
|
-
|
|
3149
|
-
|
|
3150
|
-
|
|
3151
|
-
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
|
-
EXECUTE format('
|
|
3155
|
-
CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
|
|
3156
|
-
FOR VALUES IN (FALSE);',
|
|
3157
|
-
emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || tenant_record.tenant || '_active'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || tenant_record.tenant)
|
|
3158
|
-
);
|
|
3159
|
-
|
|
3160
|
-
EXECUTE format('
|
|
3161
|
-
CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
|
|
3162
|
-
FOR VALUES IN (TRUE);',
|
|
3163
|
-
emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || tenant_record.tenant || '_archived'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || tenant_record.tenant)
|
|
3164
|
-
);
|
|
3165
|
-
|
|
3166
|
-
-- For ${SQL.plain(streamsTable.name)} table
|
|
3167
|
-
EXECUTE format('
|
|
3168
|
-
CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
|
|
3169
|
-
FOR VALUES IN (emt_sanitize_name(''%s__%s'')) PARTITION BY LIST (is_archived);',
|
|
3170
|
-
emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || tenant_record.tenant), '${SQL.plain(streamsTable.name)}', new_module, tenant_record.tenant
|
|
3171
|
-
);
|
|
3172
|
-
|
|
3173
|
-
EXECUTE format('
|
|
3174
|
-
CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
|
|
3175
|
-
FOR VALUES IN (FALSE);',
|
|
3176
|
-
emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || tenant_record.tenant || '_active'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || tenant_record.tenant)
|
|
3177
|
-
);
|
|
3178
|
-
|
|
3179
|
-
EXECUTE format('
|
|
3180
|
-
CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
|
|
3181
|
-
FOR VALUES IN (TRUE);',
|
|
3182
|
-
emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || tenant_record.tenant || '_archived'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || tenant_record.tenant)
|
|
3183
|
-
);
|
|
3184
|
-
END LOOP;
|
|
3185
|
-
END;
|
|
3186
|
-
$$ LANGUAGE plpgsql;
|
|
3187
|
-
`;
|
|
3188
|
-
const addTenantForAllModulesSQL = SQL`
|
|
3189
|
-
CREATE OR REPLACE FUNCTION add_tenant_for_all_modules(new_tenant TEXT) RETURNS void AS $$
|
|
2481
|
+
$fnpar$ LANGUAGE plpgsql;
|
|
2482
|
+
|
|
2483
|
+
-- Drop old subscriptions table if it exists
|
|
2484
|
+
DROP TABLE IF EXISTS emt_subscriptions CASCADE;
|
|
2485
|
+
|
|
2486
|
+
-- Drop old function if it exists
|
|
2487
|
+
DROP FUNCTION IF EXISTS store_subscription_checkpoint(character varying, bigint, bigint, bigint, xid8, text);
|
|
2488
|
+
|
|
2489
|
+
-- Restore clean store_processor_checkpoint (remove dual-write logic)
|
|
2490
|
+
CREATE OR REPLACE FUNCTION store_processor_checkpoint(
|
|
2491
|
+
p_processor_id TEXT,
|
|
2492
|
+
p_version BIGINT,
|
|
2493
|
+
p_position TEXT,
|
|
2494
|
+
p_check_position TEXT,
|
|
2495
|
+
p_transaction_id xid8,
|
|
2496
|
+
p_partition TEXT DEFAULT '${SQL.plain(defaultTag)}',
|
|
2497
|
+
p_processor_instance_id TEXT DEFAULT 'emt:unknown'
|
|
2498
|
+
) RETURNS INT AS $fn$
|
|
3190
2499
|
DECLARE
|
|
3191
|
-
|
|
2500
|
+
current_position TEXT;
|
|
3192
2501
|
BEGIN
|
|
3193
|
-
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3203
|
-
|
|
3204
|
-
|
|
3205
|
-
|
|
3206
|
-
|
|
3207
|
-
|
|
3208
|
-
|
|
3209
|
-
|
|
3210
|
-
|
|
3211
|
-
|
|
3212
|
-
|
|
3213
|
-
|
|
3214
|
-
|
|
3215
|
-
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
|
|
3227
|
-
|
|
3228
|
-
|
|
3229
|
-
|
|
3230
|
-
|
|
3231
|
-
|
|
3232
|
-
|
|
2502
|
+
IF p_check_position IS NOT NULL THEN
|
|
2503
|
+
UPDATE "emt_processors"
|
|
2504
|
+
SET
|
|
2505
|
+
"last_processed_checkpoint" = p_position,
|
|
2506
|
+
"last_processed_transaction_id" = p_transaction_id,
|
|
2507
|
+
"last_updated" = now()
|
|
2508
|
+
WHERE "processor_id" = p_processor_id
|
|
2509
|
+
AND "last_processed_checkpoint" = p_check_position
|
|
2510
|
+
AND "partition" = p_partition
|
|
2511
|
+
AND "version" = p_version;
|
|
2512
|
+
|
|
2513
|
+
IF FOUND THEN
|
|
2514
|
+
RETURN 1;
|
|
2515
|
+
END IF;
|
|
2516
|
+
|
|
2517
|
+
SELECT "last_processed_checkpoint" INTO current_position
|
|
2518
|
+
FROM "emt_processors"
|
|
2519
|
+
WHERE "processor_id" = p_processor_id
|
|
2520
|
+
AND "partition" = p_partition
|
|
2521
|
+
AND "version" = p_version ;
|
|
2522
|
+
|
|
2523
|
+
IF current_position = p_position THEN
|
|
2524
|
+
RETURN 0;
|
|
2525
|
+
ELSIF current_position > p_position THEN
|
|
2526
|
+
RETURN 3;
|
|
2527
|
+
ELSE
|
|
2528
|
+
RETURN 2;
|
|
2529
|
+
END IF;
|
|
2530
|
+
END IF;
|
|
2531
|
+
|
|
2532
|
+
BEGIN
|
|
2533
|
+
INSERT INTO "emt_processors"("processor_id", "version", "last_processed_checkpoint", "partition", "last_processed_transaction_id", "created_at", "last_updated")
|
|
2534
|
+
VALUES (p_processor_id, p_version, p_position, p_partition, p_transaction_id, now(), now());
|
|
2535
|
+
RETURN 1;
|
|
2536
|
+
EXCEPTION WHEN unique_violation THEN
|
|
2537
|
+
SELECT "last_processed_checkpoint" INTO current_position
|
|
2538
|
+
FROM "emt_processors"
|
|
2539
|
+
WHERE "processor_id" = p_processor_id
|
|
2540
|
+
AND "partition" = p_partition
|
|
2541
|
+
AND "version" = p_version;
|
|
2542
|
+
|
|
2543
|
+
IF current_position = p_position THEN
|
|
2544
|
+
RETURN 0;
|
|
2545
|
+
ELSE
|
|
2546
|
+
RETURN 2;
|
|
2547
|
+
END IF;
|
|
2548
|
+
END;
|
|
3233
2549
|
END;
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
|
|
2550
|
+
$fn$ LANGUAGE plpgsql;
|
|
2551
|
+
END IF;
|
|
2552
|
+
END $$;
|
|
2553
|
+
`;
|
|
2554
|
+
const migration_0_43_0_cleanupLegacySubscription = sqlMigration("emt:postgresql:eventstore:0.43.0:cleanup-legacy-subscription", [migration_0_43_0_cleanupLegacySubscriptionSQL]);
|
|
2555
|
+
|
|
2556
|
+
//#endregion
|
|
2557
|
+
//#region src/eventStore/schema/migrations/0_43_0/index.ts
|
|
2558
|
+
const migrations_0_43_0 = [migration_0_43_0_cleanupLegacySubscription];
|
|
2559
|
+
|
|
2560
|
+
//#endregion
|
|
2561
|
+
//#region src/eventStore/schema/migrations/index.ts
|
|
2562
|
+
const schemaSQL$1 = [
|
|
2563
|
+
streamsTableSQL,
|
|
2564
|
+
messagesTableSQL,
|
|
2565
|
+
projectionsTableSQL,
|
|
2566
|
+
processorsTableSQL,
|
|
2567
|
+
sanitizeNameSQL,
|
|
2568
|
+
addTablePartitions,
|
|
2569
|
+
addPartitionSQL,
|
|
2570
|
+
appendToStreamSQL,
|
|
2571
|
+
addDefaultPartitionSQL,
|
|
2572
|
+
storeSubscriptionCheckpointSQL,
|
|
2573
|
+
tryAcquireProcessorLockSQL,
|
|
2574
|
+
releaseProcessorLockSQL,
|
|
2575
|
+
registerProjectionSQL,
|
|
2576
|
+
activateProjectionSQL,
|
|
2577
|
+
deactivateProjectionSQL
|
|
2578
|
+
];
|
|
2579
|
+
const currentPostgreSQLEventStoreSchemaVersion = "0.43.0";
|
|
2580
|
+
const schemaMigration$1 = sqlMigration("emt:postgresql:eventstore:initial", schemaSQL$1);
|
|
2581
|
+
const pastEventStoreSchemaMigrations = [
|
|
2582
|
+
...migrations_0_38_7,
|
|
2583
|
+
...migrations_0_42_0,
|
|
2584
|
+
...migrations_0_43_0
|
|
2585
|
+
];
|
|
2586
|
+
const eventStoreSchemaMigrations = [...pastEventStoreSchemaMigrations, schemaMigration$1];
|
|
3237
2587
|
|
|
3238
2588
|
//#endregion
|
|
3239
2589
|
//#region src/eventStore/schema/readProcessorCheckpoint.ts
|
|
@@ -3315,12 +2665,6 @@ const schemaSQL = [
|
|
|
3315
2665
|
deactivateProjectionSQL
|
|
3316
2666
|
];
|
|
3317
2667
|
const schemaMigration = sqlMigration("emt:postgresql:eventstore:initial", schemaSQL);
|
|
3318
|
-
const eventStoreSchemaMigrations = [
|
|
3319
|
-
migration_0_38_7_and_older,
|
|
3320
|
-
migration_0_42_0_FromSubscriptionsToProcessors,
|
|
3321
|
-
migration_0_42_0_2_AddProcessorProjectionFunctions,
|
|
3322
|
-
schemaMigration
|
|
3323
|
-
];
|
|
3324
2668
|
const createEventStoreSchema = (connectionString, pool, hooks, options) => {
|
|
3325
2669
|
return pool.withTransaction(async (tx) => {
|
|
3326
2670
|
const context = await transactionToPostgreSQLProjectionHandlerContext(connectionString, pool, tx);
|
|
@@ -3371,24 +2715,32 @@ const getPostgreSQLEventStore = (connectionString, options = defaultPostgreSQLOp
|
|
|
3371
2715
|
const autoGenerateSchema = options.schema?.autoMigration === void 0 || options.schema?.autoMigration !== "None";
|
|
3372
2716
|
const inlineProjections = (options.projections ?? []).filter(({ type }) => type === "inline").map(({ projection }) => projection);
|
|
3373
2717
|
const migrate = async (migrationOptions) => {
|
|
3374
|
-
if (!migrateSchema)
|
|
3375
|
-
|
|
3376
|
-
|
|
3377
|
-
|
|
3378
|
-
|
|
3379
|
-
|
|
3380
|
-
|
|
3381
|
-
|
|
3382
|
-
|
|
3383
|
-
|
|
3384
|
-
|
|
3385
|
-
|
|
3386
|
-
|
|
3387
|
-
|
|
3388
|
-
|
|
3389
|
-
|
|
3390
|
-
|
|
3391
|
-
|
|
2718
|
+
if (!migrateSchema) {
|
|
2719
|
+
migrateSchema = createEventStoreSchema(connectionString, pool, {
|
|
2720
|
+
onBeforeSchemaCreated: async (context) => {
|
|
2721
|
+
if (options.hooks?.onBeforeSchemaCreated) await options.hooks.onBeforeSchemaCreated(context);
|
|
2722
|
+
},
|
|
2723
|
+
onAfterSchemaCreated: async (context) => {
|
|
2724
|
+
for (const projection of inlineProjections) if (projection.init) await projection.init({
|
|
2725
|
+
version: projection.version ?? 1,
|
|
2726
|
+
status: "active",
|
|
2727
|
+
registrationType: "inline",
|
|
2728
|
+
context: {
|
|
2729
|
+
...context,
|
|
2730
|
+
migrationOptions
|
|
2731
|
+
}
|
|
2732
|
+
});
|
|
2733
|
+
if (options.hooks?.onAfterSchemaCreated) await options.hooks.onAfterSchemaCreated(context);
|
|
2734
|
+
}
|
|
2735
|
+
}, migrationOptions);
|
|
2736
|
+
return migrateSchema;
|
|
2737
|
+
}
|
|
2738
|
+
const result = await migrateSchema;
|
|
2739
|
+
if (migrationOptions?.dryRun) migrateSchema = void 0;
|
|
2740
|
+
return {
|
|
2741
|
+
applied: [],
|
|
2742
|
+
skipped: result.applied.concat(result.skipped)
|
|
2743
|
+
};
|
|
3392
2744
|
};
|
|
3393
2745
|
const ensureSchemaExists = () => {
|
|
3394
2746
|
if (!autoGenerateSchema) return Promise.resolve();
|
|
@@ -3562,8 +2914,8 @@ const postgreSQLProjector = (options) => {
|
|
|
3562
2914
|
partition,
|
|
3563
2915
|
processorInstanceId,
|
|
3564
2916
|
projection: options.projection ? {
|
|
3565
|
-
name: options.projection.name ?? unknownTag
|
|
3566
|
-
kind: options.projection.kind ?? unknownTag
|
|
2917
|
+
name: options.projection.name ?? unknownTag,
|
|
2918
|
+
kind: options.projection.kind ?? unknownTag,
|
|
3567
2919
|
version: options.projection.version ?? version,
|
|
3568
2920
|
handlingType: "async"
|
|
3569
2921
|
} : void 0,
|
|
@@ -3701,10 +3053,15 @@ const postgreSQLEventStoreConsumer = (options) => {
|
|
|
3701
3053
|
reason: "No active processors"
|
|
3702
3054
|
};
|
|
3703
3055
|
return (await Promise.allSettled(activeProcessors.map(async (s) => {
|
|
3704
|
-
|
|
3705
|
-
|
|
3706
|
-
|
|
3707
|
-
|
|
3056
|
+
try {
|
|
3057
|
+
return await s.handle(messagesBatch, { connection: {
|
|
3058
|
+
connectionString: options.connectionString,
|
|
3059
|
+
pool
|
|
3060
|
+
} });
|
|
3061
|
+
} catch (error) {
|
|
3062
|
+
console.log(`Error during message batch processing for processor: ${s.id}`, error);
|
|
3063
|
+
throw error;
|
|
3064
|
+
}
|
|
3708
3065
|
}))).some((r) => r.status === "fulfilled" && r.value?.type !== "STOP") ? void 0 : { type: "STOP" };
|
|
3709
3066
|
};
|
|
3710
3067
|
const processorContext = {
|
|
@@ -3725,7 +3082,11 @@ const postgreSQLEventStoreConsumer = (options) => {
|
|
|
3725
3082
|
abortController?.abort();
|
|
3726
3083
|
await messagePuller.stop();
|
|
3727
3084
|
}
|
|
3728
|
-
|
|
3085
|
+
try {
|
|
3086
|
+
await start;
|
|
3087
|
+
} catch (error) {
|
|
3088
|
+
console.log("Error during consumer stop:", error);
|
|
3089
|
+
}
|
|
3729
3090
|
messagePuller = void 0;
|
|
3730
3091
|
abortController = null;
|
|
3731
3092
|
await stopProcessors();
|
|
@@ -3733,7 +3094,16 @@ const postgreSQLEventStoreConsumer = (options) => {
|
|
|
3733
3094
|
const init = async () => {
|
|
3734
3095
|
if (isInitialized) return;
|
|
3735
3096
|
const postgresProcessors = processors;
|
|
3736
|
-
for (const processor of postgresProcessors) if (processor.init)
|
|
3097
|
+
for (const processor of postgresProcessors) if (processor.init) try {
|
|
3098
|
+
await processor.init(processorContext);
|
|
3099
|
+
} catch (error) {
|
|
3100
|
+
console.log(`Error during processor initialization for processor: ${processor.id}. Stopping it.`, error);
|
|
3101
|
+
await processor.close(processorContext).catch((closeError) => {
|
|
3102
|
+
console.log(`Error during processor cleanup after failed initialization for processor: ${processor.id}`, closeError);
|
|
3103
|
+
});
|
|
3104
|
+
console.log(`Processor ${processor.id} stopped successfully after failed initialization.`);
|
|
3105
|
+
throw error;
|
|
3106
|
+
}
|
|
3737
3107
|
isInitialized = true;
|
|
3738
3108
|
};
|
|
3739
3109
|
return {
|
|
@@ -3760,9 +3130,13 @@ const postgreSQLEventStoreConsumer = (options) => {
|
|
|
3760
3130
|
return processor;
|
|
3761
3131
|
},
|
|
3762
3132
|
start: () => {
|
|
3763
|
-
if (isRunning)
|
|
3133
|
+
if (isRunning) {
|
|
3134
|
+
console.log("Consumer is already running. Returning the existing start promise.");
|
|
3135
|
+
return start;
|
|
3136
|
+
}
|
|
3764
3137
|
startedAwaiter.reset();
|
|
3765
3138
|
if (processors.length === 0) {
|
|
3139
|
+
console.log("Cannot start consumer without at least a single processor");
|
|
3766
3140
|
const error = new EmmettError("Cannot start consumer without at least a single processor");
|
|
3767
3141
|
startedAwaiter.reject(error);
|
|
3768
3142
|
return Promise.reject(error);
|
|
@@ -3780,16 +3154,25 @@ const postgreSQLEventStoreConsumer = (options) => {
|
|
|
3780
3154
|
pullingFrequencyInMs: pulling?.pullingFrequencyInMs ?? 50,
|
|
3781
3155
|
signal: abortController.signal
|
|
3782
3156
|
});
|
|
3783
|
-
if (!isInitialized)
|
|
3157
|
+
if (!isInitialized) {
|
|
3158
|
+
console.log("Initializing consumer before starting message pulling.");
|
|
3159
|
+
await init();
|
|
3160
|
+
}
|
|
3784
3161
|
const startFrom = zipPostgreSQLEventStoreMessageBatchPullerStartFrom(await Promise.all(processors.map(async (o) => {
|
|
3785
|
-
|
|
3786
|
-
|
|
3787
|
-
|
|
3788
|
-
|
|
3789
|
-
|
|
3790
|
-
|
|
3791
|
-
|
|
3162
|
+
try {
|
|
3163
|
+
return await o.start({
|
|
3164
|
+
execute: pool.execute,
|
|
3165
|
+
connection: {
|
|
3166
|
+
connectionString: options.connectionString,
|
|
3167
|
+
pool
|
|
3168
|
+
}
|
|
3169
|
+
});
|
|
3170
|
+
} catch (error) {
|
|
3171
|
+
console.log(`Error during processor start position retrieval for processor: ${o.id}. Stopping it.`, error);
|
|
3172
|
+
throw error;
|
|
3173
|
+
}
|
|
3792
3174
|
})));
|
|
3175
|
+
console.log(`Starting message pulling with start position: ${JSONSerializer.serialize(startFrom)}. Waiting for messages...`);
|
|
3793
3176
|
await messagePuller.start({
|
|
3794
3177
|
startFrom,
|
|
3795
3178
|
started: startedAwaiter
|
|
@@ -3831,11 +3214,11 @@ const rebuildPostgreSQLProjections = (options) => {
|
|
|
3831
3214
|
};
|
|
3832
3215
|
const projections = "projections" in options ? options.projections.map((p) => "projection" in p ? {
|
|
3833
3216
|
truncateOnStart: true,
|
|
3834
|
-
processorId: getProjectorId({ projectionName: p.projection.name ?? unknownTag
|
|
3217
|
+
processorId: getProjectorId({ projectionName: p.projection.name ?? unknownTag }),
|
|
3835
3218
|
...p
|
|
3836
3219
|
} : {
|
|
3837
3220
|
projection: p,
|
|
3838
|
-
processorId: getProjectorId({ projectionName: p.name ?? unknownTag
|
|
3221
|
+
processorId: getProjectorId({ projectionName: p.name ?? unknownTag }),
|
|
3839
3222
|
truncateOnStart: true
|
|
3840
3223
|
}) : [options];
|
|
3841
3224
|
for (const projectionDefinition of projections) consumer.projector({
|
|
@@ -3847,5 +3230,5 @@ const rebuildPostgreSQLProjections = (options) => {
|
|
|
3847
3230
|
};
|
|
3848
3231
|
|
|
3849
3232
|
//#endregion
|
|
3850
|
-
export { DefaultPostgreSQLEventStoreProcessorBatchSize, DefaultPostgreSQLEventStoreProcessorPullingFrequencyInMs, DefaultPostgreSQLProcessorLockPolicy, PostgreSQLEventStoreDefaultStreamVersion, PostgreSQLProjectionSpec, activateProjection, activateProjectionSQL, addDefaultPartitionSQL, addModuleForAllTenantsSQL, addModuleSQL, addPartitionSQL, addTablePartitions, addTenantForAllModulesSQL, addTenantSQL, appendToStream, appendToStreamSQL, assertSQLQueryResultMatches, callActivateProjection, callAppendToStream, callDeactivateProjection, callRegisterProjection, callReleaseProcessorLock, callStoreProcessorCheckpoint, callTryAcquireProcessorLock, callTryAcquireProjectionLock,
|
|
3233
|
+
export { DefaultPostgreSQLEventStoreProcessorBatchSize, DefaultPostgreSQLEventStoreProcessorPullingFrequencyInMs, DefaultPostgreSQLProcessorLockPolicy, PostgreSQLEventStoreDefaultStreamVersion, PostgreSQLProjectionSpec, activateProjection, activateProjectionSQL, addDefaultPartitionSQL, addModuleForAllTenantsSQL, addModuleSQL, addPartitionSQL, addTablePartitions, addTenantForAllModulesSQL, addTenantSQL, appendToStream, appendToStreamSQL, assertSQLQueryResultMatches, callActivateProjection, callAppendToStream, callDeactivateProjection, callRegisterProjection, callReleaseProcessorLock, callStoreProcessorCheckpoint, callTryAcquireProcessorLock, callTryAcquireProjectionLock, createEventStoreSchema, currentPostgreSQLEventStoreSchemaVersion, deactivateProjection, deactivateProjectionSQL, defaultPostgreSQLOptions, documentDoesNotExist, documentExists, documentMatchingExists, documentsAreTheSame, documentsMatchingHaveCount, eventInStream, eventStoreSchemaMigrations, eventsInStream, expectPongoDocuments, expectSQL, getPostgreSQLEventStore, handleProjections, messagesTableSQL, newEventsInStream, pastEventStoreSchemaMigrations, pongoMultiStreamProjection, pongoProjection, pongoSingleStreamProjection, postgreSQLCheckpointer, postgreSQLEventStoreConsumer, postgreSQLEventStoreMessageBatchPuller, postgreSQLProcessorLock, postgreSQLProjection, postgreSQLProjectionLock, postgreSQLProjector, postgreSQLRawBatchSQLProjection, postgreSQLRawSQLProjection, postgreSQLReactor, postgreSQLWorkflowProcessor, processorsTableSQL, projectionsTableSQL, readLastMessageGlobalPosition, readMessagesBatch, readProcessorCheckpoint, readProjectionInfo, readStream, rebuildPostgreSQLProjections, registerProjection, registerProjectionSQL, releaseProcessorLockSQL, sanitizeNameSQL, schemaMigration, schemaSQL, storeProcessorCheckpoint, storeSubscriptionCheckpointSQL, streamExists, streamsTableSQL, toProcessorLockKey, toProjectionLockKey, transactionToPostgreSQLProjectionHandlerContext, tryAcquireProcessorLockSQL, tryAcquireProjectionLockSQL, zipPostgreSQLEventStoreMessageBatchPullerStartFrom };
|
|
3851
3234
|
//# sourceMappingURL=index.js.map
|