@event-driven-io/emmett-postgresql 0.43.0-beta.15 → 0.43.0-beta.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,4 +1,4 @@
1
- import { AssertionError, EmmettError, ExpectedVersionConflictError, NO_CONCURRENCY_CHECK, STREAM_DOES_NOT_EXIST, STREAM_EXISTS, assertDeepEqual, assertEqual, assertExpectedVersionMatchesCurrent, assertFails, assertIsNotNull, assertIsNull, assertThatArray, assertTrue, asyncAwaiter, asyncRetry, bigInt, bigIntProcessorCheckpoint, defaultProcessorPartition, defaultProcessorVersion, downcastRecordedMessages, getCheckpoint, getProcessorInstanceId, getProjectorId, getWorkflowId, hashText, isBigint, isErrorConstructor, parseBigIntProcessorCheckpoint, projection, projector, reactor, reduceAsync, unknownTag as unknownTag$1, upcastRecordedMessage, workflowProcessor } from "@event-driven-io/emmett";
1
+ import { AssertionError, EmmettError, ExpectedVersionConflictError, NO_CONCURRENCY_CHECK, STREAM_DOES_NOT_EXIST, STREAM_EXISTS, assertDeepEqual, assertEqual, assertExpectedVersionMatchesCurrent, assertFails, assertIsNotNull, assertIsNull, assertThatArray, assertTrue, asyncAwaiter, asyncRetry, bigInt, bigIntProcessorCheckpoint, defaultProcessorPartition, defaultProcessorVersion, downcastRecordedMessages, getCheckpoint, getProcessorInstanceId, getProjectorId, getWorkflowId, hashText, isBigint, isErrorConstructor, parseBigIntProcessorCheckpoint, projection, projector, reactor, reduceAsync, unknownTag, upcastRecordedMessage, workflowProcessor } from "@event-driven-io/emmett";
2
2
  import { DumboError, JSONSerializer, SQL, UniqueConstraintError, dumbo, fromDatabaseDriverType, getFormatter, mapRows, runSQLMigrations, single, singleOrNull, sqlMigration } from "@event-driven-io/dumbo";
3
3
  import { v4, v7 } from "uuid";
4
4
  import { pongoClient } from "@event-driven-io/pongo";
@@ -8,7 +8,7 @@ import { pgDriver } from "@event-driven-io/pongo/pg";
8
8
  const emmettPrefix = "emt";
9
9
  const globalTag = "global";
10
10
  const defaultTag = `${"emt"}:default`;
11
- const unknownTag = `${"emt"}:unknown`;
11
+ const unknownTag$1 = `${"emt"}:unknown`;
12
12
  const globalNames = {
13
13
  module: `${"emt"}:module:${globalTag}`,
14
14
  tenant: `${"emt"}:tenant:${globalTag}`
@@ -98,36 +98,43 @@ const postgreSQLEventStoreMessageBatchPuller = ({ executor, batchSize, eachBatch
98
98
  let isRunning = false;
99
99
  let start;
100
100
  const pullMessages = async (options) => {
101
- let after;
102
101
  try {
103
- after = options.startFrom === "BEGINNING" ? 0n : options.startFrom === "END" ? (await readLastMessageGlobalPosition(executor)).currentGlobalPosition ?? 0n : parseBigIntProcessorCheckpoint(options.startFrom.lastCheckpoint);
104
- } catch (error) {
105
- options.started?.reject(error);
106
- throw error;
107
- }
108
- options.started?.resolve();
109
- const readMessagesOptions = {
110
- after,
111
- batchSize
112
- };
113
- let waitTime = 100;
114
- while (isRunning && !signal?.aborted) {
115
- const { messages, currentGlobalPosition, areMessagesLeft } = await readMessagesBatch(executor, readMessagesOptions);
116
- if (messages.length > 0) {
117
- const result = await eachBatch(messages);
118
- if (result && result.type === "STOP") {
102
+ let after;
103
+ try {
104
+ after = options.startFrom === "BEGINNING" ? 0n : options.startFrom === "END" ? (await readLastMessageGlobalPosition(executor)).currentGlobalPosition ?? 0n : parseBigIntProcessorCheckpoint(options.startFrom.lastCheckpoint);
105
+ } catch (error) {
106
+ console.log("Error occurred during reading last processor checkpoint:", error);
107
+ options.started?.reject(error);
108
+ throw error;
109
+ }
110
+ options.started?.resolve();
111
+ const readMessagesOptions = {
112
+ after,
113
+ batchSize
114
+ };
115
+ let waitTime = 100;
116
+ while (isRunning && !signal?.aborted) {
117
+ const { messages, currentGlobalPosition, areMessagesLeft } = await readMessagesBatch(executor, readMessagesOptions);
118
+ if (messages.length > 0) {
119
+ const result = await eachBatch(messages);
120
+ if (result && result.type === "STOP") {
121
+ isRunning = false;
122
+ break;
123
+ }
124
+ }
125
+ readMessagesOptions.after = currentGlobalPosition;
126
+ await new Promise((resolve) => setTimeout(resolve, waitTime));
127
+ if (stopWhen?.noMessagesLeft === true && !areMessagesLeft) {
128
+ console.log(`No messages left to process after reaching global position ${currentGlobalPosition}. Stopping the puller.`);
119
129
  isRunning = false;
120
130
  break;
121
131
  }
132
+ if (!areMessagesLeft) waitTime = Math.min(waitTime * 2, 1e3);
133
+ else waitTime = pullingFrequencyInMs;
122
134
  }
123
- readMessagesOptions.after = currentGlobalPosition;
124
- await new Promise((resolve) => setTimeout(resolve, waitTime));
125
- if (stopWhen?.noMessagesLeft === true && !areMessagesLeft) {
126
- isRunning = false;
127
- break;
128
- }
129
- if (!areMessagesLeft) waitTime = Math.min(waitTime * 2, 1e3);
130
- else waitTime = pullingFrequencyInMs;
135
+ } catch (error) {
136
+ console.log("Error occurred during message pulling:", error);
137
+ throw error;
131
138
  }
132
139
  };
133
140
  return {
@@ -239,7 +246,7 @@ CREATE OR REPLACE FUNCTION emt_try_acquire_processor_lock(
239
246
  p_processor_id TEXT,
240
247
  p_version INT,
241
248
  p_partition TEXT DEFAULT '${SQL.plain(defaultTag)}',
242
- p_processor_instance_id TEXT DEFAULT '${SQL.plain(unknownTag)}',
249
+ p_processor_instance_id TEXT DEFAULT '${SQL.plain(unknownTag$1)}',
243
250
  p_projection_name TEXT DEFAULT NULL,
244
251
  p_projection_type VARCHAR(1) DEFAULT NULL,
245
252
  p_projection_kind TEXT DEFAULT NULL,
@@ -248,6 +255,9 @@ CREATE OR REPLACE FUNCTION emt_try_acquire_processor_lock(
248
255
  RETURNS TABLE (acquired BOOLEAN, checkpoint TEXT)
249
256
  LANGUAGE plpgsql
250
257
  AS $emt_try_acquire_processor_lock$
258
+ DECLARE
259
+ current_position TEXT;
260
+ v_current_time TIMESTAMPTZ := clock_timestamp();
251
261
  BEGIN
252
262
  RETURN QUERY
253
263
  WITH lock_check AS (
@@ -265,16 +275,16 @@ BEGIN
265
275
  created_at,
266
276
  last_updated
267
277
  )
268
- SELECT p_processor_id, p_partition, p_version, p_processor_instance_id, 'running', '${SQL.plain(bigInt.toNormalizedString(0n))}', '0'::xid8, now(), now()
278
+ SELECT p_processor_id, p_partition, p_version, p_processor_instance_id, 'running', '${SQL.plain(bigInt.toNormalizedString(0n))}', '0'::xid8, v_current_time, v_current_time
269
279
  WHERE (SELECT lock_acquired FROM lock_check) = true
270
280
  ON CONFLICT (processor_id, partition, version) DO UPDATE
271
281
  SET processor_instance_id = p_processor_instance_id,
272
282
  status = 'running',
273
- last_updated = now()
283
+ last_updated = v_current_time
274
284
  WHERE ${SQL.plain(processorsTable.name)}.processor_instance_id = p_processor_instance_id
275
- OR ${SQL.plain(processorsTable.name)}.processor_instance_id = '${SQL.plain(unknownTag)}'
285
+ OR ${SQL.plain(processorsTable.name)}.processor_instance_id = '${SQL.plain(unknownTag$1)}'
276
286
  OR ${SQL.plain(processorsTable.name)}.status = 'stopped'
277
- OR ${SQL.plain(processorsTable.name)}.last_updated < now() - (p_lock_timeout_seconds || ' seconds')::interval
287
+ OR ${SQL.plain(processorsTable.name)}.last_updated < v_current_time - (p_lock_timeout_seconds || ' seconds')::interval
278
288
  RETURNING last_processed_checkpoint
279
289
  ),
280
290
  projection_status AS (
@@ -306,7 +316,7 @@ CREATE OR REPLACE FUNCTION emt_release_processor_lock(
306
316
  p_processor_id TEXT,
307
317
  p_partition TEXT,
308
318
  p_version INT,
309
- p_processor_instance_id TEXT DEFAULT '${SQL.plain(unknownTag)}',
319
+ p_processor_instance_id TEXT DEFAULT '${SQL.plain(unknownTag$1)}',
310
320
  p_projection_name TEXT DEFAULT NULL
311
321
  )
312
322
  RETURNS BOOLEAN
@@ -326,7 +336,7 @@ BEGIN
326
336
 
327
337
  UPDATE ${SQL.plain(processorsTable.name)}
328
338
  SET status = 'stopped',
329
- processor_instance_id = '${SQL.plain(unknownTag)}',
339
+ processor_instance_id = '${SQL.plain(unknownTag$1)}',
330
340
  last_updated = now()
331
341
  WHERE processor_id = p_processor_id
332
342
  AND partition = p_partition
@@ -379,6 +389,7 @@ const tryAcquireProcessorLock = async (execute, options) => {
379
389
  projectionKind: options.projection?.kind ?? null,
380
390
  lockTimeoutSeconds: options.lockTimeoutSeconds ?? 300
381
391
  })));
392
+ console.log(`Lock acquisition attempt for processor '${options.processorId}' with processor instance '${options.processorInstanceId}' and lock '${options.lockKey}' resulted in: ${acquired}`);
382
393
  return acquired ? {
383
394
  acquired: true,
384
395
  checkpoint
@@ -404,6 +415,7 @@ const releaseProcessorLock = async (execute, options) => {
404
415
  processorInstanceId: options.processorInstanceId,
405
416
  projectionName: options.projectionName ?? null
406
417
  })));
418
+ console.log(`Lock for processor '${options.processorId}' with processor instance '${options.processorInstanceId}' and lock '${options.lockKey}' released: ${result}`);
407
419
  return result;
408
420
  };
409
421
 
@@ -415,17 +427,26 @@ const postgreSQLProcessorLock = (options) => {
415
427
  const lockKey = options.lockKey ?? toProcessorLockKey(options);
416
428
  return {
417
429
  tryAcquire: async (context) => {
418
- if (acquired) return true;
430
+ if (acquired) {
431
+ console.log(`Lock for processor '${options.processorId}' is already acquired by this instance. Reusing the lock.`);
432
+ return true;
433
+ }
419
434
  const result = await tryAcquireProcessorLockWithRetry(context.execute, {
420
435
  ...options,
421
436
  lockKey
422
437
  });
423
- if (!result.acquired && options.lockAcquisitionPolicy?.type !== "skip") throw new EmmettError(`Failed to acquire lock for processor '${options.processorId}'`);
438
+ if (!result.acquired && options.lockAcquisitionPolicy?.type !== "skip") {
439
+ console.log(`Failed to acquire lock for processor '${options.processorId}' with policy '${options.lockAcquisitionPolicy?.type}'.`);
440
+ throw new EmmettError(`Failed to acquire lock for processor '${options.processorId}'`);
441
+ }
424
442
  acquired = result.acquired;
425
443
  return acquired;
426
444
  },
427
445
  release: async (context) => {
428
- if (!acquired) return;
446
+ if (!acquired) {
447
+ console.log(`Lock for processor '${options.processorId}' is not acquired by this instance. Skipping release.`);
448
+ return;
449
+ }
429
450
  const { projection, ...releaseOptions } = options;
430
451
  await releaseProcessorLock(context.execute, {
431
452
  ...releaseOptions,
@@ -1216,156 +1237,118 @@ const appendEventsRaw = (execute, streamId, streamType, messages, options) => si
1216
1237
  })));
1217
1238
 
1218
1239
  //#endregion
1219
- //#region src/eventStore/schema/migrations/0_38_7/0_38_7.migration.ts
1220
- const dropFutureConceptModuleAndTenantFunctions = SQL`
1221
- DO $$
1222
- BEGIN
1223
- -- Check and drop functions related to future concept of modules and tenants
1224
- IF EXISTS (SELECT 1 FROM pg_proc WHERE proname = 'add_module') THEN
1225
- DROP FUNCTION add_module(TEXT);
1226
- END IF;
1227
-
1228
- IF EXISTS (SELECT 1 FROM pg_proc WHERE proname = 'add_tenant') THEN
1229
- DROP FUNCTION add_tenant(TEXT, TEXT);
1230
- END IF;
1231
-
1232
- IF EXISTS (SELECT 1 FROM pg_proc WHERE proname = 'add_module_for_all_tenants') THEN
1233
- DROP FUNCTION add_module_for_all_tenants(TEXT);
1234
- END IF;
1235
-
1236
- IF EXISTS (SELECT 1 FROM pg_proc WHERE proname = 'add_tenant_for_all_modules') THEN
1237
- DROP FUNCTION add_tenant_for_all_modules(TEXT);
1238
- END IF;
1239
- END $$;
1240
- `;
1241
- const dropOldAppendToSQLWithoutGlobalPositions = SQL`
1242
- DO $$
1243
- DECLARE
1244
- v_current_return_type text;
1245
- BEGIN
1246
- -- Get the current return type definition as text
1247
- SELECT pg_get_function_result(p.oid)
1248
- INTO v_current_return_type
1249
- FROM pg_proc p
1250
- JOIN pg_namespace n ON p.pronamespace = n.oid
1251
- WHERE n.nspname = current_schema() -- or specify your schema
1252
- AND p.proname = 'emt_append_to_stream'
1253
- AND p.pronargs = 10; -- number of arguments
1254
-
1255
- -- Check if it contains the old column name
1256
- IF v_current_return_type IS NOT NULL AND
1257
- v_current_return_type LIKE '%last_global_position%' AND
1258
- v_current_return_type NOT LIKE '%global_positions%' THEN
1259
- DROP FUNCTION emt_append_to_stream(text[], jsonb[], jsonb[], text[], text[], text[], text, text, bigint, text);
1260
- RAISE NOTICE 'Old version of function dropped. Return type was: %', v_current_return_type;
1261
- END IF;
1262
- END $$;
1263
- `;
1264
- const migrationFromEventsToMessagesSQL = SQL`
1265
- DO $$
1240
+ //#region src/eventStore/schema/storeProcessorCheckpoint.ts
1241
+ const storeSubscriptionCheckpointSQL = createFunctionIfDoesNotExistSQL("store_processor_checkpoint", SQL`
1242
+ CREATE OR REPLACE FUNCTION store_processor_checkpoint(
1243
+ p_processor_id TEXT,
1244
+ p_version BIGINT,
1245
+ p_position TEXT,
1246
+ p_check_position TEXT,
1247
+ p_transaction_id xid8,
1248
+ p_partition TEXT DEFAULT '${SQL.plain(defaultTag)}',
1249
+ p_processor_instance_id TEXT DEFAULT '${SQL.plain(unknownTag$1)}'
1250
+ ) RETURNS INT AS $spc$
1266
1251
  DECLARE
1267
- partition_record RECORD;
1252
+ current_position TEXT;
1268
1253
  BEGIN
1269
- -- Rename the main table and its columns if it exists
1270
- IF EXISTS (SELECT 1 FROM pg_tables WHERE tablename = 'emt_events') THEN
1271
- -- Rename all partitions first
1272
- FOR partition_record IN
1273
- SELECT tablename
1274
- FROM pg_tables
1275
- WHERE tablename LIKE 'emt_events_%'
1276
- ORDER BY tablename DESC -- to handle child partitions first
1277
- LOOP
1278
- EXECUTE format('ALTER TABLE %I RENAME TO %I',
1279
- partition_record.tablename,
1280
- REPLACE(partition_record.tablename, 'events', 'messages'));
1281
- END LOOP;
1254
+ -- Handle the case when p_check_position is provided
1255
+ IF p_check_position IS NOT NULL THEN
1256
+ -- Try to update if the position matches p_check_position
1257
+ UPDATE "${SQL.plain(processorsTable.name)}"
1258
+ SET
1259
+ "last_processed_checkpoint" = p_position,
1260
+ "last_processed_transaction_id" = p_transaction_id,
1261
+ "last_updated" = now()
1262
+ WHERE "processor_id" = p_processor_id
1263
+ AND "last_processed_checkpoint" = p_check_position
1264
+ AND "partition" = p_partition
1265
+ AND "version" = p_version;
1282
1266
 
1283
- -- Rename the main table
1284
- ALTER TABLE emt_events RENAME TO emt_messages;
1285
-
1286
- -- Rename columns
1287
- ALTER TABLE emt_messages
1288
- RENAME COLUMN event_data TO message_data;
1289
- ALTER TABLE emt_messages
1290
- RENAME COLUMN event_metadata TO message_metadata;
1291
- ALTER TABLE emt_messages
1292
- RENAME COLUMN event_schema_version TO message_schema_version;
1293
- ALTER TABLE emt_messages
1294
- RENAME COLUMN event_type TO message_type;
1295
- ALTER TABLE emt_messages
1296
- RENAME COLUMN event_id TO message_id;
1297
- ALTER TABLE emt_messages
1298
- ADD COLUMN message_kind CHAR(1) NOT NULL DEFAULT 'E';
1267
+ IF FOUND THEN
1268
+ RETURN 1; -- Successfully updated
1269
+ END IF;
1299
1270
 
1300
- -- Rename sequence if it exists
1301
- IF EXISTS (SELECT 1 FROM pg_sequences WHERE sequencename = 'emt_global_event_position') THEN
1302
- ALTER SEQUENCE emt_global_event_position
1303
- RENAME TO emt_global_message_position;
1304
-
1305
- ALTER TABLE emt_messages
1306
- ALTER COLUMN global_position
1307
- SET DEFAULT nextval('emt_global_message_position');
1308
- END IF;
1309
- END IF;
1310
- END $$;`;
1311
- const migration_0_38_7_and_older = sqlMigration("emt:postgresql:eventstore:0.38.7:migrate-events-to-messages", [
1312
- dropFutureConceptModuleAndTenantFunctions,
1313
- dropOldAppendToSQLWithoutGlobalPositions,
1314
- migrationFromEventsToMessagesSQL
1315
- ]);
1271
+ -- Retrieve the current position
1272
+ SELECT "last_processed_checkpoint" INTO current_position
1273
+ FROM "${SQL.plain(processorsTable.name)}"
1274
+ WHERE "processor_id" = p_processor_id
1275
+ AND "partition" = p_partition
1276
+ AND "version" = p_version;
1316
1277
 
1317
- //#endregion
1318
- //#region src/eventStore/schema/migrations/0_38_7/0_38_7.snapshot.ts
1319
- const schema_0_38_7 = SQL`
1320
- DO $$
1321
- DECLARE
1322
- partition_record RECORD;
1323
- BEGIN
1324
- -- Rename the main table and its columns if it exists
1325
- IF EXISTS (SELECT 1 FROM pg_tables WHERE tablename = 'emt_events') THEN
1326
- -- Rename all partitions first
1327
- FOR partition_record IN
1328
- SELECT tablename
1329
- FROM pg_tables
1330
- WHERE tablename LIKE 'emt_events_%'
1331
- ORDER BY tablename DESC -- to handle child partitions first
1332
- LOOP
1333
- EXECUTE format('ALTER TABLE %I RENAME TO %I',
1334
- partition_record.tablename,
1335
- REPLACE(partition_record.tablename, 'events', 'messages'));
1336
- END LOOP;
1278
+ -- Return appropriate codes based on current position
1279
+ IF current_position = p_position THEN
1280
+ RETURN 0; -- Idempotent check: position already set
1281
+ ELSIF current_position > p_position THEN
1282
+ RETURN 3; -- Current ahead: another process has progressed further
1283
+ ELSE
1284
+ RETURN 2; -- Mismatch: check position doesn't match current
1285
+ END IF;
1286
+ END IF;
1337
1287
 
1338
- -- Rename the main table
1339
- ALTER TABLE emt_events RENAME TO emt_messages;
1340
-
1341
- -- Rename columns
1342
- ALTER TABLE emt_messages
1343
- RENAME COLUMN event_data TO message_data;
1344
- ALTER TABLE emt_messages
1345
- RENAME COLUMN event_metadata TO message_metadata;
1346
- ALTER TABLE emt_messages
1347
- RENAME COLUMN event_schema_version TO message_schema_version;
1348
- ALTER TABLE emt_messages
1349
- RENAME COLUMN event_type TO message_type;
1350
- ALTER TABLE emt_messages
1351
- RENAME COLUMN event_id TO message_id;
1352
- ALTER TABLE emt_messages
1353
- ADD COLUMN message_kind CHAR(1) NOT NULL DEFAULT 'E';
1288
+ -- Handle the case when p_check_position is NULL: Insert if not exists
1289
+ BEGIN
1290
+ INSERT INTO "${SQL.plain(processorsTable.name)}"("processor_id", "version", "last_processed_checkpoint", "partition", "last_processed_transaction_id", "created_at", "last_updated")
1291
+ VALUES (p_processor_id, p_version, p_position, p_partition, p_transaction_id, now(), now());
1292
+ RETURN 1; -- Successfully inserted
1293
+ EXCEPTION WHEN unique_violation THEN
1294
+ -- If insertion failed, it means the row already exists
1295
+ SELECT "last_processed_checkpoint" INTO current_position
1296
+ FROM "${SQL.plain(processorsTable.name)}"
1297
+ WHERE "processor_id" = p_processor_id
1298
+ AND "partition" = p_partition
1299
+ AND "version" = p_version;
1354
1300
 
1355
- -- Rename sequence if it exists
1356
- IF EXISTS (SELECT 1 FROM pg_sequences WHERE sequencename = 'emt_global_event_position') THEN
1357
- ALTER SEQUENCE emt_global_event_position
1358
- RENAME TO emt_global_message_position;
1359
-
1360
- ALTER TABLE emt_messages
1361
- ALTER COLUMN global_position
1362
- SET DEFAULT nextval('emt_global_message_position');
1363
- END IF;
1364
- END IF;
1365
- END $$;CREATE TABLE IF NOT EXISTS emt_streams(
1301
+ IF current_position = p_position THEN
1302
+ RETURN 0; -- Idempotent check: position already set
1303
+ ELSIF current_position > p_position THEN
1304
+ RETURN 3; -- Current ahead: another process has progressed further
1305
+ ELSE
1306
+ RETURN 2; -- Insertion failed, row already exists with different position
1307
+ END IF;
1308
+ END;
1309
+ END;
1310
+ $spc$ LANGUAGE plpgsql;
1311
+ `);
1312
+ const callStoreProcessorCheckpoint = (params) => SQL`
1313
+ SELECT store_processor_checkpoint(
1314
+ ${params.processorId},
1315
+ ${params.version},
1316
+ ${params.position},
1317
+ ${params.checkPosition},
1318
+ pg_current_xact_id(),
1319
+ ${params.partition},
1320
+ ${params.processorInstanceId}
1321
+ ) as result;`;
1322
+ const storeProcessorCheckpoint = async (execute, options) => {
1323
+ try {
1324
+ const { result } = await single(execute.command(callStoreProcessorCheckpoint({
1325
+ processorId: options.processorId,
1326
+ version: options.version ?? 1,
1327
+ position: options.newCheckpoint !== null ? options.newCheckpoint : null,
1328
+ checkPosition: options.lastProcessedCheckpoint !== null ? options.lastProcessedCheckpoint : null,
1329
+ partition: options.partition ?? defaultTag,
1330
+ processorInstanceId: options.processorInstanceId ?? unknownTag$1
1331
+ })));
1332
+ return result === 1 ? {
1333
+ success: true,
1334
+ newCheckpoint: options.newCheckpoint
1335
+ } : {
1336
+ success: false,
1337
+ reason: result === 0 ? "IGNORED" : result === 3 ? "CURRENT_AHEAD" : "MISMATCH"
1338
+ };
1339
+ } catch (error) {
1340
+ console.log(error);
1341
+ throw error;
1342
+ }
1343
+ };
1344
+
1345
+ //#endregion
1346
+ //#region src/eventStore/schema/tables.ts
1347
+ const streamsTableSQL = SQL`
1348
+ CREATE TABLE IF NOT EXISTS ${SQL.identifier(streamsTable.name)}(
1366
1349
  stream_id TEXT NOT NULL,
1367
1350
  stream_position BIGINT NOT NULL,
1368
- partition TEXT NOT NULL DEFAULT 'global',
1351
+ partition TEXT NOT NULL DEFAULT '${SQL.plain(defaultTag)}',
1369
1352
  stream_type TEXT NOT NULL,
1370
1353
  stream_metadata JSONB NOT NULL,
1371
1354
  is_archived BOOLEAN NOT NULL DEFAULT FALSE,
@@ -1373,40 +1356,62 @@ END $$;CREATE TABLE IF NOT EXISTS emt_streams(
1373
1356
  ) PARTITION BY LIST (partition);
1374
1357
 
1375
1358
  CREATE UNIQUE INDEX IF NOT EXISTS idx_streams_unique
1376
- ON emt_streams(stream_id, partition, is_archived)
1377
- INCLUDE (stream_position);
1359
+ ON ${SQL.identifier(streamsTable.name)}(stream_id, partition, is_archived)
1360
+ INCLUDE (stream_position);`;
1361
+ const messagesTableSQL = SQL`
1378
1362
  CREATE SEQUENCE IF NOT EXISTS emt_global_message_position;
1379
1363
 
1380
- CREATE TABLE IF NOT EXISTS emt_messages(
1381
- stream_id TEXT NOT NULL,
1364
+ CREATE TABLE IF NOT EXISTS ${SQL.identifier(messagesTable.name)}(
1382
1365
  stream_position BIGINT NOT NULL,
1383
- partition TEXT NOT NULL DEFAULT 'global',
1384
- message_kind CHAR(1) NOT NULL DEFAULT 'E',
1385
- message_data JSONB NOT NULL,
1386
- message_metadata JSONB NOT NULL,
1387
- message_schema_version TEXT NOT NULL,
1388
- message_type TEXT NOT NULL,
1389
- message_id TEXT NOT NULL,
1390
- is_archived BOOLEAN NOT NULL DEFAULT FALSE,
1391
1366
  global_position BIGINT DEFAULT nextval('emt_global_message_position'),
1392
1367
  transaction_id XID8 NOT NULL,
1393
1368
  created TIMESTAMPTZ NOT NULL DEFAULT now(),
1369
+ is_archived BOOLEAN NOT NULL DEFAULT FALSE,
1370
+ message_kind VARCHAR(1) NOT NULL DEFAULT 'E',
1371
+ stream_id TEXT NOT NULL,
1372
+ partition TEXT NOT NULL DEFAULT '${SQL.plain(defaultTag)}',
1373
+ message_schema_version TEXT NOT NULL,
1374
+ message_id TEXT NOT NULL,
1375
+ message_type TEXT NOT NULL,
1376
+ message_data JSONB NOT NULL,
1377
+ message_metadata JSONB NOT NULL,
1394
1378
  PRIMARY KEY (stream_id, stream_position, partition, is_archived)
1379
+ ) PARTITION BY LIST (partition);`;
1380
+ const processorsTableSQL = SQL`
1381
+ CREATE TABLE IF NOT EXISTS ${SQL.identifier(processorsTable.name)}(
1382
+ last_processed_transaction_id XID8 NOT NULL,
1383
+ version INT NOT NULL DEFAULT 1,
1384
+ processor_id TEXT NOT NULL,
1385
+ partition TEXT NOT NULL DEFAULT '${SQL.plain(defaultTag)}',
1386
+ status TEXT NOT NULL DEFAULT 'stopped',
1387
+ last_processed_checkpoint TEXT NOT NULL,
1388
+ processor_instance_id TEXT DEFAULT '${SQL.plain(unknownTag$1)}',
1389
+ created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
1390
+ last_updated TIMESTAMPTZ NOT NULL DEFAULT now(),
1391
+ PRIMARY KEY (processor_id, partition, version)
1395
1392
  ) PARTITION BY LIST (partition);
1396
- CREATE TABLE IF NOT EXISTS emt_subscriptions(
1397
- subscription_id TEXT NOT NULL,
1398
- version INT NOT NULL DEFAULT 1,
1399
- partition TEXT NOT NULL DEFAULT 'global',
1400
- last_processed_position BIGINT NOT NULL,
1401
- last_processed_transaction_id XID8 NOT NULL,
1402
- PRIMARY KEY (subscription_id, partition, version)
1393
+ `;
1394
+ const projectionsTableSQL = SQL`
1395
+ CREATE TABLE IF NOT EXISTS ${SQL.identifier(projectionsTable.name)}(
1396
+ version INT NOT NULL DEFAULT 1,
1397
+ type VARCHAR(1) NOT NULL,
1398
+ name TEXT NOT NULL,
1399
+ partition TEXT NOT NULL DEFAULT '${SQL.plain(defaultTag)}',
1400
+ kind TEXT NOT NULL,
1401
+ status TEXT NOT NULL,
1402
+ definition JSONB NOT NULL DEFAULT '{}'::jsonb,
1403
+ created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
1404
+ last_updated TIMESTAMPTZ NOT NULL DEFAULT now(),
1405
+ PRIMARY KEY (name, partition, version)
1403
1406
  ) PARTITION BY LIST (partition);
1404
- CREATE OR REPLACE FUNCTION emt_sanitize_name(input_name TEXT) RETURNS TEXT AS $$
1407
+ `;
1408
+ const sanitizeNameSQL = createFunctionIfDoesNotExistSQL("emt_sanitize_name", SQL`CREATE OR REPLACE FUNCTION emt_sanitize_name(input_name TEXT) RETURNS TEXT AS $emt_sanitize_name$
1405
1409
  BEGIN
1406
1410
  RETURN REGEXP_REPLACE(input_name, '[^a-zA-Z0-9_]', '_', 'g');
1407
1411
  END;
1408
- $$ LANGUAGE plpgsql;
1409
- CREATE OR REPLACE FUNCTION emt_add_table_partition(tableName TEXT, partition_name TEXT) RETURNS void AS $$
1412
+ $emt_sanitize_name$ LANGUAGE plpgsql;`);
1413
+ const addTablePartitions = createFunctionIfDoesNotExistSQL("emt_add_table_partition", SQL`
1414
+ CREATE OR REPLACE FUNCTION emt_add_table_partition(tableName TEXT, partition_name TEXT) RETURNS void AS $emt_add_table_partition$
1410
1415
  DECLARE
1411
1416
  v_main_partiton_name TEXT;
1412
1417
  v_active_partiton_name TEXT;
@@ -1436,20 +1441,215 @@ CREATE OR REPLACE FUNCTION emt_sanitize_name(input_name TEXT) RETURNS TEXT AS $$
1436
1441
  v_archived_partiton_name, v_main_partiton_name
1437
1442
  );
1438
1443
  END;
1439
- $$ LANGUAGE plpgsql;
1440
- CREATE OR REPLACE FUNCTION emt_add_partition(partition_name TEXT) RETURNS void AS $$
1444
+ $emt_add_table_partition$ LANGUAGE plpgsql;`);
1445
+ const addPartitionSQL = createFunctionIfDoesNotExistSQL("emt_add_partition", SQL`
1446
+ CREATE OR REPLACE FUNCTION emt_add_partition(partition_name TEXT) RETURNS void AS $emt_add_partition$
1441
1447
  BEGIN
1442
- PERFORM emt_add_table_partition('emt_messages', partition_name);
1443
- PERFORM emt_add_table_partition('emt_streams', partition_name);
1448
+ PERFORM emt_add_table_partition('${SQL.plain(messagesTable.name)}', partition_name);
1449
+ PERFORM emt_add_table_partition('${SQL.plain(streamsTable.name)}', partition_name);
1450
+
1451
+ EXECUTE format('
1452
+ CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
1453
+ FOR VALUES IN (%L);',
1454
+ emt_sanitize_name('${SQL.plain(processorsTable.name)}' || '_' || partition_name), '${SQL.plain(processorsTable.name)}', partition_name
1455
+ );
1444
1456
 
1445
1457
  EXECUTE format('
1446
1458
  CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
1447
1459
  FOR VALUES IN (%L);',
1448
- emt_sanitize_name('emt_subscriptions' || '_' || partition_name), 'emt_subscriptions', partition_name
1460
+ emt_sanitize_name('${SQL.plain(projectionsTable.name)}' || '_' || partition_name), '${SQL.plain(projectionsTable.name)}', partition_name
1449
1461
  );
1450
1462
  END;
1451
- $$ LANGUAGE plpgsql;
1463
+ $emt_add_partition$ LANGUAGE plpgsql;`);
1464
+ const addModuleSQL = SQL`
1465
+ CREATE OR REPLACE FUNCTION add_module(new_module TEXT) RETURNS void AS $$
1466
+ BEGIN
1467
+ -- For ${SQL.plain(messagesTable.name)} table
1468
+ EXECUTE format('
1469
+ CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
1470
+ FOR VALUES IN (emt_sanitize_name(%L || ''__'' || %L)) PARTITION BY LIST (is_archived);',
1471
+ emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}'), '${SQL.plain(messagesTable.name)}', new_module, '${SQL.plain(globalTag)}'
1472
+ );
1473
+
1474
+ EXECUTE format('
1475
+ CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
1476
+ FOR VALUES IN (FALSE);',
1477
+ emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}' || '_active'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}')
1478
+ );
1479
+
1480
+ EXECUTE format('
1481
+ CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
1482
+ FOR VALUES IN (TRUE);',
1483
+ emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}' || '_archived'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}')
1484
+ );
1485
+
1486
+ -- For ${SQL.plain(streamsTable.name)} table
1487
+ EXECUTE format('
1488
+ CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
1489
+ FOR VALUES IN (emt_sanitize_name(%L || ''__'' || %L)) PARTITION BY LIST (is_archived);',
1490
+ emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}'), '${SQL.plain(streamsTable.name)}', new_module, '${SQL.plain(globalTag)}'
1491
+ );
1492
+
1493
+ EXECUTE format('
1494
+ CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
1495
+ FOR VALUES IN (FALSE);',
1496
+ emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}' || '_active'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}')
1497
+ );
1498
+
1499
+ EXECUTE format('
1500
+ CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
1501
+ FOR VALUES IN (TRUE);',
1502
+ emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}' || '_archived'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}')
1503
+ );
1504
+ END;
1505
+ $$ LANGUAGE plpgsql;
1506
+ `;
1507
+ const addTenantSQL = SQL`
1508
+ CREATE OR REPLACE FUNCTION add_tenant(new_module TEXT, new_tenant TEXT) RETURNS void AS $$
1509
+ BEGIN
1510
+ -- For ${SQL.plain(messagesTable.name)} table
1511
+ EXECUTE format('
1512
+ CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
1513
+ FOR VALUES IN (emt_sanitize_name(''%s__%s'')) PARTITION BY LIST (is_archived);',
1514
+ emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || new_tenant), '${SQL.plain(messagesTable.name)}', new_module, new_tenant
1515
+ );
1516
+
1517
+ EXECUTE format('
1518
+ CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
1519
+ FOR VALUES IN (FALSE);',
1520
+ emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || new_tenant || '_active'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || new_tenant)
1521
+ );
1522
+
1523
+ EXECUTE format('
1524
+ CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
1525
+ FOR VALUES IN (TRUE);',
1526
+ emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || new_tenant || '_archived'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || new_tenant)
1527
+ );
1528
+
1529
+ -- For ${SQL.plain(streamsTable.name)} table
1530
+ EXECUTE format('
1531
+ CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
1532
+ FOR VALUES IN (emt_sanitize_name(''%s__%s'')) PARTITION BY LIST (is_archived);',
1533
+ emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || new_tenant), '${SQL.plain(streamsTable.name)}', new_module, new_tenant
1534
+ );
1535
+
1536
+ EXECUTE format('
1537
+ CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
1538
+ FOR VALUES IN (FALSE);',
1539
+ emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || new_tenant || '_active'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || new_tenant)
1540
+ );
1541
+
1542
+ EXECUTE format('
1543
+ CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
1544
+ FOR VALUES IN (TRUE);',
1545
+ emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || new_tenant || '_archived'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || new_tenant)
1546
+ );
1547
+ END;
1548
+ $$ LANGUAGE plpgsql;
1549
+ `;
1550
+ const addModuleForAllTenantsSQL = SQL`
1551
+ CREATE OR REPLACE FUNCTION add_module_for_all_tenants(new_module TEXT) RETURNS void AS $$
1552
+ DECLARE
1553
+ tenant_record RECORD;
1554
+ BEGIN
1555
+ PERFORM add_module(new_module);
1556
+
1557
+ FOR tenant_record IN SELECT DISTINCT tenant FROM ${SQL.plain(messagesTable.name)}
1558
+ LOOP
1559
+ -- For ${SQL.plain(messagesTable.name)} table
1560
+ EXECUTE format('
1561
+ CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
1562
+ FOR VALUES IN (emt_sanitize_name(''%s__%s'')) PARTITION BY LIST (is_archived);',
1563
+ emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || tenant_record.tenant), '${SQL.plain(messagesTable.name)}', new_module, tenant_record.tenant
1564
+ );
1565
+
1566
+ EXECUTE format('
1567
+ CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
1568
+ FOR VALUES IN (FALSE);',
1569
+ emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || tenant_record.tenant || '_active'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || tenant_record.tenant)
1570
+ );
1571
+
1572
+ EXECUTE format('
1573
+ CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
1574
+ FOR VALUES IN (TRUE);',
1575
+ emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || tenant_record.tenant || '_archived'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || tenant_record.tenant)
1576
+ );
1577
+
1578
+ -- For ${SQL.plain(streamsTable.name)} table
1579
+ EXECUTE format('
1580
+ CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
1581
+ FOR VALUES IN (emt_sanitize_name(''%s__%s'')) PARTITION BY LIST (is_archived);',
1582
+ emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || tenant_record.tenant), '${SQL.plain(streamsTable.name)}', new_module, tenant_record.tenant
1583
+ );
1584
+
1585
+ EXECUTE format('
1586
+ CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
1587
+ FOR VALUES IN (FALSE);',
1588
+ emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || tenant_record.tenant || '_active'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || tenant_record.tenant)
1589
+ );
1590
+
1591
+ EXECUTE format('
1592
+ CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
1593
+ FOR VALUES IN (TRUE);',
1594
+ emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || tenant_record.tenant || '_archived'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || tenant_record.tenant)
1595
+ );
1596
+ END LOOP;
1597
+ END;
1598
+ $$ LANGUAGE plpgsql;
1599
+ `;
1600
+ const addTenantForAllModulesSQL = SQL`
1601
+ CREATE OR REPLACE FUNCTION add_tenant_for_all_modules(new_tenant TEXT) RETURNS void AS $$
1602
+ DECLARE
1603
+ module_record RECORD;
1604
+ BEGIN
1605
+ FOR module_record IN SELECT DISTINCT partitionname FROM pg_partman.part_config WHERE parent_table = '${SQL.plain(messagesTable.name)}'
1606
+ LOOP
1607
+ -- For ${SQL.plain(messagesTable.name)} table
1608
+ EXECUTE format('
1609
+ CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
1610
+ FOR VALUES IN (emt_sanitize_name(''%s__%s'')) PARTITION BY LIST (is_archived);',
1611
+ emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || module_record.partitionname || '__' || new_tenant), '${SQL.plain(messagesTable.name)}', module_record.partitionname, new_tenant
1612
+ );
1613
+
1614
+ EXECUTE format('
1615
+ CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
1616
+ FOR VALUES IN (FALSE);',
1617
+ emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || module_record.partitionname || '__' || new_tenant || '_active'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || module_record.partitionname || '__' || new_tenant)
1618
+ );
1619
+
1620
+ EXECUTE format('
1621
+ CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
1622
+ FOR VALUES IN (TRUE);',
1623
+ emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || module_record.partitionname || '__' || new_tenant || '_archived'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || module_record.partitionname || '__' || new_tenant)
1624
+ );
1625
+
1626
+ -- For ${SQL.plain(streamsTable.name)} table
1627
+ EXECUTE format('
1628
+ CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
1629
+ FOR VALUES IN (emt_sanitize_name(''%s__%s'')) PARTITION BY LIST (is_archived);',
1630
+ emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || module_record.partitionname || '__' || new_tenant), '${SQL.plain(streamsTable.name)}', module_record.partitionname, new_tenant
1631
+ );
1632
+
1633
+ EXECUTE format('
1634
+ CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
1635
+ FOR VALUES IN (FALSE);',
1636
+ emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || module_record.partitionname || '__' || new_tenant || '_active'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || module_record.partitionname || '__' || new_tenant)
1637
+ );
1638
+
1639
+ EXECUTE format('
1640
+ CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
1641
+ FOR VALUES IN (TRUE);',
1642
+ emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || module_record.partitionname || '__' || new_tenant || '_archived'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || module_record.partitionname || '__' || new_tenant)
1643
+ );
1644
+ END LOOP;
1645
+ END;
1646
+ $$ LANGUAGE plpgsql;
1647
+ `;
1648
+ const addDefaultPartitionSQL = SQL`SELECT emt_add_partition('${SQL.plain(defaultTag)}');`;
1452
1649
 
1650
+ //#endregion
1651
+ //#region src/eventStore/schema/migrations/0_38_7/0_38_7.migration.ts
1652
+ const dropFutureConceptModuleAndTenantFunctions = SQL`
1453
1653
  DO $$
1454
1654
  BEGIN
1455
1655
  -- Check and drop functions related to future concept of modules and tenants
@@ -1469,9 +1669,9 @@ CREATE OR REPLACE FUNCTION emt_sanitize_name(input_name TEXT) RETURNS TEXT AS $$
1469
1669
  DROP FUNCTION add_tenant_for_all_modules(TEXT);
1470
1670
  END IF;
1471
1671
  END $$;
1472
-
1473
-
1474
- DO $$
1672
+ `;
1673
+ const dropOldAppendToSQLWithoutGlobalPositions = SQL`
1674
+ DO $$
1475
1675
  DECLARE
1476
1676
  v_current_return_type text;
1477
1677
  BEGIN
@@ -1490,158 +1690,156 @@ CREATE OR REPLACE FUNCTION emt_sanitize_name(input_name TEXT) RETURNS TEXT AS $$
1490
1690
  v_current_return_type NOT LIKE '%global_positions%' THEN
1491
1691
  DROP FUNCTION emt_append_to_stream(text[], jsonb[], jsonb[], text[], text[], text[], text, text, bigint, text);
1492
1692
  RAISE NOTICE 'Old version of function dropped. Return type was: %', v_current_return_type;
1493
- END IF;
1494
- END $$;
1495
- CREATE OR REPLACE FUNCTION emt_append_to_stream(
1496
- v_message_ids text[],
1497
- v_messages_data jsonb[],
1498
- v_messages_metadata jsonb[],
1499
- v_message_schema_versions text[],
1500
- v_message_types text[],
1501
- v_message_kinds text[],
1502
- v_stream_id text,
1503
- v_stream_type text,
1504
- v_expected_stream_position bigint DEFAULT NULL,
1505
- v_partition text DEFAULT emt_sanitize_name('default_partition')
1506
- ) RETURNS TABLE (
1507
- success boolean,
1508
- next_stream_position bigint,
1509
- global_positions bigint[],
1510
- transaction_id xid8
1511
- ) LANGUAGE plpgsql
1512
- AS $$
1513
- DECLARE
1514
- v_next_stream_position bigint;
1515
- v_position bigint;
1516
- v_updated_rows int;
1517
- v_transaction_id xid8;
1518
- v_global_positions bigint[];
1519
- BEGIN
1520
- v_transaction_id := pg_current_xact_id();
1521
-
1522
- IF v_expected_stream_position IS NULL THEN
1523
- SELECT COALESCE(
1524
- (SELECT stream_position
1525
- FROM emt_streams
1526
- WHERE stream_id = v_stream_id
1527
- AND partition = v_partition
1528
- AND is_archived = FALSE
1529
- LIMIT 1),
1530
- 0
1531
- ) INTO v_expected_stream_position;
1532
- END IF;
1533
-
1534
- v_next_stream_position := v_expected_stream_position + array_upper(v_messages_data, 1);
1535
-
1536
- IF v_expected_stream_position = 0 THEN
1537
- INSERT INTO emt_streams
1538
- (stream_id, stream_position, partition, stream_type, stream_metadata, is_archived)
1539
- VALUES
1540
- (v_stream_id, v_next_stream_position, v_partition, v_stream_type, '{}', FALSE);
1541
- ELSE
1542
- UPDATE emt_streams as s
1543
- SET stream_position = v_next_stream_position
1544
- WHERE stream_id = v_stream_id AND stream_position = v_expected_stream_position AND partition = v_partition AND is_archived = FALSE;
1545
-
1546
- get diagnostics v_updated_rows = row_count;
1547
1693
 
1548
- IF v_updated_rows = 0 THEN
1549
- RETURN QUERY SELECT FALSE, NULL::bigint, NULL::bigint[], NULL::xid8;
1550
- RETURN;
1551
- END IF;
1694
+ CREATE OR REPLACE FUNCTION emt_append_to_stream(
1695
+ v_message_ids text[],
1696
+ v_messages_data jsonb[],
1697
+ v_messages_metadata jsonb[],
1698
+ v_message_schema_versions text[],
1699
+ v_message_types text[],
1700
+ v_message_kinds text[],
1701
+ v_stream_id text,
1702
+ v_stream_type text,
1703
+ v_expected_stream_position bigint DEFAULT NULL,
1704
+ v_partition text DEFAULT emt_sanitize_name('default_partition')
1705
+ ) RETURNS TABLE (
1706
+ success boolean,
1707
+ next_stream_position bigint,
1708
+ global_positions bigint[],
1709
+ transaction_id xid8
1710
+ ) LANGUAGE plpgsql
1711
+ AS $emt_append_to_stream$
1712
+ DECLARE
1713
+ v_next_stream_position bigint;
1714
+ v_position bigint;
1715
+ v_updated_rows int;
1716
+ v_transaction_id xid8;
1717
+ v_global_positions bigint[];
1718
+ BEGIN
1719
+ v_transaction_id := pg_current_xact_id();
1720
+
1721
+ IF v_expected_stream_position IS NULL THEN
1722
+ SELECT COALESCE(
1723
+ (SELECT stream_position
1724
+ FROM emt_streams
1725
+ WHERE stream_id = v_stream_id
1726
+ AND partition = v_partition
1727
+ AND is_archived = FALSE
1728
+ LIMIT 1),
1729
+ 0
1730
+ ) INTO v_expected_stream_position;
1731
+ END IF;
1732
+
1733
+ v_next_stream_position := v_expected_stream_position + array_upper(v_messages_data, 1);
1734
+
1735
+ IF v_expected_stream_position = 0 THEN
1736
+ INSERT INTO emt_streams
1737
+ (stream_id, stream_position, partition, stream_type, stream_metadata, is_archived)
1738
+ VALUES
1739
+ (v_stream_id, v_next_stream_position, v_partition, v_stream_type, '{}', FALSE);
1740
+ ELSE
1741
+ UPDATE emt_streams as s
1742
+ SET stream_position = v_next_stream_position
1743
+ WHERE stream_id = v_stream_id AND stream_position = v_expected_stream_position AND partition = v_partition AND is_archived = FALSE;
1744
+
1745
+ get diagnostics v_updated_rows = row_count;
1746
+
1747
+ IF v_updated_rows = 0 THEN
1748
+ RETURN QUERY SELECT FALSE, NULL::bigint, NULL::bigint[], NULL::xid8;
1749
+ RETURN;
1750
+ END IF;
1751
+ END IF;
1752
+
1753
+ WITH ev AS (
1754
+ SELECT row_number() OVER () + v_expected_stream_position AS stream_position,
1755
+ message_data,
1756
+ message_metadata,
1757
+ schema_version,
1758
+ message_id,
1759
+ message_type,
1760
+ message_kind
1761
+ FROM (
1762
+ SELECT *
1763
+ FROM
1764
+ unnest(v_message_ids, v_messages_data, v_messages_metadata, v_message_schema_versions, v_message_types, v_message_kinds)
1765
+ AS message(message_id, message_data, message_metadata, schema_version, message_type, message_kind)
1766
+ ) AS message
1767
+ ),
1768
+ all_messages_insert AS (
1769
+ INSERT INTO emt_messages
1770
+ (stream_id, stream_position, partition, message_data, message_metadata, message_schema_version, message_type, message_kind, message_id, transaction_id)
1771
+ SELECT
1772
+ v_stream_id, ev.stream_position, v_partition, ev.message_data, ev.message_metadata, ev.schema_version, ev.message_type, ev.message_kind, ev.message_id, v_transaction_id
1773
+ FROM ev
1774
+ RETURNING global_position
1775
+ )
1776
+ SELECT
1777
+ array_agg(global_position ORDER BY global_position) INTO v_global_positions
1778
+ FROM
1779
+ all_messages_insert;
1780
+
1781
+ RETURN QUERY SELECT TRUE, v_next_stream_position, v_global_positions, v_transaction_id;
1782
+ END;
1783
+ $emt_append_to_stream$;
1552
1784
  END IF;
1553
-
1554
- WITH ev AS (
1555
- SELECT row_number() OVER () + v_expected_stream_position AS stream_position,
1556
- message_data,
1557
- message_metadata,
1558
- schema_version,
1559
- message_id,
1560
- message_type,
1561
- message_kind
1562
- FROM (
1563
- SELECT *
1564
- FROM
1565
- unnest(v_message_ids, v_messages_data, v_messages_metadata, v_message_schema_versions, v_message_types, v_message_kinds)
1566
- AS message(message_id, message_data, message_metadata, schema_version, message_type, message_kind)
1567
- ) AS message
1568
- ),
1569
- all_messages_insert AS (
1570
- INSERT INTO emt_messages
1571
- (stream_id, stream_position, partition, message_data, message_metadata, message_schema_version, message_type, message_kind, message_id, transaction_id)
1572
- SELECT
1573
- v_stream_id, ev.stream_position, v_partition, ev.message_data, ev.message_metadata, ev.schema_version, ev.message_type, ev.message_kind, ev.message_id, v_transaction_id
1574
- FROM ev
1575
- RETURNING global_position
1576
- )
1577
- SELECT
1578
- array_agg(global_position ORDER BY global_position) INTO v_global_positions
1579
- FROM
1580
- all_messages_insert;
1581
-
1582
- RETURN QUERY SELECT TRUE, v_next_stream_position, v_global_positions, v_transaction_id;
1583
- END;
1584
- $$;
1585
- SELECT emt_add_partition('emt:default');
1586
- CREATE OR REPLACE FUNCTION store_subscription_checkpoint(
1587
- p_subscription_id VARCHAR(100),
1588
- p_version BIGINT,
1589
- p_position BIGINT,
1590
- p_check_position BIGINT,
1591
- p_transaction_id xid8,
1592
- p_partition TEXT DEFAULT 'emt:default'
1593
- ) RETURNS INT AS $$
1785
+ END $$;
1786
+ `;
1787
+ const migrationFromEventsToMessagesSQL = SQL`
1788
+ DO $$
1594
1789
  DECLARE
1595
- current_position BIGINT;
1790
+ partition_record RECORD;
1596
1791
  BEGIN
1597
- -- Handle the case when p_check_position is provided
1598
- IF p_check_position IS NOT NULL THEN
1599
- -- Try to update if the position matches p_check_position
1600
- UPDATE "emt_subscriptions"
1601
- SET
1602
- "last_processed_position" = p_position,
1603
- "last_processed_transaction_id" = p_transaction_id
1604
- WHERE "subscription_id" = p_subscription_id AND "last_processed_position" = p_check_position AND "partition" = p_partition;
1605
-
1606
- IF FOUND THEN
1607
- RETURN 1; -- Successfully updated
1608
- END IF;
1609
-
1610
- -- Retrieve the current position
1611
- SELECT "last_processed_position" INTO current_position
1612
- FROM "emt_subscriptions"
1613
- WHERE "subscription_id" = p_subscription_id AND "partition" = p_partition;
1792
+ -- Rename the main table and its columns if it exists
1793
+ IF EXISTS (SELECT 1 FROM pg_tables WHERE tablename = 'emt_events') THEN
1794
+ -- Rename all partitions first
1795
+ FOR partition_record IN
1796
+ SELECT tablename
1797
+ FROM pg_tables
1798
+ WHERE tablename LIKE 'emt_events_%'
1799
+ ORDER BY tablename DESC -- to handle child partitions first
1800
+ LOOP
1801
+ EXECUTE format('ALTER TABLE %I RENAME TO %I',
1802
+ partition_record.tablename,
1803
+ REPLACE(partition_record.tablename, 'events', 'messages'));
1804
+ END LOOP;
1614
1805
 
1615
- -- Return appropriate codes based on current position
1616
- IF current_position = p_position THEN
1617
- RETURN 0; -- Idempotent check: position already set
1618
- ELSIF current_position > p_check_position THEN
1619
- RETURN 2; -- Failure: current position is greater
1620
- ELSE
1621
- RETURN 2; -- Default failure case for mismatched positions
1622
- END IF;
1623
- END IF;
1806
+ -- Rename the main table
1807
+ ALTER TABLE emt_events RENAME TO emt_messages;
1808
+
1809
+ -- Rename columns
1810
+ ALTER TABLE emt_messages
1811
+ RENAME COLUMN event_data TO message_data;
1812
+ ALTER TABLE emt_messages
1813
+ RENAME COLUMN event_metadata TO message_metadata;
1814
+ ALTER TABLE emt_messages
1815
+ RENAME COLUMN event_schema_version TO message_schema_version;
1816
+ ALTER TABLE emt_messages
1817
+ RENAME COLUMN event_type TO message_type;
1818
+ ALTER TABLE emt_messages
1819
+ RENAME COLUMN event_id TO message_id;
1820
+ ALTER TABLE emt_messages
1821
+ ADD COLUMN message_kind CHAR(1) NOT NULL DEFAULT 'E';
1624
1822
 
1625
- -- Handle the case when p_check_position is NULL: Insert if not exists
1626
- BEGIN
1627
- INSERT INTO "emt_subscriptions"("subscription_id", "version", "last_processed_position", "partition", "last_processed_transaction_id")
1628
- VALUES (p_subscription_id, p_version, p_position, p_partition, p_transaction_id);
1629
- RETURN 1; -- Successfully inserted
1630
- EXCEPTION WHEN unique_violation THEN
1631
- -- If insertion failed, it means the row already exists
1632
- SELECT "last_processed_position" INTO current_position
1633
- FROM "emt_subscriptions"
1634
- WHERE "subscription_id" = p_subscription_id AND "partition" = p_partition;
1823
+ -- Rename sequence if it exists
1824
+ IF EXISTS (SELECT 1 FROM pg_sequences WHERE sequencename = 'emt_global_event_position') THEN
1825
+ ALTER SEQUENCE emt_global_event_position
1826
+ RENAME TO emt_global_message_position;
1827
+
1828
+ ALTER TABLE emt_messages
1829
+ ALTER COLUMN global_position
1830
+ SET DEFAULT nextval('emt_global_message_position');
1831
+ END IF;
1832
+ END IF;
1833
+ END $$;`;
1834
+ const migration_0_38_7_and_older = sqlMigration("emt:postgresql:eventstore:0.38.7:migrate-events-to-messages:with_append_to_stream_update", [
1835
+ dropFutureConceptModuleAndTenantFunctions,
1836
+ dropOldAppendToSQLWithoutGlobalPositions,
1837
+ migrationFromEventsToMessagesSQL
1838
+ ]);
1635
1839
 
1636
- IF current_position = p_position THEN
1637
- RETURN 0; -- Idempotent check: position already set
1638
- ELSE
1639
- RETURN 2; -- Insertion failed, row already exists with different position
1640
- END IF;
1641
- END;
1642
- END;
1643
- $$ LANGUAGE plpgsql;
1644
- `;
1840
+ //#endregion
1841
+ //#region src/eventStore/schema/migrations/0_38_7/index.ts
1842
+ const migrations_0_38_7 = [migration_0_38_7_and_older];
1645
1843
 
1646
1844
  //#endregion
1647
1845
  //#region src/eventStore/schema/migrations/0_42_0/0_42_0.migration.ts
@@ -2176,1064 +2374,216 @@ END;
2176
2374
  $emt_deactivate_projection$;
2177
2375
  `;
2178
2376
  const migration_0_42_0_2_AddProcessorProjectionFunctions = sqlMigration("emt:postgresql:eventstore:0.42.0-2:add-processor-projection-functions", [migration_0_42_0_2_AddProcessorProjectionFunctionsSQL]);
2377
+ const migration_0_42_0_3_FixProcessorLockTimeoutSQL = SQL`
2378
+ CREATE OR REPLACE FUNCTION emt_try_acquire_processor_lock(
2379
+ p_lock_key BIGINT,
2380
+ p_processor_id TEXT,
2381
+ p_version INT,
2382
+ p_partition TEXT DEFAULT '${SQL.plain(defaultTag)}',
2383
+ p_processor_instance_id TEXT DEFAULT '${SQL.plain(unknownTag$1)}',
2384
+ p_projection_name TEXT DEFAULT NULL,
2385
+ p_projection_type VARCHAR(1) DEFAULT NULL,
2386
+ p_projection_kind TEXT DEFAULT NULL,
2387
+ p_lock_timeout_seconds INT DEFAULT 300
2388
+ )
2389
+ RETURNS TABLE (acquired BOOLEAN, checkpoint TEXT)
2390
+ LANGUAGE plpgsql
2391
+ AS $emt_try_acquire_processor_lock$
2392
+ DECLARE
2393
+ current_position TEXT;
2394
+ v_current_time TIMESTAMPTZ := clock_timestamp();
2395
+ BEGIN
2396
+ RETURN QUERY
2397
+ WITH lock_check AS (
2398
+ SELECT pg_try_advisory_xact_lock(p_lock_key) AS lock_acquired
2399
+ ),
2400
+ ownership_check AS (
2401
+ INSERT INTO emt_processors (
2402
+ processor_id,
2403
+ partition,
2404
+ version,
2405
+ processor_instance_id,
2406
+ status,
2407
+ last_processed_checkpoint,
2408
+ last_processed_transaction_id,
2409
+ created_at,
2410
+ last_updated
2411
+ )
2412
+ SELECT p_processor_id, p_partition, p_version, p_processor_instance_id, 'running', '0000000000000000000', '0'::xid8, v_current_time, v_current_time
2413
+ WHERE (SELECT lock_acquired FROM lock_check) = true
2414
+ ON CONFLICT (processor_id, partition, version) DO UPDATE
2415
+ SET processor_instance_id = p_processor_instance_id,
2416
+ status = 'running',
2417
+ last_updated = v_current_time
2418
+ WHERE emt_processors.processor_instance_id = p_processor_instance_id
2419
+ OR emt_processors.processor_instance_id = 'emt:unknown'
2420
+ OR emt_processors.status = 'stopped'
2421
+ OR emt_processors.last_updated < v_current_time - (p_lock_timeout_seconds || ' seconds')::interval
2422
+ RETURNING last_processed_checkpoint
2423
+ ),
2424
+ projection_status AS (
2425
+ INSERT INTO emt_projections (
2426
+ name,
2427
+ partition,
2428
+ version,
2429
+ type,
2430
+ kind,
2431
+ status,
2432
+ definition
2433
+ )
2434
+ SELECT p_projection_name, p_partition, p_version, p_projection_type, p_projection_kind, 'async_processing', '{}'::jsonb
2435
+ WHERE p_projection_name IS NOT NULL
2436
+ AND (SELECT last_processed_checkpoint FROM ownership_check) IS NOT NULL
2437
+ ON CONFLICT (name, partition, version) DO UPDATE
2438
+ SET status = 'async_processing'
2439
+ RETURNING name
2440
+ )
2441
+ SELECT
2442
+ (SELECT COUNT(*) > 0 FROM ownership_check),
2443
+ (SELECT oc.last_processed_checkpoint FROM ownership_check oc);
2444
+ END;
2445
+ $emt_try_acquire_processor_lock$;
2446
+ `;
2447
+ const migration_0_42_0_3_FixProcessorLockTimeout = sqlMigration("emt:postgresql:eventstore:0.42.0-3:fix-processor-lock-timeout", [migration_0_42_0_3_FixProcessorLockTimeoutSQL]);
2179
2448
 
2180
2449
  //#endregion
2181
- //#region src/eventStore/schema/migrations/0_42_0/0_42_0.snapshot.ts
2182
- const schema_0_42_0 = SQL`
2183
- CREATE TABLE IF NOT EXISTS emt_streams(
2184
- stream_id TEXT NOT NULL,
2185
- stream_position BIGINT NOT NULL,
2186
- partition TEXT NOT NULL DEFAULT 'emt:default',
2187
- stream_type TEXT NOT NULL,
2188
- stream_metadata JSONB NOT NULL,
2189
- is_archived BOOLEAN NOT NULL DEFAULT FALSE,
2190
- PRIMARY KEY (stream_id, partition, is_archived)
2191
- ) PARTITION BY LIST (partition);
2192
-
2193
- CREATE UNIQUE INDEX IF NOT EXISTS idx_streams_unique
2194
- ON emt_streams(stream_id, partition, is_archived)
2195
- INCLUDE (stream_position);
2196
- CREATE SEQUENCE IF NOT EXISTS emt_global_message_position;
2450
+ //#region src/eventStore/schema/migrations/0_42_0/index.ts
2451
+ const migrations_0_42_0 = [
2452
+ migration_0_42_0_FromSubscriptionsToProcessors,
2453
+ migration_0_42_0_2_AddProcessorProjectionFunctions,
2454
+ migration_0_42_0_3_FixProcessorLockTimeout
2455
+ ];
2197
2456
 
2198
- CREATE TABLE IF NOT EXISTS emt_messages(
2199
- stream_position BIGINT NOT NULL,
2200
- global_position BIGINT DEFAULT nextval('emt_global_message_position'),
2201
- transaction_id XID8 NOT NULL,
2202
- created TIMESTAMPTZ NOT NULL DEFAULT now(),
2203
- is_archived BOOLEAN NOT NULL DEFAULT FALSE,
2204
- message_kind VARCHAR(1) NOT NULL DEFAULT 'E',
2205
- stream_id TEXT NOT NULL,
2206
- partition TEXT NOT NULL DEFAULT 'emt:default',
2207
- message_schema_version TEXT NOT NULL,
2208
- message_id TEXT NOT NULL,
2209
- message_type TEXT NOT NULL,
2210
- message_data JSONB NOT NULL,
2211
- message_metadata JSONB NOT NULL,
2212
- PRIMARY KEY (stream_id, stream_position, partition, is_archived)
2213
- ) PARTITION BY LIST (partition);
2214
- CREATE TABLE IF NOT EXISTS emt_projections(
2215
- version INT NOT NULL DEFAULT 1,
2216
- type VARCHAR(1) NOT NULL,
2217
- name TEXT NOT NULL,
2218
- partition TEXT NOT NULL DEFAULT 'emt:default',
2219
- kind TEXT NOT NULL,
2220
- status TEXT NOT NULL,
2221
- definition JSONB NOT NULL DEFAULT '{}'::jsonb,
2222
- created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
2223
- last_updated TIMESTAMPTZ NOT NULL DEFAULT now(),
2224
- PRIMARY KEY (name, partition, version)
2225
- ) PARTITION BY LIST (partition);
2226
-
2227
- CREATE TABLE IF NOT EXISTS emt_processors(
2228
- last_processed_transaction_id XID8 NOT NULL,
2229
- version INT NOT NULL DEFAULT 1,
2230
- processor_id TEXT NOT NULL,
2231
- partition TEXT NOT NULL DEFAULT 'emt:default',
2232
- status TEXT NOT NULL DEFAULT 'stopped',
2233
- last_processed_checkpoint TEXT NOT NULL,
2234
- processor_instance_id TEXT DEFAULT 'emt:unknown',
2235
- created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
2236
- last_updated TIMESTAMPTZ NOT NULL DEFAULT now(),
2237
- PRIMARY KEY (processor_id, partition, version)
2238
- ) PARTITION BY LIST (partition);
2239
-
2240
- DO $$
2241
- BEGIN
2242
- IF NOT EXISTS (SELECT 1 FROM pg_proc WHERE proname = 'emt_sanitize_name') THEN
2243
- CREATE OR REPLACE FUNCTION emt_sanitize_name(input_name TEXT) RETURNS TEXT AS $emt_sanitize_name$
2244
- BEGIN
2245
- RETURN REGEXP_REPLACE(input_name, '[^a-zA-Z0-9_]', '_', 'g');
2246
- END;
2247
- $emt_sanitize_name$ LANGUAGE plpgsql;
2248
- END IF;
2249
- END $$;
2250
-
2251
- DO $$
2252
- BEGIN
2253
- IF NOT EXISTS (SELECT 1 FROM pg_proc WHERE proname = 'emt_add_table_partition') THEN
2254
-
2255
- CREATE OR REPLACE FUNCTION emt_add_table_partition(tableName TEXT, partition_name TEXT) RETURNS void AS $emt_add_table_partition$
2256
- DECLARE
2257
- v_main_partiton_name TEXT;
2258
- v_active_partiton_name TEXT;
2259
- v_archived_partiton_name TEXT;
2260
- BEGIN
2261
- v_main_partiton_name := emt_sanitize_name(tableName || '_' || partition_name);
2262
- v_active_partiton_name := emt_sanitize_name(v_main_partiton_name || '_active');
2263
- v_archived_partiton_name := emt_sanitize_name(v_main_partiton_name || '_archived');
2264
-
2265
-
2266
- -- create default partition
2267
- EXECUTE format('
2268
- CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
2269
- FOR VALUES IN (%L) PARTITION BY LIST (is_archived);',
2270
- v_main_partiton_name, tableName, partition_name
2271
- );
2272
-
2273
- EXECUTE format('
2274
- CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
2275
- FOR VALUES IN (FALSE);',
2276
- v_active_partiton_name, v_main_partiton_name
2277
- );
2278
-
2279
- EXECUTE format('
2280
- CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
2281
- FOR VALUES IN (TRUE);',
2282
- v_archived_partiton_name, v_main_partiton_name
2283
- );
2284
- END;
2285
- $emt_add_table_partition$ LANGUAGE plpgsql;
2286
- END IF;
2287
- END $$;
2288
-
2289
- DO $$
2290
- BEGIN
2291
- IF NOT EXISTS (SELECT 1 FROM pg_proc WHERE proname = 'emt_add_partition') THEN
2292
-
2293
- CREATE OR REPLACE FUNCTION emt_add_partition(partition_name TEXT) RETURNS void AS $emt_add_partition$
2294
- BEGIN
2295
- PERFORM emt_add_table_partition('emt_messages', partition_name);
2296
- PERFORM emt_add_table_partition('emt_streams', partition_name);
2297
-
2298
- EXECUTE format('
2299
- CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
2300
- FOR VALUES IN (%L);',
2301
- emt_sanitize_name('emt_processors' || '_' || partition_name), 'emt_processors', partition_name
2302
- );
2303
-
2304
- EXECUTE format('
2305
- CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
2306
- FOR VALUES IN (%L);',
2307
- emt_sanitize_name('emt_projections' || '_' || partition_name), 'emt_projections', partition_name
2308
- );
2309
- END;
2310
- $emt_add_partition$ LANGUAGE plpgsql;
2311
- END IF;
2312
- END $$;
2313
-
2314
- DO $$
2315
- BEGIN
2316
- IF NOT EXISTS (SELECT 1 FROM pg_proc WHERE proname = 'emt_append_to_stream') THEN
2317
- CREATE OR REPLACE FUNCTION emt_append_to_stream(
2318
- v_message_ids text[],
2319
- v_messages_data jsonb[],
2320
- v_messages_metadata jsonb[],
2321
- v_message_schema_versions text[],
2322
- v_message_types text[],
2323
- v_message_kinds text[],
2324
- v_stream_id text,
2325
- v_stream_type text,
2326
- v_expected_stream_position bigint DEFAULT NULL,
2327
- v_partition text DEFAULT emt_sanitize_name('default_partition')
2328
- ) RETURNS TABLE (
2329
- success boolean,
2330
- next_stream_position bigint,
2331
- global_positions bigint[],
2332
- transaction_id xid8
2333
- ) LANGUAGE plpgsql
2334
- AS $emt_append_to_stream$
2335
- DECLARE
2336
- v_next_stream_position bigint;
2337
- v_position bigint;
2338
- v_updated_rows int;
2339
- v_transaction_id xid8;
2340
- v_global_positions bigint[];
2341
- BEGIN
2342
- v_transaction_id := pg_current_xact_id();
2343
-
2344
- IF v_expected_stream_position IS NULL THEN
2345
- SELECT COALESCE(
2346
- (SELECT stream_position
2347
- FROM emt_streams
2348
- WHERE stream_id = v_stream_id
2349
- AND partition = v_partition
2350
- AND is_archived = FALSE
2351
- LIMIT 1),
2352
- 0
2353
- ) INTO v_expected_stream_position;
2354
- END IF;
2355
-
2356
- v_next_stream_position := v_expected_stream_position + array_upper(v_messages_data, 1);
2357
-
2358
- IF v_expected_stream_position = 0 THEN
2359
- INSERT INTO emt_streams
2360
- (stream_id, stream_position, partition, stream_type, stream_metadata, is_archived)
2361
- VALUES
2362
- (v_stream_id, v_next_stream_position, v_partition, v_stream_type, '{}', FALSE);
2363
- ELSE
2364
- UPDATE emt_streams as s
2365
- SET stream_position = v_next_stream_position
2366
- WHERE stream_id = v_stream_id AND stream_position = v_expected_stream_position AND partition = v_partition AND is_archived = FALSE;
2367
-
2368
- get diagnostics v_updated_rows = row_count;
2369
-
2370
- IF v_updated_rows = 0 THEN
2371
- RETURN QUERY SELECT FALSE, NULL::bigint, NULL::bigint[], NULL::xid8;
2372
- RETURN;
2373
- END IF;
2374
- END IF;
2375
-
2376
- WITH ev AS (
2377
- SELECT row_number() OVER () + v_expected_stream_position AS stream_position,
2378
- message_data,
2379
- message_metadata,
2380
- schema_version,
2381
- message_id,
2382
- message_type,
2383
- message_kind
2384
- FROM (
2385
- SELECT *
2386
- FROM
2387
- unnest(v_message_ids, v_messages_data, v_messages_metadata, v_message_schema_versions, v_message_types, v_message_kinds)
2388
- AS message(message_id, message_data, message_metadata, schema_version, message_type, message_kind)
2389
- ) AS message
2390
- ),
2391
- all_messages_insert AS (
2392
- INSERT INTO emt_messages
2393
- (stream_id, stream_position, partition, message_data, message_metadata, message_schema_version, message_type, message_kind, message_id, transaction_id)
2394
- SELECT
2395
- v_stream_id, ev.stream_position, v_partition, ev.message_data, ev.message_metadata, ev.schema_version, ev.message_type, ev.message_kind, ev.message_id, v_transaction_id
2396
- FROM ev
2397
- RETURNING global_position
2398
- )
2399
- SELECT
2400
- array_agg(global_position ORDER BY global_position) INTO v_global_positions
2401
- FROM
2402
- all_messages_insert;
2403
-
2404
- RETURN QUERY SELECT TRUE, v_next_stream_position, v_global_positions, v_transaction_id;
2405
- END;
2406
- $emt_append_to_stream$;
2407
-
2408
- END IF;
2409
- END $$;
2410
- SELECT emt_add_partition('emt:default');
2411
- DO $$
2412
- BEGIN
2413
- IF NOT EXISTS (SELECT 1 FROM pg_proc WHERE proname = 'store_processor_checkpoint') THEN
2414
-
2415
- CREATE OR REPLACE FUNCTION store_processor_checkpoint(
2416
- p_processor_id TEXT,
2417
- p_version BIGINT,
2418
- p_position TEXT,
2419
- p_check_position TEXT,
2420
- p_transaction_id xid8,
2421
- p_partition TEXT DEFAULT 'emt:default',
2422
- p_processor_instance_id TEXT DEFAULT 'emt:unknown'
2423
- ) RETURNS INT AS $spc$
2424
- DECLARE
2425
- current_position TEXT;
2426
- BEGIN
2427
- -- Handle the case when p_check_position is provided
2428
- IF p_check_position IS NOT NULL THEN
2429
- -- Try to update if the position matches p_check_position
2430
- UPDATE "emt_processors"
2431
- SET
2432
- "last_processed_checkpoint" = p_position,
2433
- "last_processed_transaction_id" = p_transaction_id,
2434
- "last_updated" = now()
2435
- WHERE "processor_id" = p_processor_id
2436
- AND "last_processed_checkpoint" = p_check_position
2437
- AND "partition" = p_partition
2438
- AND "version" = p_version;
2439
-
2440
- IF FOUND THEN
2441
- RETURN 1; -- Successfully updated
2442
- END IF;
2443
-
2444
- -- Retrieve the current position
2445
- SELECT "last_processed_checkpoint" INTO current_position
2446
- FROM "emt_processors"
2447
- WHERE "processor_id" = p_processor_id
2448
- AND "partition" = p_partition
2449
- AND "version" = p_version;
2450
-
2451
- -- Return appropriate codes based on current position
2452
- IF current_position = p_position THEN
2453
- RETURN 0; -- Idempotent check: position already set
2454
- ELSIF current_position > p_position THEN
2455
- RETURN 3; -- Current ahead: another process has progressed further
2456
- ELSE
2457
- RETURN 2; -- Mismatch: check position doesn't match current
2458
- END IF;
2459
- END IF;
2460
-
2461
- -- Handle the case when p_check_position is NULL: Insert if not exists
2462
- BEGIN
2463
- INSERT INTO "emt_processors"("processor_id", "version", "last_processed_checkpoint", "partition", "last_processed_transaction_id", "created_at", "last_updated")
2464
- VALUES (p_processor_id, p_version, p_position, p_partition, p_transaction_id, now(), now());
2465
- RETURN 1; -- Successfully inserted
2466
- EXCEPTION WHEN unique_violation THEN
2467
- -- If insertion failed, it means the row already exists
2468
- SELECT "last_processed_checkpoint" INTO current_position
2469
- FROM "emt_processors"
2470
- WHERE "processor_id" = p_processor_id
2471
- AND "partition" = p_partition
2472
- AND "version" = p_version;
2473
-
2474
- IF current_position = p_position THEN
2475
- RETURN 0; -- Idempotent check: position already set
2476
- ELSIF current_position > p_position THEN
2477
- RETURN 3; -- Current ahead: another process has progressed further
2478
- ELSE
2479
- RETURN 2; -- Insertion failed, row already exists with different position
2480
- END IF;
2481
- END;
2482
- END;
2483
- $spc$ LANGUAGE plpgsql;
2484
-
2485
- END IF;
2486
- END $$;
2487
-
2488
- CREATE OR REPLACE FUNCTION emt_try_acquire_processor_lock(
2489
- p_lock_key BIGINT,
2490
- p_processor_id TEXT,
2491
- p_version INT,
2492
- p_partition TEXT DEFAULT 'emt:default',
2493
- p_processor_instance_id TEXT DEFAULT 'emt:unknown',
2494
- p_projection_name TEXT DEFAULT NULL,
2495
- p_projection_type VARCHAR(1) DEFAULT NULL,
2496
- p_projection_kind TEXT DEFAULT NULL,
2497
- p_lock_timeout_seconds INT DEFAULT 300
2498
- )
2499
- RETURNS TABLE (acquired BOOLEAN, checkpoint TEXT)
2500
- LANGUAGE plpgsql
2501
- AS $emt_try_acquire_processor_lock$
2502
- BEGIN
2503
- RETURN QUERY
2504
- WITH lock_check AS (
2505
- SELECT pg_try_advisory_xact_lock(p_lock_key) AS lock_acquired
2506
- ),
2507
- ownership_check AS (
2508
- INSERT INTO emt_processors (
2509
- processor_id,
2510
- partition,
2511
- version,
2512
- processor_instance_id,
2513
- status,
2514
- last_processed_checkpoint,
2515
- last_processed_transaction_id,
2516
- created_at,
2517
- last_updated
2518
- )
2519
- SELECT p_processor_id, p_partition, p_version, p_processor_instance_id, 'running', '0000000000000000000', '0'::xid8, now(), now()
2520
- WHERE (SELECT lock_acquired FROM lock_check) = true
2521
- ON CONFLICT (processor_id, partition, version) DO UPDATE
2522
- SET processor_instance_id = p_processor_instance_id,
2523
- status = 'running',
2524
- last_updated = now()
2525
- WHERE emt_processors.processor_instance_id = p_processor_instance_id
2526
- OR emt_processors.processor_instance_id = 'emt:unknown'
2527
- OR emt_processors.status = 'stopped'
2528
- OR emt_processors.last_updated < now() - (p_lock_timeout_seconds || ' seconds')::interval
2529
- RETURNING last_processed_checkpoint
2530
- ),
2531
- projection_status AS (
2532
- INSERT INTO emt_projections (
2533
- name,
2534
- partition,
2535
- version,
2536
- type,
2537
- kind,
2538
- status,
2539
- definition
2540
- )
2541
- SELECT p_projection_name, p_partition, p_version, p_projection_type, p_projection_kind, 'async_processing', '{}'::jsonb
2542
- WHERE p_projection_name IS NOT NULL
2543
- AND (SELECT last_processed_checkpoint FROM ownership_check) IS NOT NULL
2544
- ON CONFLICT (name, partition, version) DO UPDATE
2545
- SET status = 'async_processing'
2546
- RETURNING name
2547
- )
2548
- SELECT
2549
- (SELECT COUNT(*) > 0 FROM ownership_check),
2550
- (SELECT oc.last_processed_checkpoint FROM ownership_check oc);
2551
- END;
2552
- $emt_try_acquire_processor_lock$;
2553
-
2554
- CREATE OR REPLACE FUNCTION emt_release_processor_lock(
2555
- p_lock_key BIGINT,
2556
- p_processor_id TEXT,
2557
- p_partition TEXT,
2558
- p_version INT,
2559
- p_processor_instance_id TEXT DEFAULT 'emt:unknown',
2560
- p_projection_name TEXT DEFAULT NULL
2561
- )
2562
- RETURNS BOOLEAN
2563
- LANGUAGE plpgsql
2564
- AS $emt_release_processor_lock$
2565
- DECLARE
2566
- v_rows_updated INT;
2567
- BEGIN
2568
- IF p_projection_name IS NOT NULL THEN
2569
- UPDATE emt_projections
2570
- SET status = 'active',
2571
- last_updated = now()
2572
- WHERE partition = p_partition
2573
- AND name = p_projection_name
2574
- AND version = p_version;
2575
- END IF;
2576
-
2577
- UPDATE emt_processors
2578
- SET status = 'stopped',
2579
- processor_instance_id = 'emt:unknown',
2580
- last_updated = now()
2581
- WHERE processor_id = p_processor_id
2582
- AND partition = p_partition
2583
- AND version = p_version
2584
- AND processor_instance_id = p_processor_instance_id;
2585
-
2586
- GET DIAGNOSTICS v_rows_updated = ROW_COUNT;
2587
-
2588
- PERFORM pg_advisory_unlock(p_lock_key);
2589
-
2590
- RETURN v_rows_updated > 0;
2591
- END;
2592
- $emt_release_processor_lock$;
2593
-
2594
- CREATE OR REPLACE FUNCTION emt_try_acquire_projection_lock(
2595
- p_lock_key BIGINT,
2596
- p_partition TEXT,
2597
- p_name TEXT,
2598
- p_version INT
2599
- )
2600
- RETURNS TABLE (acquired BOOLEAN, is_active BOOLEAN)
2601
- LANGUAGE plpgsql
2602
- AS $emt_try_acquire_projection_lock$
2603
- BEGIN
2604
- RETURN QUERY
2605
- WITH lock_check AS (
2606
- SELECT pg_try_advisory_xact_lock_shared(p_lock_key) AS acquired
2607
- ),
2608
- status_check AS (
2609
- SELECT status = 'active' AS is_active
2610
- FROM emt_projections
2611
- WHERE partition = p_partition AND name = p_name AND version = p_version
2612
- )
2613
- SELECT
2614
- COALESCE((SELECT lc.acquired FROM lock_check lc), false),
2615
- COALESCE((SELECT sc.is_active FROM status_check sc), true);
2616
- END;
2617
- $emt_try_acquire_projection_lock$;
2618
-
2619
- CREATE OR REPLACE FUNCTION emt_register_projection(
2620
- p_lock_key BIGINT,
2621
- p_name TEXT,
2622
- p_partition TEXT,
2623
- p_version INT,
2624
- p_type VARCHAR(1),
2625
- p_kind TEXT,
2626
- p_status TEXT,
2627
- p_definition JSONB
2628
- )
2629
- RETURNS BOOLEAN
2630
- LANGUAGE plpgsql
2631
- AS $emt_register_projection$
2632
- DECLARE
2633
- v_result BOOLEAN;
2634
- BEGIN
2635
- WITH lock_check AS (
2636
- SELECT pg_try_advisory_xact_lock(p_lock_key) AS lock_acquired
2637
- ),
2638
- upsert_result AS (
2639
- INSERT INTO emt_projections (
2640
- name, partition, version, type, kind, status, definition, created_at, last_updated
2641
- )
2642
- SELECT p_name, p_partition, p_version, p_type, p_kind, p_status, p_definition, now(), now()
2643
- WHERE (SELECT lock_acquired FROM lock_check) = true
2644
- ON CONFLICT (name, partition, version) DO UPDATE
2645
- SET definition = EXCLUDED.definition,
2646
- last_updated = now()
2647
- RETURNING name
2648
- )
2649
- SELECT COUNT(*) > 0 INTO v_result FROM upsert_result;
2650
-
2651
- RETURN v_result;
2652
- END;
2653
- $emt_register_projection$;
2654
-
2655
- CREATE OR REPLACE FUNCTION emt_activate_projection(
2656
- p_lock_key BIGINT,
2657
- p_name TEXT,
2658
- p_partition TEXT,
2659
- p_version INT
2660
- )
2661
- RETURNS BOOLEAN
2662
- LANGUAGE plpgsql
2663
- AS $emt_activate_projection$
2664
- DECLARE
2665
- v_result BOOLEAN;
2666
- BEGIN
2667
- WITH lock_check AS (
2668
- SELECT pg_try_advisory_xact_lock(p_lock_key) AS lock_acquired
2669
- ),
2670
- update_result AS (
2671
- UPDATE emt_projections
2672
- SET status = 'active',
2673
- last_updated = now()
2674
- WHERE name = p_name
2675
- AND partition = p_partition
2676
- AND version = p_version
2677
- AND (SELECT lock_acquired FROM lock_check) = true
2678
- RETURNING name
2679
- )
2680
- SELECT COUNT(*) > 0 INTO v_result FROM update_result;
2681
-
2682
- RETURN v_result;
2683
- END;
2684
- $emt_activate_projection$;
2685
-
2686
- CREATE OR REPLACE FUNCTION emt_deactivate_projection(
2687
- p_lock_key BIGINT,
2688
- p_name TEXT,
2689
- p_partition TEXT,
2690
- p_version INT
2691
- )
2692
- RETURNS BOOLEAN
2693
- LANGUAGE plpgsql
2694
- AS $emt_deactivate_projection$
2695
- DECLARE
2696
- v_result BOOLEAN;
2697
- BEGIN
2698
- WITH lock_check AS (
2699
- SELECT pg_try_advisory_xact_lock(p_lock_key) AS lock_acquired
2700
- ),
2701
- update_result AS (
2702
- UPDATE emt_projections
2703
- SET status = 'inactive',
2704
- last_updated = now()
2705
- WHERE name = p_name
2706
- AND partition = p_partition
2707
- AND version = p_version
2708
- AND (SELECT lock_acquired FROM lock_check) = true
2709
- RETURNING name
2710
- )
2711
- SELECT COUNT(*) > 0 INTO v_result FROM update_result;
2712
-
2713
- RETURN v_result;
2714
- END;
2715
- $emt_deactivate_projection$;
2716
- `;
2717
-
2718
- //#endregion
2719
- //#region src/eventStore/schema/storeProcessorCheckpoint.ts
2720
- const storeSubscriptionCheckpointSQL = createFunctionIfDoesNotExistSQL("store_processor_checkpoint", SQL`
2721
- CREATE OR REPLACE FUNCTION store_processor_checkpoint(
2722
- p_processor_id TEXT,
2723
- p_version BIGINT,
2724
- p_position TEXT,
2725
- p_check_position TEXT,
2726
- p_transaction_id xid8,
2727
- p_partition TEXT DEFAULT '${SQL.plain(defaultTag)}',
2728
- p_processor_instance_id TEXT DEFAULT '${SQL.plain(unknownTag)}'
2729
- ) RETURNS INT AS $spc$
2730
- DECLARE
2731
- current_position TEXT;
2732
- BEGIN
2733
- -- Handle the case when p_check_position is provided
2734
- IF p_check_position IS NOT NULL THEN
2735
- -- Try to update if the position matches p_check_position
2736
- UPDATE "${SQL.plain(processorsTable.name)}"
2737
- SET
2738
- "last_processed_checkpoint" = p_position,
2739
- "last_processed_transaction_id" = p_transaction_id,
2740
- "last_updated" = now()
2741
- WHERE "processor_id" = p_processor_id
2742
- AND "last_processed_checkpoint" = p_check_position
2743
- AND "partition" = p_partition
2744
- AND "version" = p_version;
2745
-
2746
- IF FOUND THEN
2747
- RETURN 1; -- Successfully updated
2748
- END IF;
2749
-
2750
- -- Retrieve the current position
2751
- SELECT "last_processed_checkpoint" INTO current_position
2752
- FROM "${SQL.plain(processorsTable.name)}"
2753
- WHERE "processor_id" = p_processor_id
2754
- AND "partition" = p_partition
2755
- AND "version" = p_version;
2756
-
2757
- -- Return appropriate codes based on current position
2758
- IF current_position = p_position THEN
2759
- RETURN 0; -- Idempotent check: position already set
2760
- ELSIF current_position > p_position THEN
2761
- RETURN 3; -- Current ahead: another process has progressed further
2762
- ELSE
2763
- RETURN 2; -- Mismatch: check position doesn't match current
2764
- END IF;
2765
- END IF;
2766
-
2767
- -- Handle the case when p_check_position is NULL: Insert if not exists
2768
- BEGIN
2769
- INSERT INTO "${SQL.plain(processorsTable.name)}"("processor_id", "version", "last_processed_checkpoint", "partition", "last_processed_transaction_id", "created_at", "last_updated")
2770
- VALUES (p_processor_id, p_version, p_position, p_partition, p_transaction_id, now(), now());
2771
- RETURN 1; -- Successfully inserted
2772
- EXCEPTION WHEN unique_violation THEN
2773
- -- If insertion failed, it means the row already exists
2774
- SELECT "last_processed_checkpoint" INTO current_position
2775
- FROM "${SQL.plain(processorsTable.name)}"
2776
- WHERE "processor_id" = p_processor_id
2777
- AND "partition" = p_partition
2778
- AND "version" = p_version;
2779
-
2780
- IF current_position = p_position THEN
2781
- RETURN 0; -- Idempotent check: position already set
2782
- ELSIF current_position > p_position THEN
2783
- RETURN 3; -- Current ahead: another process has progressed further
2784
- ELSE
2785
- RETURN 2; -- Insertion failed, row already exists with different position
2786
- END IF;
2787
- END;
2788
- END;
2789
- $spc$ LANGUAGE plpgsql;
2790
- `);
2791
- const callStoreProcessorCheckpoint = (params) => SQL`
2792
- SELECT store_processor_checkpoint(
2793
- ${params.processorId},
2794
- ${params.version},
2795
- ${params.position},
2796
- ${params.checkPosition},
2797
- pg_current_xact_id(),
2798
- ${params.partition},
2799
- ${params.processorInstanceId}
2800
- ) as result;`;
2801
- const storeProcessorCheckpoint = async (execute, options) => {
2802
- try {
2803
- const { result } = await single(execute.command(callStoreProcessorCheckpoint({
2804
- processorId: options.processorId,
2805
- version: options.version ?? 1,
2806
- position: options.newCheckpoint !== null ? options.newCheckpoint : null,
2807
- checkPosition: options.lastProcessedCheckpoint !== null ? options.lastProcessedCheckpoint : null,
2808
- partition: options.partition ?? defaultTag,
2809
- processorInstanceId: options.processorInstanceId ?? unknownTag
2810
- })));
2811
- return result === 1 ? {
2812
- success: true,
2813
- newCheckpoint: options.newCheckpoint
2814
- } : {
2815
- success: false,
2816
- reason: result === 0 ? "IGNORED" : result === 3 ? "CURRENT_AHEAD" : "MISMATCH"
2817
- };
2818
- } catch (error) {
2819
- console.log(error);
2820
- throw error;
2821
- }
2822
- };
2823
-
2824
- //#endregion
2825
- //#region src/eventStore/schema/migrations/0_43_0/index.ts
2826
- const migration_0_43_0_cleanupLegacySubscriptionSQL = SQL`
2827
- DO $$
2828
- BEGIN
2829
- IF EXISTS (SELECT 1 FROM pg_tables WHERE tablename = 'emt_subscriptions') THEN
2830
- -- Restore clean emt_add_partition (remove creation of emt_subscriptions partitions)
2831
- CREATE OR REPLACE FUNCTION emt_add_partition(partition_name TEXT) RETURNS void AS $fnpar$
2832
- BEGIN
2833
- PERFORM emt_add_table_partition('${SQL.plain(messagesTable.name)}', partition_name);
2834
- PERFORM emt_add_table_partition('${SQL.plain(streamsTable.name)}', partition_name);
2835
-
2836
- EXECUTE format('
2837
- CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
2838
- FOR VALUES IN (%L);',
2839
- emt_sanitize_name('${SQL.plain(processorsTable.name)}' || '_' || partition_name), '${SQL.plain(processorsTable.name)}', partition_name
2840
- );
2841
-
2842
- EXECUTE format('
2843
- CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
2844
- FOR VALUES IN (%L);',
2845
- emt_sanitize_name('${SQL.plain(projectionsTable.name)}' || '_' || partition_name), '${SQL.plain(projectionsTable.name)}', partition_name
2846
- );
2847
- END;
2848
- $fnpar$ LANGUAGE plpgsql;
2849
-
2850
- -- Drop old subscriptions table if it exists
2851
- DROP TABLE IF EXISTS emt_subscriptions CASCADE;
2852
-
2853
- -- Drop old function if it exists
2854
- DROP FUNCTION IF EXISTS store_subscription_checkpoint(character varying, bigint, bigint, bigint, xid8, text);
2855
-
2856
- -- Restore clean store_processor_checkpoint (remove dual-write logic)
2857
- CREATE OR REPLACE FUNCTION store_processor_checkpoint(
2858
- p_processor_id TEXT,
2859
- p_version BIGINT,
2860
- p_position TEXT,
2861
- p_check_position TEXT,
2862
- p_transaction_id xid8,
2863
- p_partition TEXT DEFAULT '${SQL.plain(defaultTag)}',
2864
- p_processor_instance_id TEXT DEFAULT 'emt:unknown'
2865
- ) RETURNS INT AS $fn$
2866
- DECLARE
2867
- current_position TEXT;
2868
- BEGIN
2869
- IF p_check_position IS NOT NULL THEN
2870
- UPDATE "emt_processors"
2871
- SET
2872
- "last_processed_checkpoint" = p_position,
2873
- "last_processed_transaction_id" = p_transaction_id,
2874
- "last_updated" = now()
2875
- WHERE "processor_id" = p_processor_id
2876
- AND "last_processed_checkpoint" = p_check_position
2877
- AND "partition" = p_partition
2878
- AND "version" = p_version;
2879
-
2880
- IF FOUND THEN
2881
- RETURN 1;
2882
- END IF;
2883
-
2884
- SELECT "last_processed_checkpoint" INTO current_position
2885
- FROM "emt_processors"
2886
- WHERE "processor_id" = p_processor_id
2887
- AND "partition" = p_partition
2888
- AND "version" = p_version ;
2889
-
2890
- IF current_position = p_position THEN
2891
- RETURN 0;
2892
- ELSIF current_position > p_position THEN
2893
- RETURN 3;
2894
- ELSE
2895
- RETURN 2;
2896
- END IF;
2897
- END IF;
2898
-
2899
- BEGIN
2900
- INSERT INTO "emt_processors"("processor_id", "version", "last_processed_checkpoint", "partition", "last_processed_transaction_id", "created_at", "last_updated")
2901
- VALUES (p_processor_id, p_version, p_position, p_partition, p_transaction_id, now(), now());
2902
- RETURN 1;
2903
- EXCEPTION WHEN unique_violation THEN
2904
- SELECT "last_processed_checkpoint" INTO current_position
2905
- FROM "emt_processors"
2906
- WHERE "processor_id" = p_processor_id
2907
- AND "partition" = p_partition
2908
- AND "version" = p_version;
2909
-
2910
- IF current_position = p_position THEN
2911
- RETURN 0;
2912
- ELSE
2913
- RETURN 2;
2914
- END IF;
2915
- END;
2916
- END;
2917
- $fn$ LANGUAGE plpgsql;
2918
- END IF;
2919
- END $$;
2920
- `;
2921
- const migration_0_43_0_cleanupLegacySubscription = sqlMigration("emt:postgresql:eventstore:0.43.0:cleanup-legacy-subscription", [migration_0_43_0_cleanupLegacySubscriptionSQL]);
2922
- const cleanupLegacySubscriptionTables = async (connectionString) => {
2923
- const pool = dumbo({ connectionString });
2924
- try {
2925
- await pool.withTransaction(async ({ execute }) => {
2926
- await execute.command(migration_0_43_0_cleanupLegacySubscriptionSQL);
2927
- });
2928
- } finally {
2929
- await pool.close();
2930
- }
2931
- };
2932
-
2933
- //#endregion
2934
- //#region src/eventStore/schema/tables.ts
2935
- const streamsTableSQL = SQL`
2936
- CREATE TABLE IF NOT EXISTS ${SQL.identifier(streamsTable.name)}(
2937
- stream_id TEXT NOT NULL,
2938
- stream_position BIGINT NOT NULL,
2939
- partition TEXT NOT NULL DEFAULT '${SQL.plain(defaultTag)}',
2940
- stream_type TEXT NOT NULL,
2941
- stream_metadata JSONB NOT NULL,
2942
- is_archived BOOLEAN NOT NULL DEFAULT FALSE,
2943
- PRIMARY KEY (stream_id, partition, is_archived)
2944
- ) PARTITION BY LIST (partition);
2945
-
2946
- CREATE UNIQUE INDEX IF NOT EXISTS idx_streams_unique
2947
- ON ${SQL.identifier(streamsTable.name)}(stream_id, partition, is_archived)
2948
- INCLUDE (stream_position);`;
2949
- const messagesTableSQL = SQL`
2950
- CREATE SEQUENCE IF NOT EXISTS emt_global_message_position;
2951
-
2952
- CREATE TABLE IF NOT EXISTS ${SQL.identifier(messagesTable.name)}(
2953
- stream_position BIGINT NOT NULL,
2954
- global_position BIGINT DEFAULT nextval('emt_global_message_position'),
2955
- transaction_id XID8 NOT NULL,
2956
- created TIMESTAMPTZ NOT NULL DEFAULT now(),
2957
- is_archived BOOLEAN NOT NULL DEFAULT FALSE,
2958
- message_kind VARCHAR(1) NOT NULL DEFAULT 'E',
2959
- stream_id TEXT NOT NULL,
2960
- partition TEXT NOT NULL DEFAULT '${SQL.plain(defaultTag)}',
2961
- message_schema_version TEXT NOT NULL,
2962
- message_id TEXT NOT NULL,
2963
- message_type TEXT NOT NULL,
2964
- message_data JSONB NOT NULL,
2965
- message_metadata JSONB NOT NULL,
2966
- PRIMARY KEY (stream_id, stream_position, partition, is_archived)
2967
- ) PARTITION BY LIST (partition);`;
2968
- const processorsTableSQL = SQL`
2969
- CREATE TABLE IF NOT EXISTS ${SQL.identifier(processorsTable.name)}(
2970
- last_processed_transaction_id XID8 NOT NULL,
2971
- version INT NOT NULL DEFAULT 1,
2972
- processor_id TEXT NOT NULL,
2973
- partition TEXT NOT NULL DEFAULT '${SQL.plain(defaultTag)}',
2974
- status TEXT NOT NULL DEFAULT 'stopped',
2975
- last_processed_checkpoint TEXT NOT NULL,
2976
- processor_instance_id TEXT DEFAULT '${SQL.plain(unknownTag)}',
2977
- created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
2978
- last_updated TIMESTAMPTZ NOT NULL DEFAULT now(),
2979
- PRIMARY KEY (processor_id, partition, version)
2980
- ) PARTITION BY LIST (partition);
2981
- `;
2982
- const projectionsTableSQL = SQL`
2983
- CREATE TABLE IF NOT EXISTS ${SQL.identifier(projectionsTable.name)}(
2984
- version INT NOT NULL DEFAULT 1,
2985
- type VARCHAR(1) NOT NULL,
2986
- name TEXT NOT NULL,
2987
- partition TEXT NOT NULL DEFAULT '${SQL.plain(defaultTag)}',
2988
- kind TEXT NOT NULL,
2989
- status TEXT NOT NULL,
2990
- definition JSONB NOT NULL DEFAULT '{}'::jsonb,
2991
- created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
2992
- last_updated TIMESTAMPTZ NOT NULL DEFAULT now(),
2993
- PRIMARY KEY (name, partition, version)
2994
- ) PARTITION BY LIST (partition);
2995
- `;
2996
- const sanitizeNameSQL = createFunctionIfDoesNotExistSQL("emt_sanitize_name", SQL`CREATE OR REPLACE FUNCTION emt_sanitize_name(input_name TEXT) RETURNS TEXT AS $emt_sanitize_name$
2997
- BEGIN
2998
- RETURN REGEXP_REPLACE(input_name, '[^a-zA-Z0-9_]', '_', 'g');
2999
- END;
3000
- $emt_sanitize_name$ LANGUAGE plpgsql;`);
3001
- const addTablePartitions = createFunctionIfDoesNotExistSQL("emt_add_table_partition", SQL`
3002
- CREATE OR REPLACE FUNCTION emt_add_table_partition(tableName TEXT, partition_name TEXT) RETURNS void AS $emt_add_table_partition$
3003
- DECLARE
3004
- v_main_partiton_name TEXT;
3005
- v_active_partiton_name TEXT;
3006
- v_archived_partiton_name TEXT;
3007
- BEGIN
3008
- v_main_partiton_name := emt_sanitize_name(tableName || '_' || partition_name);
3009
- v_active_partiton_name := emt_sanitize_name(v_main_partiton_name || '_active');
3010
- v_archived_partiton_name := emt_sanitize_name(v_main_partiton_name || '_archived');
3011
-
3012
-
3013
- -- create default partition
3014
- EXECUTE format('
3015
- CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
3016
- FOR VALUES IN (%L) PARTITION BY LIST (is_archived);',
3017
- v_main_partiton_name, tableName, partition_name
3018
- );
3019
-
3020
- EXECUTE format('
3021
- CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
3022
- FOR VALUES IN (FALSE);',
3023
- v_active_partiton_name, v_main_partiton_name
3024
- );
3025
-
3026
- EXECUTE format('
3027
- CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
3028
- FOR VALUES IN (TRUE);',
3029
- v_archived_partiton_name, v_main_partiton_name
3030
- );
3031
- END;
3032
- $emt_add_table_partition$ LANGUAGE plpgsql;`);
3033
- const addPartitionSQL = createFunctionIfDoesNotExistSQL("emt_add_partition", SQL`
3034
- CREATE OR REPLACE FUNCTION emt_add_partition(partition_name TEXT) RETURNS void AS $emt_add_partition$
3035
- BEGIN
3036
- PERFORM emt_add_table_partition('${SQL.plain(messagesTable.name)}', partition_name);
3037
- PERFORM emt_add_table_partition('${SQL.plain(streamsTable.name)}', partition_name);
3038
-
3039
- EXECUTE format('
3040
- CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
3041
- FOR VALUES IN (%L);',
3042
- emt_sanitize_name('${SQL.plain(processorsTable.name)}' || '_' || partition_name), '${SQL.plain(processorsTable.name)}', partition_name
3043
- );
3044
-
3045
- EXECUTE format('
3046
- CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
3047
- FOR VALUES IN (%L);',
3048
- emt_sanitize_name('${SQL.plain(projectionsTable.name)}' || '_' || partition_name), '${SQL.plain(projectionsTable.name)}', partition_name
3049
- );
3050
- END;
3051
- $emt_add_partition$ LANGUAGE plpgsql;`);
3052
- const addModuleSQL = SQL`
3053
- CREATE OR REPLACE FUNCTION add_module(new_module TEXT) RETURNS void AS $$
3054
- BEGIN
3055
- -- For ${SQL.plain(messagesTable.name)} table
3056
- EXECUTE format('
3057
- CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
3058
- FOR VALUES IN (emt_sanitize_name(%L || ''__'' || %L)) PARTITION BY LIST (is_archived);',
3059
- emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}'), '${SQL.plain(messagesTable.name)}', new_module, '${SQL.plain(globalTag)}'
3060
- );
3061
-
3062
- EXECUTE format('
3063
- CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
3064
- FOR VALUES IN (FALSE);',
3065
- emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}' || '_active'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}')
3066
- );
3067
-
3068
- EXECUTE format('
3069
- CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
3070
- FOR VALUES IN (TRUE);',
3071
- emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}' || '_archived'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}')
3072
- );
3073
-
3074
- -- For ${SQL.plain(streamsTable.name)} table
3075
- EXECUTE format('
3076
- CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
3077
- FOR VALUES IN (emt_sanitize_name(%L || ''__'' || %L)) PARTITION BY LIST (is_archived);',
3078
- emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}'), '${SQL.plain(streamsTable.name)}', new_module, '${SQL.plain(globalTag)}'
3079
- );
3080
-
3081
- EXECUTE format('
3082
- CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
3083
- FOR VALUES IN (FALSE);',
3084
- emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}' || '_active'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}')
3085
- );
3086
-
3087
- EXECUTE format('
3088
- CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
3089
- FOR VALUES IN (TRUE);',
3090
- emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}' || '_archived'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || '${SQL.plain(globalTag)}')
3091
- );
3092
- END;
3093
- $$ LANGUAGE plpgsql;
3094
- `;
3095
- const addTenantSQL = SQL`
3096
- CREATE OR REPLACE FUNCTION add_tenant(new_module TEXT, new_tenant TEXT) RETURNS void AS $$
3097
- BEGIN
3098
- -- For ${SQL.plain(messagesTable.name)} table
3099
- EXECUTE format('
3100
- CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
3101
- FOR VALUES IN (emt_sanitize_name(''%s__%s'')) PARTITION BY LIST (is_archived);',
3102
- emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || new_tenant), '${SQL.plain(messagesTable.name)}', new_module, new_tenant
3103
- );
3104
-
3105
- EXECUTE format('
3106
- CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
3107
- FOR VALUES IN (FALSE);',
3108
- emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || new_tenant || '_active'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || new_tenant)
3109
- );
3110
-
3111
- EXECUTE format('
3112
- CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
3113
- FOR VALUES IN (TRUE);',
3114
- emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || new_tenant || '_archived'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || new_tenant)
3115
- );
3116
-
3117
- -- For ${SQL.plain(streamsTable.name)} table
2457
+ //#endregion
2458
+ //#region src/eventStore/schema/migrations/0_43_0/0_43_0.migration.ts
2459
+ const migration_0_43_0_cleanupLegacySubscriptionSQL = SQL`
2460
+ DO $$
2461
+ BEGIN
2462
+ IF EXISTS (SELECT 1 FROM pg_tables WHERE tablename = 'emt_subscriptions') THEN
2463
+ -- Restore clean emt_add_partition (remove creation of emt_subscriptions partitions)
2464
+ CREATE OR REPLACE FUNCTION emt_add_partition(partition_name TEXT) RETURNS void AS $fnpar$
2465
+ BEGIN
2466
+ PERFORM emt_add_table_partition('${SQL.plain(messagesTable.name)}', partition_name);
2467
+ PERFORM emt_add_table_partition('${SQL.plain(streamsTable.name)}', partition_name);
2468
+
3118
2469
  EXECUTE format('
3119
2470
  CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
3120
- FOR VALUES IN (emt_sanitize_name(''%s__%s'')) PARTITION BY LIST (is_archived);',
3121
- emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || new_tenant), '${SQL.plain(streamsTable.name)}', new_module, new_tenant
3122
- );
3123
-
3124
- EXECUTE format('
3125
- CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
3126
- FOR VALUES IN (FALSE);',
3127
- emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || new_tenant || '_active'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || new_tenant)
2471
+ FOR VALUES IN (%L);',
2472
+ emt_sanitize_name('${SQL.plain(processorsTable.name)}' || '_' || partition_name), '${SQL.plain(processorsTable.name)}', partition_name
3128
2473
  );
3129
-
2474
+
3130
2475
  EXECUTE format('
3131
- CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
3132
- FOR VALUES IN (TRUE);',
3133
- emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || new_tenant || '_archived'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || new_tenant)
2476
+ CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
2477
+ FOR VALUES IN (%L);',
2478
+ emt_sanitize_name('${SQL.plain(projectionsTable.name)}' || '_' || partition_name), '${SQL.plain(projectionsTable.name)}', partition_name
3134
2479
  );
3135
2480
  END;
3136
- $$ LANGUAGE plpgsql;
3137
- `;
3138
- const addModuleForAllTenantsSQL = SQL`
3139
- CREATE OR REPLACE FUNCTION add_module_for_all_tenants(new_module TEXT) RETURNS void AS $$
3140
- DECLARE
3141
- tenant_record RECORD;
3142
- BEGIN
3143
- PERFORM add_module(new_module);
3144
-
3145
- FOR tenant_record IN SELECT DISTINCT tenant FROM ${SQL.plain(messagesTable.name)}
3146
- LOOP
3147
- -- For ${SQL.plain(messagesTable.name)} table
3148
- EXECUTE format('
3149
- CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
3150
- FOR VALUES IN (emt_sanitize_name(''%s__%s'')) PARTITION BY LIST (is_archived);',
3151
- emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || tenant_record.tenant), '${SQL.plain(messagesTable.name)}', new_module, tenant_record.tenant
3152
- );
3153
-
3154
- EXECUTE format('
3155
- CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
3156
- FOR VALUES IN (FALSE);',
3157
- emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || tenant_record.tenant || '_active'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || tenant_record.tenant)
3158
- );
3159
-
3160
- EXECUTE format('
3161
- CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
3162
- FOR VALUES IN (TRUE);',
3163
- emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || tenant_record.tenant || '_archived'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || new_module || '__' || tenant_record.tenant)
3164
- );
3165
-
3166
- -- For ${SQL.plain(streamsTable.name)} table
3167
- EXECUTE format('
3168
- CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
3169
- FOR VALUES IN (emt_sanitize_name(''%s__%s'')) PARTITION BY LIST (is_archived);',
3170
- emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || tenant_record.tenant), '${SQL.plain(streamsTable.name)}', new_module, tenant_record.tenant
3171
- );
3172
-
3173
- EXECUTE format('
3174
- CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
3175
- FOR VALUES IN (FALSE);',
3176
- emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || tenant_record.tenant || '_active'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || tenant_record.tenant)
3177
- );
3178
-
3179
- EXECUTE format('
3180
- CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
3181
- FOR VALUES IN (TRUE);',
3182
- emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || tenant_record.tenant || '_archived'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || new_module || '__' || tenant_record.tenant)
3183
- );
3184
- END LOOP;
3185
- END;
3186
- $$ LANGUAGE plpgsql;
3187
- `;
3188
- const addTenantForAllModulesSQL = SQL`
3189
- CREATE OR REPLACE FUNCTION add_tenant_for_all_modules(new_tenant TEXT) RETURNS void AS $$
2481
+ $fnpar$ LANGUAGE plpgsql;
2482
+
2483
+ -- Drop old subscriptions table if it exists
2484
+ DROP TABLE IF EXISTS emt_subscriptions CASCADE;
2485
+
2486
+ -- Drop old function if it exists
2487
+ DROP FUNCTION IF EXISTS store_subscription_checkpoint(character varying, bigint, bigint, bigint, xid8, text);
2488
+
2489
+ -- Restore clean store_processor_checkpoint (remove dual-write logic)
2490
+ CREATE OR REPLACE FUNCTION store_processor_checkpoint(
2491
+ p_processor_id TEXT,
2492
+ p_version BIGINT,
2493
+ p_position TEXT,
2494
+ p_check_position TEXT,
2495
+ p_transaction_id xid8,
2496
+ p_partition TEXT DEFAULT '${SQL.plain(defaultTag)}',
2497
+ p_processor_instance_id TEXT DEFAULT 'emt:unknown'
2498
+ ) RETURNS INT AS $fn$
3190
2499
  DECLARE
3191
- module_record RECORD;
2500
+ current_position TEXT;
3192
2501
  BEGIN
3193
- FOR module_record IN SELECT DISTINCT partitionname FROM pg_partman.part_config WHERE parent_table = '${SQL.plain(messagesTable.name)}'
3194
- LOOP
3195
- -- For ${SQL.plain(messagesTable.name)} table
3196
- EXECUTE format('
3197
- CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
3198
- FOR VALUES IN (emt_sanitize_name(''%s__%s'')) PARTITION BY LIST (is_archived);',
3199
- emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || module_record.partitionname || '__' || new_tenant), '${SQL.plain(messagesTable.name)}', module_record.partitionname, new_tenant
3200
- );
3201
-
3202
- EXECUTE format('
3203
- CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
3204
- FOR VALUES IN (FALSE);',
3205
- emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || module_record.partitionname || '__' || new_tenant || '_active'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || module_record.partitionname || '__' || new_tenant)
3206
- );
3207
-
3208
- EXECUTE format('
3209
- CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
3210
- FOR VALUES IN (TRUE);',
3211
- emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || module_record.partitionname || '__' || new_tenant || '_archived'), emt_sanitize_name('${SQL.plain(messagesTable.name)}_' || module_record.partitionname || '__' || new_tenant)
3212
- );
3213
-
3214
- -- For ${SQL.plain(streamsTable.name)} table
3215
- EXECUTE format('
3216
- CREATE TABLE IF NOT EXISTS %I PARTITION OF %I
3217
- FOR VALUES IN (emt_sanitize_name(''%s__%s'')) PARTITION BY LIST (is_archived);',
3218
- emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || module_record.partitionname || '__' || new_tenant), '${SQL.plain(streamsTable.name)}', module_record.partitionname, new_tenant
3219
- );
3220
-
3221
- EXECUTE format('
3222
- CREATE TABLE IF NOT EXISTS %I_active PARTITION OF %I
3223
- FOR VALUES IN (FALSE);',
3224
- emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || module_record.partitionname || '__' || new_tenant || '_active'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || module_record.partitionname || '__' || new_tenant)
3225
- );
3226
-
3227
- EXECUTE format('
3228
- CREATE TABLE IF NOT EXISTS %I_archived PARTITION OF %I
3229
- FOR VALUES IN (TRUE);',
3230
- emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || module_record.partitionname || '__' || new_tenant || '_archived'), emt_sanitize_name('${SQL.plain(streamsTable.name)}_' || module_record.partitionname || '__' || new_tenant)
3231
- );
3232
- END LOOP;
2502
+ IF p_check_position IS NOT NULL THEN
2503
+ UPDATE "emt_processors"
2504
+ SET
2505
+ "last_processed_checkpoint" = p_position,
2506
+ "last_processed_transaction_id" = p_transaction_id,
2507
+ "last_updated" = now()
2508
+ WHERE "processor_id" = p_processor_id
2509
+ AND "last_processed_checkpoint" = p_check_position
2510
+ AND "partition" = p_partition
2511
+ AND "version" = p_version;
2512
+
2513
+ IF FOUND THEN
2514
+ RETURN 1;
2515
+ END IF;
2516
+
2517
+ SELECT "last_processed_checkpoint" INTO current_position
2518
+ FROM "emt_processors"
2519
+ WHERE "processor_id" = p_processor_id
2520
+ AND "partition" = p_partition
2521
+ AND "version" = p_version ;
2522
+
2523
+ IF current_position = p_position THEN
2524
+ RETURN 0;
2525
+ ELSIF current_position > p_position THEN
2526
+ RETURN 3;
2527
+ ELSE
2528
+ RETURN 2;
2529
+ END IF;
2530
+ END IF;
2531
+
2532
+ BEGIN
2533
+ INSERT INTO "emt_processors"("processor_id", "version", "last_processed_checkpoint", "partition", "last_processed_transaction_id", "created_at", "last_updated")
2534
+ VALUES (p_processor_id, p_version, p_position, p_partition, p_transaction_id, now(), now());
2535
+ RETURN 1;
2536
+ EXCEPTION WHEN unique_violation THEN
2537
+ SELECT "last_processed_checkpoint" INTO current_position
2538
+ FROM "emt_processors"
2539
+ WHERE "processor_id" = p_processor_id
2540
+ AND "partition" = p_partition
2541
+ AND "version" = p_version;
2542
+
2543
+ IF current_position = p_position THEN
2544
+ RETURN 0;
2545
+ ELSE
2546
+ RETURN 2;
2547
+ END IF;
2548
+ END;
3233
2549
  END;
3234
- $$ LANGUAGE plpgsql;
3235
- `;
3236
- const addDefaultPartitionSQL = SQL`SELECT emt_add_partition('${SQL.plain(defaultTag)}');`;
2550
+ $fn$ LANGUAGE plpgsql;
2551
+ END IF;
2552
+ END $$;
2553
+ `;
2554
+ const migration_0_43_0_cleanupLegacySubscription = sqlMigration("emt:postgresql:eventstore:0.43.0:cleanup-legacy-subscription", [migration_0_43_0_cleanupLegacySubscriptionSQL]);
2555
+
2556
+ //#endregion
2557
+ //#region src/eventStore/schema/migrations/0_43_0/index.ts
2558
+ const migrations_0_43_0 = [migration_0_43_0_cleanupLegacySubscription];
2559
+
2560
+ //#endregion
2561
+ //#region src/eventStore/schema/migrations/index.ts
2562
+ const schemaSQL$1 = [
2563
+ streamsTableSQL,
2564
+ messagesTableSQL,
2565
+ projectionsTableSQL,
2566
+ processorsTableSQL,
2567
+ sanitizeNameSQL,
2568
+ addTablePartitions,
2569
+ addPartitionSQL,
2570
+ appendToStreamSQL,
2571
+ addDefaultPartitionSQL,
2572
+ storeSubscriptionCheckpointSQL,
2573
+ tryAcquireProcessorLockSQL,
2574
+ releaseProcessorLockSQL,
2575
+ registerProjectionSQL,
2576
+ activateProjectionSQL,
2577
+ deactivateProjectionSQL
2578
+ ];
2579
+ const currentPostgreSQLEventStoreSchemaVersion = "0.43.0";
2580
+ const schemaMigration$1 = sqlMigration("emt:postgresql:eventstore:initial", schemaSQL$1);
2581
+ const pastEventStoreSchemaMigrations = [
2582
+ ...migrations_0_38_7,
2583
+ ...migrations_0_42_0,
2584
+ ...migrations_0_43_0
2585
+ ];
2586
+ const eventStoreSchemaMigrations = [...pastEventStoreSchemaMigrations, schemaMigration$1];
3237
2587
 
3238
2588
  //#endregion
3239
2589
  //#region src/eventStore/schema/readProcessorCheckpoint.ts
@@ -3315,12 +2665,6 @@ const schemaSQL = [
3315
2665
  deactivateProjectionSQL
3316
2666
  ];
3317
2667
  const schemaMigration = sqlMigration("emt:postgresql:eventstore:initial", schemaSQL);
3318
- const eventStoreSchemaMigrations = [
3319
- migration_0_38_7_and_older,
3320
- migration_0_42_0_FromSubscriptionsToProcessors,
3321
- migration_0_42_0_2_AddProcessorProjectionFunctions,
3322
- schemaMigration
3323
- ];
3324
2668
  const createEventStoreSchema = (connectionString, pool, hooks, options) => {
3325
2669
  return pool.withTransaction(async (tx) => {
3326
2670
  const context = await transactionToPostgreSQLProjectionHandlerContext(connectionString, pool, tx);
@@ -3371,24 +2715,32 @@ const getPostgreSQLEventStore = (connectionString, options = defaultPostgreSQLOp
3371
2715
  const autoGenerateSchema = options.schema?.autoMigration === void 0 || options.schema?.autoMigration !== "None";
3372
2716
  const inlineProjections = (options.projections ?? []).filter(({ type }) => type === "inline").map(({ projection }) => projection);
3373
2717
  const migrate = async (migrationOptions) => {
3374
- if (!migrateSchema) migrateSchema = createEventStoreSchema(connectionString, pool, {
3375
- onBeforeSchemaCreated: async (context) => {
3376
- if (options.hooks?.onBeforeSchemaCreated) await options.hooks.onBeforeSchemaCreated(context);
3377
- },
3378
- onAfterSchemaCreated: async (context) => {
3379
- for (const projection of inlineProjections) if (projection.init) await projection.init({
3380
- version: projection.version ?? 1,
3381
- status: "active",
3382
- registrationType: "inline",
3383
- context: {
3384
- ...context,
3385
- migrationOptions
3386
- }
3387
- });
3388
- if (options.hooks?.onAfterSchemaCreated) await options.hooks.onAfterSchemaCreated(context);
3389
- }
3390
- }, migrationOptions);
3391
- return migrateSchema;
2718
+ if (!migrateSchema) {
2719
+ migrateSchema = createEventStoreSchema(connectionString, pool, {
2720
+ onBeforeSchemaCreated: async (context) => {
2721
+ if (options.hooks?.onBeforeSchemaCreated) await options.hooks.onBeforeSchemaCreated(context);
2722
+ },
2723
+ onAfterSchemaCreated: async (context) => {
2724
+ for (const projection of inlineProjections) if (projection.init) await projection.init({
2725
+ version: projection.version ?? 1,
2726
+ status: "active",
2727
+ registrationType: "inline",
2728
+ context: {
2729
+ ...context,
2730
+ migrationOptions
2731
+ }
2732
+ });
2733
+ if (options.hooks?.onAfterSchemaCreated) await options.hooks.onAfterSchemaCreated(context);
2734
+ }
2735
+ }, migrationOptions);
2736
+ return migrateSchema;
2737
+ }
2738
+ const result = await migrateSchema;
2739
+ if (migrationOptions?.dryRun) migrateSchema = void 0;
2740
+ return {
2741
+ applied: [],
2742
+ skipped: result.applied.concat(result.skipped)
2743
+ };
3392
2744
  };
3393
2745
  const ensureSchemaExists = () => {
3394
2746
  if (!autoGenerateSchema) return Promise.resolve();
@@ -3562,8 +2914,8 @@ const postgreSQLProjector = (options) => {
3562
2914
  partition,
3563
2915
  processorInstanceId,
3564
2916
  projection: options.projection ? {
3565
- name: options.projection.name ?? unknownTag$1,
3566
- kind: options.projection.kind ?? unknownTag$1,
2917
+ name: options.projection.name ?? unknownTag,
2918
+ kind: options.projection.kind ?? unknownTag,
3567
2919
  version: options.projection.version ?? version,
3568
2920
  handlingType: "async"
3569
2921
  } : void 0,
@@ -3701,10 +3053,15 @@ const postgreSQLEventStoreConsumer = (options) => {
3701
3053
  reason: "No active processors"
3702
3054
  };
3703
3055
  return (await Promise.allSettled(activeProcessors.map(async (s) => {
3704
- return await s.handle(messagesBatch, { connection: {
3705
- connectionString: options.connectionString,
3706
- pool
3707
- } });
3056
+ try {
3057
+ return await s.handle(messagesBatch, { connection: {
3058
+ connectionString: options.connectionString,
3059
+ pool
3060
+ } });
3061
+ } catch (error) {
3062
+ console.log(`Error during message batch processing for processor: ${s.id}`, error);
3063
+ throw error;
3064
+ }
3708
3065
  }))).some((r) => r.status === "fulfilled" && r.value?.type !== "STOP") ? void 0 : { type: "STOP" };
3709
3066
  };
3710
3067
  const processorContext = {
@@ -3725,7 +3082,11 @@ const postgreSQLEventStoreConsumer = (options) => {
3725
3082
  abortController?.abort();
3726
3083
  await messagePuller.stop();
3727
3084
  }
3728
- await start;
3085
+ try {
3086
+ await start;
3087
+ } catch (error) {
3088
+ console.log("Error during consumer stop:", error);
3089
+ }
3729
3090
  messagePuller = void 0;
3730
3091
  abortController = null;
3731
3092
  await stopProcessors();
@@ -3733,7 +3094,16 @@ const postgreSQLEventStoreConsumer = (options) => {
3733
3094
  const init = async () => {
3734
3095
  if (isInitialized) return;
3735
3096
  const postgresProcessors = processors;
3736
- for (const processor of postgresProcessors) if (processor.init) await processor.init(processorContext);
3097
+ for (const processor of postgresProcessors) if (processor.init) try {
3098
+ await processor.init(processorContext);
3099
+ } catch (error) {
3100
+ console.log(`Error during processor initialization for processor: ${processor.id}. Stopping it.`, error);
3101
+ await processor.close(processorContext).catch((closeError) => {
3102
+ console.log(`Error during processor cleanup after failed initialization for processor: ${processor.id}`, closeError);
3103
+ });
3104
+ console.log(`Processor ${processor.id} stopped successfully after failed initialization.`);
3105
+ throw error;
3106
+ }
3737
3107
  isInitialized = true;
3738
3108
  };
3739
3109
  return {
@@ -3760,9 +3130,13 @@ const postgreSQLEventStoreConsumer = (options) => {
3760
3130
  return processor;
3761
3131
  },
3762
3132
  start: () => {
3763
- if (isRunning) return start;
3133
+ if (isRunning) {
3134
+ console.log("Consumer is already running. Returning the existing start promise.");
3135
+ return start;
3136
+ }
3764
3137
  startedAwaiter.reset();
3765
3138
  if (processors.length === 0) {
3139
+ console.log("Cannot start consumer without at least a single processor");
3766
3140
  const error = new EmmettError("Cannot start consumer without at least a single processor");
3767
3141
  startedAwaiter.reject(error);
3768
3142
  return Promise.reject(error);
@@ -3780,16 +3154,25 @@ const postgreSQLEventStoreConsumer = (options) => {
3780
3154
  pullingFrequencyInMs: pulling?.pullingFrequencyInMs ?? 50,
3781
3155
  signal: abortController.signal
3782
3156
  });
3783
- if (!isInitialized) await init();
3157
+ if (!isInitialized) {
3158
+ console.log("Initializing consumer before starting message pulling.");
3159
+ await init();
3160
+ }
3784
3161
  const startFrom = zipPostgreSQLEventStoreMessageBatchPullerStartFrom(await Promise.all(processors.map(async (o) => {
3785
- return await o.start({
3786
- execute: pool.execute,
3787
- connection: {
3788
- connectionString: options.connectionString,
3789
- pool
3790
- }
3791
- });
3162
+ try {
3163
+ return await o.start({
3164
+ execute: pool.execute,
3165
+ connection: {
3166
+ connectionString: options.connectionString,
3167
+ pool
3168
+ }
3169
+ });
3170
+ } catch (error) {
3171
+ console.log(`Error during processor start position retrieval for processor: ${o.id}. Stopping it.`, error);
3172
+ throw error;
3173
+ }
3792
3174
  })));
3175
+ console.log(`Starting message pulling with start position: ${JSONSerializer.serialize(startFrom)}. Waiting for messages...`);
3793
3176
  await messagePuller.start({
3794
3177
  startFrom,
3795
3178
  started: startedAwaiter
@@ -3831,11 +3214,11 @@ const rebuildPostgreSQLProjections = (options) => {
3831
3214
  };
3832
3215
  const projections = "projections" in options ? options.projections.map((p) => "projection" in p ? {
3833
3216
  truncateOnStart: true,
3834
- processorId: getProjectorId({ projectionName: p.projection.name ?? unknownTag$1 }),
3217
+ processorId: getProjectorId({ projectionName: p.projection.name ?? unknownTag }),
3835
3218
  ...p
3836
3219
  } : {
3837
3220
  projection: p,
3838
- processorId: getProjectorId({ projectionName: p.name ?? unknownTag$1 }),
3221
+ processorId: getProjectorId({ projectionName: p.name ?? unknownTag }),
3839
3222
  truncateOnStart: true
3840
3223
  }) : [options];
3841
3224
  for (const projectionDefinition of projections) consumer.projector({
@@ -3847,5 +3230,5 @@ const rebuildPostgreSQLProjections = (options) => {
3847
3230
  };
3848
3231
 
3849
3232
  //#endregion
3850
- export { DefaultPostgreSQLEventStoreProcessorBatchSize, DefaultPostgreSQLEventStoreProcessorPullingFrequencyInMs, DefaultPostgreSQLProcessorLockPolicy, PostgreSQLEventStoreDefaultStreamVersion, PostgreSQLProjectionSpec, activateProjection, activateProjectionSQL, addDefaultPartitionSQL, addModuleForAllTenantsSQL, addModuleSQL, addPartitionSQL, addTablePartitions, addTenantForAllModulesSQL, addTenantSQL, appendToStream, appendToStreamSQL, assertSQLQueryResultMatches, callActivateProjection, callAppendToStream, callDeactivateProjection, callRegisterProjection, callReleaseProcessorLock, callStoreProcessorCheckpoint, callTryAcquireProcessorLock, callTryAcquireProjectionLock, cleanupLegacySubscriptionTables, createEventStoreSchema, deactivateProjection, deactivateProjectionSQL, defaultPostgreSQLOptions, defaultTag, documentDoesNotExist, documentExists, documentMatchingExists, documentsAreTheSame, documentsMatchingHaveCount, emmettPrefix, eventInStream, eventStoreSchemaMigrations, eventsInStream, expectPongoDocuments, expectSQL, getPostgreSQLEventStore, globalNames, globalTag, handleProjections, messagesTable, messagesTableSQL, newEventsInStream, pongoMultiStreamProjection, pongoProjection, pongoSingleStreamProjection, postgreSQLCheckpointer, postgreSQLEventStoreConsumer, postgreSQLEventStoreMessageBatchPuller, postgreSQLProcessorLock, postgreSQLProjection, postgreSQLProjectionLock, postgreSQLProjector, postgreSQLRawBatchSQLProjection, postgreSQLRawSQLProjection, postgreSQLReactor, postgreSQLWorkflowProcessor, processorsTable, processorsTableSQL, projectionsTable, projectionsTableSQL, readLastMessageGlobalPosition, readMessagesBatch, readProcessorCheckpoint, readProjectionInfo, readStream, rebuildPostgreSQLProjections, registerProjection, registerProjectionSQL, releaseProcessorLockSQL, sanitizeNameSQL, schemaMigration, schemaSQL, storeProcessorCheckpoint, storeSubscriptionCheckpointSQL, streamExists, streamsTable, streamsTableSQL, toProcessorLockKey, toProjectionLockKey, transactionToPostgreSQLProjectionHandlerContext, tryAcquireProcessorLockSQL, tryAcquireProjectionLockSQL, unknownTag, zipPostgreSQLEventStoreMessageBatchPullerStartFrom };
3233
+ export { DefaultPostgreSQLEventStoreProcessorBatchSize, DefaultPostgreSQLEventStoreProcessorPullingFrequencyInMs, DefaultPostgreSQLProcessorLockPolicy, PostgreSQLEventStoreDefaultStreamVersion, PostgreSQLProjectionSpec, activateProjection, activateProjectionSQL, addDefaultPartitionSQL, addModuleForAllTenantsSQL, addModuleSQL, addPartitionSQL, addTablePartitions, addTenantForAllModulesSQL, addTenantSQL, appendToStream, appendToStreamSQL, assertSQLQueryResultMatches, callActivateProjection, callAppendToStream, callDeactivateProjection, callRegisterProjection, callReleaseProcessorLock, callStoreProcessorCheckpoint, callTryAcquireProcessorLock, callTryAcquireProjectionLock, createEventStoreSchema, currentPostgreSQLEventStoreSchemaVersion, deactivateProjection, deactivateProjectionSQL, defaultPostgreSQLOptions, documentDoesNotExist, documentExists, documentMatchingExists, documentsAreTheSame, documentsMatchingHaveCount, eventInStream, eventStoreSchemaMigrations, eventsInStream, expectPongoDocuments, expectSQL, getPostgreSQLEventStore, handleProjections, messagesTableSQL, newEventsInStream, pastEventStoreSchemaMigrations, pongoMultiStreamProjection, pongoProjection, pongoSingleStreamProjection, postgreSQLCheckpointer, postgreSQLEventStoreConsumer, postgreSQLEventStoreMessageBatchPuller, postgreSQLProcessorLock, postgreSQLProjection, postgreSQLProjectionLock, postgreSQLProjector, postgreSQLRawBatchSQLProjection, postgreSQLRawSQLProjection, postgreSQLReactor, postgreSQLWorkflowProcessor, processorsTableSQL, projectionsTableSQL, readLastMessageGlobalPosition, readMessagesBatch, readProcessorCheckpoint, readProjectionInfo, readStream, rebuildPostgreSQLProjections, registerProjection, registerProjectionSQL, releaseProcessorLockSQL, sanitizeNameSQL, schemaMigration, schemaSQL, storeProcessorCheckpoint, storeSubscriptionCheckpointSQL, streamExists, streamsTableSQL, toProcessorLockKey, toProjectionLockKey, transactionToPostgreSQLProjectionHandlerContext, tryAcquireProcessorLockSQL, tryAcquireProjectionLockSQL, zipPostgreSQLEventStoreMessageBatchPullerStartFrom };
3851
3234
  //# sourceMappingURL=index.js.map