@event-driven-io/emmett-postgresql 0.35.0-alpha.2 → 0.36.0

package/dist/index.js

@@ -176,6 +176,108 @@ var JSONParser = {
     return options?.map ? options.map(parsed) : parsed;
   }
 };
+var MessageProcessorType = {
+  PROJECTOR: "projector",
+  REACTOR: "reactor"
+};
+var defaultProcessingMessageProcessingScope = (handler, partialContext) => handler(partialContext);
+var reactor = (options) => {
+  const eachMessage = "eachMessage" in options && options.eachMessage ? options.eachMessage : () => Promise.resolve();
+  let isActive = true;
+  const { checkpoints, processorId, partition } = options;
+  const processingScope = options.processingScope ?? defaultProcessingMessageProcessingScope;
+  return {
+    id: options.processorId,
+    type: options.type ?? MessageProcessorType.REACTOR,
+    close: () => options.hooks?.onClose ? options.hooks?.onClose() : Promise.resolve(),
+    start: async (startOptions) => {
+      isActive = true;
+      return await processingScope(async (context) => {
+        if (options.hooks?.onStart) {
+          await options.hooks?.onStart(context);
+        }
+        if (options.startFrom !== "CURRENT" && options.startFrom)
+          return options.startFrom;
+        let lastCheckpoint = null;
+        if (checkpoints) {
+          const readResult = await checkpoints?.read(
+            {
+              processorId,
+              partition
+            },
+            startOptions
+          );
+          lastCheckpoint = readResult.lastCheckpoint;
+        }
+        if (lastCheckpoint === null) return "BEGINNING";
+        return {
+          lastCheckpoint
+        };
+      }, startOptions);
+    },
+    get isActive() {
+      return isActive;
+    },
+    handle: async (messages, partialContext) => {
+      if (!isActive) return Promise.resolve();
+      return await processingScope(async (context) => {
+        let result = void 0;
+        let lastCheckpoint = null;
+        for (const message2 of messages) {
+          const messageProcessingResult = await eachMessage(message2, context);
+          if (checkpoints) {
+            const storeCheckpointResult = await checkpoints.store(
+              {
+                processorId: options.processorId,
+                version: options.version,
+                message: message2,
+                lastCheckpoint,
+                partition: options.partition
+              },
+              context
+            );
+            if (storeCheckpointResult && storeCheckpointResult.success) {
+              lastCheckpoint = storeCheckpointResult.newCheckpoint;
+            }
+          }
+          if (messageProcessingResult && messageProcessingResult.type === "STOP") {
+            isActive = false;
+            result = messageProcessingResult;
+            break;
+          }
+          if (options.stopAfter && options.stopAfter(message2)) {
+            isActive = false;
+            result = { type: "STOP", reason: "Stop condition reached" };
+            break;
+          }
+          if (messageProcessingResult && messageProcessingResult.type === "SKIP")
+            continue;
+        }
+        return result;
+      }, partialContext);
+    }
+  };
+};
+var projector = (options) => {
+  const { projection: projection2, ...rest } = options;
+  return reactor({
+    ...rest,
+    type: MessageProcessorType.PROJECTOR,
+    processorId: options.processorId ?? `projection:${projection2.name}`,
+    hooks: {
+      onStart: options.truncateOnStart && options.projection.truncate || options.hooks?.onStart ? async (context) => {
+        if (options.truncateOnStart && options.projection.truncate)
+          await options.projection.truncate(context);
+        if (options.hooks?.onStart) await options.hooks?.onStart(context);
+      } : void 0,
+      onClose: options.hooks?.onClose
+    },
+    eachMessage: async (event2, context) => {
+      if (!projection2.canHandle.includes(event2.type)) return;
+      await projection2.handle([event2], context);
+    }
+  });
+};
 var projection = (definition) => definition;
 var filter = (filter2) => new TransformStream3({
   transform(chunk, controller) {
@@ -502,7 +604,7 @@ var readMessagesBatch = async (execute, options) => {
   const fromCondition = from !== -0n ? `AND global_position >= ${from}` : "";
   const toCondition = "to" in options ? `AND global_position <= ${options.to}` : "";
   const limitCondition = "batchSize" in options ? `LIMIT ${options.batchSize}` : "";
-  const events = await mapRows(
+  const messages = await mapRows(
     execute.query(
       sql2(
         `SELECT stream_id, stream_position, global_position, message_data, message_metadata, message_schema_version, message_type, message_id
@@ -533,14 +635,14 @@ var readMessagesBatch = async (execute, options) => {
       };
     }
   );
-  return events.length > 0 ? {
-    currentGlobalPosition: events[events.length - 1].metadata.globalPosition,
-    messages: events,
-    areEventsLeft: events.length === batchSize
+  return messages.length > 0 ? {
+    currentGlobalPosition: messages[messages.length - 1].metadata.globalPosition,
+    messages,
+    areMessagesLeft: messages.length === batchSize
   } : {
     currentGlobalPosition: "from" in options ? options.from : "after" in options ? options.after : 0n,
     messages: [],
-    areEventsLeft: false
+    areMessagesLeft: false
   };
 };
 
@@ -551,21 +653,22 @@ var postgreSQLEventStoreMessageBatchPuller = ({
   executor,
   batchSize,
   eachBatch,
-  pullingFrequencyInMs
+  pullingFrequencyInMs,
+  stopWhen
 }) => {
   let isRunning = false;
   let start;
   const pullMessages = async (options) => {
-    const after = options.startFrom === "BEGINNING" ? 0n : options.startFrom === "END" ? (await readLastMessageGlobalPosition(executor)).currentGlobalPosition ?? 0n : options.startFrom.globalPosition;
+    const after = options.startFrom === "BEGINNING" ? 0n : options.startFrom === "END" ? (await readLastMessageGlobalPosition(executor)).currentGlobalPosition ?? 0n : options.startFrom.lastCheckpoint;
     const readMessagesOptions = {
       after,
       batchSize
     };
     let waitTime = 100;
     do {
-      const { messages, currentGlobalPosition, areEventsLeft } = await readMessagesBatch(executor, readMessagesOptions);
+      const { messages, currentGlobalPosition, areMessagesLeft } = await readMessagesBatch(executor, readMessagesOptions);
       if (messages.length > 0) {
-        const result = await eachBatch({ messages });
+        const result = await eachBatch(messages);
         if (result && result.type === "STOP") {
           isRunning = false;
           break;
@@ -573,7 +676,11 @@ var postgreSQLEventStoreMessageBatchPuller = ({
       }
       readMessagesOptions.after = currentGlobalPosition;
       await new Promise((resolve) => setTimeout(resolve, waitTime));
-      if (!areEventsLeft) {
+      if (stopWhen?.noMessagesLeft === true && !areMessagesLeft) {
+        isRunning = false;
+        break;
+      }
+      if (!areMessagesLeft) {
         waitTime = Math.min(waitTime * 2, 1e3);
       } else {
         waitTime = pullingFrequencyInMs;
@@ -608,6 +715,7 @@ var zipPostgreSQLEventStoreMessageBatchPullerStartFrom = (options) => {
 
 // src/eventStore/consumers/postgreSQLEventStoreConsumer.ts
 import { dumbo as dumbo2 } from "@event-driven-io/dumbo";
+import { v7 as uuid6 } from "uuid";
 
 // src/eventStore/consumers/postgreSQLProcessor.ts
 import {
@@ -866,7 +974,7 @@ BEGIN
 END;
 $$ LANGUAGE plpgsql;
 `);
-async function storeProcessorCheckpoint(execute, options) {
+var storeProcessorCheckpoint = async (execute, options) => {
   try {
     const { result } = await single2(
       execute.command(
@@ -885,7 +993,7 @@ async function storeProcessorCheckpoint(execute, options) {
     console.log(error);
     throw error;
   }
-}
+};
 
 // src/eventStore/schema/tables.ts
 import { rawSql as rawSql2 } from "@event-driven-io/dumbo";
@@ -1324,120 +1432,114 @@ var createEventStoreSchema = async (pool) => {
 };
 
 // src/eventStore/consumers/postgreSQLProcessor.ts
-var PostgreSQLProcessor = {
-  result: {
-    skip: (options) => ({
-      type: "SKIP",
-      ...options ?? {}
-    }),
-    stop: (options) => ({
-      type: "STOP",
-      ...options ?? {}
-    })
+var postgreSQLCheckpointer = () => ({
+  read: async (options, context) => {
+    const result = await readProcessorCheckpoint(context.execute, options);
+    return { lastCheckpoint: result?.lastProcessedPosition };
+  },
+  store: async (options, context) => {
+    const result = await storeProcessorCheckpoint(context.execute, {
+      lastProcessedPosition: options.lastCheckpoint,
+      newPosition: options.message.metadata.globalPosition,
+      processorId: options.processorId,
+      partition: options.partition,
+      version: options.version
+    });
+    return result.success ? { success: true, newCheckpoint: result.newPosition } : result;
   }
-};
-var genericPostgreSQLProcessor = (options) => {
-  const { eachMessage } = options;
-  let isActive = true;
-  const poolOptions = {
-    ...options.connectionOptions ? options.connectionOptions : {}
-  };
-  const processorConnectionString = "connectionString" in poolOptions ? poolOptions.connectionString : null;
-  const processorPool = "dumbo" in poolOptions ? poolOptions.dumbo : processorConnectionString ? dumbo({
-    connectionString: processorConnectionString,
-    ...poolOptions
-  }) : null;
-  const getPool = (context) => {
-    const connectionString = processorConnectionString ?? context.connectionString;
+});
+var postgreSQLProcessingScope = (options) => {
+  const processorConnectionString = options.connectionString;
+  const processorPool = options.pool;
+  const processingScope = async (handler, partialContext) => {
+    const connection = partialContext?.connection;
+    const connectionString = processorConnectionString ?? connection?.connectionString;
     if (!connectionString)
       throw new EmmettError(
         `PostgreSQL processor '${options.processorId}' is missing connection string. Ensure that you passed it through options`
       );
-    const pool = (!processorConnectionString || connectionString == processorConnectionString ? context?.pool : processorPool) ?? processorPool;
+    const pool = (!processorConnectionString || connectionString == processorConnectionString ? connection?.pool : processorPool) ?? processorPool;
     if (!pool)
       throw new EmmettError(
         `PostgreSQL processor '${options.processorId}' is missing connection string. Ensure that you passed it through options`
       );
-    return {
-      connectionString,
-      pool
-    };
-  };
-  return {
-    id: options.processorId,
-    start: async (execute) => {
-      isActive = true;
-      if (options.startFrom !== "CURRENT") return options.startFrom;
-      const { lastProcessedPosition } = await readProcessorCheckpoint(execute, {
-        processorId: options.processorId,
-        partition: options.partition
-      });
-      if (lastProcessedPosition === null) return "BEGINNING";
-      return { globalPosition: lastProcessedPosition };
-    },
-    get isActive() {
-      return isActive;
-    },
-    handle: async ({ messages }, context) => {
-      if (!isActive) return;
-      const { pool, connectionString } = getPool(context);
-      return pool.withTransaction(async (transaction) => {
-        let result = void 0;
-        let lastProcessedPosition = null;
-        for (const message of messages) {
-          const typedMessage = message;
-          const client = await transaction.connection.open();
-          const messageProcessingResult = await eachMessage(typedMessage, {
-            execute: transaction.execute,
-            connection: {
-              connectionString,
-              pool,
-              transaction,
-              client
-            }
-          });
-          await storeProcessorCheckpoint(transaction.execute, {
-            processorId: options.processorId,
-            version: options.version,
-            lastProcessedPosition,
-            newPosition: typedMessage.metadata.globalPosition,
-            partition: options.partition
-          });
-          lastProcessedPosition = typedMessage.metadata.globalPosition;
-          if (messageProcessingResult && messageProcessingResult.type === "STOP") {
-            isActive = false;
-            result = messageProcessingResult;
-            break;
-          }
-          if (options.stopAfter && options.stopAfter(typedMessage)) {
-            isActive = false;
-            result = { type: "STOP", reason: "Stop condition reached" };
-            break;
-          }
-          if (messageProcessingResult && messageProcessingResult.type === "SKIP")
-            continue;
+    return pool.withTransaction(async (transaction) => {
+      const client = await transaction.connection.open();
+      return handler({
+        execute: transaction.execute,
+        connection: {
+          connectionString,
+          pool,
+          client,
+          transaction
         }
-        return result;
       });
-    }
+    });
   };
+  return processingScope;
 };
-var postgreSQLProjectionProcessor = (options) => {
-  const projection2 = options.projection;
-  return genericPostgreSQLProcessor({
-    processorId: options.processorId ?? `projection:${projection2.name}`,
-    eachMessage: async (event, context) => {
-      if (!projection2.canHandle.includes(event.type)) return;
-      await projection2.handle([event], context);
-    },
-    ...options
+var getProcessorPool = (options) => {
+  const poolOptions = {
+    ...options.connectionOptions ? options.connectionOptions : {}
+  };
+  const processorConnectionString = "connectionString" in poolOptions ? poolOptions.connectionString ?? null : null;
+  const processorPool = "dumbo" in poolOptions ? poolOptions.dumbo : processorConnectionString ? dumbo({
+    connectionString: processorConnectionString,
+    ...poolOptions
+  }) : null;
+  return {
+    pool: processorPool,
+    connectionString: processorConnectionString,
+    close: processorPool != null && !("dumbo" in poolOptions) ? processorPool.close : void 0
+  };
+};
+var postgreSQLProjector = (options) => {
+  const { pool, connectionString, close } = getProcessorPool(options);
+  const hooks = {
+    onStart: options.hooks?.onStart,
+    onClose: options.hooks?.onClose || close ? async () => {
+      if (options.hooks?.onClose) await options.hooks?.onClose();
+      if (close) await close();
+    } : void 0
+  };
+  return projector({
+    ...options,
+    hooks,
+    processingScope: postgreSQLProcessingScope({
+      pool,
+      connectionString,
+      processorId: options.processorId ?? `projection:${options.projection.name}`
+    }),
+    checkpoints: postgreSQLCheckpointer()
+  });
+};
+var postgreSQLReactor = (options) => {
+  const { pool, connectionString, close } = getProcessorPool(options);
+  const hooks = {
+    onStart: options.hooks?.onStart,
+    onClose: options.hooks?.onClose || close ? async () => {
+      if (options.hooks?.onClose) await options.hooks?.onClose();
+      if (close) await close();
+    } : void 0
+  };
+  return reactor({
+    ...options,
+    hooks,
+    processingScope: postgreSQLProcessingScope({
+      pool,
+      connectionString,
+      processorId: options.processorId
+    }),
+    checkpoints: postgreSQLCheckpointer()
   });
 };
-var postgreSQLProcessor = (options) => {
+var postgreSQLMessageProcessor = (options) => {
   if ("projection" in options) {
-    return postgreSQLProjectionProcessor(options);
+    return postgreSQLProjector(
+      options
+    );
   }
-  return genericPostgreSQLProcessor(options);
+  return postgreSQLReactor(options);
 };
 
 // src/eventStore/consumers/postgreSQLEventStoreConsumer.ts
@@ -1458,8 +1560,10 @@ var postgreSQLEventStoreConsumer = (options) => {
     const result = await Promise.allSettled(
       activeProcessors.map((s) => {
         return s.handle(messagesBatch, {
-          pool,
-          connectionString: options.connectionString
+          connection: {
+            connectionString: options.connectionString,
+            pool
+          }
         });
       })
     );
@@ -1470,6 +1574,7 @@ var postgreSQLEventStoreConsumer = (options) => {
     };
   };
   const messagePooler = currentMessagePuller = postgreSQLEventStoreMessageBatchPuller({
+    stopWhen: options.stopWhen,
     executor: pool.execute,
     eachBatch,
     batchSize: pulling?.batchSize ?? DefaultPostgreSQLEventStoreProcessorBatchSize,
@@ -1483,15 +1588,20 @@ var postgreSQLEventStoreConsumer = (options) => {
       currentMessagePuller = void 0;
     }
     await start;
+    await Promise.all(processors.map((p) => p.close()));
   };
   return {
-    processors,
+    consumerId: options.consumerId ?? uuid6(),
     get isRunning() {
       return isRunning;
     },
+    processors,
     processor: (options2) => {
-      const processor = postgreSQLProcessor(options2);
-      processors.push(processor);
+      const processor = postgreSQLMessageProcessor(options2);
+      processors.push(
+        // TODO: change that
+        processor
+      );
       return processor;
     },
     start: () => {
@@ -1505,7 +1615,18 @@ var postgreSQLEventStoreConsumer = (options) => {
           );
         isRunning = true;
         const startFrom = zipPostgreSQLEventStoreMessageBatchPullerStartFrom(
-          await Promise.all(processors.map((o) => o.start(pool.execute)))
+          await Promise.all(
+            processors.map(async (o) => {
+              const result = await o.start({
+                execute: pool.execute,
+                connection: {
+                  connectionString: options.connectionString,
+                  pool
+                }
+              });
+              return result;
+            })
+          )
         );
         return messagePooler.start({ startFrom });
       })();
@@ -1519,18 +1640,114 @@ var postgreSQLEventStoreConsumer = (options) => {
   };
 };
 
-// src/eventStore/projections/index.ts
-import "@event-driven-io/dumbo";
+// src/eventStore/consumers/rebuildPostgreSQLProjections.ts
+import { v7 as uuid7 } from "uuid";
+var rebuildPostgreSQLProjections = (options) => {
+  const consumer = postgreSQLEventStoreConsumer({
+    ...options,
+    stopWhen: { noMessagesLeft: true }
+  });
+  const projections = "projections" in options ? options.projections.map(
+    (p) => "projection" in p ? {
+      ...p,
+      processorId: `projection:${p.projection.name ?? uuid7()}-rebuild`,
+      truncateOnStart: p.truncateOnStart ?? true
+    } : {
+      projection: p,
+      processorId: `projection:${p.name ?? uuid7()}-rebuild`,
+      truncateOnStart: true
+    }
+  ) : [options];
+  for (const projectionDefinition of projections) {
+    consumer.processor({
+      ...projectionDefinition,
+      processorId: projectionDefinition.processorId ?? `projection:${projectionDefinition.projection.name ?? uuid7()}-rebuild`,
+      truncateOnStart: projectionDefinition.truncateOnStart ?? true
+    });
+  }
+  return consumer;
+};
+
+// src/eventStore/projections/pongo/pongoProjections.ts
+import {
+  pongoClient
+} from "@event-driven-io/pongo";
+var pongoProjection = ({
+  truncate,
+  handle,
+  canHandle
+}) => postgreSQLProjection({
+  canHandle,
+  handle: async (events, context) => {
+    const {
+      connection: { connectionString, client, pool }
+    } = context;
+    const pongo = pongoClient(connectionString, {
+      connectionOptions: { client, pool }
+    });
+    await handle(events, {
+      ...context,
+      pongo
+    });
+  },
+  truncate: truncate ? (context) => {
+    const {
+      connection: { connectionString, client, pool }
+    } = context;
+    const pongo = pongoClient(connectionString, {
+      connectionOptions: { client, pool }
+    });
+    return truncate({
+      ...context,
+      pongo
+    });
+  } : void 0
+});
+var pongoMultiStreamProjection = (options) => {
+  const { collectionName, getDocumentId, canHandle } = options;
+  return pongoProjection({
+    handle: async (events, { pongo }) => {
+      const collection = pongo.db().collection(collectionName);
+      for (const event of events) {
+        await collection.handle(getDocumentId(event), async (document) => {
+          return "initialState" in options ? await options.evolve(
+            document ?? options.initialState(),
+            event
+          ) : await options.evolve(
+            document,
+            event
+          );
+        });
+      }
+    },
+    canHandle,
+    truncate: async (context) => {
+      const {
+        connection: { connectionString, client, pool }
+      } = context;
+      const pongo = pongoClient(connectionString, {
+        connectionOptions: { client, pool }
+      });
+      await pongo.db().collection(collectionName).deleteMany();
+    }
+  });
+};
+var pongoSingleStreamProjection = (options) => {
+  return pongoMultiStreamProjection({
+    ...options,
+    getDocumentId: options.getDocumentId ?? ((event) => event.metadata.streamName)
+  });
+};
 
 // src/eventStore/projections/pongo/pongoProjectionSpec.ts
 import "@event-driven-io/dumbo";
 import {
-  pongoClient
+  pongoClient as pongoClient2
 } from "@event-driven-io/pongo";
 var withCollection = (handle, options) => {
   const { pool, connectionString, inDatabase, inCollection } = options;
   return pool.withConnection(async (connection) => {
-    const pongo = pongoClient(connectionString, {
+    const pongo = pongoClient2(connectionString, {
       connectionOptions: { connection }
     });
     try {
@@ -1658,60 +1875,11 @@ var expectPongoDocuments = {
   }
 };
 
-// src/eventStore/projections/pongo/projections.ts
-import {
-  pongoClient as pongoClient2
-} from "@event-driven-io/pongo";
-var pongoProjection = ({
-  handle,
-  canHandle
-}) => postgreSQLProjection({
-  canHandle,
-  handle: async (events, context) => {
-    const {
-      connection: { connectionString, client }
-    } = context;
-    const pongo = pongoClient2(connectionString, {
-      connectionOptions: { client }
-    });
-    await handle(events, {
-      ...context,
-      pongo
-    });
-  }
-});
-var pongoMultiStreamProjection = (options) => {
-  const { collectionName, getDocumentId, canHandle } = options;
-  return pongoProjection({
-    handle: async (events, { pongo }) => {
-      const collection = pongo.db().collection(collectionName);
-      for (const event of events) {
-        await collection.handle(getDocumentId(event), async (document) => {
-          return "initialState" in options ? await options.evolve(
-            document ?? options.initialState(),
-            event
-          ) : await options.evolve(
-            document,
-            event
-          );
-        });
-      }
-    },
-    canHandle
-  });
-};
-var pongoSingleStreamProjection = (options) => {
-  return pongoMultiStreamProjection({
-    ...options,
-    getDocumentId: options.getDocumentId ?? ((event) => event.metadata.streamName)
-  });
-};
-
 // src/eventStore/projections/postgresProjectionSpec.ts
 import {
   dumbo as dumbo3
 } from "@event-driven-io/dumbo";
-import { v4 as uuid6 } from "uuid";
+import { v4 as uuid8 } from "uuid";
 var PostgreSQLProjectionSpec = {
   for: (options) => {
     {
@@ -1731,8 +1899,8 @@ var PostgreSQLProjectionSpec = {
                 const metadata = {
                   globalPosition: ++globalPosition,
                   streamPosition: globalPosition,
-                  streamName: `test-${uuid6()}`,
-                  messageId: uuid6()
+                  streamName: `test-${uuid8()}`,
+                  messageId: uuid8()
                 };
                 allEvents.push({
                   ...event,
@@ -1830,7 +1998,8 @@ var expectSQL = {
   })
 };
 
-// src/eventStore/projections/index.ts
+// src/eventStore/projections/postgreSQLProjection.ts
+import "@event-driven-io/dumbo";
 var handleProjections = async (options) => {
   const {
     projections: allProjections,
@@ -1995,7 +2164,6 @@ export {
   DefaultPostgreSQLEventStoreProcessorBatchSize,
   DefaultPostgreSQLEventStoreProcessorPullingFrequencyInMs,
   PostgreSQLEventStoreDefaultStreamVersion,
-  PostgreSQLProcessor,
   PostgreSQLProjectionSpec,
   addDefaultPartitionSQL,
   addModuleForAllTenantsSQL,
@@ -2031,17 +2199,20 @@ export {
   pongoMultiStreamProjection,
   pongoProjection,
   pongoSingleStreamProjection,
+  postgreSQLCheckpointer,
   postgreSQLEventStoreConsumer,
   postgreSQLEventStoreMessageBatchPuller,
-  postgreSQLProcessor,
+  postgreSQLMessageProcessor,
   postgreSQLProjection,
-  postgreSQLProjectionProcessor,
+  postgreSQLProjector,
   postgreSQLRawBatchSQLProjection,
   postgreSQLRawSQLProjection,
+  postgreSQLReactor,
   readLastMessageGlobalPosition,
   readMessagesBatch,
   readProcessorCheckpoint,
   readStream,
+  rebuildPostgreSQLProjections,
   sanitizeNameSQL,
   schemaSQL,
   storeProcessorCheckpoint,