@boostercloud/framework-provider-azure 2.9.2 → 2.10.0

package/dist/index.js

@@ -19,6 +19,7 @@ const events_stream_producer_adapter_1 = require("./library/events-stream-produc
 const event_hubs_1 = require("@azure/event-hubs");
 const events_stream_consumer_adapter_1 = require("./library/events-stream-consumer-adapter");
 const health_adapter_1 = require("./library/health-adapter");
+const events_store_adapter_1 = require("./library/events-store-adapter");
 let cosmosClient;
 if (typeof process.env[constants_1.environmentVarNames.cosmosDbConnectionString] === 'undefined') {
     cosmosClient = {};
@@ -67,7 +68,7 @@ const Provider = (rockets) => ({
         rawStreamToEnvelopes: events_stream_consumer_adapter_1.rawEventsStreamToEnvelopes,
         dedupEventStream: events_stream_consumer_adapter_1.dedupEventStream.bind(null, cosmosClient),
         produce: events_stream_producer_adapter_1.produceEventsStream.bind(null, producer),
-        store: events_adapter_1.storeEvents.bind(null, cosmosClient),
+        store: events_store_adapter_1.storeEvents.bind(null, cosmosClient),
         storeSnapshot: events_adapter_1.storeSnapshot.bind(null, cosmosClient),
         forEntitySince: events_adapter_1.readEntityEventsSince.bind(null, cosmosClient),
         latestEntitySnapshot: events_adapter_1.readEntityLatestSnapshot.bind(null, cosmosClient),

package/dist/library/events-adapter.d.ts

@@ -1,9 +1,8 @@
 import { CosmosClient } from '@azure/cosmos';
-import { BoosterConfig, EntitySnapshotEnvelope, EventEnvelope, NonPersistedEntitySnapshotEnvelope, NonPersistedEventEnvelope, UUID } from '@boostercloud/framework-types';
+import { BoosterConfig, EntitySnapshotEnvelope, EventEnvelope, NonPersistedEntitySnapshotEnvelope, UUID } from '@boostercloud/framework-types';
 import { Context } from '@azure/functions';
 export declare function rawEventsToEnvelopes(context: Context): Array<EventEnvelope>;
 export declare function readEntityEventsSince(cosmosDb: CosmosClient, config: BoosterConfig, entityTypeName: string, entityID: UUID, since?: string): Promise<Array<EventEnvelope>>;
 export declare function readEntityLatestSnapshot(cosmosDb: CosmosClient, config: BoosterConfig, entityTypeName: string, entityID: UUID): Promise<EntitySnapshotEnvelope | undefined>;
-export declare function storeEvents(cosmosDb: CosmosClient, eventEnvelopes: Array<NonPersistedEventEnvelope>, config: BoosterConfig): Promise<Array<EventEnvelope>>;
 export declare function storeSnapshot(cosmosDb: CosmosClient, snapshotEnvelope: NonPersistedEntitySnapshotEnvelope, config: BoosterConfig): Promise<EntitySnapshotEnvelope>;
 export declare function storeDispatchedEvent(cosmosDb: CosmosClient, eventEnvelope: EventEnvelope, config: BoosterConfig): Promise<boolean>;

package/dist/library/events-adapter.js

@@ -1,10 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.storeDispatchedEvent = exports.storeSnapshot = exports.storeEvents = exports.readEntityLatestSnapshot = exports.readEntityEventsSince = exports.rawEventsToEnvelopes = void 0;
+exports.storeDispatchedEvent = exports.storeSnapshot = exports.readEntityLatestSnapshot = exports.readEntityEventsSince = exports.rawEventsToEnvelopes = void 0;
 const framework_common_helpers_1 = require("@boostercloud/framework-common-helpers");
 const constants_1 = require("../constants");
 const partition_keys_1 = require("./partition-keys");
-// eslint-disable-next-line @typescript-eslint/no-magic-numbers
 const originOfTime = new Date(0).toISOString();
 function rawEventsToEnvelopes(context) {
     return context.bindings.rawEvent;
@@ -61,32 +60,9 @@ async function readEntityLatestSnapshot(cosmosDb, config, entityTypeName, entity
     }
 }
 exports.readEntityLatestSnapshot = readEntityLatestSnapshot;
-async function storeEvents(cosmosDb, eventEnvelopes, config) {
-    const logger = (0, framework_common_helpers_1.getLogger)(config, 'events-adapter#storeEvents');
-    logger.debug('[EventsAdapter#storeEvents] Storing EventEnvelopes with eventEnvelopes:', eventEnvelopes);
-    const persistableEvents = [];
-    for (const eventEnvelope of eventEnvelopes) {
-        const persistableEvent = {
-            ...eventEnvelope,
-            createdAt: new Date().toISOString(),
-        };
-        await cosmosDb
-            .database(config.resourceNames.applicationStack)
-            .container(config.resourceNames.eventsStore)
-            .items.create({
-            ...persistableEvent,
-            [constants_1.eventsStoreAttributes.partitionKey]: (0, partition_keys_1.partitionKeyForEvent)(eventEnvelope.entityTypeName, eventEnvelope.entityID),
-            [constants_1.eventsStoreAttributes.sortKey]: persistableEvent.createdAt,
-        });
-        persistableEvents.push(persistableEvent);
-    }
-    logger.debug('[EventsAdapter#storeEvents] EventEnvelope stored');
-    return persistableEvents;
-}
-exports.storeEvents = storeEvents;
 async function storeSnapshot(cosmosDb, snapshotEnvelope, config) {
     const logger = (0, framework_common_helpers_1.getLogger)(config, 'events-adapter#storeSnapshot');
-    logger.debug('[EventsAdapter#storeSnapshot] Storing snapshot with snapshotEnvelope:', snapshotEnvelope);
+    logger.debug('Storing snapshot with snapshotEnvelope:', snapshotEnvelope);
     const partitionKey = (0, partition_keys_1.partitionKeyForSnapshot)(snapshotEnvelope.entityTypeName, snapshotEnvelope.entityID);
     /**
      * The sort key of the snapshot matches the sort key of the last event that generated it.

package/dist/library/events-store-adapter.d.ts

@@ -0,0 +1,15 @@
+import { BoosterConfig, EventEnvelope, NonPersistedEventEnvelope } from '@boostercloud/framework-types';
+import { CosmosClient } from '@azure/cosmos';
+/**
+ * Limits: The Azure Cosmos DB request size limit constrains the size of the TransactionalBatch payload to not exceed 2 MB,
+ * and the maximum execution time is 5 seconds. There's a current limit of 100 operations per TransactionalBatch to ensure
+ * the performance is as expected and within SLAs.
+ *
+ * Errors: If there's a failure, the failed operation will have a status code of its corresponding error. All the other
+ * operations will have a 424 status code (failed dependency). The status code enables one to identify the cause of transaction failure.
+ *
+ * @param cosmosDb
+ * @param eventEnvelopes
+ * @param config
+ */
+export declare function storeEvents(cosmosDb: CosmosClient, eventEnvelopes: Array<NonPersistedEventEnvelope>, config: BoosterConfig): Promise<Array<EventEnvelope>>;

package/dist/library/events-store-adapter.js

@@ -0,0 +1,112 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.storeEvents = void 0;
+const framework_common_helpers_1 = require("@boostercloud/framework-common-helpers");
+const partition_keys_1 = require("./partition-keys");
+const constants_1 = require("../constants");
+const DEFAULT_CHUNK_SIZE = 100;
+/**
+ * Limits: The Azure Cosmos DB request size limit constrains the size of the TransactionalBatch payload to not exceed 2 MB,
+ * and the maximum execution time is 5 seconds. There's a current limit of 100 operations per TransactionalBatch to ensure
+ * the performance is as expected and within SLAs.
+ *
+ * Errors: If there's a failure, the failed operation will have a status code of its corresponding error. All the other
+ * operations will have a 424 status code (failed dependency). The status code enables one to identify the cause of transaction failure.
+ *
+ * @param cosmosDb
+ * @param eventEnvelopes
+ * @param config
+ */
+async function storeEvents(cosmosDb, eventEnvelopes, config) {
+    const logger = (0, framework_common_helpers_1.getLogger)(config, 'store-events-adapter#storeEvents');
+    logger.debug('Storing EventEnvelopes with eventEnvelopes:', eventEnvelopes);
+    const envelopesWithCreatedAt = [];
+    const eventsPerPartitionKey = eventEnvelopes.reduce(groupByPartitionKey, {});
+    for (const [partitionKey, eventsInPartitionKey] of Object.entries(eventsPerPartitionKey)) {
+        const chunksOfEventsInPartitionKey = chunkEvents(eventsInPartitionKey, DEFAULT_CHUNK_SIZE);
+        for (const eventListInChunk of chunksOfEventsInPartitionKey) {
+            const eventEnvelopesChunkWithCreatedAt = toEventEnvelopes(eventListInChunk);
+            envelopesWithCreatedAt.push(...eventEnvelopesChunkWithCreatedAt);
+            const inputOperations = toInputOperations(eventEnvelopesChunkWithCreatedAt, config);
+            const batchResponse = await batchEvents(cosmosDb, config, inputOperations, partitionKey);
+            // Batch is transactional and will roll back all operations if one fails
+            if ((batchResponse === null || batchResponse === void 0 ? void 0 : batchResponse.code) !== 200) {
+                logger.error(`An error ocurred storing a batch of events. Batch response: ${JSON.stringify(batchResponse)}`);
+                throw new Error(`Error ${batchResponse.substatus} storing events: ${JSON.stringify(batchResponse)}`);
+            }
+            const result = batchResponse.result;
+            logger.debug(`EventEnvelopes with ${partitionKey} stored: ${result}`);
+        }
+    }
+    return envelopesWithCreatedAt;
+}
+exports.storeEvents = storeEvents;
+function toEventEnvelopes(events) {
+    return events.map((eventEnvelope) => ({
+        ...eventEnvelope,
+        createdAt: new Date().toISOString(),
+    }));
+}
+function toInputOperations(eventEnvelopesChunkWithCreatedAt, config) {
+    return eventEnvelopesChunkWithCreatedAt.map((eventEnvelopeWithCreatedAt) => {
+        const body = storableResource(config, eventEnvelopeWithCreatedAt);
+        return {
+            operationType: 'Create',
+            resourceBody: body,
+        };
+    });
+}
+async function batchEvents(cosmosDb, config, inputOperations, partitionKey) {
+    const logger = (0, framework_common_helpers_1.getLogger)(config, 'store-events-adapter#batchEvents');
+    try {
+        logger.debug('Storing EventEnvelopes with inputOperations:', inputOperations);
+        return await cosmosDb
+            .database(config.resourceNames.applicationStack)
+            .container(config.resourceNames.eventsStore)
+            .items.batch(inputOperations, partitionKey);
+    }
+    catch (e) {
+        logger.error('Unexpected error storing events', e);
+        throw e;
+    }
+}
+function groupByPartitionKey(eventsPerPartitionKey, event) {
+    const partitionKey = (0, partition_keys_1.partitionKeyForEvent)(event.entityTypeName, event.entityID);
+    if (!eventsPerPartitionKey[partitionKey]) {
+        eventsPerPartitionKey[partitionKey] = [];
+    }
+    eventsPerPartitionKey[partitionKey].push(event);
+    return eventsPerPartitionKey;
+}
+/**
+ * Batch method expects a JSONObject. JSONObject valid types are: boolean | number | string | null | JSONArray | JSONObject
+ * This method tries to build a JSONObject from an EventEnvelope.
+ * It could fail if value contains a circular reference or a BigInt value is encountered.
+ * @param config
+ * @param eventEnvelope
+ */
+function storableResource(config, eventEnvelope) {
+    const logger = (0, framework_common_helpers_1.getLogger)(config, 'store-events-adapter#storableResource');
+    try {
+        const partitionKey = (0, partition_keys_1.partitionKeyForEvent)(eventEnvelope.entityTypeName, eventEnvelope.entityID);
+        const eventEnvelopeWithEntityID = {
+            ...eventEnvelope,
+            [constants_1.eventsStoreAttributes.partitionKey]: partitionKey,
+            [constants_1.eventsStoreAttributes.sortKey]: eventEnvelope.createdAt,
+        };
+        return eventEnvelopeWithEntityID;
+    }
+    catch (e) {
+        logger.error(`Could not parse eventEnvelope ${eventEnvelope} to JSONObject`, e);
+        throw e;
+    }
+}
+/**
+ * Split events in chunks of size DEFAULT_CHUNK_SIZE
+ *
+ * @param arr
+ * @param size
+ */
+function chunkEvents(arr, size) {
+    return Array.from({ length: Math.ceil(arr.length / size) }, (_, i) => arr.slice(i * size, i * size + size));
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@boostercloud/framework-provider-azure",
-  "version": "2.9.2",
+  "version": "2.10.0",
   "description": "Handle Booster's integration with Azure",
   "keywords": [
     "framework-provider-azure"
@@ -27,14 +27,14 @@
     "@azure/functions": "^1.2.2",
     "@azure/identity": "~2.1.0",
     "@azure/event-hubs": "5.11.1",
-    "@boostercloud/framework-common-helpers": "^2.9.2",
-    "@boostercloud/framework-types": "^2.9.2",
+    "@boostercloud/framework-common-helpers": "^2.10.0",
+    "@boostercloud/framework-types": "^2.10.0",
     "tslib": "^2.4.0",
     "@effect-ts/core": "^0.60.4",
     "@azure/web-pubsub": "~1.1.0"
   },
   "devDependencies": {
-    "@boostercloud/eslint-config": "^2.9.2",
+    "@boostercloud/eslint-config": "^2.10.0",
     "@types/chai": "4.2.18",
     "@types/chai-as-promised": "7.1.4",
     "@types/faker": "5.1.5",