@boostercloud/framework-provider-azure 2.1.0 → 2.2.0

package/dist/constants.d.ts

@@ -15,10 +15,18 @@ export declare const connectionsStoreAttributes: {
     readonly partitionKey: "connectionID";
     readonly ttl: "expirationTime";
 };
+export declare const dedupAttributes: {
+    readonly partitionKey: "primaryKey";
+    readonly ttl: "expirationTime";
+};
 export declare const environmentVarNames: {
     readonly restAPIURL: "BOOSTER_REST_API_URL";
     readonly websocketAPIURL: "BOOSTER_WEBSOCKET_API_URL";
     readonly cosmosDbConnectionString: "COSMOSDB_CONNECTION_STRING";
+    readonly eventHubConnectionString: "EVENTHUB_CONNECTION_STRING";
+    readonly eventHubName: "EVENTHUB_NAME";
+    readonly eventHubMaxRetries: "EVENTHUB_MAX_RETRIES";
+    readonly eventHubMode: "EVENTHUB_MODE";
 };
 export declare const AZURE_CONFLICT_ERROR_CODE = 409;
 export declare const AZURE_PRECONDITION_FAILED_ERROR = 412;

package/dist/constants.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.AZURE_PRECONDITION_FAILED_ERROR = exports.AZURE_CONFLICT_ERROR_CODE = exports.environmentVarNames = exports.connectionsStoreAttributes = exports.subscriptionsStoreAttributes = exports.eventsStoreAttributes = void 0;
+exports.AZURE_PRECONDITION_FAILED_ERROR = exports.AZURE_CONFLICT_ERROR_CODE = exports.environmentVarNames = exports.dedupAttributes = exports.connectionsStoreAttributes = exports.subscriptionsStoreAttributes = exports.eventsStoreAttributes = void 0;
 exports.eventsStoreAttributes = {
     partitionKey: 'entityTypeName_entityID_kind',
     sortKey: 'createdAt',
@@ -17,10 +17,18 @@ exports.connectionsStoreAttributes = {
     partitionKey: 'connectionID',
     ttl: 'expirationTime',
 };
+exports.dedupAttributes = {
+    partitionKey: 'primaryKey',
+    ttl: 'expirationTime',
+};
 exports.environmentVarNames = {
     restAPIURL: 'BOOSTER_REST_API_URL',
     websocketAPIURL: 'BOOSTER_WEBSOCKET_API_URL',
     cosmosDbConnectionString: 'COSMOSDB_CONNECTION_STRING',
+    eventHubConnectionString: 'EVENTHUB_CONNECTION_STRING',
+    eventHubName: 'EVENTHUB_NAME',
+    eventHubMaxRetries: 'EVENTHUB_MAX_RETRIES',
+    eventHubMode: 'EVENTHUB_MODE',
 };
 // Azure special error codes
 exports.AZURE_CONFLICT_ERROR_CODE = 409;

package/dist/index.js

@@ -1,4 +1,5 @@
 "use strict";
+var _a;
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Provider = exports.loadInfrastructurePackage = void 0;
 const tslib_1 = require("tslib");
@@ -14,6 +15,9 @@ const events_searcher_adapter_1 = require("./library/events-searcher-adapter");
 const subscription_adapter_1 = require("./library/subscription-adapter");
 const connections_adapter_1 = require("./library/connections-adapter");
 const rocket_adapter_1 = require("./library/rocket-adapter");
+const events_stream_producer_adapter_1 = require("./library/events-stream-producer-adapter");
+const event_hubs_1 = require("@azure/event-hubs");
+const events_stream_consumer_adapter_1 = require("./library/events-stream-consumer-adapter");
 const health_adapter_1 = require("./library/health-adapter");
 let cosmosClient;
 if (typeof process.env[constants_1.environmentVarNames.cosmosDbConnectionString] === 'undefined') {
@@ -22,6 +26,32 @@ if (typeof process.env[constants_1.environmentVarNames.cosmosDbConnectionString]
 else {
     cosmosClient = new cosmos_1.CosmosClient(process.env[constants_1.environmentVarNames.cosmosDbConnectionString]);
 }
+let producer;
+const eventHubConnectionString = process.env[constants_1.environmentVarNames.eventHubConnectionString];
+const eventHubName = process.env[constants_1.environmentVarNames.eventHubName];
+const DEFAULT_MAX_RETRY = 5;
+const DEFAULT_EVENT_HUB_MODE = event_hubs_1.RetryMode.Exponential;
+if (typeof eventHubConnectionString === 'undefined' ||
+    typeof eventHubName === 'undefined' ||
+    eventHubConnectionString === '' ||
+    eventHubName === '') {
+    producer = {};
+}
+else {
+    const maxRetries = process.env[constants_1.environmentVarNames.eventHubMaxRetries]
+        ? Number(process.env[constants_1.environmentVarNames.eventHubMaxRetries])
+        : DEFAULT_MAX_RETRY;
+    const mode = process.env[constants_1.environmentVarNames.eventHubMaxRetries] &&
+        ((_a = process.env[constants_1.environmentVarNames.eventHubMode]) === null || _a === void 0 ? void 0 : _a.toUpperCase()) === 'FIXED'
+        ? event_hubs_1.RetryMode.Fixed
+        : DEFAULT_EVENT_HUB_MODE;
+    producer = new event_hubs_1.EventHubProducerClient(eventHubConnectionString, eventHubName, {
+        retryOptions: {
+            maxRetries: maxRetries,
+            mode: mode,
+        },
+    });
+}
 /* We load the infrastructure package dynamically here to avoid including it in the
  * dependencies that are deployed in the lambda functions. The infrastructure
  * package is only used during the deploy.
@@ -34,6 +64,9 @@ const Provider = (rockets) => ({
     // ProviderEventsLibrary
     events: {
         rawToEnvelopes: events_adapter_1.rawEventsToEnvelopes,
+        rawStreamToEnvelopes: events_stream_consumer_adapter_1.rawEventsStreamToEnvelopes,
+        dedupEventStream: events_stream_consumer_adapter_1.dedupEventStream.bind(null, cosmosClient),
+        produce: events_stream_producer_adapter_1.produceEventsStream.bind(null, producer),
         store: events_adapter_1.storeEvents.bind(null, cosmosClient),
         storeSnapshot: events_adapter_1.storeSnapshot.bind(null, cosmosClient),
         forEntitySince: events_adapter_1.readEntityEventsSince.bind(null, cosmosClient),

package/dist/library/events-stream-consumer-adapter.d.ts

@@ -0,0 +1,5 @@
+import { BoosterConfig, EventEnvelope, EventStream } from '@boostercloud/framework-types';
+import { Context } from '@azure/functions';
+import { CosmosClient } from '@azure/cosmos';
+export declare function dedupEventStream(cosmosDb: CosmosClient, config: BoosterConfig, context: Context): Promise<EventStream>;
+export declare function rawEventsStreamToEnvelopes(config: BoosterConfig, context: Context, dedupEventStream: EventStream): Array<EventEnvelope>;

package/dist/library/events-stream-consumer-adapter.js

@@ -0,0 +1,54 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.rawEventsStreamToEnvelopes = exports.dedupEventStream = void 0;
+const framework_common_helpers_1 = require("@boostercloud/framework-common-helpers");
+const constants_1 = require("../constants");
+const DEFAULT_DEDUP_TTL = 86400;
+async function dedupEventStream(cosmosDb, config, context) {
+    var _a, _b;
+    const logger = (0, framework_common_helpers_1.getLogger)(config, 'dedup-events-stream#dedupEventsStream');
+    const events = context.bindings.eventHubMessages || [];
+    logger.debug(`Dedup ${events.length} events`);
+    const resources = [];
+    for (const event of events) {
+        const rawParsed = JSON.parse(event);
+        const eventTag = {
+            primaryKey: rawParsed._etag,
+            createdAt: new Date().toISOString(),
+            ttl: (_b = (_a = config.eventStreamConfiguration.parameters) === null || _a === void 0 ? void 0 : _a.dedupTtl) !== null && _b !== void 0 ? _b : DEFAULT_DEDUP_TTL,
+        };
+        try {
+            const { resource } = await cosmosDb
+                .database(config.resourceNames.applicationStack)
+                .container(config.resourceNames.eventsDedup)
+                .items.create(eventTag);
+            if (resource) {
+                resources.push(event);
+            }
+        }
+        catch (error) {
+            if (error.code !== constants_1.AZURE_CONFLICT_ERROR_CODE) {
+                throw error;
+            }
+            logger.warn(`Ignoring duplicated event with etag ${eventTag}.`);
+        }
+    }
+    return resources;
+}
+exports.dedupEventStream = dedupEventStream;
+function rawEventsStreamToEnvelopes(config, context, dedupEventStream) {
+    const logger = (0, framework_common_helpers_1.getLogger)(config, 'events-adapter#rawEventsStreamToEnvelopes');
+    logger.debug(`Mapping ${dedupEventStream.length} events`);
+    const bindingData = context.bindingData;
+    return dedupEventStream.map((message, index) => {
+        const rawParsed = JSON.parse(message);
+        const instance = process.env.WEBSITE_INSTANCE_ID;
+        const partitionKeyArrayElement = bindingData.partitionKeyArray[index];
+        const offset = bindingData.offsetArray[index];
+        const sequence = bindingData.sequenceNumberArray[index];
+        const time = bindingData.enqueuedTimeUtcArray[index];
+        logger.debug(`CONSUMED_EVENT:${instance}#${partitionKeyArrayElement}=>(${index}#${offset}#${sequence}#${time})`);
+        return rawParsed;
+    });
+}
+exports.rawEventsStreamToEnvelopes = rawEventsStreamToEnvelopes;

package/dist/library/events-stream-producer-adapter.d.ts

@@ -0,0 +1,3 @@
+import { BoosterConfig, EventEnvelope } from '@boostercloud/framework-types';
+import { EventHubProducerClient } from '@azure/event-hubs';
+export declare function produceEventsStream(producer: EventHubProducerClient, entityName: string, entityID: string, eventEnvelopes: Array<EventEnvelope>, config: BoosterConfig): Promise<void>;

package/dist/library/events-stream-producer-adapter.js

@@ -0,0 +1,43 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.produceEventsStream = void 0;
+const framework_common_helpers_1 = require("@boostercloud/framework-common-helpers");
+const partition_keys_1 = require("./partition-keys");
+async function produceEventsStream(producer, entityName, entityID, eventEnvelopes, config) {
+    const logger = (0, framework_common_helpers_1.getLogger)(config, 'events-stream-producer#produceEventsStream');
+    logger.debug('Producing eventEnvelopes', eventEnvelopes);
+    const batchOptions = {
+        partitionKey: (0, partition_keys_1.partitionKeyForEvent)(entityName, entityID),
+    };
+    let batch = await producer.createBatch(batchOptions);
+    let numEventsSent = 0;
+    let i = 0;
+    while (i < eventEnvelopes.length) {
+        // messages can fail to be added to the batch if they exceed the maximum size configured for the EventHub.
+        const eventEnvelope = eventEnvelopes[i];
+        const isAdded = batch.tryAdd({ body: eventEnvelope });
+        if (isAdded) {
+            logger.info(`Added ${JSON.stringify(eventEnvelope)} to the batch`);
+            ++i;
+            continue;
+        }
+        if (batch.count === 0) {
+            throw new Error(`Message was too large and can't be sent until it's made smaller. ${JSON.stringify(eventEnvelope)}`);
+        }
+        // We reached the batch size limit
+        logger.info(`Sending ${batch.count} messages`);
+        await producer.sendBatch(batch);
+        numEventsSent += batch.count;
+        batch = await producer.createBatch(batchOptions);
+    }
+    if (batch.count > 0) {
+        logger.info(`Sending remaining ${batch.count} messages`);
+        await producer.sendBatch(batch);
+        numEventsSent += batch.count;
+    }
+    logger.info(`Sent ${numEventsSent} events`);
+    if (numEventsSent !== eventEnvelopes.length) {
+        throw new Error(`Not all messages were sent (${numEventsSent}/${eventEnvelopes.length})`);
+    }
+}
+exports.produceEventsStream = produceEventsStream;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@boostercloud/framework-provider-azure",
-  "version": "2.1.0",
+  "version": "2.2.0",
   "description": "Handle Booster's integration with Azure",
   "keywords": [
     "framework-provider-azure"
@@ -26,14 +26,15 @@
     "@azure/cosmos": "^4.0.0",
     "@azure/functions": "^1.2.2",
     "@azure/identity": "~2.1.0",
-    "@boostercloud/framework-common-helpers": "^2.1.0",
-    "@boostercloud/framework-types": "^2.1.0",
+    "@azure/event-hubs": "5.11.1",
+    "@boostercloud/framework-common-helpers": "^2.2.0",
+    "@boostercloud/framework-types": "^2.2.0",
     "tslib": "^2.4.0",
     "@effect-ts/core": "^0.60.4",
     "@azure/web-pubsub": "~1.1.0"
   },
   "devDependencies": {
-    "@boostercloud/eslint-config": "^2.1.0",
+    "@boostercloud/eslint-config": "^2.2.0",
     "@types/chai": "4.2.18",
     "@types/chai-as-promised": "7.1.4",
     "@types/faker": "5.1.5",