@boostercloud/framework-provider-azure 2.0.0 → 2.2.0

package/dist/constants.d.ts

@@ -15,10 +15,18 @@ export declare const connectionsStoreAttributes: {
     readonly partitionKey: "connectionID";
     readonly ttl: "expirationTime";
 };
+export declare const dedupAttributes: {
+    readonly partitionKey: "primaryKey";
+    readonly ttl: "expirationTime";
+};
 export declare const environmentVarNames: {
     readonly restAPIURL: "BOOSTER_REST_API_URL";
     readonly websocketAPIURL: "BOOSTER_WEBSOCKET_API_URL";
     readonly cosmosDbConnectionString: "COSMOSDB_CONNECTION_STRING";
+    readonly eventHubConnectionString: "EVENTHUB_CONNECTION_STRING";
+    readonly eventHubName: "EVENTHUB_NAME";
+    readonly eventHubMaxRetries: "EVENTHUB_MAX_RETRIES";
+    readonly eventHubMode: "EVENTHUB_MODE";
 };
 export declare const AZURE_CONFLICT_ERROR_CODE = 409;
 export declare const AZURE_PRECONDITION_FAILED_ERROR = 412;

package/dist/constants.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.AZURE_PRECONDITION_FAILED_ERROR = exports.AZURE_CONFLICT_ERROR_CODE = exports.environmentVarNames = exports.connectionsStoreAttributes = exports.subscriptionsStoreAttributes = exports.eventsStoreAttributes = void 0;
+exports.AZURE_PRECONDITION_FAILED_ERROR = exports.AZURE_CONFLICT_ERROR_CODE = exports.environmentVarNames = exports.dedupAttributes = exports.connectionsStoreAttributes = exports.subscriptionsStoreAttributes = exports.eventsStoreAttributes = void 0;
 exports.eventsStoreAttributes = {
     partitionKey: 'entityTypeName_entityID_kind',
     sortKey: 'createdAt',
@@ -17,10 +17,18 @@ exports.connectionsStoreAttributes = {
     partitionKey: 'connectionID',
     ttl: 'expirationTime',
 };
+exports.dedupAttributes = {
+    partitionKey: 'primaryKey',
+    ttl: 'expirationTime',
+};
 exports.environmentVarNames = {
     restAPIURL: 'BOOSTER_REST_API_URL',
     websocketAPIURL: 'BOOSTER_WEBSOCKET_API_URL',
     cosmosDbConnectionString: 'COSMOSDB_CONNECTION_STRING',
+    eventHubConnectionString: 'EVENTHUB_CONNECTION_STRING',
+    eventHubName: 'EVENTHUB_NAME',
+    eventHubMaxRetries: 'EVENTHUB_MAX_RETRIES',
+    eventHubMode: 'EVENTHUB_MODE',
 };
 // Azure special error codes
 exports.AZURE_CONFLICT_ERROR_CODE = 409;

package/dist/index.js

@@ -1,4 +1,5 @@
 "use strict";
+var _a;
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Provider = exports.loadInfrastructurePackage = void 0;
 const tslib_1 = require("tslib");
@@ -14,6 +15,10 @@ const events_searcher_adapter_1 = require("./library/events-searcher-adapter");
 const subscription_adapter_1 = require("./library/subscription-adapter");
 const connections_adapter_1 = require("./library/connections-adapter");
 const rocket_adapter_1 = require("./library/rocket-adapter");
+const events_stream_producer_adapter_1 = require("./library/events-stream-producer-adapter");
+const event_hubs_1 = require("@azure/event-hubs");
+const events_stream_consumer_adapter_1 = require("./library/events-stream-consumer-adapter");
+const health_adapter_1 = require("./library/health-adapter");
 let cosmosClient;
 if (typeof process.env[constants_1.environmentVarNames.cosmosDbConnectionString] === 'undefined') {
     cosmosClient = {};
@@ -21,6 +26,32 @@ if (typeof process.env[constants_1.environmentVarNames.cosmosDbConnectionString]
 else {
     cosmosClient = new cosmos_1.CosmosClient(process.env[constants_1.environmentVarNames.cosmosDbConnectionString]);
 }
+let producer;
+const eventHubConnectionString = process.env[constants_1.environmentVarNames.eventHubConnectionString];
+const eventHubName = process.env[constants_1.environmentVarNames.eventHubName];
+const DEFAULT_MAX_RETRY = 5;
+const DEFAULT_EVENT_HUB_MODE = event_hubs_1.RetryMode.Exponential;
+if (typeof eventHubConnectionString === 'undefined' ||
+    typeof eventHubName === 'undefined' ||
+    eventHubConnectionString === '' ||
+    eventHubName === '') {
+    producer = {};
+}
+else {
+    const maxRetries = process.env[constants_1.environmentVarNames.eventHubMaxRetries]
+        ? Number(process.env[constants_1.environmentVarNames.eventHubMaxRetries])
+        : DEFAULT_MAX_RETRY;
+    const mode = process.env[constants_1.environmentVarNames.eventHubMaxRetries] &&
+        ((_a = process.env[constants_1.environmentVarNames.eventHubMode]) === null || _a === void 0 ? void 0 : _a.toUpperCase()) === 'FIXED'
+        ? event_hubs_1.RetryMode.Fixed
+        : DEFAULT_EVENT_HUB_MODE;
+    producer = new event_hubs_1.EventHubProducerClient(eventHubConnectionString, eventHubName, {
+        retryOptions: {
+            maxRetries: maxRetries,
+            mode: mode,
+        },
+    });
+}
 /* We load the infrastructure package dynamically here to avoid including it in the
  * dependencies that are deployed in the lambda functions. The infrastructure
  * package is only used during the deploy.
@@ -33,6 +64,9 @@ const Provider = (rockets) => ({
     // ProviderEventsLibrary
     events: {
         rawToEnvelopes: events_adapter_1.rawEventsToEnvelopes,
+        rawStreamToEnvelopes: events_stream_consumer_adapter_1.rawEventsStreamToEnvelopes,
+        dedupEventStream: events_stream_consumer_adapter_1.dedupEventStream.bind(null, cosmosClient),
+        produce: events_stream_producer_adapter_1.produceEventsStream.bind(null, producer),
         store: events_adapter_1.storeEvents.bind(null, cosmosClient),
         storeSnapshot: events_adapter_1.storeSnapshot.bind(null, cosmosClient),
         forEntitySince: events_adapter_1.readEntityEventsSince.bind(null, cosmosClient),
@@ -75,6 +109,16 @@ const Provider = (rockets) => ({
     rockets: {
         rawToEnvelopes: rocket_adapter_1.rawRocketInputToEnvelope,
     },
+    sensor: {
+        databaseEventsHealthDetails: health_adapter_1.databaseEventsHealthDetails.bind(null, cosmosClient),
+        databaseReadModelsHealthDetails: health_adapter_1.databaseReadModelsHealthDetails.bind(null, cosmosClient),
+        isDatabaseEventUp: health_adapter_1.isDatabaseEventUp.bind(null, cosmosClient),
+        areDatabaseReadModelsUp: health_adapter_1.areDatabaseReadModelsUp.bind(null, cosmosClient),
+        databaseUrls: health_adapter_1.databaseUrl.bind(null, cosmosClient),
+        graphQLFunctionUrl: health_adapter_1.graphqlFunctionUrl,
+        isGraphQLFunctionUp: health_adapter_1.isGraphQLFunctionUp,
+        rawRequestToHealthEnvelope: health_adapter_1.rawRequestToSensorHealth,
+    },
     // ProviderInfrastructureGetter
     infrastructure: () => {
         const infrastructurePackageName = require('../package.json').name + '-infrastructure';

package/dist/library/events-stream-consumer-adapter.d.ts

@@ -0,0 +1,5 @@
+import { BoosterConfig, EventEnvelope, EventStream } from '@boostercloud/framework-types';
+import { Context } from '@azure/functions';
+import { CosmosClient } from '@azure/cosmos';
+export declare function dedupEventStream(cosmosDb: CosmosClient, config: BoosterConfig, context: Context): Promise<EventStream>;
+export declare function rawEventsStreamToEnvelopes(config: BoosterConfig, context: Context, dedupEventStream: EventStream): Array<EventEnvelope>;

package/dist/library/events-stream-consumer-adapter.js

@@ -0,0 +1,54 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.rawEventsStreamToEnvelopes = exports.dedupEventStream = void 0;
+const framework_common_helpers_1 = require("@boostercloud/framework-common-helpers");
+const constants_1 = require("../constants");
+const DEFAULT_DEDUP_TTL = 86400;
+async function dedupEventStream(cosmosDb, config, context) {
+    var _a, _b;
+    const logger = (0, framework_common_helpers_1.getLogger)(config, 'dedup-events-stream#dedupEventsStream');
+    const events = context.bindings.eventHubMessages || [];
+    logger.debug(`Dedup ${events.length} events`);
+    const resources = [];
+    for (const event of events) {
+        const rawParsed = JSON.parse(event);
+        const eventTag = {
+            primaryKey: rawParsed._etag,
+            createdAt: new Date().toISOString(),
+            ttl: (_b = (_a = config.eventStreamConfiguration.parameters) === null || _a === void 0 ? void 0 : _a.dedupTtl) !== null && _b !== void 0 ? _b : DEFAULT_DEDUP_TTL,
+        };
+        try {
+            const { resource } = await cosmosDb
+                .database(config.resourceNames.applicationStack)
+                .container(config.resourceNames.eventsDedup)
+                .items.create(eventTag);
+            if (resource) {
+                resources.push(event);
+            }
+        }
+        catch (error) {
+            if (error.code !== constants_1.AZURE_CONFLICT_ERROR_CODE) {
+                throw error;
+            }
+            logger.warn(`Ignoring duplicated event with etag ${eventTag}.`);
+        }
+    }
+    return resources;
+}
+exports.dedupEventStream = dedupEventStream;
+function rawEventsStreamToEnvelopes(config, context, dedupEventStream) {
+    const logger = (0, framework_common_helpers_1.getLogger)(config, 'events-adapter#rawEventsStreamToEnvelopes');
+    logger.debug(`Mapping ${dedupEventStream.length} events`);
+    const bindingData = context.bindingData;
+    return dedupEventStream.map((message, index) => {
+        const rawParsed = JSON.parse(message);
+        const instance = process.env.WEBSITE_INSTANCE_ID;
+        const partitionKeyArrayElement = bindingData.partitionKeyArray[index];
+        const offset = bindingData.offsetArray[index];
+        const sequence = bindingData.sequenceNumberArray[index];
+        const time = bindingData.enqueuedTimeUtcArray[index];
+        logger.debug(`CONSUMED_EVENT:${instance}#${partitionKeyArrayElement}=>(${index}#${offset}#${sequence}#${time})`);
+        return rawParsed;
+    });
+}
+exports.rawEventsStreamToEnvelopes = rawEventsStreamToEnvelopes;

package/dist/library/events-stream-producer-adapter.d.ts

@@ -0,0 +1,3 @@
+import { BoosterConfig, EventEnvelope } from '@boostercloud/framework-types';
+import { EventHubProducerClient } from '@azure/event-hubs';
+export declare function produceEventsStream(producer: EventHubProducerClient, entityName: string, entityID: string, eventEnvelopes: Array<EventEnvelope>, config: BoosterConfig): Promise<void>;

package/dist/library/events-stream-producer-adapter.js

@@ -0,0 +1,43 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.produceEventsStream = void 0;
+const framework_common_helpers_1 = require("@boostercloud/framework-common-helpers");
+const partition_keys_1 = require("./partition-keys");
+async function produceEventsStream(producer, entityName, entityID, eventEnvelopes, config) {
+    const logger = (0, framework_common_helpers_1.getLogger)(config, 'events-stream-producer#produceEventsStream');
+    logger.debug('Producing eventEnvelopes', eventEnvelopes);
+    const batchOptions = {
+        partitionKey: (0, partition_keys_1.partitionKeyForEvent)(entityName, entityID),
+    };
+    let batch = await producer.createBatch(batchOptions);
+    let numEventsSent = 0;
+    let i = 0;
+    while (i < eventEnvelopes.length) {
+        // messages can fail to be added to the batch if they exceed the maximum size configured for the EventHub.
+        const eventEnvelope = eventEnvelopes[i];
+        const isAdded = batch.tryAdd({ body: eventEnvelope });
+        if (isAdded) {
+            logger.info(`Added ${JSON.stringify(eventEnvelope)} to the batch`);
+            ++i;
+            continue;
+        }
+        if (batch.count === 0) {
+            throw new Error(`Message was too large and can't be sent until it's made smaller. ${JSON.stringify(eventEnvelope)}`);
+        }
+        // We reached the batch size limit
+        logger.info(`Sending ${batch.count} messages`);
+        await producer.sendBatch(batch);
+        numEventsSent += batch.count;
+        batch = await producer.createBatch(batchOptions);
+    }
+    if (batch.count > 0) {
+        logger.info(`Sending remaining ${batch.count} messages`);
+        await producer.sendBatch(batch);
+        numEventsSent += batch.count;
+    }
+    logger.info(`Sent ${numEventsSent} events`);
+    if (numEventsSent !== eventEnvelopes.length) {
+        throw new Error(`Not all messages were sent (${numEventsSent}/${eventEnvelopes.length})`);
+    }
+}
+exports.produceEventsStream = produceEventsStream;

package/dist/library/health-adapter.d.ts

@@ -0,0 +1,15 @@
+import { BoosterConfig, HealthEnvelope } from '@boostercloud/framework-types';
+import { Container, CosmosClient } from '@azure/cosmos';
+import { Context } from '@azure/functions';
+export declare function databaseUrl(cosmosDb: CosmosClient, config: BoosterConfig): Promise<Array<string>>;
+export declare function getContainer(cosmosDb: CosmosClient, config: BoosterConfig, containerName: string): Container;
+export declare function isContainerUp(cosmosDb: CosmosClient, config: BoosterConfig, containerName: string): Promise<boolean>;
+export declare function countAll(container: Container): Promise<number>;
+export declare function databaseEventsHealthDetails(cosmosDb: CosmosClient, config: BoosterConfig): Promise<unknown>;
+export declare function graphqlFunctionUrl(): Promise<string>;
+export declare function isDatabaseEventUp(cosmosDb: CosmosClient, config: BoosterConfig): Promise<boolean>;
+export declare function areDatabaseReadModelsUp(cosmosDb: CosmosClient, config: BoosterConfig): Promise<boolean>;
+export declare function isGraphQLFunctionUp(): Promise<boolean>;
+export declare function rawRequestToSensorHealthComponentPath(rawRequest: Context): string;
+export declare function rawRequestToSensorHealth(context: Context): HealthEnvelope;
+export declare function databaseReadModelsHealthDetails(cosmosDb: CosmosClient, config: BoosterConfig): Promise<unknown>;

package/dist/library/health-adapter.js

@@ -0,0 +1,111 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.databaseReadModelsHealthDetails = exports.rawRequestToSensorHealth = exports.rawRequestToSensorHealthComponentPath = exports.isGraphQLFunctionUp = exports.areDatabaseReadModelsUp = exports.isDatabaseEventUp = exports.graphqlFunctionUrl = exports.databaseEventsHealthDetails = exports.countAll = exports.isContainerUp = exports.getContainer = exports.databaseUrl = void 0;
+const constants_1 = require("../constants");
+const framework_common_helpers_1 = require("@boostercloud/framework-common-helpers");
+async function databaseUrl(cosmosDb, config) {
+    const database = cosmosDb.database(config.resourceNames.applicationStack);
+    return [database.url];
+}
+exports.databaseUrl = databaseUrl;
+function getContainer(cosmosDb, config, containerName) {
+    return cosmosDb.database(config.resourceNames.applicationStack).container(containerName);
+}
+exports.getContainer = getContainer;
+async function isContainerUp(cosmosDb, config, containerName) {
+    const container = getContainer(cosmosDb, config, containerName);
+    const { resources } = await container.items.query('SELECT TOP 1 1 FROM c', { maxItemCount: -1 }).fetchAll();
+    return resources !== undefined;
+}
+exports.isContainerUp = isContainerUp;
+async function countAll(container) {
+    const { resources } = await container.items.query('SELECT VALUE COUNT(1) FROM c', { maxItemCount: -1 }).fetchAll();
+    return resources ? resources[0] : 0;
+}
+exports.countAll = countAll;
+async function databaseEventsHealthDetails(cosmosDb, config) {
+    const container = getContainer(cosmosDb, config, config.resourceNames.eventsStore);
+    const url = container.url;
+    const count = await countAll(container);
+    return {
+        url: url,
+        count: count,
+    };
+}
+exports.databaseEventsHealthDetails = databaseEventsHealthDetails;
+async function graphqlFunctionUrl() {
+    try {
+        const basePath = process.env[constants_1.environmentVarNames.restAPIURL];
+        return `${basePath}/graphql`;
+    }
+    catch (e) {
+        return '';
+    }
+}
+exports.graphqlFunctionUrl = graphqlFunctionUrl;
+async function isDatabaseEventUp(cosmosDb, config) {
+    return await isContainerUp(cosmosDb, config, config.resourceNames.eventsStore);
+}
+exports.isDatabaseEventUp = isDatabaseEventUp;
+async function areDatabaseReadModelsUp(cosmosDb, config) {
+    const promises = Object.values(config.readModels).map((readModel) => {
+        const name = readModel.class.name;
+        const container = config.resourceNames.forReadModel(name);
+        return isContainerUp(cosmosDb, config, container);
+    });
+    const containersUp = await Promise.all(promises);
+    return containersUp.every((isContainerUp) => isContainerUp);
+}
+exports.areDatabaseReadModelsUp = areDatabaseReadModelsUp;
+async function isGraphQLFunctionUp() {
+    try {
+        const restAPIUrl = await graphqlFunctionUrl();
+        const response = await (0, framework_common_helpers_1.request)(restAPIUrl, 'POST');
+        return response.status === 200;
+    }
+    catch (e) {
+        return false;
+    }
+}
+exports.isGraphQLFunctionUp = isGraphQLFunctionUp;
+function rawRequestToSensorHealthComponentPath(rawRequest) {
+    var _a;
+    const parameters = (_a = rawRequest.req) === null || _a === void 0 ? void 0 : _a.url.replace(/^.*sensor\/health\/?/, '');
+    return parameters !== null && parameters !== void 0 ? parameters : '';
+}
+exports.rawRequestToSensorHealthComponentPath = rawRequestToSensorHealthComponentPath;
+function rawRequestToSensorHealth(context) {
+    var _a, _b, _c, _d;
+    const componentPath = rawRequestToSensorHealthComponentPath(context);
+    const requestID = context.executionContext.invocationId;
+    return {
+        requestID: requestID,
+        context: {
+            request: {
+                headers: (_a = context.req) === null || _a === void 0 ? void 0 : _a.headers,
+                body: (_b = context.req) === null || _b === void 0 ? void 0 : _b.body,
+            },
+            rawContext: context,
+        },
+        componentPath: componentPath,
+        token: (_d = (_c = context.req) === null || _c === void 0 ? void 0 : _c.headers) === null || _d === void 0 ? void 0 : _d.authorization,
+    };
+}
+exports.rawRequestToSensorHealth = rawRequestToSensorHealth;
+async function databaseReadModelsHealthDetails(cosmosDb, config) {
+    const readModels = Object.values(config.readModels);
+    const result = [];
+    for (const readModel of readModels) {
+        const name = readModel.class.name;
+        const containerName = config.resourceNames.forReadModel(name);
+        const container = getContainer(cosmosDb, config, containerName);
+        const url = container.url;
+        const count = await countAll(container);
+        result.push({
+            url: url,
+            count: count,
+        });
+    }
+    return result;
+}
+exports.databaseReadModelsHealthDetails = databaseReadModelsHealthDetails;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@boostercloud/framework-provider-azure",
-  "version": "2.0.0",
+  "version": "2.2.0",
   "description": "Handle Booster's integration with Azure",
   "keywords": [
     "framework-provider-azure"
@@ -26,14 +26,15 @@
     "@azure/cosmos": "^4.0.0",
     "@azure/functions": "^1.2.2",
     "@azure/identity": "~2.1.0",
-    "@boostercloud/framework-common-helpers": "^2.0.0",
-    "@boostercloud/framework-types": "^2.0.0",
+    "@azure/event-hubs": "5.11.1",
+    "@boostercloud/framework-common-helpers": "^2.2.0",
+    "@boostercloud/framework-types": "^2.2.0",
     "tslib": "^2.4.0",
     "@effect-ts/core": "^0.60.4",
     "@azure/web-pubsub": "~1.1.0"
   },
   "devDependencies": {
-    "@boostercloud/eslint-config": "^2.0.0",
+    "@boostercloud/eslint-config": "^2.2.0",
     "@types/chai": "4.2.18",
     "@types/chai-as-promised": "7.1.4",
     "@types/faker": "5.1.5",