@electrolux-oss/plugin-infrawallet-backend 0.1.3 → 0.1.4

package/config.d.ts

@@ -1,9 +1,6 @@
 export interface Config {
   backend: {
     infraWallet: {
-      /**
-       * @deepVisibility secret
-       */
       integrations: {
         azure?: [
           {
@@ -11,6 +8,9 @@ export interface Config {
             subscriptionId: string;
             clientId: string;
             tenantId: string;
+            /**
+             * @visibility secret
+             */
             clientSecret: string;
             tags?: string[];
           },
@@ -20,7 +20,13 @@ export interface Config {
             name: string;
             accountId: string;
             assumedRoleName: string;
+            /**
+             * @visibility secret
+             */
             accessKeyId?: string;
+            /**
+             * @visibility secret
+             */
             accessKeySecret?: string;
             tags?: string[];
           },

package/dist/index.cjs.js

@@ -11,8 +11,8 @@ var clientSts = require('@aws-sdk/client-sts');
 var lodash = require('lodash');
 var moment = require('moment');
 var armCostmanagement = require('@azure/arm-costmanagement');
-var identity = require('@azure/identity');
 var coreRestPipeline = require('@azure/core-rest-pipeline');
+var identity = require('@azure/identity');
 var bigquery = require('@google-cloud/bigquery');
 
 function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e : { default: e }; }
@@ -24,34 +24,76 @@ var moment__default = /*#__PURE__*/_interopDefaultCompat(moment);
 async function getCategoryMappings(database, provider) {
   const result = {};
   const client = await database.getClient();
-  const mappings = await client.where({ provider }).select().from("category_mappings");
-  mappings.forEach((mapping) => {
+  const default_mappings = await client.where({ provider: provider.toLowerCase() }).select().from("category_mappings_default");
+  default_mappings.forEach((mapping) => {
+    if (typeof mapping.cloud_service_names === "string") {
+      JSON.parse(mapping.cloud_service_names).forEach((service) => {
+        result[service] = mapping.category;
+      });
+    } else {
+      mapping.cloud_service_names.forEach((service) => {
+        result[service] = mapping.category;
+      });
+    }
+  });
+  const override_mappings = await client.where({ provider }).select().from("category_mappings_override");
+  override_mappings.forEach((mapping) => {
     if (typeof mapping.cloud_service_names === "string") {
-      result[mapping.category] = JSON.parse(mapping.cloud_service_names);
+      JSON.parse(mapping.cloud_service_names).forEach((service) => {
+        result[service] = mapping.category;
+      });
     } else {
-      result[mapping.category] = mapping.cloud_service_names;
+      mapping.cloud_service_names.forEach((service) => {
+        result[service] = mapping.category;
+      });
     }
   });
   return result;
 }
 function getCategoryByServiceName(serviceName, categoryMappings) {
-  for (const key of Object.keys(categoryMappings)) {
-    const serviceNames = categoryMappings[key];
-    if (serviceNames && serviceNames.includes(serviceName)) {
-      return key;
-    }
+  if (serviceName in categoryMappings) {
+    return categoryMappings[serviceName];
   }
   return "Uncategorized";
 }
+async function getReportsFromCache(cache, provider, configKey, query) {
+  const cacheKey = [
+    provider,
+    configKey,
+    query.filters,
+    query.groups,
+    query.granularity,
+    query.startTime,
+    query.endTime
+  ].join("_");
+  const cachedCosts = await cache.get(cacheKey);
+  return cachedCosts;
+}
+async function setReportsToCache(cache, reports, provider, configKey, query, ttl) {
+  const cacheKey = [
+    provider,
+    configKey,
+    query.filters,
+    query.groups,
+    query.granularity,
+    query.startTime,
+    query.endTime
+  ].join("_");
+  await cache.set(cacheKey, reports, {
+    ttl: ttl 
+  });
+}
 
 class AwsClient {
-  constructor(config, database, logger) {
+  constructor(providerName, config, database, cache, logger) {
+    this.providerName = providerName;
     this.config = config;
     this.database = database;
+    this.cache = cache;
     this.logger = logger;
   }
-  static create(config, database, logger) {
-    return new AwsClient(config, database, logger);
+  static create(config, database, cache, logger) {
+    return new AwsClient("AWS", config, database, cache, logger);
   }
   convertServiceName(serviceName) {
     let convertedName = serviceName;
@@ -61,16 +103,13 @@ class AwsClient {
       ["Virtual Private Cloud", "VPC (Virtual Private Cloud)"],
       ["Relational Database Service", "RDS (Relational Database Service)"],
       ["Simple Storage Service", "S3 (Simple Storage Service)"],
-      [
-        "Managed Streaming for Apache Kafka",
-        "MSK (Managed Streaming for Apache Kafka)"
-      ],
-      [
-        "Elastic Container Service for Kubernetes",
-        "EKS (Elastic Container Service for Kubernetes)"
-      ],
+      ["Managed Streaming for Apache Kafka", "MSK (Managed Streaming for Apache Kafka)"],
+      ["Elastic Container Service for Kubernetes", "EKS (Elastic Container Service for Kubernetes)"],
+      ["Elastic Container Service", "ECS (Elastic Container Service)"],
+      ["EC2 Container Registry (ECR)", "ECR (Elastic Container Registry)"],
       ["Simple Queue Service", "SQS (Simple Queue Service)"],
-      ["Simple Notification Service", "SNS (Simple Notification Service)"]
+      ["Simple Notification Service", "SNS (Simple Notification Service)"],
+      ["Database Migration Service", "DMS (Database Migration Service)"]
     ]);
     for (const prefix of prefixes) {
       if (serviceName.startsWith(prefix)) {
@@ -80,24 +119,31 @@ class AwsClient {
     if (aliases.has(convertedName)) {
       convertedName = aliases.get(convertedName) || convertedName;
     }
-    return `AWS/${convertedName}`;
+    return `${this.providerName}/${convertedName}`;
   }
   async fetchCostsFromCloud(query) {
-    const conf = this.config.getOptionalConfigArray(
-      "backend.infraWallet.integrations.aws"
-    );
+    const conf = this.config.getOptionalConfigArray("backend.infraWallet.integrations.aws");
     if (!conf) {
-      return [];
+      return { reports: [], errors: [] };
     }
     const promises = [];
     const results = [];
+    const errors = [];
     query.groups.split(",").forEach((group) => {
       if (group.includes(":")) {
         group.split(":");
       }
     });
     for (const c of conf) {
-      const name = c.getString("name");
+      const accountName = c.getString("name");
+      const cachedCosts = await getReportsFromCache(this.cache, this.providerName, accountName, query);
+      if (cachedCosts) {
+        this.logger.debug(`${this.providerName}/${accountName} costs from cache`);
+        cachedCosts.map((cost) => {
+          results.push(cost);
+        });
+        continue;
+      }
       const accountId = c.getString("accountId");
       const assumedRoleName = c.getString("assumedRoleName");
       const accessKeyId = c.getOptionalString("accessKeyId");
@@ -108,7 +154,7 @@ class AwsClient {
         const [k, v] = tag.split(":");
         tagKeyValues[k.trim()] = v.trim();
       });
-      const categoryMappings = await getCategoryMappings(this.database, "aws");
+      const categoryMappings = await getCategoryMappings(this.database, this.providerName);
       let stsParams = {};
       if (accessKeyId && accessKeySecret) {
         stsParams = {
@@ -178,26 +224,21 @@ class AwsClient {
                 row.Groups.forEach((group) => {
                   var _a3;
                   const serviceName = group.Keys ? group.Keys[0] : "";
-                  const keyName = `${name}_${serviceName}`;
+                  const keyName = `${accountName}_${serviceName}`;
                   if (!accumulator[keyName]) {
                     accumulator[keyName] = {
                       id: keyName,
-                      name: `AWS/${name}`,
+                      name: `${this.providerName}/${accountName}`,
                       service: this.convertServiceName(serviceName),
-                      category: getCategoryByServiceName(
-                        serviceName,
-                        categoryMappings
-                      ),
-                      provider: "AWS",
+                      category: getCategoryByServiceName(serviceName, categoryMappings),
+                      provider: this.providerName,
                       reports: {},
                       ...tagKeyValues
                     };
                   }
                   const groupMetrics = group.Metrics;
                   if (groupMetrics !== void 0) {
-                    accumulator[keyName].reports[period] = parseFloat(
-                      (_a3 = groupMetrics.UnblendedCost.Amount) != null ? _a3 : "0.0"
-                    );
+                    accumulator[keyName].reports[period] = parseFloat((_a3 = groupMetrics.UnblendedCost.Amount) != null ? _a3 : "0.0");
                   }
                 });
               }
@@ -205,28 +246,46 @@ class AwsClient {
             },
             {}
           );
+          await setReportsToCache(
+            this.cache,
+            Object.values(transformedData),
+            this.providerName,
+            accountName,
+            query,
+            60 * 60 * 2 * 1e3
+          );
           Object.values(transformedData).map((value) => {
             results.push(value);
           });
         } catch (e) {
           this.logger.error(e);
+          errors.push({
+            provider: this.providerName,
+            name: `${this.providerName}/${accountName}`,
+            error: e.message
+          });
         }
       })();
       promises.push(promise);
     }
     await Promise.all(promises);
-    return results;
+    return {
+      reports: results,
+      errors
+    };
   }
 }
 
 class AzureClient {
-  constructor(config, database, logger) {
+  constructor(providerName, config, database, cache, logger) {
+    this.providerName = providerName;
     this.config = config;
     this.database = database;
+    this.cache = cache;
     this.logger = logger;
   }
-  static create(config, database, logger) {
-    return new AzureClient(config, database, logger);
+  static create(config, database, cache, logger) {
+    return new AzureClient("Azure", config, database, cache, logger);
   }
   convertServiceName(serviceName) {
     let convertedName = serviceName;
@@ -236,7 +295,7 @@ class AzureClient {
         convertedName = serviceName.slice(prefix.length).trim();
       }
     }
-    return `Azure/${convertedName}`;
+    return `${this.providerName}/${convertedName}`;
   }
   formatDate(dateNumber) {
     const dateString = dateNumber.toString();
@@ -263,7 +322,10 @@ class AzureClient {
       if (response.status === 200) {
         return JSON.parse(response.bodyAsText || "{}");
       } else if (response.status === 429) {
-        const retryAfter = parseInt(response.headers.get("x-ms-ratelimit-microsoft.costmanagement-entity-retry-after") || "60", 10);
+        const retryAfter = parseInt(
+          response.headers.get("x-ms-ratelimit-microsoft.costmanagement-entity-retry-after") || "60",
+          10
+        );
         this.logger.warn(`Hit Azure rate limit, retrying after ${retryAfter} seconds...`);
         await new Promise((resolve) => setTimeout(resolve, retryAfter * 1e3));
         retries++;
@@ -297,30 +359,30 @@ class AzureClient {
     return allResults;
   }
   async fetchCostsFromCloud(query) {
-    const conf = this.config.getOptionalConfigArray(
-      "backend.infraWallet.integrations.azure"
-    );
+    const conf = this.config.getOptionalConfigArray("backend.infraWallet.integrations.azure");
     if (!conf) {
-      return [];
+      return { reports: [], errors: [] };
     }
-    const categoryMappings = await getCategoryMappings(
-      this.database,
-      "azure"
-    );
+    const categoryMappings = await getCategoryMappings(this.database, this.providerName);
     const promises = [];
     const results = [];
+    const errors = [];
     const groupPairs = [{ type: "Dimension", name: "ServiceName" }];
     for (const c of conf) {
-      const name = c.getString("name");
+      const accountName = c.getString("name");
+      const cachedCosts = await getReportsFromCache(this.cache, this.providerName, accountName, query);
+      if (cachedCosts) {
+        this.logger.debug(`${this.providerName}/${accountName} costs from cache`);
+        cachedCosts.map((cost) => {
+          results.push(cost);
+        });
+        continue;
+      }
       const subscriptionId = c.getString("subscriptionId");
       const tenantId = c.getString("tenantId");
       const clientId = c.getString("clientId");
       const clientSecret = c.getString("clientSecret");
-      const credential = new identity.ClientSecretCredential(
-        tenantId,
-        clientId,
-        clientSecret
-      );
+      const credential = new identity.ClientSecretCredential(tenantId, clientId, clientSecret);
       const client = new armCostmanagement.CostManagementClient(credential);
       const tags = c.getOptionalStringArray("tags");
       const tagKeyValues = {};
@@ -347,17 +409,17 @@ class AzureClient {
               if (query.granularity.toUpperCase() === "DAILY") {
                 date = this.formatDate(date);
               }
-              let keyName = name;
+              let keyName = accountName;
               for (let i = 0; i < groupPairs.length; i++) {
                 keyName += `->${row[i + 2]}`;
               }
               if (!accumulator[keyName]) {
                 accumulator[keyName] = {
                   id: keyName,
-                  name: `Azure/${name}`,
+                  name: `${this.providerName}/${accountName}`,
                   service: this.convertServiceName(serviceName),
                   category: getCategoryByServiceName(serviceName, categoryMappings),
-                  provider: "Azure",
+                  provider: this.providerName,
                   reports: {},
                   ...tagKeyValues
                 };
@@ -374,38 +436,56 @@ class AzureClient {
             },
             {}
           );
+          await setReportsToCache(
+            this.cache,
+            Object.values(transformedData),
+            this.providerName,
+            accountName,
+            query,
+            60 * 60 * 2 * 1e3
+          );
           Object.values(transformedData).map((value) => {
             results.push(value);
           });
         } catch (e) {
           this.logger.error(e);
+          errors.push({
+            provider: this.providerName,
+            name: `${this.providerName}/${accountName}`,
+            error: e.message
+          });
         }
       })();
       promises.push(promise);
     }
     await Promise.all(promises);
-    return results;
+    return {
+      reports: results,
+      errors
+    };
   }
 }
 
 class GCPClient {
-  constructor(config, database, logger) {
+  constructor(providerName, config, database, cache, logger) {
+    this.providerName = providerName;
     this.config = config;
     this.database = database;
+    this.cache = cache;
     this.logger = logger;
   }
-  static create(config, database, logger) {
-    return new GCPClient(config, database, logger);
+  static create(config, database, cache, logger) {
+    return new GCPClient("GCP", config, database, cache, logger);
   }
   convertServiceName(serviceName) {
     let convertedName = serviceName;
-    const prefixes = ["GCP"];
+    const prefixes = ["Google Cloud"];
     for (const prefix of prefixes) {
       if (serviceName.startsWith(prefix)) {
         convertedName = serviceName.slice(prefix.length).trim();
       }
     }
-    return `GCP/${convertedName}`;
+    return `${this.providerName}/${convertedName}`;
   }
   async queryBigQuery(keyFilePath, projectId, datasetId, tableId, query) {
     const options = {
@@ -443,16 +523,23 @@ class GCPClient {
     }
   }
   async fetchCostsFromCloud(query) {
-    const conf = this.config.getOptionalConfigArray(
-      "backend.infraWallet.integrations.gcp"
-    );
+    const conf = this.config.getOptionalConfigArray("backend.infraWallet.integrations.gcp");
     if (!conf) {
-      return [];
+      return { reports: [], errors: [] };
     }
     const promises = [];
     const results = [];
+    const errors = [];
     for (const c of conf) {
-      const name = c.getString("name");
+      const accountName = c.getString("name");
+      const cachedCosts = await getReportsFromCache(this.cache, this.providerName, accountName, query);
+      if (cachedCosts) {
+        this.logger.debug(`${this.providerName}/${accountName} costs from cache`);
+        cachedCosts.map((cost) => {
+          results.push(cost);
+        });
+        continue;
+      }
       const keyFilePath = c.getString("keyFilePath");
       const projectId = c.getString("projectId");
       const datasetId = c.getString("datasetId");
@@ -463,31 +550,22 @@ class GCPClient {
         const [k, v] = tag.split(":");
         tagKeyValues[k.trim()] = v.trim();
       });
-      const categoryMappings = await getCategoryMappings(this.database, "gcp");
+      const categoryMappings = await getCategoryMappings(this.database, this.providerName);
       const promise = (async () => {
         try {
-          const costResponse = await this.queryBigQuery(
-            keyFilePath,
-            projectId,
-            datasetId,
-            tableId,
-            query
-          );
+          const costResponse = await this.queryBigQuery(keyFilePath, projectId, datasetId, tableId, query);
           const transformedData = lodash.reduce(
             costResponse,
             (acc, row) => {
               const period = row.period;
-              const keyName = `${name}_${row.project}_${row.service}`;
+              const keyName = `${accountName}_${row.project}_${row.service}`;
               if (!acc[keyName]) {
                 acc[keyName] = {
                   id: keyName,
-                  name: `GCP/${name}`,
+                  name: `${this.providerName}/${accountName}`,
                   service: this.convertServiceName(row.service),
-                  category: getCategoryByServiceName(
-                    row.service,
-                    categoryMappings
-                  ),
-                  provider: "GCP",
+                  category: getCategoryByServiceName(row.service, categoryMappings),
+                  provider: this.providerName,
                   reports: {},
                   ...{ project: row.project },
                   // TODO: how should we handle the project field? for now, we add project name as a field in the report
@@ -500,51 +578,56 @@ class GCPClient {
             },
             {}
           );
+          await setReportsToCache(
+            this.cache,
+            Object.values(transformedData),
+            this.providerName,
+            accountName,
+            query,
+            60 * 60 * 2 * 1e3
+          );
           Object.values(transformedData).map((value) => {
             results.push(value);
           });
         } catch (e) {
           this.logger.error(e);
+          errors.push({
+            provider: this.providerName,
+            name: `${this.providerName}/${accountName}`,
+            error: e.message
+          });
         }
       })();
       promises.push(promise);
     }
     await Promise.all(promises);
-    return results;
+    return {
+      reports: results,
+      errors
+    };
   }
 }
 
 async function setUpDatabase(database) {
   var _a;
   const client = await database.getClient();
-  const migrationsDir = backendPluginApi.resolvePackagePath(
-    "@electrolux-oss/plugin-infrawallet-backend",
-    "migrations"
-  );
+  const migrationsDir = backendPluginApi.resolvePackagePath("@electrolux-oss/plugin-infrawallet-backend", "migrations");
   if (!((_a = database.migrations) == null ? void 0 : _a.skip)) {
     await client.migrate.latest({
       directory: migrationsDir
     });
   }
-  const category_mappings_count = await client("category_mappings").count(
-    "id as c"
-  );
-  if (category_mappings_count[0].c === 0 || category_mappings_count[0].c === "0") {
-    const seedsDir = backendPluginApi.resolvePackagePath(
-      "@electrolux-oss/plugin-infrawallet-backend",
-      "seeds"
-    );
-    await client.seed.run({ directory: seedsDir });
-  }
+  const seedsDir = backendPluginApi.resolvePackagePath("@electrolux-oss/plugin-infrawallet-backend", "seeds");
+  await client.seed.run({ directory: seedsDir });
 }
 async function createRouter(options) {
   const { logger, config, cache, database } = options;
   await setUpDatabase(database);
   const router = Router__default.default();
   router.use(express__default.default.json());
-  const azureClient = AzureClient.create(config, database, logger);
-  const awsClient = AwsClient.create(config, database, logger);
-  const gcpClient = GCPClient.create(config, database, logger);
+  const azureClient = AzureClient.create(config, database, cache, logger);
+  const awsClient = AwsClient.create(config, database, cache, logger);
+  const gcpClient = GCPClient.create(config, database, cache, logger);
   const cloudClients = [azureClient, awsClient, gcpClient];
   router.get("/health", (_, response) => {
     logger.info("PONG!");
@@ -558,46 +641,40 @@ async function createRouter(options) {
     const endTime = request.query.endTime;
     const promises = [];
     const results = [];
+    const errors = [];
     cloudClients.forEach(async (client) => {
       const fetchCloudCosts = (async () => {
-        const cacheKey = [
-          client.constructor.name,
-          filters,
-          groups,
-          granularity,
-          startTime,
-          endTime
-        ].join("_");
-        const cachedCosts = await cache.get(cacheKey);
-        if (cachedCosts) {
-          logger.debug(`${client.constructor.name} costs from cache`);
-          cachedCosts.forEach((cost) => {
+        try {
+          const clientResponse = await client.fetchCostsFromCloud({
+            filters,
+            groups,
+            granularity,
+            startTime,
+            endTime
+          });
+          clientResponse.errors.forEach((e) => {
+            errors.push(e);
+          });
+          clientResponse.reports.forEach((cost) => {
             results.push(cost);
           });
-        } else {
-          try {
-            const costs = await client.fetchCostsFromCloud({
-              filters,
-              groups,
-              granularity,
-              startTime,
-              endTime
-            });
-            await cache.set(cacheKey, costs, {
-              ttl: 60 * 60 * 2 * 1e3
-            });
-            costs.forEach((cost) => {
-              results.push(cost);
-            });
-          } catch (e) {
-            logger.error(e);
-          }
+        } catch (e) {
+          logger.error(e);
+          errors.push({
+            provider: client.constructor.name,
+            name: client.constructor.name,
+            error: e.message
+          });
         }
       })();
       promises.push(fetchCloudCosts);
     });
     await Promise.all(promises);
-    response.json({ data: results, status: "ok" });
+    if (errors.length > 0) {
+      response.status(207).json({ data: results, errors, status: 207 });
+    } else {
+      response.json({ data: results, errors, status: 200 });
+    }
   });
   router.use(backendCommon.errorHandler());
   return router;