@electrolux-oss/plugin-infrawallet-backend 0.1.8 → 0.1.9

package/config.d.ts

@@ -41,6 +41,11 @@ export interface Config {
             tags?: string[];
           },
         ];
+        mock?: [
+          {
+            name: string;
+          },
+        ];
       };
       metricProviders?: {
         datadog?: [
@@ -68,6 +73,11 @@ export interface Config {
             token: string;
           },
         ];
+        mock?: [
+          {
+            name: string;
+          },
+        ];
       };
     };
   };

package/dist/index.cjs.js

@@ -13,16 +13,37 @@ var moment = require('moment');
 var armCostmanagement = require('@azure/arm-costmanagement');
 var coreRestPipeline = require('@azure/core-rest-pipeline');
 var identity = require('@azure/identity');
-var datadogApiClient = require('@datadog/datadog-api-client');
 var bigquery = require('@google-cloud/bigquery');
+var datadogApiClient = require('@datadog/datadog-api-client');
 var fetch = require('node-fetch');
+var fs = require('fs');
+var upath = require('upath');
 
 function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e : { default: e }; }
 
+function _interopNamespaceCompat(e) {
+  if (e && typeof e === 'object' && 'default' in e) return e;
+  var n = Object.create(null);
+  if (e) {
+    Object.keys(e).forEach(function (k) {
+      if (k !== 'default') {
+        var d = Object.getOwnPropertyDescriptor(e, k);
+        Object.defineProperty(n, k, d.get ? d : {
+          enumerable: true,
+          get: function () { return e[k]; }
+        });
+      }
+    });
+  }
+  n.default = e;
+  return Object.freeze(n);
+}
+
 var express__default = /*#__PURE__*/_interopDefaultCompat(express);
 var Router__default = /*#__PURE__*/_interopDefaultCompat(Router);
 var moment__default = /*#__PURE__*/_interopDefaultCompat(moment);
 var fetch__default = /*#__PURE__*/_interopDefaultCompat(fetch);
+var upath__namespace = /*#__PURE__*/_interopNamespaceCompat(upath);
 
 async function getWallet(database, walletName) {
   const client = await database.getClient();
@@ -54,29 +75,25 @@ async function deleteWalletMetricSetting(database, walletSetting) {
 async function getCategoryMappings(database, provider) {
   const result = {};
   const client = await database.getClient();
-  const default_mappings = await client.where({ provider: provider.toLowerCase() }).select().from("category_mappings_default");
-  default_mappings.forEach((mapping) => {
-    if (typeof mapping.cloud_service_names === "string") {
-      JSON.parse(mapping.cloud_service_names).forEach((service) => {
-        result[service] = mapping.category;
-      });
-    } else {
-      mapping.cloud_service_names.forEach((service) => {
-        result[service] = mapping.category;
-      });
+  const defaultMappings = await client.where({ provider: provider.toLowerCase() }).select().from("category_mappings_default");
+  defaultMappings.forEach((mapping) => {
+    let services = mapping.cloud_service_names;
+    if (typeof services === "string") {
+      services = JSON.parse(services);
     }
+    services.forEach((service) => {
+      result[service] = mapping.category;
+    });
   });
-  const override_mappings = await client.where({ provider }).select().from("category_mappings_override");
-  override_mappings.forEach((mapping) => {
-    if (typeof mapping.cloud_service_names === "string") {
-      JSON.parse(mapping.cloud_service_names).forEach((service) => {
-        result[service] = mapping.category;
-      });
-    } else {
-      mapping.cloud_service_names.forEach((service) => {
-        result[service] = mapping.category;
-      });
+  const overrideMappings = await client.where({ provider }).select().from("category_mappings_override");
+  overrideMappings.forEach((mapping) => {
+    let services = mapping.cloud_service_names;
+    if (typeof services === "string") {
+      services = JSON.parse(services);
     }
+    services.forEach((service) => {
+      result[service] = mapping.category;
+    });
   });
   return result;
 }
@@ -84,6 +101,14 @@ function getCategoryByServiceName(serviceName, categoryMappings) {
   if (serviceName in categoryMappings) {
     return categoryMappings[serviceName];
   }
+  for (const service in categoryMappings) {
+    if (Object.hasOwn(categoryMappings, service)) {
+      const regex = new RegExp(service);
+      if (regex.test(serviceName)) {
+        return categoryMappings[service];
+      }
+    }
+  }
   return "Uncategorized";
 }
 async function getReportsFromCache(cache, provider, configKey, query) {
@@ -109,7 +134,8 @@ async function getMetricsFromCache(cache, provider, configKey, query) {
     query.startTime,
     query.endTime
   ].join("_");
-  const cachedMetrics = await cache.get(cacheKey);
+  const crypto = require("crypto");
+  const cachedMetrics = await cache.get(crypto.createHash("md5").update(cacheKey).digest("hex"));
   return cachedMetrics;
 }
 async function setReportsToCache(cache, reports, provider, configKey, query, ttl) {
@@ -507,6 +533,98 @@ class AzureClient extends InfraWalletClient {
   }
 }
 
+class GCPClient extends InfraWalletClient {
+  static create(config, database, cache, logger) {
+    return new GCPClient("GCP", config, database, cache, logger);
+  }
+  convertServiceName(serviceName) {
+    let convertedName = serviceName;
+    const prefixes = ["Google Cloud"];
+    for (const prefix of prefixes) {
+      if (serviceName.startsWith(prefix)) {
+        convertedName = serviceName.slice(prefix.length).trim();
+      }
+    }
+    return `${this.providerName}/${convertedName}`;
+  }
+  async initCloudClient(subAccountConfig) {
+    const keyFilePath = subAccountConfig.getString("keyFilePath");
+    const projectId = subAccountConfig.getString("projectId");
+    const options = {
+      keyFilename: keyFilePath,
+      projectId
+    };
+    const bigqueryClient = new bigquery.BigQuery(options);
+    return bigqueryClient;
+  }
+  async fetchCostsFromCloud(subAccountConfig, client, query) {
+    const projectId = subAccountConfig.getString("projectId");
+    const datasetId = subAccountConfig.getString("datasetId");
+    const tableId = subAccountConfig.getString("tableId");
+    try {
+      const periodFormat = query.granularity.toUpperCase() === "MONTHLY" ? "%Y-%m" : "%Y-%m-%d";
+      const sql = `
+        SELECT
+          project.name AS project,
+          service.description AS service,
+          FORMAT_TIMESTAMP('${periodFormat}', usage_start_time) AS period,
+          (SUM(CAST(cost AS NUMERIC)) + SUM(IFNULL((SELECT SUM(CAST(c.amount AS NUMERIC)) FROM UNNEST(credits) AS c), 0))) AS total_cost
+        FROM
+          \`${projectId}.${datasetId}.${tableId}\`
+        WHERE
+          project.name IS NOT NULL
+          AND cost > 0
+          AND usage_start_time >= TIMESTAMP_MILLIS(${query.startTime})
+          AND usage_start_time <= TIMESTAMP_MILLIS(${query.endTime})
+        GROUP BY
+          project, service, period
+        ORDER BY
+          project, period, total_cost DESC`;
+      const [job] = await client.createQueryJob({
+        query: sql
+      });
+      const [rows] = await job.getQueryResults();
+      return rows;
+    } catch (err) {
+      throw new Error(err.message);
+    }
+  }
+  async transformCostsData(subAccountConfig, _query, costResponse, categoryMappings) {
+    const accountName = subAccountConfig.getString("name");
+    const tags = subAccountConfig.getOptionalStringArray("tags");
+    const tagKeyValues = {};
+    tags == null ? void 0 : tags.forEach((tag) => {
+      const [k, v] = tag.split(":");
+      tagKeyValues[k.trim()] = v.trim();
+    });
+    const transformedData = lodash.reduce(
+      costResponse,
+      (acc, row) => {
+        const period = row.period;
+        const keyName = `${accountName}_${row.project}_${row.service}`;
+        if (!acc[keyName]) {
+          acc[keyName] = {
+            id: keyName,
+            name: `${this.providerName}/${accountName}`,
+            service: this.convertServiceName(row.service),
+            category: getCategoryByServiceName(row.service, categoryMappings),
+            provider: this.providerName,
+            reports: {},
+            ...{ project: row.project },
+            // TODO: how should we handle the project field? for now, we add project name as a field in the report
+            ...tagKeyValues
+            // note that if there is a tag `project:foo` in config, it overrides the project field set above
+          };
+        }
+        acc[keyName].reports[period] = parseFloat(row.total_cost);
+        return acc;
+      },
+      {}
+    );
+    return Object.values(transformedData);
+  }
+}
+
 class MetricProvider {
   constructor(providerName, config, database, cache, logger) {
     this.providerName = providerName;
@@ -637,98 +755,6 @@ class DatadogProvider extends MetricProvider {
   }
 }
 
-class GCPClient extends InfraWalletClient {
-  static create(config, database, cache, logger) {
-    return new GCPClient("GCP", config, database, cache, logger);
-  }
-  convertServiceName(serviceName) {
-    let convertedName = serviceName;
-    const prefixes = ["Google Cloud"];
-    for (const prefix of prefixes) {
-      if (serviceName.startsWith(prefix)) {
-        convertedName = serviceName.slice(prefix.length).trim();
-      }
-    }
-    return `${this.providerName}/${convertedName}`;
-  }
-  async initCloudClient(subAccountConfig) {
-    const keyFilePath = subAccountConfig.getString("keyFilePath");
-    const projectId = subAccountConfig.getString("projectId");
-    const options = {
-      keyFilename: keyFilePath,
-      projectId
-    };
-    const bigqueryClient = new bigquery.BigQuery(options);
-    return bigqueryClient;
-  }
-  async fetchCostsFromCloud(subAccountConfig, client, query) {
-    const projectId = subAccountConfig.getString("projectId");
-    const datasetId = subAccountConfig.getString("datasetId");
-    const tableId = subAccountConfig.getString("tableId");
-    try {
-      const periodFormat = query.granularity.toUpperCase() === "MONTHLY" ? "%Y-%m" : "%Y-%m-%d";
-      const sql = `
-        SELECT
-          project.name AS project,
-          service.description AS service,
-          FORMAT_TIMESTAMP('${periodFormat}', usage_start_time) AS period,
-          (SUM(CAST(cost AS NUMERIC)) + SUM(IFNULL((SELECT SUM(CAST(c.amount AS NUMERIC)) FROM UNNEST(credits) AS c), 0))) AS total_cost
-        FROM
-          \`${projectId}.${datasetId}.${tableId}\`
-        WHERE
-          project.name IS NOT NULL
-          AND cost > 0
-          AND usage_start_time >= TIMESTAMP_MILLIS(${query.startTime})
-          AND usage_start_time <= TIMESTAMP_MILLIS(${query.endTime})
-        GROUP BY
-          project, service, period
-        ORDER BY
-          project, period, total_cost DESC`;
-      const [job] = await client.createQueryJob({
-        query: sql
-      });
-      const [rows] = await job.getQueryResults();
-      return rows;
-    } catch (err) {
-      throw new Error(err.message);
-    }
-  }
-  async transformCostsData(subAccountConfig, _query, costResponse, categoryMappings) {
-    const accountName = subAccountConfig.getString("name");
-    const tags = subAccountConfig.getOptionalStringArray("tags");
-    const tagKeyValues = {};
-    tags == null ? void 0 : tags.forEach((tag) => {
-      const [k, v] = tag.split(":");
-      tagKeyValues[k.trim()] = v.trim();
-    });
-    const transformedData = lodash.reduce(
-      costResponse,
-      (acc, row) => {
-        const period = row.period;
-        const keyName = `${accountName}_${row.project}_${row.service}`;
-        if (!acc[keyName]) {
-          acc[keyName] = {
-            id: keyName,
-            name: `${this.providerName}/${accountName}`,
-            service: this.convertServiceName(row.service),
-            category: getCategoryByServiceName(row.service, categoryMappings),
-            provider: this.providerName,
-            reports: {},
-            ...{ project: row.project },
-            // TODO: how should we handle the project field? for now, we add project name as a field in the report
-            ...tagKeyValues
-            // note that if there is a tag `project:foo` in config, it overrides the project field set above
-          };
-        }
-        acc[keyName].reports[period] = parseFloat(row.total_cost);
-        return acc;
-      },
-      {}
-    );
-    return Object.values(transformedData);
-  }
-}
-
 class GrafanaCloudProvider extends MetricProvider {
   static create(config, database, cache, logger) {
     return new GrafanaCloudProvider("GrafanaCloud", config, database, cache, logger);
@@ -787,14 +813,117 @@ class GrafanaCloudProvider extends MetricProvider {
   }
 }
 
+class MockProvider extends MetricProvider {
+  static create(config, database, cache, logger) {
+    return new MockProvider("Mock", config, database, cache, logger);
+  }
+  async initProviderClient(_config) {
+    return null;
+  }
+  async fetchMetrics(_metricProviderConfig, _client, _query) {
+    return null;
+  }
+  async transformMetricData(_metricProviderConfig, query, _metricResponse) {
+    var _a, _b;
+    const transformedData = [];
+    const metricName = query.name;
+    let mockSettings = {};
+    try {
+      mockSettings = JSON.parse(query.query);
+    } catch (e) {
+    }
+    const minValue = (_a = mockSettings.min) != null ? _a : 0;
+    const maxValue = (_b = mockSettings.max) != null ? _b : 1e3;
+    const metric = {
+      id: metricName,
+      provider: this.providerName,
+      name: metricName,
+      reports: {}
+    };
+    let cursor = moment__default.default(parseInt(query.startTime, 10));
+    while (cursor <= moment__default.default(parseInt(query.endTime, 10))) {
+      const period = cursor.format(query.granularity === "daily" ? "YYYY-MM-DD" : "YYYY-MM");
+      metric.reports[period] = Math.floor(Math.random() * (maxValue - minValue) + minValue);
+      cursor = cursor.add(1, query.granularity === "daily" ? "days" : "months");
+    }
+    transformedData.push(metric);
+    return transformedData;
+  }
+}
+
+class MockClient extends InfraWalletClient {
+  static create(config, database, cache, logger) {
+    return new MockClient("mock", config, database, cache, logger);
+  }
+  async initCloudClient(config) {
+    this.logger.debug(`MockClient.initCloudClient called with config: ${JSON.stringify(config)}`);
+    return null;
+  }
+  async fetchCostsFromCloud(_subAccountConfig, _client, _query) {
+    return null;
+  }
+  async transformCostsData(_subAccountConfig, query, _costResponse, _categoryMappings) {
+    try {
+      const startD = moment__default.default.unix(Number(query.startTime) / 1e3);
+      let endD = moment__default.default.unix(Number(query.endTime) / 1e3);
+      const mockDir = backendPluginApi.resolvePackagePath("@electrolux-oss/plugin-infrawallet-backend", "mock");
+      const mockFilePath = upath__namespace.join(mockDir, "mock_response.json");
+      const data = await fs.promises.readFile(mockFilePath, "utf8");
+      const jsonData = JSON.parse(data);
+      const currentDate = moment__default.default();
+      if (endD.isAfter(currentDate)) {
+        this.logger.warn("End Date is in the future, adjusting to current date.");
+        endD = currentDate.clone();
+        endD.add(1, "day");
+      }
+      const processedData = await Promise.all(
+        jsonData.map(async (item) => {
+          item.reports = {};
+          const StartDate = moment__default.default(startD);
+          let step;
+          let dateFormat = "YYYY-MM";
+          if (query.granularity.toLowerCase() === "monthly") {
+            step = "months";
+            dateFormat = "YYYY-MM";
+          } else if (query.granularity.toLowerCase() === "daily") {
+            step = "days";
+            dateFormat = "YYYY-MM-DD";
+          }
+          while (StartDate.isBefore(endD)) {
+            const dateString = StartDate.format(dateFormat);
+            if (query.granularity.toLowerCase() === "monthly") {
+              item.reports[dateString] = this.getRandomValue(0.4 * 30, 33.3 * 30);
+            } else {
+              item.reports[dateString] = this.getRandomValue(0.4, 33.3);
+            }
+            StartDate.add(1, step);
+          }
+          return item;
+        })
+      );
+      return processedData;
+    } catch (err) {
+      this.logger.error("Error while reading a file", err);
+      throw err;
+    }
+  }
+  getRandomValue(min, max) {
+    const random = Math.random();
+    const amplifiedRandom = Math.pow(random, 3);
+    return amplifiedRandom * (max - min) + min;
+  }
+}
+
 const COST_CLIENT_MAPPINGS = {
   aws: AwsClient,
   azure: AzureClient,
-  gcp: GCPClient
+  gcp: GCPClient,
+  mock: MockClient
 };
 const METRIC_PROVIDER_MAPPINGS = {
   datadog: DatadogProvider,
-  grafanacloud: GrafanaCloudProvider
+  grafanacloud: GrafanaCloudProvider,
+  mock: MockProvider
 };
 
 async function setUpDatabase(database) {