@electrolux-oss/plugin-infrawallet-backend 0.1.7 → 0.1.9

package/config.d.ts

@@ -41,6 +41,43 @@ export interface Config {
             tags?: string[];
           },
         ];
+        mock?: [
+          {
+            name: string;
+          },
+        ];
+      };
+      metricProviders?: {
+        datadog?: [
+          {
+            name: string;
+            /**
+             * @visibility secret
+             */
+            apiKey: string;
+            /**
+             * @visibility secret
+             */
+            applicationKey: string;
+            ddSite: string;
+          },
+        ];
+        grafanaCloud?: [
+          {
+            name: string;
+            url: string;
+            datasourceUid: string;
+            /**
+             * @visibility secret
+             */
+            token: string;
+          },
+        ];
+        mock?: [
+          {
+            name: string;
+          },
+        ];
       };
     };
   };

package/dist/index.cjs.js

@@ -14,39 +14,86 @@ var armCostmanagement = require('@azure/arm-costmanagement');
 var coreRestPipeline = require('@azure/core-rest-pipeline');
 var identity = require('@azure/identity');
 var bigquery = require('@google-cloud/bigquery');
+var datadogApiClient = require('@datadog/datadog-api-client');
+var fetch = require('node-fetch');
+var fs = require('fs');
+var upath = require('upath');
 
 function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e : { default: e }; }
 
+function _interopNamespaceCompat(e) {
+  if (e && typeof e === 'object' && 'default' in e) return e;
+  var n = Object.create(null);
+  if (e) {
+    Object.keys(e).forEach(function (k) {
+      if (k !== 'default') {
+        var d = Object.getOwnPropertyDescriptor(e, k);
+        Object.defineProperty(n, k, d.get ? d : {
+          enumerable: true,
+          get: function () { return e[k]; }
+        });
+      }
+    });
+  }
+  n.default = e;
+  return Object.freeze(n);
+}
+
 var express__default = /*#__PURE__*/_interopDefaultCompat(express);
 var Router__default = /*#__PURE__*/_interopDefaultCompat(Router);
 var moment__default = /*#__PURE__*/_interopDefaultCompat(moment);
+var fetch__default = /*#__PURE__*/_interopDefaultCompat(fetch);
+var upath__namespace = /*#__PURE__*/_interopNamespaceCompat(upath);
+
+async function getWallet(database, walletName) {
+  const client = await database.getClient();
+  const result = await client("wallets").where("name", walletName).first();
+  return result;
+}
+async function getWalletMetricSettings(database, walletName) {
+  const client = await database.getClient();
+  const metricSettings = await client.select("business_metrics.*").from("business_metrics").where("wallets.name", walletName).join("wallets", "business_metrics.wallet_id", "=", "wallets.id");
+  return metricSettings;
+}
+async function updateOrInsertWalletMetricSetting(database, walletSetting) {
+  const client = await database.getClient();
+  const result = await client("business_metrics").insert(walletSetting).onConflict("id").merge();
+  if (result[0] > 0) {
+    return true;
+  }
+  return false;
+}
+async function deleteWalletMetricSetting(database, walletSetting) {
+  const client = await database.getClient();
+  const result = await client("business_metrics").where("id", walletSetting.id).del();
+  if (result > 0) {
+    return true;
+  }
+  return false;
+}
 
 async function getCategoryMappings(database, provider) {
   const result = {};
   const client = await database.getClient();
-  const default_mappings = await client.where({ provider: provider.toLowerCase() }).select().from("category_mappings_default");
-  default_mappings.forEach((mapping) => {
-    if (typeof mapping.cloud_service_names === "string") {
-      JSON.parse(mapping.cloud_service_names).forEach((service) => {
-        result[service] = mapping.category;
-      });
-    } else {
-      mapping.cloud_service_names.forEach((service) => {
-        result[service] = mapping.category;
-      });
+  const defaultMappings = await client.where({ provider: provider.toLowerCase() }).select().from("category_mappings_default");
+  defaultMappings.forEach((mapping) => {
+    let services = mapping.cloud_service_names;
+    if (typeof services === "string") {
+      services = JSON.parse(services);
     }
+    services.forEach((service) => {
+      result[service] = mapping.category;
+    });
   });
-  const override_mappings = await client.where({ provider }).select().from("category_mappings_override");
-  override_mappings.forEach((mapping) => {
-    if (typeof mapping.cloud_service_names === "string") {
-      JSON.parse(mapping.cloud_service_names).forEach((service) => {
-        result[service] = mapping.category;
-      });
-    } else {
-      mapping.cloud_service_names.forEach((service) => {
-        result[service] = mapping.category;
-      });
+  const overrideMappings = await client.where({ provider }).select().from("category_mappings_override");
+  overrideMappings.forEach((mapping) => {
+    let services = mapping.cloud_service_names;
+    if (typeof services === "string") {
+      services = JSON.parse(services);
     }
+    services.forEach((service) => {
+      result[service] = mapping.category;
+    });
   });
   return result;
 }
@@ -54,6 +101,14 @@ function getCategoryByServiceName(serviceName, categoryMappings) {
   if (serviceName in categoryMappings) {
     return categoryMappings[serviceName];
   }
+  for (const service in categoryMappings) {
+    if (Object.hasOwn(categoryMappings, service)) {
+      const regex = new RegExp(service);
+      if (regex.test(serviceName)) {
+        return categoryMappings[service];
+      }
+    }
+  }
   return "Uncategorized";
 }
 async function getReportsFromCache(cache, provider, configKey, query) {
@@ -69,6 +124,20 @@ async function getReportsFromCache(cache, provider, configKey, query) {
   const cachedCosts = await cache.get(cacheKey);
   return cachedCosts;
 }
+async function getMetricsFromCache(cache, provider, configKey, query) {
+  const cacheKey = [
+    provider,
+    configKey,
+    query.name,
+    query.query,
+    query.granularity,
+    query.startTime,
+    query.endTime
+  ].join("_");
+  const crypto = require("crypto");
+  const cachedMetrics = await cache.get(crypto.createHash("md5").update(cacheKey).digest("hex"));
+  return cachedMetrics;
+}
 async function setReportsToCache(cache, reports, provider, configKey, query, ttl) {
   const cacheKey = [
     provider,
@@ -83,6 +152,21 @@ async function setReportsToCache(cache, reports, provider, configKey, query, ttl
     ttl: ttl 
   });
 }
+async function setMetricsToCache(cache, metrics, provider, configKey, query, ttl) {
+  const cacheKey = [
+    provider,
+    configKey,
+    query.name,
+    query.query,
+    query.granularity,
+    query.startTime,
+    query.endTime
+  ].join("_");
+  const crypto = require("crypto");
+  await cache.set(crypto.createHash("md5").update(cacheKey).digest("hex"), metrics, {
+    ttl: ttl 
+  });
+}
 
 class InfraWalletClient {
   constructor(providerName, config, database, cache, logger) {
@@ -541,10 +625,305 @@ class GCPClient extends InfraWalletClient {
   }
 }
 
-const PROVIDER_CLIENT_MAPPINGS = {
+class MetricProvider {
+  constructor(providerName, config, database, cache, logger) {
+    this.providerName = providerName;
+    this.config = config;
+    this.database = database;
+    this.cache = cache;
+    this.logger = logger;
+  }
+  async getMetrics(query) {
+    const conf = this.config.getOptionalConfigArray(
+      `backend.infraWallet.metricProviders.${this.providerName.toLowerCase()}`
+    );
+    if (!conf) {
+      return { metrics: [], errors: [] };
+    }
+    const promises = [];
+    const results = [];
+    const errors = [];
+    for (const c of conf) {
+      const configName = c.getString("name");
+      const client = await this.initProviderClient(c);
+      const dbClient = await this.database.getClient();
+      const metricSettings = await dbClient.where({
+        "wallets.name": query.walletName,
+        "business_metrics.metric_provider": this.providerName.toLowerCase(),
+        "business_metrics.config_name": configName
+      }).select("business_metrics.*").from("business_metrics").join("wallets", "business_metrics.wallet_id", "=", "wallets.id");
+      for (const metric of metricSettings || []) {
+        const promise = (async () => {
+          try {
+            const fullQuery = {
+              name: metric.metric_name,
+              query: metric.query,
+              ...query
+            };
+            const cachedMetrics = await getMetricsFromCache(this.cache, this.providerName, configName, fullQuery);
+            if (cachedMetrics) {
+              this.logger.debug(`${this.providerName}/${configName}/${fullQuery.name} metrics from cache`);
+              cachedMetrics.map((m) => {
+                results.push(m);
+              });
+              return;
+            }
+            const metricResponse = await this.fetchMetrics(c, client, fullQuery);
+            const transformedMetrics = await this.transformMetricData(c, fullQuery, metricResponse);
+            await setMetricsToCache(
+              this.cache,
+              transformedMetrics,
+              this.providerName,
+              configName,
+              fullQuery,
+              60 * 60 * 2 * 1e3
+            );
+            transformedMetrics.map((value) => {
+              results.push(value);
+            });
+          } catch (e) {
+            this.logger.error(e);
+            errors.push({
+              provider: this.providerName,
+              name: `${this.providerName}/${configName}/${metric.getString("metricName")}`,
+              error: e.message
+            });
+          }
+        })();
+        promises.push(promise);
+      }
+    }
+    await Promise.all(promises);
+    return {
+      metrics: results,
+      errors
+    };
+  }
+}
+
+class DatadogProvider extends MetricProvider {
+  static create(config, database, cache, logger) {
+    return new DatadogProvider("Datadog", config, database, cache, logger);
+  }
+  async initProviderClient(config) {
+    const apiKey = config.getString("apiKey");
+    const applicationKey = config.getString("applicationKey");
+    const ddSite = config.getString("ddSite");
+    const configuration = datadogApiClient.client.createConfiguration({
+      baseServer: new datadogApiClient.client.BaseServerConfiguration(ddSite, {}),
+      authMethods: {
+        apiKeyAuth: apiKey,
+        appKeyAuth: applicationKey
+      }
+    });
+    const client = new datadogApiClient.v1.MetricsApi(configuration);
+    return client;
+  }
+  async fetchMetrics(_metricProviderConfig, client, query) {
+    var _a;
+    const params = {
+      from: parseInt(query.startTime, 10) / 1e3,
+      to: parseInt(query.endTime, 10) / 1e3,
+      query: (_a = query.query) == null ? void 0 : _a.replaceAll("IW_INTERVAL", query.granularity === "daily" ? "86400" : "2592000")
+    };
+    return client.queryMetrics(params).then((data) => {
+      if (data.status === "ok") {
+        return data;
+      }
+      throw new Error(data.error);
+    });
+  }
+  async transformMetricData(_metricProviderConfig, query, metricResponse) {
+    const transformedData = [];
+    for (const series of metricResponse.series) {
+      const metricName = query.name;
+      const tagSet = series.tagSet;
+      const metric = {
+        id: `${metricName} ${tagSet.length === 0 ? "" : tagSet}`,
+        provider: this.providerName,
+        name: metricName,
+        reports: {}
+      };
+      for (const point of series.pointlist) {
+        const period = moment__default.default(point[0]).format(query.granularity === "daily" ? "YYYY-MM-DD" : "YYYY-MM");
+        const value = point[1];
+        metric.reports[period] = value;
+      }
+      transformedData.push(metric);
+    }
+    return transformedData;
+  }
+}
+
+class GrafanaCloudProvider extends MetricProvider {
+  static create(config, database, cache, logger) {
+    return new GrafanaCloudProvider("GrafanaCloud", config, database, cache, logger);
+  }
+  async initProviderClient(_config) {
+    return null;
+  }
+  async fetchMetrics(metricProviderConfig, _client, query) {
+    var _a;
+    const url = metricProviderConfig.getString("url");
+    const datasourceUid = metricProviderConfig.getString("datasourceUid");
+    const token = metricProviderConfig.getString("token");
+    const headers = {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${token}`
+    };
+    const payload = {
+      queries: [
+        {
+          datasource: {
+            uid: datasourceUid
+          },
+          expr: (_a = query.query) == null ? void 0 : _a.replaceAll("IW_INTERVAL", query.granularity === "daily" ? "1d" : "30d"),
+          refId: "A"
+        }
+      ],
+      from: query.startTime,
+      to: query.endTime
+    };
+    const response = await fetch__default.default(`${url}/api/ds/query`, {
+      method: "post",
+      body: JSON.stringify(payload),
+      headers
+    });
+    const data = await response.json();
+    return data;
+  }
+  async transformMetricData(_metricProviderConfig, query, metricResponse) {
+    const transformedData = [];
+    const metricName = query.name;
+    const metric = {
+      id: metricName,
+      provider: this.providerName,
+      name: metricName,
+      reports: {}
+    };
+    const periods = metricResponse.results.A.frames[0].data.values[0];
+    const values = metricResponse.results.A.frames[0].data.values[1];
+    for (let i = 0; i < periods.length; i++) {
+      const period = moment__default.default(periods[i]).format(query.granularity === "daily" ? "YYYY-MM-DD" : "YYYY-MM");
+      const value = values[i];
+      metric.reports[period] = value;
+    }
+    transformedData.push(metric);
+    return transformedData;
+  }
+}
+
+class MockProvider extends MetricProvider {
+  static create(config, database, cache, logger) {
+    return new MockProvider("Mock", config, database, cache, logger);
+  }
+  async initProviderClient(_config) {
+    return null;
+  }
+  async fetchMetrics(_metricProviderConfig, _client, _query) {
+    return null;
+  }
+  async transformMetricData(_metricProviderConfig, query, _metricResponse) {
+    var _a, _b;
+    const transformedData = [];
+    const metricName = query.name;
+    let mockSettings = {};
+    try {
+      mockSettings = JSON.parse(query.query);
+    } catch (e) {
+    }
+    const minValue = (_a = mockSettings.min) != null ? _a : 0;
+    const maxValue = (_b = mockSettings.max) != null ? _b : 1e3;
+    const metric = {
+      id: metricName,
+      provider: this.providerName,
+      name: metricName,
+      reports: {}
+    };
+    let cursor = moment__default.default(parseInt(query.startTime, 10));
+    while (cursor <= moment__default.default(parseInt(query.endTime, 10))) {
+      const period = cursor.format(query.granularity === "daily" ? "YYYY-MM-DD" : "YYYY-MM");
+      metric.reports[period] = Math.floor(Math.random() * (maxValue - minValue) + minValue);
+      cursor = cursor.add(1, query.granularity === "daily" ? "days" : "months");
+    }
+    transformedData.push(metric);
+    return transformedData;
+  }
+}
+
+class MockClient extends InfraWalletClient {
+  static create(config, database, cache, logger) {
+    return new MockClient("mock", config, database, cache, logger);
+  }
+  async initCloudClient(config) {
+    this.logger.debug(`MockClient.initCloudClient called with config: ${JSON.stringify(config)}`);
+    return null;
+  }
+  async fetchCostsFromCloud(_subAccountConfig, _client, _query) {
+    return null;
+  }
+  async transformCostsData(_subAccountConfig, query, _costResponse, _categoryMappings) {
+    try {
+      const startD = moment__default.default.unix(Number(query.startTime) / 1e3);
+      let endD = moment__default.default.unix(Number(query.endTime) / 1e3);
+      const mockDir = backendPluginApi.resolvePackagePath("@electrolux-oss/plugin-infrawallet-backend", "mock");
+      const mockFilePath = upath__namespace.join(mockDir, "mock_response.json");
+      const data = await fs.promises.readFile(mockFilePath, "utf8");
+      const jsonData = JSON.parse(data);
+      const currentDate = moment__default.default();
+      if (endD.isAfter(currentDate)) {
+        this.logger.warn("End Date is in the future, adjusting to current date.");
+        endD = currentDate.clone();
+        endD.add(1, "day");
+      }
+      const processedData = await Promise.all(
+        jsonData.map(async (item) => {
+          item.reports = {};
+          const StartDate = moment__default.default(startD);
+          let step;
+          let dateFormat = "YYYY-MM";
+          if (query.granularity.toLowerCase() === "monthly") {
+            step = "months";
+            dateFormat = "YYYY-MM";
+          } else if (query.granularity.toLowerCase() === "daily") {
+            step = "days";
+            dateFormat = "YYYY-MM-DD";
+          }
+          while (StartDate.isBefore(endD)) {
+            const dateString = StartDate.format(dateFormat);
+            if (query.granularity.toLowerCase() === "monthly") {
+              item.reports[dateString] = this.getRandomValue(0.4 * 30, 33.3 * 30);
+            } else {
+              item.reports[dateString] = this.getRandomValue(0.4, 33.3);
+            }
+            StartDate.add(1, step);
+          }
+          return item;
+        })
+      );
+      return processedData;
+    } catch (err) {
+      this.logger.error("Error while reading a file", err);
+      throw err;
+    }
+  }
+  getRandomValue(min, max) {
+    const random = Math.random();
+    const amplifiedRandom = Math.pow(random, 3);
+    return amplifiedRandom * (max - min) + min;
+  }
+}
+
+const COST_CLIENT_MAPPINGS = {
   aws: AwsClient,
   azure: AzureClient,
-  gcp: GCPClient
+  gcp: GCPClient,
+  mock: MockClient
+};
+const METRIC_PROVIDER_MAPPINGS = {
+  datadog: DatadogProvider,
+  grafanacloud: GrafanaCloudProvider,
+  mock: MockProvider
 };
 
 async function setUpDatabase(database) {
@@ -579,8 +958,8 @@ async function createRouter(options) {
     const errors = [];
     const conf = config.getConfig("backend.infraWallet.integrations");
     conf.keys().forEach((provider) => {
-      if (provider in PROVIDER_CLIENT_MAPPINGS) {
-        const client = PROVIDER_CLIENT_MAPPINGS[provider].create(config, database, cache, logger);
+      if (provider in COST_CLIENT_MAPPINGS) {
+        const client = COST_CLIENT_MAPPINGS[provider].create(config, database, cache, logger);
         const fetchCloudCosts = (async () => {
           try {
             const clientResponse = await client.getCostReports({
@@ -615,6 +994,98 @@ async function createRouter(options) {
       response.json({ data: results, errors, status: 200 });
     }
   });
+  router.get("/:walletName/metrics", async (request, response) => {
+    const walletName = request.params.walletName;
+    const granularity = request.query.granularity;
+    const startTime = request.query.startTime;
+    const endTime = request.query.endTime;
+    const promises = [];
+    const results = [];
+    const errors = [];
+    const conf = config.getConfig("backend.infraWallet.metricProviders");
+    conf.keys().forEach((provider) => {
+      if (provider in METRIC_PROVIDER_MAPPINGS) {
+        const client = METRIC_PROVIDER_MAPPINGS[provider].create(config, database, cache, logger);
+        const fetchMetrics = (async () => {
+          try {
+            const metricResponse = await client.getMetrics({
+              walletName,
+              granularity,
+              startTime,
+              endTime
+            });
+            metricResponse.errors.forEach((e) => {
+              errors.push(e);
+            });
+            metricResponse.metrics.forEach((metric) => {
+              results.push(metric);
+            });
+          } catch (e) {
+            logger.error(e);
+            errors.push({
+              provider: client.constructor.name,
+              name: client.constructor.name,
+              error: e.message
+            });
+          }
+        })();
+        promises.push(fetchMetrics);
+      }
+    });
+    await Promise.all(promises);
+    if (errors.length > 0) {
+      response.status(207).json({ data: results, errors, status: 207 });
+    } else {
+      response.json({ data: results, errors, status: 200 });
+    }
+  });
+  router.get("/:walletName", async (request, response) => {
+    const walletName = request.params.walletName;
+    const wallet = await getWallet(database, walletName);
+    if (wallet === void 0) {
+      response.status(404).json({ error: "Wallet not found", status: 404 });
+      return;
+    }
+    response.json({ data: wallet, status: 200 });
+  });
+  router.get("/:walletName/metrics_setting", async (request, response) => {
+    const walletName = request.params.walletName;
+    const metricSettings = await getWalletMetricSettings(database, walletName);
+    response.json({ data: metricSettings, status: 200 });
+  });
+  router.get("/metric/metric_configs", async (_request, response) => {
+    const conf = config.getConfig("backend.infraWallet.metricProviders");
+    const configNames = [];
+    conf.keys().forEach((provider) => {
+      const configs = conf.getOptionalConfigArray(provider);
+      if (configs) {
+        configs.forEach((c) => {
+          configNames.push({ metric_provider: provider, config_name: c.getString("name") });
+        });
+      }
+    });
+    response.json({ data: configNames, status: 200 });
+  });
+  router.put("/:walletName/metrics_setting", async (request, response) => {
+    var _a;
+    const readOnly = (_a = config.getOptionalBoolean("infraWallet.settings.readOnly")) != null ? _a : false;
+    if (readOnly) {
+      response.status(403).json({ error: "API not enabled in read-only mode", status: 403 });
+      return;
+    }
+    const updatedMetricSetting = await updateOrInsertWalletMetricSetting(database, request.body);
+    response.json({ updated: updatedMetricSetting, status: 200 });
+  });
+  router.delete("/:walletName/metrics_setting", async (request, response) => {
+    var _a;
+    const readOnly = (_a = config.getOptionalBoolean("infraWallet.settings.readOnly")) != null ? _a : false;
+    if (readOnly) {
+      response.status(403).json({ error: "API not enabled in read-only mode", status: 403 });
+      return;
+    }
+    const deletedMetricSetting = await deleteWalletMetricSetting(database, request.body);
+    response.json({ deleted: deletedMetricSetting, status: 200 });
+  });
   router.use(backendCommon.errorHandler());
   return router;
 }