@electrolux-oss/plugin-infrawallet-backend 0.1.2 → 0.1.4

package/dist/index.cjs.js

@@ -11,7 +11,9 @@ var clientSts = require('@aws-sdk/client-sts');
 var lodash = require('lodash');
 var moment = require('moment');
 var armCostmanagement = require('@azure/arm-costmanagement');
+var coreRestPipeline = require('@azure/core-rest-pipeline');
 var identity = require('@azure/identity');
+var bigquery = require('@google-cloud/bigquery');
 
 function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e : { default: e }; }
 
@@ -22,50 +24,126 @@ var moment__default = /*#__PURE__*/_interopDefaultCompat(moment);
 async function getCategoryMappings(database, provider) {
   const result = {};
   const client = await database.getClient();
-  const mappings = await client.where({ provider }).select().from("category_mappings");
-  mappings.forEach((mapping) => {
+  const default_mappings = await client.where({ provider: provider.toLowerCase() }).select().from("category_mappings_default");
+  default_mappings.forEach((mapping) => {
     if (typeof mapping.cloud_service_names === "string") {
-      result[mapping.category] = JSON.parse(mapping.cloud_service_names);
+      JSON.parse(mapping.cloud_service_names).forEach((service) => {
+        result[service] = mapping.category;
+      });
+    } else {
+      mapping.cloud_service_names.forEach((service) => {
+        result[service] = mapping.category;
+      });
+    }
+  });
+  const override_mappings = await client.where({ provider }).select().from("category_mappings_override");
+  override_mappings.forEach((mapping) => {
+    if (typeof mapping.cloud_service_names === "string") {
+      JSON.parse(mapping.cloud_service_names).forEach((service) => {
+        result[service] = mapping.category;
+      });
     } else {
-      result[mapping.category] = mapping.cloud_service_names;
+      mapping.cloud_service_names.forEach((service) => {
+        result[service] = mapping.category;
+      });
     }
   });
   return result;
 }
 function getCategoryByServiceName(serviceName, categoryMappings) {
-  for (const key of Object.keys(categoryMappings)) {
-    const serviceNames = categoryMappings[key];
-    if (serviceNames && serviceNames.includes(serviceName)) {
-      return key;
-    }
+  if (serviceName in categoryMappings) {
+    return categoryMappings[serviceName];
   }
   return "Uncategorized";
 }
+async function getReportsFromCache(cache, provider, configKey, query) {
+  const cacheKey = [
+    provider,
+    configKey,
+    query.filters,
+    query.groups,
+    query.granularity,
+    query.startTime,
+    query.endTime
+  ].join("_");
+  const cachedCosts = await cache.get(cacheKey);
+  return cachedCosts;
+}
+async function setReportsToCache(cache, reports, provider, configKey, query, ttl) {
+  const cacheKey = [
+    provider,
+    configKey,
+    query.filters,
+    query.groups,
+    query.granularity,
+    query.startTime,
+    query.endTime
+  ].join("_");
+  await cache.set(cacheKey, reports, {
+    ttl: ttl 
+  });
+}
 
 class AwsClient {
-  constructor(config, database) {
+  constructor(providerName, config, database, cache, logger) {
+    this.providerName = providerName;
     this.config = config;
     this.database = database;
+    this.cache = cache;
+    this.logger = logger;
   }
-  static create(config, database) {
-    return new AwsClient(config, database);
+  static create(config, database, cache, logger) {
+    return new AwsClient("AWS", config, database, cache, logger);
+  }
+  convertServiceName(serviceName) {
+    let convertedName = serviceName;
+    const prefixes = ["Amazon", "AWS"];
+    const aliases = /* @__PURE__ */ new Map([
+      ["Elastic Compute Cloud - Compute", "EC2 - Instances"],
+      ["Virtual Private Cloud", "VPC (Virtual Private Cloud)"],
+      ["Relational Database Service", "RDS (Relational Database Service)"],
+      ["Simple Storage Service", "S3 (Simple Storage Service)"],
+      ["Managed Streaming for Apache Kafka", "MSK (Managed Streaming for Apache Kafka)"],
+      ["Elastic Container Service for Kubernetes", "EKS (Elastic Container Service for Kubernetes)"],
+      ["Elastic Container Service", "ECS (Elastic Container Service)"],
+      ["EC2 Container Registry (ECR)", "ECR (Elastic Container Registry)"],
+      ["Simple Queue Service", "SQS (Simple Queue Service)"],
+      ["Simple Notification Service", "SNS (Simple Notification Service)"],
+      ["Database Migration Service", "DMS (Database Migration Service)"]
+    ]);
+    for (const prefix of prefixes) {
+      if (serviceName.startsWith(prefix)) {
+        convertedName = serviceName.slice(prefix.length).trim();
+      }
+    }
+    if (aliases.has(convertedName)) {
+      convertedName = aliases.get(convertedName) || convertedName;
+    }
+    return `${this.providerName}/${convertedName}`;
   }
   async fetchCostsFromCloud(query) {
-    const conf = this.config.getOptionalConfigArray(
-      "backend.infraWallet.integrations.aws"
-    );
+    const conf = this.config.getOptionalConfigArray("backend.infraWallet.integrations.aws");
     if (!conf) {
-      return [];
+      return { reports: [], errors: [] };
     }
     const promises = [];
     const results = [];
+    const errors = [];
     query.groups.split(",").forEach((group) => {
       if (group.includes(":")) {
         group.split(":");
       }
     });
     for (const c of conf) {
-      const name = c.getString("name");
+      const accountName = c.getString("name");
+      const cachedCosts = await getReportsFromCache(this.cache, this.providerName, accountName, query);
+      if (cachedCosts) {
+        this.logger.debug(`${this.providerName}/${accountName} costs from cache`);
+        cachedCosts.map((cost) => {
+          results.push(cost);
+        });
+        continue;
+      }
       const accountId = c.getString("accountId");
       const assumedRoleName = c.getString("assumedRoleName");
       const accessKeyId = c.getOptionalString("accessKeyId");
@@ -76,7 +154,7 @@ class AwsClient {
         const [k, v] = tag.split(":");
         tagKeyValues[k.trim()] = v.trim();
       });
-      const categoryMappings = await getCategoryMappings(this.database, "aws");
+      const categoryMappings = await getCategoryMappings(this.database, this.providerName);
       let stsParams = {};
       if (accessKeyId && accessKeySecret) {
         stsParams = {
@@ -93,92 +171,172 @@ class AwsClient {
       }
       const promise = (async () => {
         var _a, _b, _c;
-        const client = new clientSts.STSClient(stsParams);
-        const commandInput = {
-          // AssumeRoleRequest
-          RoleArn: `arn:aws:iam::${accountId}:role/${assumedRoleName}`,
-          RoleSessionName: "AssumeRoleSession1"
-        };
-        const assumeRoleCommand = new clientSts.AssumeRoleCommand(commandInput);
-        const assumeRoleResponse = await client.send(assumeRoleCommand);
-        const awsCeClient = new clientCostExplorer.CostExplorerClient({
-          region: "us-east-1",
-          credentials: {
-            accessKeyId: (_a = assumeRoleResponse.Credentials) == null ? void 0 : _a.AccessKeyId,
-            secretAccessKey: (_b = assumeRoleResponse.Credentials) == null ? void 0 : _b.SecretAccessKey,
-            sessionToken: (_c = assumeRoleResponse.Credentials) == null ? void 0 : _c.SessionToken
-          }
-        });
-        const input = {
-          TimePeriod: {
-            Start: moment__default.default(parseInt(query.startTime, 10)).format("YYYY-MM-DD"),
-            End: moment__default.default(parseInt(query.endTime, 10)).format("YYYY-MM-DD")
-          },
-          Granularity: query.granularity.toUpperCase(),
-          Filter: { Dimensions: { Key: "RECORD_TYPE", Values: ["Usage"] } },
-          GroupBy: [{ Type: "DIMENSION", Key: "SERVICE" }],
-          Metrics: ["UnblendedCost"]
-        };
-        const getCostCommand = new clientCostExplorer.GetCostAndUsageCommand(input);
-        const costAndusageResponse = await awsCeClient.send(getCostCommand);
-        const transformedData = lodash.reduce(
-          costAndusageResponse.ResultsByTime,
-          (acc, row) => {
-            var _a2;
-            const rowTime = (_a2 = row.TimePeriod) == null ? void 0 : _a2.Start;
-            const period = rowTime ? rowTime.substring(0, 7) : "unknown";
-            if (row.Groups) {
-              row.Groups.forEach((group) => {
-                var _a3;
-                const groupKeys = group.Keys ? group.Keys[0] : "";
-                const keyName = `${name}_${groupKeys}`;
-                if (!acc[keyName]) {
-                  acc[keyName] = {
-                    id: keyName,
-                    name,
-                    service: `${groupKeys} (AWS)`,
-                    category: getCategoryByServiceName(
-                      groupKeys,
-                      categoryMappings
-                    ),
-                    provider: "AWS",
-                    reports: {},
-                    ...tagKeyValues
-                  };
-                }
-                const groupMetrics = group.Metrics;
-                if (groupMetrics !== void 0) {
-                  acc[keyName].reports[period] = parseFloat(
-                    (_a3 = groupMetrics.UnblendedCost.Amount) != null ? _a3 : "0.0"
-                  );
-                }
-              });
+        try {
+          const client = new clientSts.STSClient(stsParams);
+          const commandInput = {
+            // AssumeRoleRequest
+            RoleArn: `arn:aws:iam::${accountId}:role/${assumedRoleName}`,
+            RoleSessionName: "AssumeRoleSession1"
+          };
+          const assumeRoleCommand = new clientSts.AssumeRoleCommand(commandInput);
+          const assumeRoleResponse = await client.send(assumeRoleCommand);
+          const awsCeClient = new clientCostExplorer.CostExplorerClient({
+            region: "us-east-1",
+            credentials: {
+              accessKeyId: (_a = assumeRoleResponse.Credentials) == null ? void 0 : _a.AccessKeyId,
+              secretAccessKey: (_b = assumeRoleResponse.Credentials) == null ? void 0 : _b.SecretAccessKey,
+              sessionToken: (_c = assumeRoleResponse.Credentials) == null ? void 0 : _c.SessionToken
             }
-            return acc;
-          },
-          {}
-        );
-        Object.values(transformedData).map((value) => {
-          results.push(value);
-        });
+          });
+          let costAndUsageResults = [];
+          let nextPageToken = void 0;
+          do {
+            const input = {
+              TimePeriod: {
+                Start: moment__default.default(parseInt(query.startTime, 10)).format("YYYY-MM-DD"),
+                End: moment__default.default(parseInt(query.endTime, 10)).format("YYYY-MM-DD")
+              },
+              Granularity: query.granularity.toUpperCase(),
+              Filter: { Dimensions: { Key: "RECORD_TYPE", Values: ["Usage"] } },
+              GroupBy: [{ Type: "DIMENSION", Key: "SERVICE" }],
+              Metrics: ["UnblendedCost"],
+              NextPageToken: nextPageToken
+            };
+            const getCostCommand = new clientCostExplorer.GetCostAndUsageCommand(input);
+            const costAndUsageResponse = await awsCeClient.send(getCostCommand);
+            costAndUsageResults = costAndUsageResults.concat(costAndUsageResponse.ResultsByTime);
+            nextPageToken = costAndUsageResponse.NextPageToken;
+          } while (nextPageToken);
+          const transformedData = lodash.reduce(
+            costAndUsageResults,
+            (accumulator, row) => {
+              var _a2;
+              const rowTime = (_a2 = row.TimePeriod) == null ? void 0 : _a2.Start;
+              let period = "unknown";
+              if (rowTime) {
+                if (query.granularity.toUpperCase() === "MONTHLY") {
+                  period = rowTime.substring(0, 7);
+                } else {
+                  period = rowTime;
+                }
+              }
+              if (row.Groups) {
+                row.Groups.forEach((group) => {
+                  var _a3;
+                  const serviceName = group.Keys ? group.Keys[0] : "";
+                  const keyName = `${accountName}_${serviceName}`;
+                  if (!accumulator[keyName]) {
+                    accumulator[keyName] = {
+                      id: keyName,
+                      name: `${this.providerName}/${accountName}`,
+                      service: this.convertServiceName(serviceName),
+                      category: getCategoryByServiceName(serviceName, categoryMappings),
+                      provider: this.providerName,
+                      reports: {},
+                      ...tagKeyValues
+                    };
+                  }
+                  const groupMetrics = group.Metrics;
+                  if (groupMetrics !== void 0) {
+                    accumulator[keyName].reports[period] = parseFloat((_a3 = groupMetrics.UnblendedCost.Amount) != null ? _a3 : "0.0");
+                  }
+                });
+              }
+              return accumulator;
+            },
+            {}
+          );
+          await setReportsToCache(
+            this.cache,
+            Object.values(transformedData),
+            this.providerName,
+            accountName,
+            query,
+            60 * 60 * 2 * 1e3
+          );
+          Object.values(transformedData).map((value) => {
+            results.push(value);
+          });
+        } catch (e) {
+          this.logger.error(e);
+          errors.push({
+            provider: this.providerName,
+            name: `${this.providerName}/${accountName}`,
+            error: e.message
+          });
+        }
       })();
       promises.push(promise);
     }
     await Promise.all(promises);
-    return results;
+    return {
+      reports: results,
+      errors
+    };
   }
 }
 
 class AzureClient {
-  constructor(config, database) {
+  constructor(providerName, config, database, cache, logger) {
+    this.providerName = providerName;
     this.config = config;
     this.database = database;
+    this.cache = cache;
+    this.logger = logger;
+  }
+  static create(config, database, cache, logger) {
+    return new AzureClient("Azure", config, database, cache, logger);
+  }
+  convertServiceName(serviceName) {
+    let convertedName = serviceName;
+    const prefixes = ["Azure"];
+    for (const prefix of prefixes) {
+      if (serviceName.startsWith(prefix)) {
+        convertedName = serviceName.slice(prefix.length).trim();
+      }
+    }
+    return `${this.providerName}/${convertedName}`;
+  }
+  formatDate(dateNumber) {
+    const dateString = dateNumber.toString();
+    if (dateString.length !== 8) {
+      return null;
+    }
+    const year = dateString.slice(0, 4);
+    const month = dateString.slice(4, 6);
+    const day = dateString.slice(6);
+    return `${year}-${month}-${day}`;
   }
-  static create(config, database) {
-    return new AzureClient(config, database);
+  async fetchDataWithRetry(client, url, body, maxRetries = 5) {
+    let retries = 0;
+    while (retries < maxRetries) {
+      const request = coreRestPipeline.createPipelineRequest({
+        url,
+        method: "POST",
+        body: JSON.stringify(body),
+        headers: coreRestPipeline.createHttpHeaders({
+          "Content-Type": "application/json"
+        })
+      });
+      const response = await client.pipeline.sendRequest(client, request);
+      if (response.status === 200) {
+        return JSON.parse(response.bodyAsText || "{}");
+      } else if (response.status === 429) {
+        const retryAfter = parseInt(
+          response.headers.get("x-ms-ratelimit-microsoft.costmanagement-entity-retry-after") || "60",
+          10
+        );
+        this.logger.warn(`Hit Azure rate limit, retrying after ${retryAfter} seconds...`);
+        await new Promise((resolve) => setTimeout(resolve, retryAfter * 1e3));
+        retries++;
+      } else {
+        throw new Error(response.bodyAsText);
+      }
+    }
+    throw new Error("Max retries exceeded");
   }
   async queryAzureCostExplorer(azureClient, subscription, granularity, groups, startDate, endDate) {
-    const scope = `/subscriptions/${subscription}`;
+    const url = `https://management.azure.com/subscriptions/${subscription}/providers/Microsoft.CostManagement/query?api-version=2022-10-01`;
     const query = {
       type: "ActualCost",
       dataset: {
@@ -188,34 +346,43 @@ class AzureClient {
       },
       timeframe: "Custom",
       timePeriod: {
-        from: startDate.toISOString(),
-        to: endDate.toISOString()
+        from: startDate.toDate(),
+        to: endDate.toDate()
       }
     };
-    const result = await azureClient.query.usage(scope, query);
-    return result;
+    let result = await this.fetchDataWithRetry(azureClient, url, query);
+    let allResults = result.properties.rows;
+    while (result.properties.nextLink) {
+      result = await this.fetchDataWithRetry(azureClient, result.properties.nextLink, query);
+      allResults = allResults.concat(result.properties.rows);
+    }
+    return allResults;
   }
   async fetchCostsFromCloud(query) {
-    const conf = this.config.getOptionalConfigArray(
-      "backend.infraWallet.integrations.azure"
-    );
+    const conf = this.config.getOptionalConfigArray("backend.infraWallet.integrations.azure");
     if (!conf) {
-      return [];
+      return { reports: [], errors: [] };
     }
+    const categoryMappings = await getCategoryMappings(this.database, this.providerName);
     const promises = [];
     const results = [];
+    const errors = [];
     const groupPairs = [{ type: "Dimension", name: "ServiceName" }];
     for (const c of conf) {
-      const name = c.getString("name");
+      const accountName = c.getString("name");
+      const cachedCosts = await getReportsFromCache(this.cache, this.providerName, accountName, query);
+      if (cachedCosts) {
+        this.logger.debug(`${this.providerName}/${accountName} costs from cache`);
+        cachedCosts.map((cost) => {
+          results.push(cost);
+        });
+        continue;
+      }
       const subscriptionId = c.getString("subscriptionId");
       const tenantId = c.getString("tenantId");
       const clientId = c.getString("clientId");
       const clientSecret = c.getString("clientSecret");
-      const credential = new identity.ClientSecretCredential(
-        tenantId,
-        clientId,
-        clientSecret
-      );
+      const credential = new identity.ClientSecretCredential(tenantId, clientId, clientSecret);
       const client = new armCostmanagement.CostManagementClient(credential);
       const tags = c.getOptionalStringArray("tags");
       const tagKeyValues = {};
@@ -223,10 +390,6 @@ class AzureClient {
         const [k, v] = tag.split(":");
         tagKeyValues[k.trim()] = v.trim();
       });
-      const categoryMappings = await getCategoryMappings(
-        this.database,
-        "azure"
-      );
       const promise = (async () => {
         try {
           const costResponse = await this.queryAzureCostExplorer(
@@ -238,77 +401,234 @@ class AzureClient {
             moment__default.default(parseInt(query.endTime, 10))
           );
           const transformedData = lodash.reduce(
-            costResponse.rows,
-            (acc, row) => {
-              let keyName = name;
+            costResponse,
+            (accumulator, row) => {
+              const cost = row[0];
+              let date = row[1];
+              const serviceName = row[2];
+              if (query.granularity.toUpperCase() === "DAILY") {
+                date = this.formatDate(date);
+              }
+              let keyName = accountName;
               for (let i = 0; i < groupPairs.length; i++) {
                 keyName += `->${row[i + 2]}`;
               }
+              if (!accumulator[keyName]) {
+                accumulator[keyName] = {
+                  id: keyName,
+                  name: `${this.providerName}/${accountName}`,
+                  service: this.convertServiceName(serviceName),
+                  category: getCategoryByServiceName(serviceName, categoryMappings),
+                  provider: this.providerName,
+                  reports: {},
+                  ...tagKeyValues
+                };
+              }
+              if (!moment__default.default(date).isBefore(moment__default.default(parseInt(query.startTime, 10)))) {
+                if (query.granularity.toUpperCase() === "MONTHLY") {
+                  const yearMonth = date.substring(0, 7);
+                  accumulator[keyName].reports[yearMonth] = parseFloat(cost);
+                } else {
+                  accumulator[keyName].reports[date] = parseFloat(cost);
+                }
+              }
+              return accumulator;
+            },
+            {}
+          );
+          await setReportsToCache(
+            this.cache,
+            Object.values(transformedData),
+            this.providerName,
+            accountName,
+            query,
+            60 * 60 * 2 * 1e3
+          );
+          Object.values(transformedData).map((value) => {
+            results.push(value);
+          });
+        } catch (e) {
+          this.logger.error(e);
+          errors.push({
+            provider: this.providerName,
+            name: `${this.providerName}/${accountName}`,
+            error: e.message
+          });
+        }
+      })();
+      promises.push(promise);
+    }
+    await Promise.all(promises);
+    return {
+      reports: results,
+      errors
+    };
+  }
+}
+
+class GCPClient {
+  constructor(providerName, config, database, cache, logger) {
+    this.providerName = providerName;
+    this.config = config;
+    this.database = database;
+    this.cache = cache;
+    this.logger = logger;
+  }
+  static create(config, database, cache, logger) {
+    return new GCPClient("GCP", config, database, cache, logger);
+  }
+  convertServiceName(serviceName) {
+    let convertedName = serviceName;
+    const prefixes = ["Google Cloud"];
+    for (const prefix of prefixes) {
+      if (serviceName.startsWith(prefix)) {
+        convertedName = serviceName.slice(prefix.length).trim();
+      }
+    }
+    return `${this.providerName}/${convertedName}`;
+  }
+  async queryBigQuery(keyFilePath, projectId, datasetId, tableId, query) {
+    const options = {
+      keyFilename: keyFilePath,
+      projectId
+    };
+    const bigquery$1 = new bigquery.BigQuery(options);
+    try {
+      const periodFormat = query.granularity.toUpperCase() === "MONTHLY" ? "%Y-%m" : "%Y-%m-%d";
+      const sql = `
+        SELECT
+          project.name AS project,
+          service.description AS service,
+          FORMAT_TIMESTAMP('${periodFormat}', usage_start_time) AS period,
+          SUM(cost) AS total_cost
+        FROM
+          \`${projectId}.${datasetId}.${tableId}\`
+        WHERE
+          project.name IS NOT NULL
+          AND cost > 0
+          AND usage_start_time >= TIMESTAMP_MILLIS(${query.startTime})
+          AND usage_start_time <= TIMESTAMP_MILLIS(${query.endTime})
+        GROUP BY
+          project, service, period
+        ORDER BY
+          project, period, total_cost DESC`;
+      const [job] = await bigquery$1.createQueryJob({
+        query: sql,
+        location: "US"
+      });
+      const [rows] = await job.getQueryResults();
+      return rows;
+    } catch (err) {
+      throw new Error(err.message);
+    }
+  }
+  async fetchCostsFromCloud(query) {
+    const conf = this.config.getOptionalConfigArray("backend.infraWallet.integrations.gcp");
+    if (!conf) {
+      return { reports: [], errors: [] };
+    }
+    const promises = [];
+    const results = [];
+    const errors = [];
+    for (const c of conf) {
+      const accountName = c.getString("name");
+      const cachedCosts = await getReportsFromCache(this.cache, this.providerName, accountName, query);
+      if (cachedCosts) {
+        this.logger.debug(`${this.providerName}/${accountName} costs from cache`);
+        cachedCosts.map((cost) => {
+          results.push(cost);
+        });
+        continue;
+      }
+      const keyFilePath = c.getString("keyFilePath");
+      const projectId = c.getString("projectId");
+      const datasetId = c.getString("datasetId");
+      const tableId = c.getString("tableId");
+      const tags = c.getOptionalStringArray("tags");
+      const tagKeyValues = {};
+      tags == null ? void 0 : tags.forEach((tag) => {
+        const [k, v] = tag.split(":");
+        tagKeyValues[k.trim()] = v.trim();
+      });
+      const categoryMappings = await getCategoryMappings(this.database, this.providerName);
+      const promise = (async () => {
+        try {
+          const costResponse = await this.queryBigQuery(keyFilePath, projectId, datasetId, tableId, query);
+          const transformedData = lodash.reduce(
+            costResponse,
+            (acc, row) => {
+              const period = row.period;
+              const keyName = `${accountName}_${row.project}_${row.service}`;
               if (!acc[keyName]) {
                 acc[keyName] = {
                   id: keyName,
-                  name,
-                  service: `${row[2]} (Azure)`,
-                  category: getCategoryByServiceName(row[2], categoryMappings),
-                  provider: "Azure",
+                  name: `${this.providerName}/${accountName}`,
+                  service: this.convertServiceName(row.service),
+                  category: getCategoryByServiceName(row.service, categoryMappings),
+                  provider: this.providerName,
                   reports: {},
+                  ...{ project: row.project },
+                  // TODO: how should we handle the project field? for now, we add project name as a field in the report
                   ...tagKeyValues
+                  // note that if there is a tag `project:foo` in config, it overrides the project field set above
                 };
               }
-              if (!moment__default.default(row[1]).isBefore(moment__default.default(parseInt(query.startTime, 10)))) {
-                acc[keyName].reports[row[1].substring(0, 7)] = parseFloat(
-                  row[0]
-                );
-              }
+              acc[keyName].reports[period] = row.total_cost;
               return acc;
             },
             {}
           );
+          await setReportsToCache(
+            this.cache,
+            Object.values(transformedData),
+            this.providerName,
+            accountName,
+            query,
+            60 * 60 * 2 * 1e3
+          );
           Object.values(transformedData).map((value) => {
             results.push(value);
           });
         } catch (e) {
-          throw new Error(e.message);
+          this.logger.error(e);
+          errors.push({
+            provider: this.providerName,
+            name: `${this.providerName}/${accountName}`,
+            error: e.message
+          });
         }
       })();
       promises.push(promise);
     }
     await Promise.all(promises);
-    return results;
+    return {
+      reports: results,
+      errors
+    };
   }
 }
 
 async function setUpDatabase(database) {
   var _a;
   const client = await database.getClient();
-  const migrationsDir = backendPluginApi.resolvePackagePath(
-    "@electrolux-oss/plugin-infrawallet-backend",
-    "migrations"
-  );
+  const migrationsDir = backendPluginApi.resolvePackagePath("@electrolux-oss/plugin-infrawallet-backend", "migrations");
   if (!((_a = database.migrations) == null ? void 0 : _a.skip)) {
     await client.migrate.latest({
       directory: migrationsDir
     });
   }
-  const category_mappings_count = await client("category_mappings").count(
-    "id as c"
-  );
-  if (category_mappings_count[0].c === 0 || category_mappings_count[0].c === "0") {
-    const seedsDir = backendPluginApi.resolvePackagePath(
-      "@electrolux-oss/plugin-infrawallet-backend",
-      "seeds"
-    );
-    await client.seed.run({ directory: seedsDir });
-  }
+  const seedsDir = backendPluginApi.resolvePackagePath("@electrolux-oss/plugin-infrawallet-backend", "seeds");
+  await client.seed.run({ directory: seedsDir });
 }
 async function createRouter(options) {
   const { logger, config, cache, database } = options;
   await setUpDatabase(database);
   const router = Router__default.default();
   router.use(express__default.default.json());
-  const azureClient = AzureClient.create(config, database);
-  const awsClient = AwsClient.create(config, database);
-  const cloudClients = [azureClient, awsClient];
+  const azureClient = AzureClient.create(config, database, cache, logger);
+  const awsClient = AwsClient.create(config, database, cache, logger);
+  const gcpClient = GCPClient.create(config, database, cache, logger);
+  const cloudClients = [azureClient, awsClient, gcpClient];
   router.get("/health", (_, response) => {
     logger.info("PONG!");
     response.json({ status: "ok" });
@@ -321,42 +641,40 @@ async function createRouter(options) {
     const endTime = request.query.endTime;
     const promises = [];
     const results = [];
+    const errors = [];
     cloudClients.forEach(async (client) => {
       const fetchCloudCosts = (async () => {
-        const cacheKey = [
-          client.constructor.name,
-          filters,
-          groups,
-          granularity,
-          startTime,
-          endTime
-        ].join("_");
-        const cachedCosts = await cache.get(cacheKey);
-        if (cachedCosts) {
-          logger.debug(`${client.constructor.name} costs from cache`);
-          cachedCosts.forEach((cost) => {
-            results.push(cost);
-          });
-        } else {
-          const costs = await client.fetchCostsFromCloud({
+        try {
+          const clientResponse = await client.fetchCostsFromCloud({
             filters,
             groups,
             granularity,
             startTime,
             endTime
           });
-          await cache.set(cacheKey, costs, {
-            ttl: 60 * 60 * 2 * 1e3
+          clientResponse.errors.forEach((e) => {
+            errors.push(e);
           });
-          costs.forEach((cost) => {
+          clientResponse.reports.forEach((cost) => {
             results.push(cost);
           });
+        } catch (e) {
+          logger.error(e);
+          errors.push({
+            provider: client.constructor.name,
+            name: client.constructor.name,
+            error: e.message
+          });
         }
       })();
       promises.push(fetchCloudCosts);
     });
     await Promise.all(promises);
-    response.json({ data: results, status: "ok" });
+    if (errors.length > 0) {
+      response.status(207).json({ data: results, errors, status: 207 });
+    } else {
+      response.json({ data: results, errors, status: 200 });
+    }
   });
   router.use(backendCommon.errorHandler());
   return router;