@meyicloud/meyi-cost-server 1.8.5 → 1.8.7

package/README.md

@@ -48,7 +48,7 @@ flowchart LR
   Router --> Controller[Cost controller]
   Controller --> Customer[Tenant CUR metadata]
   Controller --> SaaS[SaaS Athena context]
-  SaaS --> Athena[Central tenant-partitioned CUR]
+  SaaS --> Athena[Central tenant and connection-partitioned CUR]
   Controller --> Budget[Budget services]
   Budget --> DB[(PostgreSQL)]
 ```
@@ -319,6 +319,7 @@ failed and do not change CUR, standard reports, or onboarding.
 | `AWS_REGION` | `us-east-1` | No | Used as the CUR region fallback. |
 | `COST_CUR_TENANT_COLUMN` | `tenant_id` | No | CUR column used to isolate the Hive-style tenant partition. |
 | `COST_CUR_TENANT_PARTITION` | Request tenant ID | No | Overrides the partition value. Avoid a global override in multi-tenant production unless every request is intentionally mapped to that partition. |
+| `COST_CUR_CONNECTION_COLUMN` | `connection_id` | No | CUR column used to isolate the selected AWS source connection. The connection value is resolved from onboarding data and is not a global environment override. |
 | `COST_CUR_MAX_ROWS` | `1000` | No | Resource-report row cap, clamped between 1 and 5000. |
 | `COST_CUR_ROLE_ARN` | Empty | No | Dedicated SaaS-side role assumed only for central Athena/Glue/S3 queries. |
 | `COST_CUR_EXTERNAL_ID` | Empty | No | External ID for the dedicated SaaS CUR role. |
@@ -344,6 +345,7 @@ COST_CUR_OUTPUT_LOCATION=s3://meyi-saas-athena-results/
 COST_CUR_REGION=us-east-1
 COST_CUR_WORKGROUP=meyi-cost
 COST_CUR_TENANT_COLUMN=tenant_id
+COST_CUR_CONNECTION_COLUMN=connection_id
 COST_CUR_MAX_ROWS=1000
 COST_CUR_ROLE_ARN=arn:aws:iam::SAAS_ACCOUNT_ID:role/meyi-cur-query
 COST_CUR_ALLOW_TENANT_CATALOG=false

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@meyicloud/meyi-cost-server",
-  "version": "1.8.5",
+  "version": "1.8.7",
   "description": "Tenant-aware AWS CUR and Athena cost plugin server for MeyiConnect",
   "type": "module",
   "main": "./index.js",

package/src/cur-discovery/cur-discovery.aws.js

@@ -27,8 +27,9 @@ export class CurDiscoveryAws {
     this.env = env;
     this.region = String(env.COST_CUR_REGION || env.AWS_REGION || "us-east-1");
     this.database = String(env.COST_CUR_DATABASE || "").trim();
-    this.tableOverride = String(env.COST_CUR_TABLE || "").trim();
-    this.tenantColumn = String(env.COST_CUR_TENANT_COLUMN || "tenant_id").trim();
+    this.tableOverride = String(env.COST_CUR_TABLE || "").trim();
+    this.tenantColumn = String(env.COST_CUR_TENANT_COLUMN || "tenant_id").trim();
+    this.connectionColumn = String(env.COST_CUR_CONNECTION_COLUMN || "connection_id").trim();
     const credentials = centralCredentials(env, credentialProvider);
     this.glue = glueClient || new GlueClient({ region: this.region, credentials });
     this.s3 = s3Client || new S3Client({ region: this.region, credentials });
@@ -60,9 +61,10 @@ export class CurDiscoveryAws {
         const columns = new Set((table.StorageDescriptor?.Columns || []).map((item) => String(item.Name || "").toLowerCase()));
         const partitions = new Set((table.PartitionKeys || []).map((item) => String(item.Name || "").toLowerCase()));
         const pathMatches = location && (tenantUri.startsWith(location) || location.startsWith(tenantUri));
-        const tenantSafe = partitions.has(this.tenantColumn.toLowerCase()) || columns.has(this.tenantColumn.toLowerCase());
-        const curSchema = columns.has("line_item_usage_start_date");
-        if (pathMatches && tenantSafe && curSchema) {
+        const tenantSafe = partitions.has(this.tenantColumn.toLowerCase()) || columns.has(this.tenantColumn.toLowerCase());
+        const connectionSafe = partitions.has(this.connectionColumn.toLowerCase()) || columns.has(this.connectionColumn.toLowerCase());
+        const curSchema = columns.has("line_item_usage_start_date");
+        if (pathMatches && tenantSafe && connectionSafe && curSchema) {
           candidates.push({
             database: this.database,
             name: table.Name,

package/src/cur-discovery/cur-discovery.service.js

@@ -61,7 +61,8 @@ export class CurDiscoveryService {
           curSourceBucket: job.cur_bucket,
           curSourcePrefix: job.cur_prefix,
           curSourceRegion: job.cur_region,
-          curIngestionMode: "central",
+          curIngestionMode: "central",
+          connectionId: job.connection_id,
         },
       };
       const context = this.athenaContextService.resolve(customerContext);

package/src/services/cost-analyser.service.js

@@ -201,6 +201,8 @@ export class CostAnalyserService {
         COST_CUR_REGION: athena.region,
         COST_CUR_TENANT_COLUMN: athena.tenantColumn,
         COST_CUR_TENANT_PARTITION: athena.tenantPartition || tenantId,
+        COST_CUR_CONNECTION_COLUMN: required(athena.connectionColumn, "COST_CUR_CONNECTION_COLUMN"),
+        COST_CUR_CONNECTION_PARTITION: required(athena.connectionPartition, "COST_CUR_CONNECTION_PARTITION"),
         AI_PROVIDER_SECRET_ARN: providerSecretArn,
         AI_PROVIDER_SECRET_REGION: this.region,
         TOP_N_SERVICES: this.topServices,

package/src/services/cur.service.js

@@ -27,9 +27,11 @@ export function getCurConfig(env = process.env, metadata = {}) {
     table: tableName ? identifier(tableName, "table", true) : "",
     outputLocation,
     region: String(metadata.curRegion || env.COST_CUR_REGION || env.AWS_REGION || "us-east-1"),
-    workgroup: identifier(metadata.curWorkgroup || env.COST_CUR_WORKGROUP || "meyi-cost", "workgroup", true),
-    tenantColumn: identifier(metadata.curTenantColumn || env.COST_CUR_TENANT_COLUMN || "tenant_id", "tenant column"),
-    tenantPartition: String(metadata.curTenantPartition || env.COST_CUR_TENANT_PARTITION || "").trim(),
+    workgroup: identifier(metadata.curWorkgroup || env.COST_CUR_WORKGROUP || "meyi-cost", "workgroup", true),
+    tenantColumn: identifier(metadata.curTenantColumn || env.COST_CUR_TENANT_COLUMN || "tenant_id", "tenant column"),
+    tenantPartition: String(metadata.curTenantPartition || env.COST_CUR_TENANT_PARTITION || "").trim(),
+    connectionColumn: identifier(metadata.curConnectionColumn || env.COST_CUR_CONNECTION_COLUMN || "connection_id", "connection column"),
+    connectionPartition: String(metadata.curConnectionPartition || "").trim(),
     maxRows: Math.min(Math.max(Number(env.COST_CUR_MAX_ROWS || 1000), 1), 5000),
   };
 }
@@ -82,10 +84,13 @@ function table(config) {
   return `"${config.database}"."${config.table}"`;
 }
 
-function tenantWhere(config, tenant) {
-  const partition = config.tenantPartition || tenant;
-  return `"${config.tenantColumn}" = ${sqlString(partition)}`;
-}
+function sourceWhere(config, tenant) {
+  const partition = config.tenantPartition || tenant;
+  if (!config.connectionPartition) {
+    throw new Error("CUR connection partition is required for source-isolated queries");
+  }
+  return `"${config.tenantColumn}" = ${sqlString(partition)}\n      AND "${config.connectionColumn}" = ${sqlString(config.connectionPartition)}`;
+}
 
 export async function getCurDataStatus({ client, config, tenant }) {
   const statusRows = await execute(client, config, `
@@ -93,7 +98,7 @@ export async function getCurDataStatus({ client, config, tenant }) {
       CAST(COUNT(*) AS BIGINT) AS record_count,
       CAST(MAX(CAST(line_item_usage_start_date AS TIMESTAMP)) AS VARCHAR) AS last_data_at
     FROM ${table(config)}
-    WHERE ${tenantWhere(config, tenant)}
+    WHERE ${sourceWhere(config, tenant)}
   `);
   const status = statusRows[0] || {};
   return {
@@ -107,27 +112,37 @@ function rangeWhere(range) {
     AND CAST(line_item_usage_start_date AS TIMESTAMP) < TIMESTAMP ${sqlString(`${range.End} 00:00:00`)}`;
 }
 
-function inCondition(column, values = []) {
+function inCondition(column, values = []) {
   if (!values.length) return "";
   return `AND CAST("${column}" AS VARCHAR) IN (${values.map(sqlString).join(", ")})`;
-}
-
-function cur2TagExpression(tagKey) {
-  const key = String(tagKey || "Name").trim();
-  if (!key) throw new Error("Invalid CUR tag key");
-  return `COALESCE(element_at(resource_tags, ${sqlString(`user:${key.replace(/^user:/, "")}`)}), element_at(resource_tags, ${sqlString(key.replace(/^user:/, ""))}))`;
-}
+}
+
+function cur2TagExpression(tagKey) {
+  const key = String(tagKey || "").trim();
+  if (!key) throw new Error("Invalid CUR tag key");
+  const plainKey = key.replace(/^(?:user:|resourceTags\/)/, "");
+  const candidates = key.includes("/")
+    ? [key]
+    : [`resourceTags/${plainKey}`, `user:${plainKey}`, plainKey];
+  return `COALESCE(${[...new Set(candidates)].map((candidate) => `element_at(resource_tags, ${sqlString(candidate)})`).join(", ")})`;
+}
+
+const regionExpression = `COALESCE(
+  NULLIF(TRIM(CAST(product_region_code AS VARCHAR)), ''),
+  NULLIF(regexp_extract(COALESCE(line_item_resource_id, ''), '^arn:[^:]+:[^:]+:([^:]+):', 1), ''),
+  'Region unavailable in CUR'
+)`;
 
 function inExpression(expression, values = []) {
   if (!values.length) return "";
   return `AND CAST(${expression} AS VARCHAR) IN (${values.map(sqlString).join(", ")})`;
 }
 
-function filterWhere(filters = {}) {
-  const conditions = [
-    inCondition("product_product_name", filters.services),
-    inCondition("product_region_code", filters.regions),
-    inCondition("line_item_usage_account_id", filters.accountIds),
+function filterWhere(filters = {}) {
+  const conditions = [
+    inCondition("product_product_name", filters.services),
+    inExpression(regionExpression, filters.regions),
+    inCondition("line_item_usage_account_id", filters.accountIds),
   ];
   if (filters.tagKey && filters.tagValues?.length) {
     conditions.push(inExpression(cur2TagExpression(filters.tagKey), filters.tagValues));
@@ -136,14 +151,15 @@ function filterWhere(filters = {}) {
 }
 
 function dimensionExpression(groupBy, tagKey) {
-  const dimensions = {
-    service: "product_product_name",
-    account: "line_item_usage_account_id",
-    region: "product_region_code",
-    resource: "line_item_resource_id",
-  };
-  if (groupBy === "tag") return cur2TagExpression(tagKey);
-  return dimensions[groupBy] ? `"${dimensions[groupBy]}"` : undefined;
+  const dimensions = {
+    service: "product_product_name",
+    account: "line_item_usage_account_id",
+    region: regionExpression,
+    resource: "line_item_resource_id",
+  };
+  if (groupBy === "tag") return cur2TagExpression(tagKey);
+  if (groupBy === "region") return dimensions.region;
+  return dimensions[groupBy] ? `"${dimensions[groupBy]}"` : undefined;
 }
 
 function summarize(periods, accountNames = new Map()) {
@@ -161,7 +177,7 @@ function summarize(periods, accountNames = new Map()) {
     .sort((a, b) => b.amount - a.amount);
 }
 
-export async function getCurReport({ client, config, tenant, range, groupBy, granularity = "MONTHLY", tagKey, accountNames = new Map(), filters = {}, includeBreakdown = false }) {
+export async function getCurReport({ client, config, tenant, range, groupBy, granularity = "MONTHLY", tagKey, accountNames = new Map(), filters = {}, includeBreakdown = false }) {
   const column = dimensionExpression(groupBy, tagKey);
   if (!column) throw new Error("Unsupported CUR report group");
   const bucket = granularity === "DAILY" ? "day" : "month";
@@ -172,10 +188,11 @@ export async function getCurReport({ client, config, tenant, range, groupBy, gra
       COALESCE(NULLIF(CAST(${column} AS VARCHAR), ''), 'Unallocated') AS item_key,
       CAST(SUM(line_item_unblended_cost) AS DOUBLE) AS amount
     FROM ${table(config)}
-    WHERE ${tenantWhere(config, tenant)}
-      AND ${rangeWhere(range)}
-      AND line_item_unblended_cost > 0
-      ${filterWhere(filters)}
+    WHERE ${sourceWhere(config, tenant)}
+      AND ${rangeWhere(range)}
+      AND line_item_unblended_cost > 0
+      ${groupBy === "tag" ? `AND NULLIF(TRIM(CAST(${column} AS VARCHAR)), '') IS NOT NULL` : ""}
+      ${filterWhere(filters)}
     GROUP BY 1, 2
     ORDER BY 1, 3 DESC
     ${limit}
@@ -183,18 +200,19 @@ export async function getCurReport({ client, config, tenant, range, groupBy, gra
   const rows = await execute(client, config, query);
   const periods = rows.map((row) => ({ start: String(row.period_start).slice(0, 10), key: row.item_key || "Unallocated", amount: amount(row.amount) }));
   let summary = summarize(periods, accountNames);
-  if (includeBreakdown && groupBy !== "resource") {
+  if (includeBreakdown) {
     const relationQuery = `
       SELECT
         COALESCE(NULLIF(CAST(${column} AS VARCHAR), ''), 'Unallocated') AS item_key,
         COALESCE(NULLIF(CAST(line_item_usage_account_id AS VARCHAR), ''), 'Unallocated') AS account_key,
-        COALESCE(NULLIF(CAST(product_region_code AS VARCHAR), ''), 'GLOBAL') AS region_key,
+        ${regionExpression} AS region_key,
         CAST(SUM(line_item_unblended_cost) AS DOUBLE) AS amount
       FROM ${table(config)}
-      WHERE ${tenantWhere(config, tenant)}
-        AND ${rangeWhere(range)}
-        AND line_item_unblended_cost > 0
-        ${filterWhere(filters)}
+      WHERE ${sourceWhere(config, tenant)}
+        AND ${rangeWhere(range)}
+        AND line_item_unblended_cost > 0
+        ${groupBy === "tag" ? `AND NULLIF(TRIM(CAST(${column} AS VARCHAR)), '') IS NOT NULL` : ""}
+        ${filterWhere(filters)}
       GROUP BY 1, 2, 3
     `;
     const relationRows = await execute(client, config, relationQuery);
@@ -208,7 +226,7 @@ export async function getCurReport({ client, config, tenant, range, groupBy, gra
         values.set(relationKey, (values.get(relationKey) || 0) + amount(row.amount));
       };
       add(accountsByItem, row.account_key || "Unallocated");
-      add(regionsByItem, row.region_key || "GLOBAL");
+      add(regionsByItem, row.region_key || "Region unavailable in CUR");
     }
     const relations = (target, key, type) => [...(target.get(key) || new Map()).entries()].map(([value, cost]) => ({ key: value, label: type === "account" ? accountNames.get(value) || value : value, amount: round(cost) })).sort((a, b) => b.amount - a.amount);
     summary = summary.map((item) => ({ ...item, accounts: relations(accountsByItem, item.key, "account"), regions: relations(regionsByItem, item.key, "region") }));
@@ -246,7 +264,7 @@ export async function getCurOverview({ client, config, tenant, range, trendRange
       CAST(SUM(line_item_unblended_cost) AS DOUBLE) AS net_cost,
       CAST(COUNT(DISTINCT CASE WHEN line_item_unblended_cost > 0 AND line_item_resource_id IS NOT NULL AND line_item_resource_id <> '' THEN line_item_resource_id END) AS BIGINT) AS active_resources
     FROM ${table(config)}
-    WHERE ${tenantWhere(config, tenant)} AND ${rangeWhere(range)}
+    WHERE ${sourceWhere(config, tenant)} AND ${rangeWhere(range)}
   `;
   const [metricsRows, services, accounts, regions, trendReport] = await Promise.all([
     execute(client, config, metricsQuery),
@@ -297,35 +315,44 @@ export async function getCurOverview({ client, config, tenant, range, trendRange
   };
 }
 
-export async function getCurTags({ client, config, tenant }) {
+export async function getCurTags({ client, config, tenant }) {
   if (!tenant) throw new Error("A tenant is required to list CUR tags");
   const query = `
     SELECT DISTINCT tag_key
     FROM ${table(config)}
     CROSS JOIN UNNEST(map_keys(resource_tags)) AS tags(tag_key)
-    WHERE ${tenantWhere(config, tenant)}
+    WHERE ${sourceWhere(config, tenant)}
       AND tag_key IS NOT NULL
       AND TRIM(tag_key) <> ''
     ORDER BY tag_key
   `;
   const rows = await execute(client, config, query);
-  return rows.map((row) => String(row.tag_key || "").replace(/^user:/, "")).filter(Boolean);
-}
+  return rows
+    .map((row) => String(row.tag_key || "").replace(/^(?:user:|resourceTags\/)/, ""))
+    .filter(Boolean);
+}
 
 export async function getCurFilterOptions({ client, config, tenant, range, tagKey, accountNames = new Map() }) {
   const distinct = async (column) => execute(client, config, `
     SELECT DISTINCT CAST("${column}" AS VARCHAR) AS value
     FROM ${table(config)}
-    WHERE ${tenantWhere(config, tenant)}
+    WHERE ${sourceWhere(config, tenant)}
       AND ${rangeWhere(range)}
       AND line_item_unblended_cost > 0
       AND "${column}" IS NOT NULL
       AND TRIM(CAST("${column}" AS VARCHAR)) <> ''
     ORDER BY 1
   `);
-  const [services, regions, accounts, tags] = await Promise.all([
-    distinct("product_product_name"),
-    distinct("product_region_code"),
+  const [services, regions, accounts, tags] = await Promise.all([
+    distinct("product_product_name"),
+    execute(client, config, `
+      SELECT DISTINCT CAST(${regionExpression} AS VARCHAR) AS value
+      FROM ${table(config)}
+      WHERE ${sourceWhere(config, tenant)}
+        AND ${rangeWhere(range)}
+        AND line_item_unblended_cost > 0
+      ORDER BY 1
+    `),
     distinct("line_item_usage_account_id"),
     getCurTags({ client, config, tenant }),
   ]);
@@ -334,7 +361,7 @@ export async function getCurFilterOptions({ client, config, tenant, range, tagKe
     tagValues = (await execute(client, config, `
       SELECT DISTINCT CAST(${cur2TagExpression(tagKey)} AS VARCHAR) AS value
       FROM ${table(config)}
-      WHERE ${tenantWhere(config, tenant)}
+      WHERE ${sourceWhere(config, tenant)}
         AND ${rangeWhere(range)}
         AND line_item_unblended_cost > 0
         AND ${cur2TagExpression(tagKey)} IS NOT NULL

package/src/services/saas-athena-context.service.js

@@ -23,7 +23,11 @@ export class SaasAthenaContextService {
       this.env.COST_CUR_TENANT_PARTITION
       || ingestion.tenantPartition,
     ).trim();
-    const resolvedConfig = getCurConfig(this.env, { ...catalogMeta, curTenantPartition: tenantPartition });
+    const resolvedConfig = getCurConfig(this.env, {
+      ...catalogMeta,
+      curTenantPartition: tenantPartition,
+      curConnectionPartition: tenantMeta.connectionId,
+    });
     const roleArn = String(this.env.COST_CUR_ROLE_ARN || "").trim();
     const config = resolvedConfig;
     const credentials = roleArn ? this.credentialProvider({

package/src/workers/cost-analysis-report.worker.js

@@ -7,7 +7,13 @@ export class CostAnalysisReportWorker {
     this.logger = logger;
     this.enabled = String(env.COST_AI_ENABLED || "").toLowerCase() === "true";
     this.pollMs = Math.max(Number(env.COST_AI_REPORT_POLL_INTERVAL_MS || 60_000), 10_000);
-    this.leaseMs = Math.max(Number(env.COST_AI_REPORT_LEASE_MS || 30 * 60_000), 60_000);
+    // Must exceed COST_AI_ANALYSER_TIMEOUT_MS (45 min). claimDue holds a
+    // schedule by pushing next_run_at forward by the lease; run() releases it
+    // on every exit path, so the value only matters when the process dies
+    // mid-run. A lease shorter than the analyser timeout lets a second worker
+    // re-claim a schedule whose Fargate task is still running - a duplicate
+    // report row and a duplicate billed model run.
+    this.leaseMs = Math.max(Number(env.COST_AI_REPORT_LEASE_MS || 60 * 60_000), 60_000);
     this.batchSize = Math.min(Math.max(Number(env.COST_AI_REPORT_BATCH_SIZE || 5), 1), 25);
     this.running = false;
     this.timer = null;