meyi-cost-server 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/AGENTS.md +179 -0
  2. package/README.md +474 -0
  3. package/cur.js +2 -0
  4. package/index.js +1 -0
  5. package/package.json +36 -0
  6. package/src/controllers/budget.controller.js +39 -0
  7. package/src/controllers/cost.controller.js +261 -0
  8. package/src/cur-discovery/cur-discovery.aws.js +82 -0
  9. package/src/cur-discovery/cur-discovery.repository.js +177 -0
  10. package/src/cur-discovery/cur-discovery.service.js +112 -0
  11. package/src/cur-discovery/cur-discovery.worker.js +57 -0
  12. package/src/cur-discovery/schema.js +48 -0
  13. package/src/lib/cost-utils.js +98 -0
  14. package/src/models/budget.model.js +26 -0
  15. package/src/models/cur-data-status.model.js +16 -0
  16. package/src/models/cur-ingestion.model.js +9 -0
  17. package/src/models/customer-aws-context.model.js +20 -0
  18. package/src/models/saas-cur-context.model.js +10 -0
  19. package/src/plugin.js +72 -0
  20. package/src/repositories/aws-onboarding.repository.js +134 -0
  21. package/src/repositories/budget-alert.repository.js +37 -0
  22. package/src/repositories/budget.repository.js +33 -0
  23. package/src/routes/index.js +22 -0
  24. package/src/schema/cost-budget.schema.js +9 -0
  25. package/src/services/aws-context.service.js +1 -0
  26. package/src/services/budget-alert.service.js +47 -0
  27. package/src/services/budget.service.js +32 -0
  28. package/src/services/cost-explorer.service.js +27 -0
  29. package/src/services/cur-provider.service.js +91 -0
  30. package/src/services/cur.service.js +355 -0
  31. package/src/services/customer-aws-context.service.js +62 -0
  32. package/src/services/saas-athena-context.service.js +54 -0
@@ -0,0 +1,39 @@
1
+ export class BudgetController {
2
+ constructor({ contextService, budgetService, budgetAlertService }) {
3
+ this.contextService = contextService;
4
+ this.budgetService = budgetService;
5
+ this.budgetAlertService = budgetAlertService;
6
+ }
7
+
8
+ async list(req, res, next) {
9
+ try { return res.json({ budgets: await this.budgetService.list(this.contextService.tenantId(req)) }); }
10
+ catch (error) { return next(error); }
11
+ }
12
+
13
+ async create(req, res, next) {
14
+ try { return res.status(201).json(await this.budgetService.create(this.contextService.tenantId(req), req.body)); }
15
+ catch (error) {
16
+ if (error.statusCode === 400) return res.status(400).json({ error: error.message });
17
+ return next(error);
18
+ }
19
+ }
20
+
21
+ async delete(req, res, next) {
22
+ try { await this.budgetService.delete(this.contextService.tenantId(req), req.params.id); return res.status(204).end(); }
23
+ catch (error) { return next(error); }
24
+ }
25
+
26
+ async listDismissals(req, res, next) {
27
+ try {
28
+ const dismissals = await this.budgetAlertService.listDismissals(this.contextService.tenantId(req), req.user?.id, req.query.period);
29
+ return res.json({ dismissals });
30
+ } catch (error) { return next(error); }
31
+ }
32
+
33
+ async dismissAlert(req, res, next) {
34
+ try {
35
+ const dismissal = await this.budgetAlertService.dismiss(this.contextService.tenantId(req), req.user?.id, req.body);
36
+ return res.status(201).json(dismissal);
37
+ } catch (error) { return next(error); }
38
+ }
39
+ }
@@ -0,0 +1,261 @@
1
+ import {
2
+ GetCostAndUsageCommand,
3
+ GetCostForecastCommand,
4
+ GetDimensionValuesCommand,
5
+ GetTagsCommand,
6
+ } from "@aws-sdk/client-cost-explorer";
7
+ import {
8
+ getCurFilterOptions,
9
+ getCurOverview,
10
+ getCurReport,
11
+ getCurTags,
12
+ } from "../services/cur.service.js";
13
+ import {
14
+ accountFilter,
15
+ amount,
16
+ dateRange,
17
+ grossCostFilter,
18
+ iso,
19
+ listParam,
20
+ parseGroups,
21
+ parseRelationships,
22
+ round,
23
+ } from "../lib/cost-utils.js";
24
+
25
+ export class CostController {
26
+ constructor({ contextService, curProvider, explorerService, discoveryRepository = null, logger = console }) {
27
+ this.contextService = contextService;
28
+ this.curProvider = curProvider;
29
+ this.explorerService = explorerService;
30
+ this.discoveryRepository = discoveryRepository;
31
+ this.logger = logger;
32
+ }
33
+
34
+ async accounts(req, res, next) {
35
+ try {
36
+ const ctx = await this.contextService.resolve(req);
37
+ const { tenant, accounts, client, queryEnabled, accessMode } = ctx;
38
+ const names = new Map(accounts.map((item) => [item.id, item.name]));
39
+ const cur = await this.curProvider.run(ctx, (athena, config) => getCurReport({ client: athena, config, tenant, range: dateRange(req.query), groupBy: "account", accountNames: names }));
40
+ if (cur) return res.json({ accounts: cur.summary.map((item) => ({ id: item.accountId || item.key, name: item.label, region: "global", status: "active" })), dataSource: cur.dataSource, accessMode: cur.accessMode });
41
+ if (accounts.length || !queryEnabled) return res.json({ accounts, dataSource: queryEnabled ? "aws-cost-explorer" : "onboarding", accessMode });
42
+ const response = await client.send(new GetDimensionValuesCommand({ TimePeriod: dateRange(), Dimension: "LINKED_ACCOUNT" }));
43
+ const discovered = (response.DimensionValues || []).map((item) => ({ id: item.Value, name: item.Attributes?.description || item.Value, region: "global", status: "active" }));
44
+ return res.json({ accounts: discovered, dataSource: "aws-cost-explorer", accessMode });
45
+ } catch (error) { return next(error); }
46
+ }
47
+
48
+ async overview(req, res, next) {
49
+ try {
50
+ const ctx = await this.contextService.resolve(req);
51
+ const { tenant, accounts, client, queryEnabled, accessMode } = ctx;
52
+ const range = dateRange(req.query);
53
+ const trendMode = String(req.query.period || "monthly").toLowerCase() === "yearly" ? "yearly" : "monthly";
54
+ const trendStart = trendMode === "monthly" ? new Date(`${range.End}T00:00:00Z`) : new Date(`${range.Start}T00:00:00Z`);
55
+ if (trendMode === "monthly") trendStart.setUTCMonth(0, 1);
56
+ const trendRange = { Start: iso(trendStart), End: range.End };
57
+ const names = new Map(accounts.map((item) => [item.id, item.name]));
58
+ const cur = await this.curProvider.run(ctx, (athena, config) => getCurOverview({ client: athena, config, tenant, range, trendRange, trendMode, accountNames: names }));
59
+ if (cur) return res.json(cur);
60
+ if (!queryEnabled) return res.json({ tenantId: tenant, accounts: [], currency: "USD", totalCost: 0, activeAccounts: 0, trends: [], topServices: [], topAccounts: [], topRegions: [], dataSource: "onboarding", accessMode, message: "No selected AWS accounts or test credentials were found. Complete AWS onboarding or configure the local credential fallback." });
61
+ const accountIds = accounts.map((item) => item.id);
62
+ const [trendResponse, netTrendResponse, servicePeriods, accountPeriods, regionPeriods, accountDirectory] = await Promise.all([
63
+ client.send(new GetCostAndUsageCommand({ TimePeriod: trendRange, Granularity: "MONTHLY", Metrics: ["UnblendedCost"], Filter: grossCostFilter(accountIds) })),
64
+ client.send(new GetCostAndUsageCommand({ TimePeriod: range, Granularity: "MONTHLY", Metrics: ["UnblendedCost"], Filter: accountFilter(accountIds) })),
65
+ this.explorerService.grouped(client, accounts, range, "SERVICE"),
66
+ this.explorerService.grouped(client, accounts, range, "LINKED_ACCOUNT"),
67
+ this.explorerService.grouped(client, accounts, range, "REGION"),
68
+ accounts.length ? Promise.resolve(null) : client.send(new GetDimensionValuesCommand({ TimePeriod: range, Dimension: "LINKED_ACCOUNT" })),
69
+ ]);
70
+ for (const item of accountDirectory?.DimensionValues || []) if (item.Value) names.set(item.Value, item.Attributes?.description || item.Value);
71
+ let trends = (trendResponse.ResultsByTime || []).map((period) => ({ month: period.TimePeriod?.Start, amount: round(amount(period.Total?.UnblendedCost)), currency: "USD" }));
72
+ if (trendMode === "yearly") {
73
+ const years = new Map();
74
+ for (const item of trends) {
75
+ const year = String(item.month || "").slice(0, 4);
76
+ years.set(year, round((years.get(year) || 0) + item.amount));
77
+ }
78
+ const firstYear = trendStart.getUTCFullYear();
79
+ const trendEnd = new Date(`${range.End}T00:00:00Z`); trendEnd.setUTCDate(trendEnd.getUTCDate() - 1);
80
+ const lastYear = trendEnd.getUTCFullYear();
81
+ trends = Array.from({ length: lastYear - firstYear + 1 }, (_, offset) => {
82
+ const year = String(firstYear + offset);
83
+ return { month: `${year}-01-01`, amount: years.get(year) || 0, currency: "USD" };
84
+ });
85
+ }
86
+ const totalCost = round((trendResponse.ResultsByTime || []).filter((period) => period.TimePeriod?.Start >= range.Start).reduce((total, period) => total + amount(period.Total?.UnblendedCost), 0));
87
+ const netCost = round((netTrendResponse.ResultsByTime || []).reduce((total, period) => total + amount(period.Total?.UnblendedCost), 0));
88
+ const creditsAndAdjustments = round(netCost - totalCost);
89
+ const activeResources = null;
90
+ const topResources = [];
91
+ let forecast = null;
92
+ if (!req.query.start && !req.query.end) {
93
+ try {
94
+ const now = new Date();
95
+ const tomorrow = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() + 1));
96
+ const nextMonth = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() + 1, 1));
97
+ if (tomorrow < nextMonth) {
98
+ const result = await client.send(new GetCostForecastCommand({ TimePeriod: { Start: iso(tomorrow), End: iso(nextMonth) }, Metric: "UNBLENDED_COST", Granularity: "MONTHLY" }));
99
+ forecast = round(totalCost + Number(result.Total?.Amount || 0));
100
+ }
101
+ } catch (error) { this.logger.warn?.("[Cost] Forecast unavailable", error.message); }
102
+ }
103
+ const topAccounts = parseGroups(accountPeriods, names, "account");
104
+ const resolvedAccounts = accounts.length ? accounts : topAccounts.map((item) => ({ id: item.accountId || item.key, name: item.label, region: "global", status: "active" }));
105
+ return res.json({ tenantId: tenant, accounts: resolvedAccounts, currency: "USD", period: range, totalCost, grossCost: totalCost, netCost, creditsAndAdjustments, forecast, activeAccounts: resolvedAccounts.length, activeResources, activeResourcesMessage: "Exact active-resource totals require CUR/Athena resource IDs.", trends, topServices: parseGroups(servicePeriods, names, "service").slice(0, 10), topAccounts, topRegions: parseGroups(regionPeriods).slice(0, 10), topResources, costBasis: "gross-positive-unblended", dataSource: "aws-cost-explorer", accessMode });
106
+ } catch (error) { return next(error); }
107
+ }
108
+
109
+ async filterOptions(req, res, next) {
110
+ try {
111
+ const ctx = await this.contextService.resolve(req);
112
+ const { accounts, client, queryEnabled, accessMode } = ctx;
113
+ const range = dateRange(req.query);
114
+ const tagKey = String(req.query.tagKey || "").trim();
115
+ const names = new Map(accounts.map((item) => [item.id, item.name]));
116
+ const cur = await this.curProvider.run(ctx, (athena, config) => getCurFilterOptions({ client: athena, config, tenant: ctx.tenant, range, tagKey, accountNames: names }));
117
+ if (cur) return res.json({ ...cur, tenantId: ctx.tenant, updatedAt: new Date().toISOString() });
118
+ if (!queryEnabled) return res.json({ services: [], regions: [], accounts, tagKeys: [], tagValues: [], accessMode, updatedAt: new Date().toISOString() });
119
+ const baseFilter = grossCostFilter(accounts.map((item) => item.id));
120
+ const getDimensions = async (dimension) => {
121
+ const values = [];
122
+ let token;
123
+ do {
124
+ const page = await client.send(new GetDimensionValuesCommand({ TimePeriod: range, Dimension: dimension, Filter: baseFilter, NextPageToken: token }));
125
+ values.push(...(page.DimensionValues || [])); token = page.NextPageToken;
126
+ } while (token);
127
+ return values;
128
+ };
129
+ const getTagValues = async () => {
130
+ const values = [];
131
+ let token;
132
+ do {
133
+ const page = await client.send(new GetTagsCommand({ TimePeriod: range, Filter: accountFilter(accounts.map((item) => item.id)), TagKey: tagKey || undefined, NextPageToken: token }));
134
+ values.push(...(page.Tags || [])); token = page.NextPageToken;
135
+ } while (token);
136
+ return [...new Set(values)];
137
+ };
138
+ const [serviceValues, regionValues, accountValues, tags] = await Promise.all([getDimensions("SERVICE"), getDimensions("REGION"), getDimensions("LINKED_ACCOUNT"), getTagValues()]);
139
+ const options = (items = []) => items.map((item) => ({ value: item.Value, label: item.Attributes?.description || item.Value })).filter((item) => item.value);
140
+ return res.json({ services: options(serviceValues), regions: options(regionValues), accounts: options(accountValues), tagKeys: tagKey ? [] : tags, tagValues: tagKey ? tags : [], accessMode, updatedAt: new Date().toISOString() });
141
+ } catch (error) { return next(error); }
142
+ }
143
+
144
+ async reports(req, res, next) {
145
+ try {
146
+ const ctx = await this.contextService.resolve(req);
147
+ const { tenant, accounts, client, queryEnabled, accessMode } = ctx;
148
+ const range = dateRange(req.query);
149
+ const groupBy = String(req.query.groupBy || "service").toLowerCase();
150
+ const map = { service: "SERVICE", account: "LINKED_ACCOUNT", region: "REGION", resource: "RESOURCE_ID" };
151
+ const dimension = groupBy === "tag" ? `tag:${String(req.query.tagKey || "Name")}` : map[groupBy];
152
+ if (!dimension) return res.status(400).json({ error: "groupBy must be service, account, region, resource, or tag" });
153
+ const names = new Map(accounts.map((item) => [item.id, item.name]));
154
+ if (!accounts.length && queryEnabled) {
155
+ try {
156
+ const directory = await client.send(new GetDimensionValuesCommand({ TimePeriod: range, Dimension: "LINKED_ACCOUNT" }));
157
+ for (const item of directory.DimensionValues || []) if (item.Value) names.set(item.Value, item.Attributes?.description || item.Value);
158
+ } catch (error) { this.logger.warn?.("[Cost] Account-name lookup unavailable", error.message); }
159
+ }
160
+ const granularity = String(req.query.granularity || "MONTHLY").toUpperCase();
161
+ const selections = { services: listParam(req.query.services), regions: listParam(req.query.regions), accountIds: listParam(req.query.accountIds), tagKey: String(req.query.filterTagKey || "").trim(), tagValues: listParam(req.query.tagValues) };
162
+ const cur = await this.curProvider.run(ctx, (athena, config) => getCurReport({ client: athena, config, tenant, range, groupBy, granularity, tagKey: req.query.tagKey, accountNames: names, filters: selections, includeBreakdown: true }));
163
+ if (cur) return res.json({ ...cur, tenantId: tenant, updatedAt: new Date().toISOString() });
164
+ let periods = [];
165
+ let accountRelationPeriods = [];
166
+ let regionRelationPeriods = [];
167
+ let message;
168
+ if (queryEnabled) {
169
+ try {
170
+ const primaryGranularity = granularity === "DAILY" ? "DAILY" : "MONTHLY";
171
+ [periods, accountRelationPeriods, regionRelationPeriods] = await Promise.all([
172
+ this.explorerService.grouped(client, accounts, range, dimension, primaryGranularity, selections),
173
+ groupBy === "resource" || groupBy === "account" ? Promise.resolve([]) : this.explorerService.grouped(client, accounts, range, [dimension, "LINKED_ACCOUNT"], primaryGranularity, selections),
174
+ groupBy === "resource" || groupBy === "region" ? Promise.resolve([]) : this.explorerService.grouped(client, accounts, range, [dimension, "REGION"], primaryGranularity, selections),
175
+ ]);
176
+ } catch (error) {
177
+ if (groupBy !== "resource" || error.name !== "ValidationException") throw error;
178
+ message = "Resource-level Cost Explorer data is not enabled for this payer account.";
179
+ this.logger.warn?.("[Cost] Resource report unavailable", error.message);
180
+ }
181
+ }
182
+ const parseType = groupBy === "service" ? "service" : groupBy === "account" ? "account" : undefined;
183
+ const accountRelationships = parseRelationships(accountRelationPeriods, parseType, "account", names);
184
+ const regionRelationships = parseRelationships(regionRelationPeriods, parseType, "region", names);
185
+ const summary = parseGroups(periods, names, parseType).map((item) => ({ ...item, accounts: groupBy === "account" ? [{ key: item.accountId || item.key, label: item.label, amount: item.amount }] : accountRelationships.get(item.key) || [], regions: groupBy === "region" ? [{ key: item.key, label: item.label, amount: item.amount }] : regionRelationships.get(item.key) || [] }));
186
+ const timeline = periods.map((period) => ({ start: period.TimePeriod?.Start, end: period.TimePeriod?.End, total: round((period.Groups || []).reduce((sum, group) => sum + amount(group.Metrics?.UnblendedCost), 0)), groups: parseGroups([period], names, parseType) }));
187
+ return res.json({ tenantId: tenant, groupBy, range, currency: "USD", totalCost: round(summary.reduce((sum, item) => sum + item.amount, 0)), count: summary.length, summary, timeline, filters: selections, updatedAt: new Date().toISOString(), costBasis: queryEnabled ? "gross-positive-unblended" : undefined, dataSource: queryEnabled ? "aws-cost-explorer" : "onboarding", accessMode, message });
188
+ } catch (error) { return next(error); }
189
+ }
190
+
191
+ async tags(req, res, next) {
192
+ try {
193
+ const ctx = await this.contextService.resolve(req);
194
+ const { accounts, client, queryEnabled, accessMode } = ctx;
195
+ const cur = await this.curProvider.run(ctx, (athena, config) => getCurTags({ client: athena, config, tenant: ctx.tenant }));
196
+ if (cur) return res.json({ tags: cur, dataSource: "aws-cur-athena", accessMode: "cur-readonly" });
197
+ if (!queryEnabled) return res.json({ tags: [], dataSource: "onboarding", accessMode });
198
+ const response = await client.send(new GetTagsCommand({ TimePeriod: dateRange(req.query), Filter: accountFilter(accounts.map((item) => item.id)) }));
199
+ return res.json({ tags: response.Tags || [], dataSource: "aws-cost-explorer", accessMode });
200
+ } catch (error) { return next(error); }
201
+ }
202
+
203
+ async curDiscovery(req, res, next) {
204
+ try {
205
+ const ctx = await this.contextService.resolve(req);
206
+ if (!this.discoveryRepository) return res.json({ tenantId: ctx.tenant, jobs: [], logs: [] });
207
+ const { jobs, logs } = await this.discoveryRepository.statusForTenant(ctx.tenant, req.query.logLimit);
208
+ return res.json({
209
+ tenantId: ctx.tenant,
210
+ jobs: jobs.map((job) => ({
211
+ id: String(job.id),
212
+ connectionId: job.connection_id,
213
+ status: job.status,
214
+ attemptCount: Number(job.attempt_count || 0),
215
+ bucket: job.cur_bucket,
216
+ prefix: job.cur_prefix,
217
+ region: job.cur_region,
218
+ tenantPartition: job.tenant_partition,
219
+ database: job.glue_database,
220
+ table: job.glue_table,
221
+ tableLocation: job.table_location,
222
+ curS3Uri: job.cur_s3_uri,
223
+ sampleObjectKey: job.sample_object_key,
224
+ lastDataAt: job.last_data_at,
225
+ lastError: job.last_error,
226
+ nextRunAt: job.next_run_at,
227
+ lastStartedAt: job.last_started_at,
228
+ lastFinishedAt: job.last_finished_at,
229
+ updatedAt: job.updated_at,
230
+ })),
231
+ logs: logs.map((entry) => ({
232
+ id: String(entry.id),
233
+ jobId: String(entry.job_id),
234
+ level: entry.level,
235
+ event: entry.event,
236
+ message: entry.message,
237
+ details: entry.details || {},
238
+ createdAt: entry.created_at,
239
+ })),
240
+ });
241
+ } catch (error) { return next(error); }
242
+ }
243
+
244
+ async dataStatus(req, res, next) {
245
+ try {
246
+ const ctx = await this.contextService.resolve(req);
247
+ const status = await this.curProvider.status(ctx);
248
+ try {
249
+ await this.contextService.recordCurStatus(ctx.tenant, status);
250
+ } catch (error) {
251
+ this.logger.warn?.("[Cost] Failed to persist CUR readiness", error.message);
252
+ }
253
+ return res.json({
254
+ tenantId: ctx.tenant,
255
+ customerAccessMode: ctx.accessMode,
256
+ ...status,
257
+ });
258
+ } catch (error) { return next(error); }
259
+ }
260
+
261
+ }
@@ -0,0 +1,82 @@
1
+ import { GetTablesCommand, GlueClient } from "@aws-sdk/client-glue";
2
+ import { ListObjectsV2Command, S3Client } from "@aws-sdk/client-s3";
3
+ import { fromTemporaryCredentials } from "@aws-sdk/credential-providers";
4
+
5
+ const cleanPrefix = (value) => String(value || "").replace(/^\/+|\/+$/g, "");
6
+ const normalizeS3Uri = (value) => `${String(value || "").replace(/\/+$/g, "")}/`;
7
+
8
+ function centralCredentials(env, credentialProvider) {
9
+ const roleArn = String(env.COST_CUR_ROLE_ARN || "").trim();
10
+ if (!roleArn) return undefined;
11
+ return credentialProvider({
12
+ params: {
13
+ RoleArn: roleArn,
14
+ RoleSessionName: "meyi-cur-discovery",
15
+ ExternalId: env.COST_CUR_EXTERNAL_ID || undefined,
16
+ },
17
+ });
18
+ }
19
+
20
+ export class CurDiscoveryAws {
21
+ constructor({
22
+ env = process.env,
23
+ credentialProvider = fromTemporaryCredentials,
24
+ glueClient,
25
+ s3Client,
26
+ } = {}) {
27
+ this.env = env;
28
+ this.region = String(env.COST_CUR_REGION || env.AWS_REGION || "us-east-1");
29
+ this.database = String(env.COST_CUR_DATABASE || "").trim();
30
+ this.tableOverride = String(env.COST_CUR_TABLE || "").trim();
31
+ this.tenantColumn = String(env.COST_CUR_TENANT_COLUMN || "tenant_id").trim();
32
+ const credentials = centralCredentials(env, credentialProvider);
33
+ this.glue = glueClient || new GlueClient({ region: this.region, credentials });
34
+ this.s3 = s3Client || new S3Client({ region: this.region, credentials });
35
+ }
36
+
37
+ async firstTenantObject(job) {
38
+ const prefix = `${cleanPrefix(job.cur_prefix)}/`;
39
+ const response = await this.s3.send(new ListObjectsV2Command({
40
+ Bucket: job.cur_bucket,
41
+ Prefix: prefix,
42
+ MaxKeys: 1,
43
+ }));
44
+ const object = response.Contents?.[0];
45
+ return object ? { key: object.Key, lastModified: object.LastModified || null } : null;
46
+ }
47
+
48
+ async findCompatibleTable(job) {
49
+ if (!this.database) return null;
50
+ const tenantUri = normalizeS3Uri(`s3://${job.cur_bucket}/${cleanPrefix(job.cur_prefix)}`);
51
+ const candidates = [];
52
+ let nextToken;
53
+ do {
54
+ const response = await this.glue.send(new GetTablesCommand({
55
+ DatabaseName: this.database,
56
+ NextToken: nextToken,
57
+ }));
58
+ for (const table of response.TableList || []) {
59
+ const location = normalizeS3Uri(table.StorageDescriptor?.Location);
60
+ const columns = new Set((table.StorageDescriptor?.Columns || []).map((item) => String(item.Name || "").toLowerCase()));
61
+ const partitions = new Set((table.PartitionKeys || []).map((item) => String(item.Name || "").toLowerCase()));
62
+ const pathMatches = location && (tenantUri.startsWith(location) || location.startsWith(tenantUri));
63
+ const tenantSafe = partitions.has(this.tenantColumn.toLowerCase()) || columns.has(this.tenantColumn.toLowerCase());
64
+ const curSchema = columns.has("line_item_usage_start_date");
65
+ if (pathMatches && tenantSafe && curSchema) {
66
+ candidates.push({
67
+ database: this.database,
68
+ name: table.Name,
69
+ location,
70
+ tenantColumn: this.tenantColumn,
71
+ });
72
+ }
73
+ }
74
+ nextToken = response.NextToken;
75
+ } while (nextToken);
76
+
77
+ if (this.tableOverride) {
78
+ return candidates.find((table) => table.name === this.tableOverride) || null;
79
+ }
80
+ return candidates.length === 1 ? candidates[0] : null;
81
+ }
82
+ }
@@ -0,0 +1,177 @@
1
+ import { sql } from "drizzle-orm";
2
+ import { rows } from "../lib/cost-utils.js";
3
+
4
+ export class CurDiscoveryRepository {
5
+ constructor({ db, schema, qSchema }) {
6
+ this.db = db;
7
+ this.schema = schema;
8
+ this.jobsTable = `${qSchema}.cost_cur_discovery_jobs`;
9
+ this.logsTable = `${qSchema}.cost_cur_discovery_job_logs`;
10
+ this.curConfigTable = `${qSchema}.cost_cur_config`;
11
+ this.connectionsTable = `${qSchema}.aws_connections`;
12
+ this.pluginsTable = `${qSchema}.plugins`;
13
+ }
14
+
15
+ async seedEligibleJobs() {
16
+ const relations = rows(await this.db.execute(sql`
17
+ SELECT
18
+ to_regclass(${`${this.schema}.cost_cur_config`}) AS cur_config_table,
19
+ to_regclass(${`${this.schema}.aws_connections`}) AS connections_table,
20
+ to_regclass(${`${this.schema}.plugins`}) AS plugins_table
21
+ `))[0] || {};
22
+ if (!relations.cur_config_table || !relations.connections_table || !relations.plugins_table) return 0;
23
+
24
+ const result = await this.db.execute(sql.raw(`
25
+ INSERT INTO ${this.jobsTable} (
26
+ tenant_id, connection_id, cur_bucket, cur_prefix, cur_region, tenant_partition, next_run_at
27
+ )
28
+ SELECT
29
+ c.tenant_id::text, c.connection_id, c.bucket, c.prefix, c.region, c.tenant_partition, now()
30
+ FROM ${this.curConfigTable} c
31
+ JOIN ${this.connectionsTable} a
32
+ ON a.connection_id = c.connection_id
33
+ AND a.tenant_id::text = c.tenant_id::text
34
+ JOIN ${this.pluginsTable} p
35
+ ON p.tenant_id::text = c.tenant_id::text
36
+ AND lower(p.name) = 'cost'
37
+ WHERE a.status = 'CONNECTED'
38
+ AND a.plugins ? 'Cost'
39
+ AND COALESCE(p.is_entitled, false) = true
40
+ AND COALESCE(p.is_installed, false) = true
41
+ AND COALESCE(p.is_enabled, false) = true
42
+ ON CONFLICT (tenant_id, connection_id) DO UPDATE SET
43
+ cur_bucket = EXCLUDED.cur_bucket,
44
+ cur_prefix = EXCLUDED.cur_prefix,
45
+ cur_region = EXCLUDED.cur_region,
46
+ tenant_partition = EXCLUDED.tenant_partition,
47
+ updated_at = now()
48
+ RETURNING id
49
+ `));
50
+ return rows(result).length;
51
+ }
52
+
53
+ async dueJobs(limit = 25) {
54
+ return rows(await this.db.execute(sql.raw(`
55
+ SELECT j.*
56
+ FROM ${this.jobsTable} j
57
+ JOIN ${this.curConfigTable} c
58
+ ON c.tenant_id::text = j.tenant_id
59
+ AND c.connection_id = j.connection_id
60
+ JOIN ${this.connectionsTable} a
61
+ ON a.connection_id = j.connection_id
62
+ AND a.tenant_id::text = j.tenant_id
63
+ JOIN ${this.pluginsTable} p
64
+ ON p.tenant_id::text = j.tenant_id
65
+ AND lower(p.name) = 'cost'
66
+ WHERE j.status <> 'READY'
67
+ AND (j.next_run_at IS NULL OR j.next_run_at <= now())
68
+ AND a.status = 'CONNECTED'
69
+ AND a.plugins ? 'Cost'
70
+ AND COALESCE(p.is_entitled, false) = true
71
+ AND COALESCE(p.is_installed, false) = true
72
+ AND COALESCE(p.is_enabled, false) = true
73
+ ORDER BY COALESCE(j.next_run_at, j.created_at), j.id
74
+ LIMIT ${Math.min(Math.max(Number(limit) || 25, 1), 100)}
75
+ `)));
76
+ }
77
+
78
+ async claim(jobId, leaseMs = 1_200_000) {
79
+ const leaseUntil = new Date(Date.now() + leaseMs);
80
+ const staleBefore = new Date(Date.now() - leaseMs);
81
+ return rows(await this.db.execute(sql`
82
+ UPDATE ${sql.raw(this.jobsTable)}
83
+ SET status = 'RUNNING',
84
+ attempt_count = attempt_count + 1,
85
+ last_started_at = now(),
86
+ next_run_at = ${leaseUntil},
87
+ last_error = NULL,
88
+ updated_at = now()
89
+ WHERE id = ${jobId}
90
+ AND status <> 'READY'
91
+ AND (next_run_at IS NULL OR next_run_at <= now())
92
+ AND (status <> 'RUNNING' OR last_started_at IS NULL OR last_started_at <= ${staleBefore})
93
+ RETURNING *
94
+ `))[0] || null;
95
+ }
96
+
97
+ async log(job, level, event, message, details = {}) {
98
+ await this.db.execute(sql`
99
+ INSERT INTO ${sql.raw(this.logsTable)} (job_id, tenant_id, level, event, message, details)
100
+ VALUES (${job.id}, ${job.tenant_id}, ${level}, ${event}, ${message}, ${JSON.stringify(details)}::jsonb)
101
+ `);
102
+ }
103
+
104
+ async wait(job, { status, intervalMs, error = null, sampleObjectKey = null, table = null }) {
105
+ const nextRunAt = new Date(Date.now() + intervalMs);
106
+ await this.db.execute(sql`
107
+ UPDATE ${sql.raw(this.jobsTable)}
108
+ SET status = ${status},
109
+ next_run_at = ${nextRunAt},
110
+ last_finished_at = now(),
111
+ sample_object_key = COALESCE(${sampleObjectKey}, sample_object_key),
112
+ glue_database = COALESCE(${table?.database || null}, glue_database),
113
+ glue_table = COALESCE(${table?.name || null}, glue_table),
114
+ table_location = COALESCE(${table?.location || null}, table_location),
115
+ cur_s3_uri = ${`s3://${job.cur_bucket}/${String(job.cur_prefix).replace(/^\/+|\/+$/g, "")}/`},
116
+ last_error = ${error},
117
+ updated_at = now()
118
+ WHERE id = ${job.id}
119
+ `);
120
+ await this.updateCurConfig(job, status === "FAILED" ? "FAILED" : "WAITING_FOR_DATA", null);
121
+ }
122
+
123
+ async ready(job, { table, sampleObjectKey, lastDataAt }) {
124
+ await this.db.execute(sql`
125
+ UPDATE ${sql.raw(this.jobsTable)}
126
+ SET status = 'READY',
127
+ next_run_at = NULL,
128
+ last_finished_at = now(),
129
+ glue_database = ${table.database},
130
+ glue_table = ${table.name},
131
+ table_location = ${table.location},
132
+ cur_s3_uri = ${`s3://${job.cur_bucket}/${String(job.cur_prefix).replace(/^\/+|\/+$/g, "")}/`},
133
+ sample_object_key = ${sampleObjectKey},
134
+ last_data_at = ${lastDataAt || null},
135
+ last_error = NULL,
136
+ updated_at = now()
137
+ WHERE id = ${job.id}
138
+ `);
139
+ await this.updateCurConfig(job, "READY", lastDataAt);
140
+ }
141
+
142
+ async updateCurConfig(job, status, lastDataAt) {
143
+ await this.db.execute(sql`
144
+ UPDATE ${sql.raw(this.curConfigTable)}
145
+ SET status = ${status},
146
+ last_data_at = COALESCE(${lastDataAt || null}, last_data_at),
147
+ verified_at = now(),
148
+ updated_at = now()
149
+ WHERE tenant_id::text = ${job.tenant_id}
150
+ AND connection_id = ${job.connection_id}
151
+ `);
152
+ }
153
+ /**
154
+ * Discovery jobs for a tenant with their most recent log lines, for the
155
+ * Cost > CUR Discovery screen. Read-only: the scheduler owns every write.
156
+ */
157
+ async statusForTenant(tenant, logLimit = 50) {
158
+ const jobs = rows(await this.db.execute(sql`
159
+ SELECT id, tenant_id, connection_id, cur_bucket, cur_prefix, cur_region,
160
+ tenant_partition, status, attempt_count, next_run_at, last_started_at,
161
+ last_finished_at, glue_database, glue_table, table_location, cur_s3_uri,
162
+ sample_object_key, last_data_at, last_error, created_at, updated_at
163
+ FROM ${sql.raw(this.jobsTable)}
164
+ WHERE tenant_id = ${tenant}
165
+ ORDER BY updated_at DESC
166
+ `));
167
+ if (!jobs.length) return { jobs: [], logs: [] };
168
+ const logs = rows(await this.db.execute(sql`
169
+ SELECT id, job_id, level, event, message, details, created_at
170
+ FROM ${sql.raw(this.logsTable)}
171
+ WHERE tenant_id = ${tenant}
172
+ ORDER BY created_at DESC, id DESC
173
+ LIMIT ${Math.min(Math.max(Number(logLimit) || 50, 1), 500)}
174
+ `));
175
+ return { jobs, logs };
176
+ }
177
+ }
@@ -0,0 +1,112 @@
1
+ import { getCurDataStatus } from "../services/cur.service.js";
2
+
3
+ const safeError = (error) => String(error?.message || error || "Unknown CUR discovery error").slice(0, 2000);
4
+
5
+ export class CurDiscoveryService {
6
+ constructor({ repository, aws, athenaContextService, logger = console, intervalMs = 3_600_000 }) {
7
+ this.repository = repository;
8
+ this.aws = aws;
9
+ this.athenaContextService = athenaContextService;
10
+ this.logger = logger;
11
+ this.intervalMs = intervalMs;
12
+ }
13
+
14
+ write(job, level, event, message, details = {}) {
15
+ const prefix = `[Cost CUR Discovery][tenant=${job.tenant_id}][job=${job.id}]`;
16
+ const method = level === "ERROR" ? "error" : level === "WARN" ? "warn" : "log";
17
+ this.logger[method]?.(`${prefix} ${event}: ${message}`, details);
18
+ return this.repository.log(job, level, event, message, details);
19
+ }
20
+
21
+ async run(rawJob) {
22
+ const job = await this.repository.claim(rawJob.id, this.intervalMs);
23
+ if (!job) return;
24
+ await this.write(job, "INFO", "STARTED", "Checking tenant CUR delivery and catalog state", {
25
+ connectionId: job.connection_id,
26
+ bucket: job.cur_bucket,
27
+ prefix: job.cur_prefix,
28
+ attempt: job.attempt_count,
29
+ });
30
+
31
+ try {
32
+ const object = await this.aws.firstTenantObject(job);
33
+ if (!object) {
34
+ await this.repository.wait(job, { status: "WAITING_FOR_DATA", intervalMs: this.intervalMs });
35
+ await this.write(job, "INFO", "S3_DATA_PENDING", "No CUR object is available for this tenant; retry scheduled", {
36
+ nextCheckMinutes: Math.round(this.intervalMs / 60000),
37
+ });
38
+ return;
39
+ }
40
+
41
+ const table = await this.aws.findCompatibleTable(job);
42
+ if (!table) {
43
+ await this.repository.wait(job, {
44
+ status: "WAITING_FOR_TABLE",
45
+ intervalMs: this.intervalMs,
46
+ sampleObjectKey: object.key,
47
+ });
48
+ await this.write(job, "INFO", "GLUE_TABLE_PENDING", "Tenant CUR data exists but no unique compatible Glue table was found", {
49
+ database: this.aws.database,
50
+ sampleObjectKey: object.key,
51
+ nextCheckMinutes: Math.round(this.intervalMs / 60000),
52
+ });
53
+ return;
54
+ }
55
+
56
+ const customerContext = {
57
+ tenant: job.tenant_id,
58
+ meta: {
59
+ curDiscoveredTable: table.name,
60
+ curTenantPartition: job.tenant_partition,
61
+ curSourceBucket: job.cur_bucket,
62
+ curSourcePrefix: job.cur_prefix,
63
+ curSourceRegion: job.cur_region,
64
+ curIngestionMode: "central",
65
+ },
66
+ };
67
+ const context = this.athenaContextService.resolve(customerContext);
68
+ if (!context.config.enabled || !context.client) {
69
+ throw new Error(context.config.disabledReason || "Central CUR/Athena is not fully configured");
70
+ }
71
+ const readiness = await getCurDataStatus({
72
+ client: context.client,
73
+ config: context.config,
74
+ tenant: job.tenant_id,
75
+ });
76
+ if (readiness.recordCount < 1) {
77
+ await this.repository.wait(job, {
78
+ status: "WAITING_FOR_DATA",
79
+ intervalMs: this.intervalMs,
80
+ sampleObjectKey: object.key,
81
+ table,
82
+ });
83
+ await this.write(job, "INFO", "ATHENA_DATA_PENDING", "Glue table exists but Athena has no rows for this tenant partition", {
84
+ database: table.database,
85
+ table: table.name,
86
+ tableLocation: table.location,
87
+ nextCheckMinutes: Math.round(this.intervalMs / 60000),
88
+ });
89
+ return;
90
+ }
91
+
92
+ await this.repository.ready(job, {
93
+ table,
94
+ sampleObjectKey: object.key,
95
+ lastDataAt: readiness.lastDataAt,
96
+ });
97
+ await this.write(job, "INFO", "READY", "Tenant CUR discovery completed; automatic polling stopped for this job", {
98
+ database: table.database,
99
+ table: table.name,
100
+ tableLocation: table.location,
101
+ recordCount: readiness.recordCount,
102
+ lastDataAt: readiness.lastDataAt,
103
+ });
104
+ } catch (error) {
105
+ const message = safeError(error);
106
+ await this.repository.wait(job, { status: "FAILED", intervalMs: this.intervalMs, error: message });
107
+ await this.write(job, "ERROR", "FAILED", message, {
108
+ nextCheckMinutes: Math.round(this.intervalMs / 60000),
109
+ });
110
+ }
111
+ }
112
+ }