meyi-cost-server 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +179 -0
- package/README.md +474 -0
- package/cur.js +2 -0
- package/index.js +1 -0
- package/package.json +36 -0
- package/src/controllers/budget.controller.js +39 -0
- package/src/controllers/cost.controller.js +261 -0
- package/src/cur-discovery/cur-discovery.aws.js +82 -0
- package/src/cur-discovery/cur-discovery.repository.js +177 -0
- package/src/cur-discovery/cur-discovery.service.js +112 -0
- package/src/cur-discovery/cur-discovery.worker.js +57 -0
- package/src/cur-discovery/schema.js +48 -0
- package/src/lib/cost-utils.js +98 -0
- package/src/models/budget.model.js +26 -0
- package/src/models/cur-data-status.model.js +16 -0
- package/src/models/cur-ingestion.model.js +9 -0
- package/src/models/customer-aws-context.model.js +20 -0
- package/src/models/saas-cur-context.model.js +10 -0
- package/src/plugin.js +72 -0
- package/src/repositories/aws-onboarding.repository.js +134 -0
- package/src/repositories/budget-alert.repository.js +37 -0
- package/src/repositories/budget.repository.js +33 -0
- package/src/routes/index.js +22 -0
- package/src/schema/cost-budget.schema.js +9 -0
- package/src/services/aws-context.service.js +1 -0
- package/src/services/budget-alert.service.js +47 -0
- package/src/services/budget.service.js +32 -0
- package/src/services/cost-explorer.service.js +27 -0
- package/src/services/cur-provider.service.js +91 -0
- package/src/services/cur.service.js +355 -0
- package/src/services/customer-aws-context.service.js +62 -0
- package/src/services/saas-athena-context.service.js +54 -0
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { truthy } from "../lib/cost-utils.js";
|
|
2
|
+
|
|
3
|
+
export class CurDiscoveryWorker {
|
|
4
|
+
constructor({ repository, service, logger = console, env = process.env }) {
|
|
5
|
+
this.repository = repository;
|
|
6
|
+
this.service = service;
|
|
7
|
+
this.logger = logger;
|
|
8
|
+
this.enabled = env.COST_CUR_DISCOVERY_ENABLED === undefined
|
|
9
|
+
? true
|
|
10
|
+
: truthy(env.COST_CUR_DISCOVERY_ENABLED);
|
|
11
|
+
// Poll more frequently than the per-job retry interval. A worker tick can
|
|
12
|
+
// occur milliseconds before next_run_at; using the retry interval here
|
|
13
|
+
// would defer that job for another complete interval.
|
|
14
|
+
this.pollIntervalMs = Math.max(Number(env.COST_CUR_DISCOVERY_POLL_INTERVAL_MS || 60_000), 10_000);
|
|
15
|
+
this.batchSize = Math.min(Math.max(Number(env.COST_CUR_DISCOVERY_BATCH_SIZE || 25), 1), 100);
|
|
16
|
+
this.running = false;
|
|
17
|
+
this.timer = null;
|
|
18
|
+
this.initialTimer = null;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
async tick() {
|
|
22
|
+
if (!this.enabled || this.running) return;
|
|
23
|
+
this.running = true;
|
|
24
|
+
try {
|
|
25
|
+
await this.repository.seedEligibleJobs();
|
|
26
|
+
const jobs = await this.repository.dueJobs(this.batchSize);
|
|
27
|
+
const concurrency = 5;
|
|
28
|
+
for (let index = 0; index < jobs.length; index += concurrency) {
|
|
29
|
+
await Promise.all(jobs.slice(index, index + concurrency).map((job) => this.service.run(job)));
|
|
30
|
+
}
|
|
31
|
+
} catch (error) {
|
|
32
|
+
this.logger.error?.("[Cost CUR Discovery] Scheduler tick failed", error);
|
|
33
|
+
} finally {
|
|
34
|
+
this.running = false;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
start() {
|
|
39
|
+
if (!this.enabled || this.timer) {
|
|
40
|
+
if (!this.enabled) this.logger.log?.("[Cost CUR Discovery] Disabled");
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
this.logger.log?.(`[Cost CUR Discovery] Started; poll=${Math.round(this.pollIntervalMs / 1000)}s batch=${this.batchSize}`);
|
|
44
|
+
this.initialTimer = setTimeout(() => void this.tick(), 1_000);
|
|
45
|
+
this.initialTimer.unref?.();
|
|
46
|
+
this.timer = setInterval(() => void this.tick(), this.pollIntervalMs);
|
|
47
|
+
this.timer.unref?.();
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
stop() {
|
|
51
|
+
if (this.initialTimer) clearTimeout(this.initialTimer);
|
|
52
|
+
if (this.timer) clearInterval(this.timer);
|
|
53
|
+
this.initialTimer = null;
|
|
54
|
+
this.timer = null;
|
|
55
|
+
this.logger.log?.("[Cost CUR Discovery] Stopped");
|
|
56
|
+
}
|
|
57
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { sql } from "drizzle-orm";
|
|
2
|
+
|
|
3
|
+
export async function installCurDiscoverySchema(db, qSchema) {
|
|
4
|
+
await db.execute(sql.raw(`CREATE TABLE IF NOT EXISTS ${qSchema}.cost_cur_discovery_jobs (
|
|
5
|
+
id bigserial PRIMARY KEY,
|
|
6
|
+
tenant_id text NOT NULL,
|
|
7
|
+
connection_id text NOT NULL,
|
|
8
|
+
cur_bucket text NOT NULL,
|
|
9
|
+
cur_prefix text NOT NULL,
|
|
10
|
+
cur_region text NOT NULL,
|
|
11
|
+
tenant_partition text NOT NULL,
|
|
12
|
+
status text NOT NULL DEFAULT 'PENDING' CHECK (status IN ('PENDING', 'RUNNING', 'WAITING_FOR_DATA', 'WAITING_FOR_TABLE', 'READY', 'FAILED')),
|
|
13
|
+
attempt_count integer NOT NULL DEFAULT 0,
|
|
14
|
+
next_run_at timestamptz,
|
|
15
|
+
last_started_at timestamptz,
|
|
16
|
+
last_finished_at timestamptz,
|
|
17
|
+
glue_database text,
|
|
18
|
+
glue_table text,
|
|
19
|
+
table_location text,
|
|
20
|
+
cur_s3_uri text,
|
|
21
|
+
sample_object_key text,
|
|
22
|
+
last_data_at timestamptz,
|
|
23
|
+
last_error text,
|
|
24
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
25
|
+
updated_at timestamptz NOT NULL DEFAULT now(),
|
|
26
|
+
UNIQUE (tenant_id, connection_id)
|
|
27
|
+
)`));
|
|
28
|
+
// Early CUR builds used uuid here. The discovery API treats tenant IDs as
|
|
29
|
+
// opaque strings, so normalize upgraded databases to the current schema.
|
|
30
|
+
await db.execute(sql.raw(`ALTER TABLE ${qSchema}.cost_cur_discovery_jobs
|
|
31
|
+
ALTER COLUMN tenant_id TYPE text USING tenant_id::text`));
|
|
32
|
+
await db.execute(sql.raw(`CREATE INDEX IF NOT EXISTS cost_cur_discovery_jobs_due_idx ON ${qSchema}.cost_cur_discovery_jobs (status, next_run_at)`));
|
|
33
|
+
await db.execute(sql.raw(`CREATE INDEX IF NOT EXISTS cost_cur_discovery_jobs_tenant_idx ON ${qSchema}.cost_cur_discovery_jobs (tenant_id, connection_id)`));
|
|
34
|
+
await db.execute(sql.raw(`CREATE TABLE IF NOT EXISTS ${qSchema}.cost_cur_discovery_job_logs (
|
|
35
|
+
id bigserial PRIMARY KEY,
|
|
36
|
+
job_id bigint NOT NULL REFERENCES ${qSchema}.cost_cur_discovery_jobs(id) ON DELETE CASCADE,
|
|
37
|
+
tenant_id text NOT NULL,
|
|
38
|
+
level text NOT NULL CHECK (level IN ('INFO', 'WARN', 'ERROR')),
|
|
39
|
+
event text NOT NULL,
|
|
40
|
+
message text NOT NULL,
|
|
41
|
+
details jsonb NOT NULL DEFAULT '{}'::jsonb,
|
|
42
|
+
created_at timestamptz NOT NULL DEFAULT now()
|
|
43
|
+
)`));
|
|
44
|
+
await db.execute(sql.raw(`ALTER TABLE ${qSchema}.cost_cur_discovery_job_logs
|
|
45
|
+
ALTER COLUMN tenant_id TYPE text USING tenant_id::text`));
|
|
46
|
+
await db.execute(sql.raw(`CREATE INDEX IF NOT EXISTS cost_cur_discovery_job_logs_job_idx ON ${qSchema}.cost_cur_discovery_job_logs (job_id, created_at DESC)`));
|
|
47
|
+
await db.execute(sql.raw(`CREATE INDEX IF NOT EXISTS cost_cur_discovery_job_logs_tenant_idx ON ${qSchema}.cost_cur_discovery_job_logs (tenant_id, created_at DESC)`));
|
|
48
|
+
}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
const GROSS_COST_RECORD_TYPES = [
|
|
2
|
+
"Usage", "Fee", "Tax", "Support", "Upfront", "Recurring", "RIFee",
|
|
3
|
+
"DiscountedUsage", "SavingsPlanCoveredUsage", "SavingsPlanRecurringFee",
|
|
4
|
+
"SavingsPlanUpfrontFee",
|
|
5
|
+
];
|
|
6
|
+
|
|
7
|
+
export const truthy = (value) => ["1", "true", "yes", "on"].includes(String(value || "").toLowerCase());
|
|
8
|
+
export const rows = (result) => Array.isArray(result) ? result : result?.rows || [];
|
|
9
|
+
export const iso = (date) => date.toISOString().slice(0, 10);
|
|
10
|
+
export const amount = (metric) => Number(metric?.Amount || 0);
|
|
11
|
+
export const round = (value) => Math.round((Number(value) + Number.EPSILON) * 100) / 100;
|
|
12
|
+
|
|
13
|
+
export function safeSchema(value) {
|
|
14
|
+
const schema = String(value || "meyiconnect");
|
|
15
|
+
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(schema)) throw new Error("Invalid DB schema");
|
|
16
|
+
return schema;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function dateRange(query = {}) {
|
|
20
|
+
const end = query.end ? new Date(`${query.end}T00:00:00Z`) : new Date();
|
|
21
|
+
const start = query.start ? new Date(`${query.start}T00:00:00Z`) : new Date(Date.UTC(end.getUTCFullYear(), end.getUTCMonth(), 1));
|
|
22
|
+
const exclusiveEnd = new Date(end);
|
|
23
|
+
exclusiveEnd.setUTCDate(exclusiveEnd.getUTCDate() + (query.end ? 1 : 0));
|
|
24
|
+
if (!query.end) exclusiveEnd.setUTCDate(exclusiveEnd.getUTCDate() + 1);
|
|
25
|
+
if (!Number.isFinite(start.getTime()) || !Number.isFinite(exclusiveEnd.getTime()) || start >= exclusiveEnd) {
|
|
26
|
+
const error = new Error("Invalid cost date range");
|
|
27
|
+
error.statusCode = 400;
|
|
28
|
+
throw error;
|
|
29
|
+
}
|
|
30
|
+
return { Start: iso(start), End: iso(exclusiveEnd) };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function accountFilter(accountIds) {
|
|
34
|
+
return accountIds.length ? { Dimensions: { Key: "LINKED_ACCOUNT", Values: accountIds } } : undefined;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function combineFilters(...values) {
|
|
38
|
+
const filters = values.flat().filter(Boolean);
|
|
39
|
+
return filters.length === 0 ? undefined : filters.length === 1 ? filters[0] : { And: filters };
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function listParam(value) {
|
|
43
|
+
const values = Array.isArray(value) ? value : String(value || "").split(",");
|
|
44
|
+
return [...new Set(values.map((item) => String(item).trim()).filter(Boolean))].slice(0, 100);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function grossCostFilter(accountIds, selections = {}) {
|
|
48
|
+
const filters = [];
|
|
49
|
+
const accounts = accountFilter(accountIds);
|
|
50
|
+
if (accounts) filters.push(accounts);
|
|
51
|
+
filters.push({ Dimensions: { Key: "RECORD_TYPE", Values: GROSS_COST_RECORD_TYPES } });
|
|
52
|
+
if (selections.services?.length) filters.push({ Dimensions: { Key: "SERVICE", Values: selections.services } });
|
|
53
|
+
if (selections.regions?.length) filters.push({ Dimensions: { Key: "REGION", Values: selections.regions } });
|
|
54
|
+
if (selections.accountIds?.length) filters.push({ Dimensions: { Key: "LINKED_ACCOUNT", Values: selections.accountIds } });
|
|
55
|
+
if (selections.tagKey && selections.tagValues?.length) filters.push({ Tags: { Key: selections.tagKey, Values: selections.tagValues } });
|
|
56
|
+
return combineFilters(filters);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export function normalizeServiceName(value) {
|
|
60
|
+
const name = String(value || "Other");
|
|
61
|
+
return name === "Amazon Elastic Compute Cloud - Compute" || name === "EC2 - Other"
|
|
62
|
+
? "Amazon Elastic Compute Cloud"
|
|
63
|
+
: name;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function parseGroups(results = [], accountNames = new Map(), groupType) {
|
|
67
|
+
const totals = new Map();
|
|
68
|
+
for (const period of results) {
|
|
69
|
+
for (const group of period.Groups || []) {
|
|
70
|
+
const rawKey = group.Keys?.join(" / ") || "Other";
|
|
71
|
+
const key = groupType === "service" ? normalizeServiceName(rawKey) : rawKey;
|
|
72
|
+
totals.set(key, (totals.get(key) || 0) + amount(group.Metrics?.UnblendedCost));
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
const sum = [...totals.values()].reduce((total, value) => total + value, 0);
|
|
76
|
+
return [...totals.entries()]
|
|
77
|
+
.filter(([, value]) => Math.abs(value) > 0.000001)
|
|
78
|
+
.map(([key, value]) => ({ key, label: accountNames.get(key) || key, accountId: groupType === "account" ? key : undefined, amount: round(value), percentage: sum ? round(value / sum * 100) : 0 }))
|
|
79
|
+
.sort((a, b) => b.amount - a.amount);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export function parseRelationships(results = [], primaryType, relatedType, accountNames = new Map()) {
|
|
83
|
+
const groupedItems = new Map();
|
|
84
|
+
for (const period of results) {
|
|
85
|
+
for (const group of period.Groups || []) {
|
|
86
|
+
let primary = group.Keys?.[0] || "Other";
|
|
87
|
+
if (primaryType === "service") primary = normalizeServiceName(primary);
|
|
88
|
+
const related = group.Keys?.[1] || "Unallocated";
|
|
89
|
+
if (!groupedItems.has(primary)) groupedItems.set(primary, new Map());
|
|
90
|
+
const values = groupedItems.get(primary);
|
|
91
|
+
values.set(related, (values.get(related) || 0) + amount(group.Metrics?.UnblendedCost));
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return new Map([...groupedItems.entries()].map(([key, values]) => [key, [...values.entries()]
|
|
95
|
+
.map(([value, cost]) => ({ key: value, label: relatedType === "account" ? accountNames.get(value) || value : value, amount: round(cost) }))
|
|
96
|
+
.filter((item) => Math.abs(item.amount) > 0.000001)
|
|
97
|
+
.sort((a, b) => b.amount - a.amount)]));
|
|
98
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export class CostBudget {
|
|
2
|
+
constructor({ id, name, amount, currency = "USD", period = "monthly", spent = 0, alertThreshold = 80, provider = "aws", createdAt, updatedAt }) {
|
|
3
|
+
this.id = id;
|
|
4
|
+
this.name = name;
|
|
5
|
+
this.amount = Number(amount);
|
|
6
|
+
this.currency = currency;
|
|
7
|
+
this.period = period;
|
|
8
|
+
this.spent = Number(spent || 0);
|
|
9
|
+
this.alert_threshold = Number(alertThreshold);
|
|
10
|
+
this.provider = provider;
|
|
11
|
+
this.createdAt = createdAt;
|
|
12
|
+
if (updatedAt !== undefined) this.updatedAt = updatedAt;
|
|
13
|
+
Object.freeze(this);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export class BudgetAlertDismissal {
|
|
18
|
+
constructor({ budgetId, period, status, dismissedAt, dismissed }) {
|
|
19
|
+
this.budgetId = budgetId;
|
|
20
|
+
this.period = period;
|
|
21
|
+
this.status = status;
|
|
22
|
+
if (dismissedAt !== undefined) this.dismissedAt = dismissedAt;
|
|
23
|
+
if (dismissed !== undefined) this.dismissed = Boolean(dismissed);
|
|
24
|
+
Object.freeze(this);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export class CurDataStatus {
|
|
2
|
+
constructor({ configured, required = false, ready = false, state, credentialMode = "saas-runtime", ingestionMode = "central", sourceConfigured = false, lastDataAt = null, recordCount = 0, message = null, discovery = null }) {
|
|
3
|
+
this.configured = Boolean(configured);
|
|
4
|
+
this.required = Boolean(required);
|
|
5
|
+
this.ready = Boolean(ready);
|
|
6
|
+
this.state = state || (this.ready ? "ready" : this.configured ? "pending" : "not_configured");
|
|
7
|
+
this.credentialMode = credentialMode;
|
|
8
|
+
this.ingestionMode = ingestionMode;
|
|
9
|
+
this.sourceConfigured = Boolean(sourceConfigured);
|
|
10
|
+
this.lastDataAt = lastDataAt;
|
|
11
|
+
this.recordCount = Number(recordCount || 0);
|
|
12
|
+
this.message = message;
|
|
13
|
+
this.discovery = discovery ? Object.freeze({ ...discovery }) : null;
|
|
14
|
+
Object.freeze(this);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export class CurIngestionDescriptor {
|
|
2
|
+
constructor({ tenant, metadata = {}, env = process.env }) {
|
|
3
|
+
this.mode = String(metadata.curIngestionMode || env.COST_CUR_INGESTION_MODE || "central").trim() || "central";
|
|
4
|
+
this.sourceConfigured = Boolean(metadata.curSourceBucket);
|
|
5
|
+
this.sourceRegion = metadata.curSourceRegion || null;
|
|
6
|
+
this.tenantPartition = String(env.COST_CUR_TENANT_PARTITION || metadata.curTenantPartition || tenant).trim();
|
|
7
|
+
Object.freeze(this);
|
|
8
|
+
}
|
|
9
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
const normalizeAccount = (account = {}) => Object.freeze({
|
|
2
|
+
id: String(account.id || account.aws_account_id || "").trim(),
|
|
3
|
+
name: String(account.name || account.id || account.aws_account_id || "").trim(),
|
|
4
|
+
region: String(account.region || "global").trim() || "global",
|
|
5
|
+
status: String(account.status || "active").trim() || "active",
|
|
6
|
+
});
|
|
7
|
+
|
|
8
|
+
export class CustomerAwsContext {
|
|
9
|
+
constructor({ tenant, accounts = [], client, credentials, meta = {}, roleArn, queryEnabled = false, accessMode = "not-configured" }) {
|
|
10
|
+
this.tenant = String(tenant || "default").trim() || "default";
|
|
11
|
+
this.accounts = Object.freeze(accounts.map(normalizeAccount).filter((account) => account.id));
|
|
12
|
+
this.client = client;
|
|
13
|
+
this.credentials = credentials;
|
|
14
|
+
this.meta = Object.freeze({ ...meta });
|
|
15
|
+
this.roleArn = roleArn || undefined;
|
|
16
|
+
this.queryEnabled = Boolean(queryEnabled);
|
|
17
|
+
this.accessMode = accessMode;
|
|
18
|
+
Object.freeze(this);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export class SaasCurContext {
|
|
2
|
+
constructor({ tenant, config, client, credentialMode, ingestion }) {
|
|
3
|
+
this.tenant = String(tenant || "default").trim() || "default";
|
|
4
|
+
this.config = Object.freeze({ ...config });
|
|
5
|
+
this.client = client;
|
|
6
|
+
this.credentialMode = credentialMode;
|
|
7
|
+
this.ingestion = ingestion;
|
|
8
|
+
Object.freeze(this);
|
|
9
|
+
}
|
|
10
|
+
}
|
package/src/plugin.js
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { AwsOnboardingRepository } from "./repositories/aws-onboarding.repository.js";
|
|
2
|
+
import { BudgetRepository } from "./repositories/budget.repository.js";
|
|
3
|
+
import { BudgetAlertRepository } from "./repositories/budget-alert.repository.js";
|
|
4
|
+
import { CustomerAwsContextService } from "./services/customer-aws-context.service.js";
|
|
5
|
+
import { SaasAthenaContextService } from "./services/saas-athena-context.service.js";
|
|
6
|
+
import { BudgetService } from "./services/budget.service.js";
|
|
7
|
+
import { BudgetAlertService } from "./services/budget-alert.service.js";
|
|
8
|
+
import { CostExplorerService } from "./services/cost-explorer.service.js";
|
|
9
|
+
import { CurProviderService } from "./services/cur-provider.service.js";
|
|
10
|
+
import { CostController } from "./controllers/cost.controller.js";
|
|
11
|
+
import { BudgetController } from "./controllers/budget.controller.js";
|
|
12
|
+
import { createCostRouter } from "./routes/index.js";
|
|
13
|
+
import { installCostBudgetSchema } from "./schema/cost-budget.schema.js";
|
|
14
|
+
import { installCurDiscoverySchema } from "./cur-discovery/schema.js";
|
|
15
|
+
import { CurDiscoveryRepository } from "./cur-discovery/cur-discovery.repository.js";
|
|
16
|
+
import { CurDiscoveryAws } from "./cur-discovery/cur-discovery.aws.js";
|
|
17
|
+
import { CurDiscoveryService } from "./cur-discovery/cur-discovery.service.js";
|
|
18
|
+
import { CurDiscoveryWorker } from "./cur-discovery/cur-discovery.worker.js";
|
|
19
|
+
import { safeSchema } from "./lib/cost-utils.js";
|
|
20
|
+
|
|
21
|
+
export function createInsightCost({ app, db, apiBaseUri = "/api/v1", logger = console } = {}) {
|
|
22
|
+
if (!db) throw new Error("db is required");
|
|
23
|
+
const schema = safeSchema(process.env.DB_SCHEMA || "meyiconnect");
|
|
24
|
+
const qSchema = `"${schema}"`;
|
|
25
|
+
const defaultTenant = String(process.env.DEFAULT_TENANT_ID || "default").trim() || "default";
|
|
26
|
+
const onboardingRepository = new AwsOnboardingRepository({ db, schema });
|
|
27
|
+
const contextService = new CustomerAwsContextService({ repository: onboardingRepository, defaultTenant });
|
|
28
|
+
const budgetRepository = new BudgetRepository({ db, qSchema });
|
|
29
|
+
const budgetAlertRepository = new BudgetAlertRepository({ db, qSchema });
|
|
30
|
+
const budgetService = new BudgetService({ repository: budgetRepository });
|
|
31
|
+
const budgetAlertService = new BudgetAlertService({ repository: budgetAlertRepository });
|
|
32
|
+
const explorerService = new CostExplorerService();
|
|
33
|
+
const athenaContextService = new SaasAthenaContextService();
|
|
34
|
+
const curProvider = new CurProviderService({ athenaContextService, logger });
|
|
35
|
+
const discoveryRepository = new CurDiscoveryRepository({ db, schema, qSchema });
|
|
36
|
+
const discoveryAws = new CurDiscoveryAws();
|
|
37
|
+
const discoveryIntervalMs = Math.max(Number(process.env.COST_CUR_DISCOVERY_INTERVAL_MS || 3_600_000), 60_000);
|
|
38
|
+
const discoveryService = new CurDiscoveryService({
|
|
39
|
+
repository: discoveryRepository,
|
|
40
|
+
aws: discoveryAws,
|
|
41
|
+
athenaContextService,
|
|
42
|
+
logger,
|
|
43
|
+
intervalMs: discoveryIntervalMs,
|
|
44
|
+
});
|
|
45
|
+
const discoveryWorker = new CurDiscoveryWorker({ repository: discoveryRepository, service: discoveryService, logger });
|
|
46
|
+
const costController = new CostController({ contextService, explorerService, curProvider, discoveryRepository, logger });
|
|
47
|
+
const budgetController = new BudgetController({ contextService, budgetService, budgetAlertService });
|
|
48
|
+
const router = createCostRouter({ costController, budgetController }, logger);
|
|
49
|
+
let mounted = false;
|
|
50
|
+
|
|
51
|
+
return {
|
|
52
|
+
async install() {
|
|
53
|
+
await installCostBudgetSchema(db, qSchema);
|
|
54
|
+
await installCurDiscoverySchema(db, qSchema);
|
|
55
|
+
logger.log?.("[Cost] Database migration verified");
|
|
56
|
+
},
|
|
57
|
+
async start() {
|
|
58
|
+
if (app && !mounted) {
|
|
59
|
+
app.use(`${apiBaseUri}/cost`, router);
|
|
60
|
+
mounted = true;
|
|
61
|
+
}
|
|
62
|
+
discoveryWorker.start();
|
|
63
|
+
logger.log?.(`[Cost] Routes active at ${apiBaseUri}/cost/*`);
|
|
64
|
+
},
|
|
65
|
+
async stop() {
|
|
66
|
+
discoveryWorker.stop();
|
|
67
|
+
},
|
|
68
|
+
router,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export default createInsightCost;
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import { sql } from "drizzle-orm";
|
|
2
|
+
import { rows } from "../lib/cost-utils.js";
|
|
3
|
+
|
|
4
|
+
export class AwsOnboardingRepository {
|
|
5
|
+
constructor({ db, schema }) {
|
|
6
|
+
this.db = db;
|
|
7
|
+
this.schema = schema;
|
|
8
|
+
this.connectionsTable = `"${schema}".aws_connections`;
|
|
9
|
+
this.accountsTable = `"${schema}".aws_accounts`;
|
|
10
|
+
this.curConfigTable = `"${schema}".cost_cur_config`;
|
|
11
|
+
this.curDiscoveryTable = `"${schema}".cost_cur_discovery_jobs`;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
async findCostAccess(tenant) {
|
|
15
|
+
const relations = rows(await this.db.execute(sql`
|
|
16
|
+
SELECT
|
|
17
|
+
to_regclass(${`${this.schema}.aws_connections`}) AS connections_table,
|
|
18
|
+
to_regclass(${`${this.schema}.aws_accounts`}) AS accounts_table,
|
|
19
|
+
to_regclass(${`${this.schema}.cost_cur_config`}) AS cur_config_table,
|
|
20
|
+
to_regclass(${`${this.schema}.cost_cur_discovery_jobs`}) AS cur_discovery_table
|
|
21
|
+
`))[0] || {};
|
|
22
|
+
|
|
23
|
+
if (!relations.connections_table) return { accounts: [], meta: {} };
|
|
24
|
+
|
|
25
|
+
const connection = rows(await this.db.execute(sql`
|
|
26
|
+
SELECT
|
|
27
|
+
connection_id,
|
|
28
|
+
role_arn,
|
|
29
|
+
external_id
|
|
30
|
+
FROM ${sql.raw(this.connectionsTable)}
|
|
31
|
+
WHERE tenant_id::text = ${tenant}
|
|
32
|
+
AND status = 'CONNECTED'
|
|
33
|
+
AND plugins ? 'Cost'
|
|
34
|
+
AND COALESCE(verification_checks->>'costAccess', 'false') = 'true'
|
|
35
|
+
ORDER BY connected_at DESC NULLS LAST, created_at DESC
|
|
36
|
+
LIMIT 1
|
|
37
|
+
`))[0];
|
|
38
|
+
|
|
39
|
+
if (!connection) return { accounts: [], meta: {} };
|
|
40
|
+
|
|
41
|
+
const curConfig = relations.cur_config_table ? rows(await this.db.execute(sql`
|
|
42
|
+
SELECT export_arn, bucket, prefix, region, tenant_partition, status, last_data_at
|
|
43
|
+
FROM ${sql.raw(this.curConfigTable)}
|
|
44
|
+
WHERE tenant_id::text = ${tenant}
|
|
45
|
+
AND connection_id = ${connection.connection_id}
|
|
46
|
+
LIMIT 1
|
|
47
|
+
`))[0] || {} : {};
|
|
48
|
+
|
|
49
|
+
const curDiscovery = relations.cur_discovery_table ? rows(await this.db.execute(sql`
|
|
50
|
+
SELECT id, status, attempt_count, glue_database, glue_table, table_location,
|
|
51
|
+
cur_s3_uri, last_data_at, last_error, last_started_at, last_finished_at, next_run_at
|
|
52
|
+
FROM ${sql.raw(this.curDiscoveryTable)}
|
|
53
|
+
WHERE tenant_id = ${tenant}
|
|
54
|
+
AND connection_id = ${connection.connection_id}
|
|
55
|
+
LIMIT 1
|
|
56
|
+
`))[0] || {} : {};
|
|
57
|
+
|
|
58
|
+
const accounts = relations.accounts_table ? rows(await this.db.execute(sql`
|
|
59
|
+
SELECT account_id, account_name, status
|
|
60
|
+
FROM ${sql.raw(this.accountsTable)}
|
|
61
|
+
WHERE tenant_id::text = ${tenant}
|
|
62
|
+
AND connection_id = ${connection.connection_id}
|
|
63
|
+
AND LOWER(status) = 'active'
|
|
64
|
+
ORDER BY account_name
|
|
65
|
+
`)) : [];
|
|
66
|
+
|
|
67
|
+
return {
|
|
68
|
+
accounts: accounts.map((item) => ({
|
|
69
|
+
id: item.account_id,
|
|
70
|
+
name: item.account_name || item.account_id,
|
|
71
|
+
region: "global",
|
|
72
|
+
status: item.status,
|
|
73
|
+
})),
|
|
74
|
+
meta: {
|
|
75
|
+
payerRoleArn: connection.role_arn,
|
|
76
|
+
externalId: connection.external_id,
|
|
77
|
+
connectionId: connection.connection_id,
|
|
78
|
+
curExportArn: curConfig.export_arn,
|
|
79
|
+
curSourceBucket: curConfig.bucket,
|
|
80
|
+
curSourcePrefix: curConfig.prefix,
|
|
81
|
+
curSourceRegion: curConfig.region,
|
|
82
|
+
curTenantPartition: curConfig.tenant_partition || tenant,
|
|
83
|
+
curIngestionMode: "central",
|
|
84
|
+
curStatus: curConfig.status,
|
|
85
|
+
curLastDataAt: curConfig.last_data_at,
|
|
86
|
+
curDiscoveredTable: curDiscovery.status === "READY" ? curDiscovery.glue_table : undefined,
|
|
87
|
+
curDiscovery: curDiscovery.id ? {
|
|
88
|
+
jobId: String(curDiscovery.id),
|
|
89
|
+
status: curDiscovery.status,
|
|
90
|
+
attemptCount: Number(curDiscovery.attempt_count || 0),
|
|
91
|
+
database: curDiscovery.glue_database,
|
|
92
|
+
table: curDiscovery.glue_table,
|
|
93
|
+
tableLocation: curDiscovery.table_location,
|
|
94
|
+
curS3Uri: curDiscovery.cur_s3_uri,
|
|
95
|
+
lastDataAt: curDiscovery.last_data_at,
|
|
96
|
+
lastError: curDiscovery.last_error,
|
|
97
|
+
lastStartedAt: curDiscovery.last_started_at,
|
|
98
|
+
lastFinishedAt: curDiscovery.last_finished_at,
|
|
99
|
+
nextRunAt: curDiscovery.next_run_at,
|
|
100
|
+
} : null,
|
|
101
|
+
},
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
async updateCurStatus(tenant, status) {
|
|
106
|
+
const relation = rows(await this.db.execute(sql`
|
|
107
|
+
SELECT to_regclass(${`${this.schema}.cost_cur_config`}) AS cur_config_table
|
|
108
|
+
`))[0];
|
|
109
|
+
if (!relation?.cur_config_table) return;
|
|
110
|
+
const curStatus = status.ready
|
|
111
|
+
? "READY"
|
|
112
|
+
: status.state === "pending_data"
|
|
113
|
+
? "WAITING_FOR_DATA"
|
|
114
|
+
: status.state === "unavailable"
|
|
115
|
+
? "FAILED"
|
|
116
|
+
: "PROVISIONING";
|
|
117
|
+
await this.db.execute(sql`
|
|
118
|
+
UPDATE ${sql.raw(this.curConfigTable)}
|
|
119
|
+
SET status = ${curStatus},
|
|
120
|
+
last_data_at = ${status.lastDataAt || null},
|
|
121
|
+
verified_at = now(),
|
|
122
|
+
updated_at = now()
|
|
123
|
+
WHERE connection_id = (
|
|
124
|
+
SELECT connection_id
|
|
125
|
+
FROM ${sql.raw(this.connectionsTable)}
|
|
126
|
+
WHERE tenant_id::text = ${tenant}
|
|
127
|
+
AND status = 'CONNECTED'
|
|
128
|
+
AND plugins ? 'Cost'
|
|
129
|
+
ORDER BY connected_at DESC NULLS LAST, created_at DESC
|
|
130
|
+
LIMIT 1
|
|
131
|
+
)
|
|
132
|
+
`);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { sql } from "drizzle-orm";
|
|
2
|
+
import { rows } from "../lib/cost-utils.js";
|
|
3
|
+
|
|
4
|
+
export class BudgetAlertRepository {
|
|
5
|
+
constructor({ db, qSchema }) {
|
|
6
|
+
this.db = db;
|
|
7
|
+
this.budgetTable = `${qSchema}.cost_budgets`;
|
|
8
|
+
this.dismissalTable = `${qSchema}.cost_budget_alert_dismissals`;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
list(tenantId, userId, period) {
|
|
12
|
+
return this.db.execute(sql`
|
|
13
|
+
SELECT budget_id, period, status, dismissed_at
|
|
14
|
+
FROM ${sql.raw(this.dismissalTable)}
|
|
15
|
+
WHERE tenant_id = ${tenantId} AND user_id = ${userId} AND period = ${period}
|
|
16
|
+
`).then(rows);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
async budgetExists(tenantId, budgetId) {
|
|
20
|
+
const result = rows(await this.db.execute(sql`
|
|
21
|
+
SELECT id FROM ${sql.raw(this.budgetTable)}
|
|
22
|
+
WHERE id = ${budgetId} AND tenant_id = ${tenantId}
|
|
23
|
+
LIMIT 1
|
|
24
|
+
`));
|
|
25
|
+
return result.length > 0;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
async dismiss({ tenantId, userId, budgetId, period, status }) {
|
|
29
|
+
await this.db.execute(sql`
|
|
30
|
+
INSERT INTO ${sql.raw(this.dismissalTable)}
|
|
31
|
+
(tenant_id, user_id, budget_id, period, status)
|
|
32
|
+
VALUES
|
|
33
|
+
(${tenantId}, ${userId}, ${budgetId}, ${period}, ${status})
|
|
34
|
+
ON CONFLICT (tenant_id, user_id, budget_id, period, status) DO NOTHING
|
|
35
|
+
`);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { sql } from "drizzle-orm";
|
|
2
|
+
import { rows } from "../lib/cost-utils.js";
|
|
3
|
+
|
|
4
|
+
export class BudgetRepository {
|
|
5
|
+
constructor({ db, qSchema }) {
|
|
6
|
+
this.db = db;
|
|
7
|
+
this.table = `${qSchema}.cost_budgets`;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
list(tenantId) {
|
|
11
|
+
return this.db.execute(sql`
|
|
12
|
+
SELECT * FROM ${sql.raw(this.table)}
|
|
13
|
+
WHERE tenant_id = ${tenantId}
|
|
14
|
+
ORDER BY created_at DESC
|
|
15
|
+
`).then(rows);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
async create({ id, tenantId, name, amount, currency, period, alertThreshold }) {
|
|
19
|
+
await this.db.execute(sql`
|
|
20
|
+
INSERT INTO ${sql.raw(this.table)}
|
|
21
|
+
(id, tenant_id, name, amount, currency, period, alert_threshold)
|
|
22
|
+
VALUES
|
|
23
|
+
(${id}, ${tenantId}, ${name}, ${amount}, ${currency}, ${period}, ${alertThreshold})
|
|
24
|
+
`);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
async delete(tenantId, id) {
|
|
28
|
+
await this.db.execute(sql`
|
|
29
|
+
DELETE FROM ${sql.raw(this.table)}
|
|
30
|
+
WHERE id = ${id} AND tenant_id = ${tenantId}
|
|
31
|
+
`);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { Router } from "express";
|
|
2
|
+
|
|
3
|
+
export function createCostRouter({ costController, budgetController }, logger = console) {
|
|
4
|
+
const router = Router();
|
|
5
|
+
router.get("/accounts", costController.accounts.bind(costController));
|
|
6
|
+
router.get("/overview", costController.overview.bind(costController));
|
|
7
|
+
router.get("/filter-options", costController.filterOptions.bind(costController));
|
|
8
|
+
router.get("/reports", costController.reports.bind(costController));
|
|
9
|
+
router.get("/tags", costController.tags.bind(costController));
|
|
10
|
+
router.get("/data-status", costController.dataStatus.bind(costController));
|
|
11
|
+
router.get("/cur-discovery", costController.curDiscovery.bind(costController));
|
|
12
|
+
router.get("/budgets", budgetController.list.bind(budgetController));
|
|
13
|
+
router.post("/budgets", budgetController.create.bind(budgetController));
|
|
14
|
+
router.delete("/budgets/:id", budgetController.delete.bind(budgetController));
|
|
15
|
+
router.get("/budget-alert-dismissals", budgetController.listDismissals.bind(budgetController));
|
|
16
|
+
router.post("/budget-alert-dismissals", budgetController.dismissAlert.bind(budgetController));
|
|
17
|
+
router.use((error, _req, res, _next) => {
|
|
18
|
+
logger.error?.("[Cost] Request failed", error);
|
|
19
|
+
res.status(error.statusCode || 502).json({ error: error.message || "AWS Cost Explorer request failed", code: error.name });
|
|
20
|
+
});
|
|
21
|
+
return router;
|
|
22
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { sql } from "drizzle-orm";
|
|
2
|
+
|
|
3
|
+
export async function installCostBudgetSchema(db, qSchema) {
|
|
4
|
+
await db.execute(sql.raw(`CREATE SCHEMA IF NOT EXISTS ${qSchema}`));
|
|
5
|
+
await db.execute(sql.raw(`CREATE TABLE IF NOT EXISTS ${qSchema}.cost_budgets (id text PRIMARY KEY, tenant_id text NOT NULL, name text NOT NULL, amount numeric(18,2) NOT NULL, currency text NOT NULL DEFAULT 'USD', period text NOT NULL DEFAULT 'monthly', spent numeric(18,2) NOT NULL DEFAULT 0, alert_threshold numeric(5,2) NOT NULL DEFAULT 80, created_at timestamptz NOT NULL DEFAULT now(), updated_at timestamptz NOT NULL DEFAULT now(), UNIQUE (tenant_id, name))`));
|
|
6
|
+
await db.execute(sql.raw(`CREATE INDEX IF NOT EXISTS cost_budgets_tenant_idx ON ${qSchema}.cost_budgets (tenant_id)`));
|
|
7
|
+
await db.execute(sql.raw(`CREATE TABLE IF NOT EXISTS ${qSchema}.cost_budget_alert_dismissals (tenant_id text NOT NULL, user_id text NOT NULL, budget_id text NOT NULL REFERENCES ${qSchema}.cost_budgets(id) ON DELETE CASCADE, period text NOT NULL, status text NOT NULL CHECK (status IN ('near_limit', 'over_budget')), dismissed_at timestamptz NOT NULL DEFAULT now(), PRIMARY KEY (tenant_id, user_id, budget_id, period, status))`));
|
|
8
|
+
await db.execute(sql.raw(`CREATE INDEX IF NOT EXISTS cost_budget_alert_dismissals_user_idx ON ${qSchema}.cost_budget_alert_dismissals (tenant_id, user_id, period)`));
|
|
9
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { CustomerAwsContextService as AwsContextService } from "./customer-aws-context.service.js";
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { BudgetAlertDismissal } from "../models/budget.model.js";
|
|
2
|
+
const validPeriod = (value) => /^\d{4}-(0[1-9]|1[0-2])$/.test(String(value || ""));
|
|
3
|
+
const validStatus = (value) => value === "near_limit" || value === "over_budget";
|
|
4
|
+
|
|
5
|
+
export class BudgetAlertService {
|
|
6
|
+
constructor({ repository }) {
|
|
7
|
+
this.repository = repository;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
async listDismissals(tenantId, userId, period) {
|
|
11
|
+
if (!userId) {
|
|
12
|
+
const error = new Error("Authenticated user is required");
|
|
13
|
+
error.statusCode = 401;
|
|
14
|
+
throw error;
|
|
15
|
+
}
|
|
16
|
+
if (!validPeriod(period)) {
|
|
17
|
+
const error = new Error("period must use YYYY-MM format");
|
|
18
|
+
error.statusCode = 400;
|
|
19
|
+
throw error;
|
|
20
|
+
}
|
|
21
|
+
const result = await this.repository.list(tenantId, userId, period);
|
|
22
|
+
return result.map((item) => new BudgetAlertDismissal({ budgetId: item.budget_id, period: item.period, status: item.status, dismissedAt: item.dismissed_at }));
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
async dismiss(tenantId, userId, payload = {}) {
|
|
26
|
+
const budgetId = String(payload.budgetId || "").trim();
|
|
27
|
+
const period = String(payload.period || "").trim();
|
|
28
|
+
const status = String(payload.status || "").trim();
|
|
29
|
+
if (!userId) {
|
|
30
|
+
const error = new Error("Authenticated user is required");
|
|
31
|
+
error.statusCode = 401;
|
|
32
|
+
throw error;
|
|
33
|
+
}
|
|
34
|
+
if (!budgetId || !validPeriod(period) || !validStatus(status)) {
|
|
35
|
+
const error = new Error("budgetId, a YYYY-MM period, and a valid status are required");
|
|
36
|
+
error.statusCode = 400;
|
|
37
|
+
throw error;
|
|
38
|
+
}
|
|
39
|
+
if (!await this.repository.budgetExists(tenantId, budgetId)) {
|
|
40
|
+
const error = new Error("Budget not found");
|
|
41
|
+
error.statusCode = 404;
|
|
42
|
+
throw error;
|
|
43
|
+
}
|
|
44
|
+
await this.repository.dismiss({ tenantId, userId, budgetId, period, status });
|
|
45
|
+
return new BudgetAlertDismissal({ budgetId, period, status, dismissed: true });
|
|
46
|
+
}
|
|
47
|
+
}
|