@meyicloud/meyi-cost-server 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +11 -25
- package/README.md +28 -53
- package/index.js +0 -1
- package/package.json +3 -4
- package/src/controllers/cost-analysis.controller.js +11 -1
- package/src/plugin.js +8 -14
- package/src/repositories/aws-onboarding.repository.js +6 -4
- package/src/repositories/cost-analysis-report.repository.js +21 -3
- package/src/routes/index.js +1 -0
- package/src/schema/cost-analysis-report.schema.js +10 -0
- package/src/services/cost-analyser.service.js +158 -0
- package/src/services/cost-analysis-report.service.js +21 -5
- package/src/services/cost-report-artifact.service.js +22 -0
- package/src/workers/cost-analysis-report.worker.js +10 -9
- package/src/lib/cost-analysis.js +0 -116
- package/src/lib/llm-provider.js +0 -239
- package/src/repositories/cost-analysis.repository.js +0 -50
- package/src/schema/cost-analysis.schema.js +0 -7
- package/src/services/cost-analysis-data.service.js +0 -39
- package/src/services/cost-analysis-pdf.service.js +0 -42
- package/src/services/cost-analysis.service.js +0 -190
package/AGENTS.md
CHANGED
|
@@ -35,23 +35,13 @@ Keep HTTP concerns in routes/controllers, AWS and database behavior in services,
|
|
|
35
35
|
schema creation in `schema`, and pure calculations in `lib`. `index.js` and
|
|
36
36
|
`cur.js` must remain thin compatibility entry points.
|
|
37
37
|
|
|
38
|
-
##
|
|
38
|
+
## External cost analyser
|
|
39
39
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
"AWS Bedrock" resolve correctly.
|
|
46
|
-
|
|
47
|
-
`CostAnalysisService.resolveProvider(tenantId)` picks between a host-supplied
|
|
48
|
-
`providerResolver` and the `COST_AI_*` environment defaults. Two rules matter:
|
|
49
|
-
the analysis fingerprint is computed from the **effective** model so switching
|
|
50
|
-
provider cannot serve a cached answer from the previous one, and a resolver
|
|
51
|
-
failure degrades to the environment rather than failing the request.
|
|
52
|
-
|
|
53
|
-
When adding a provider, add it here and to `test/llm-provider.test.js`; no
|
|
54
|
-
other file should need to change.
|
|
40
|
+
The backend package never calls an LLM directly. `CostAnalysisReportWorker`
|
|
41
|
+
claims due tenant schedules and delegates execution to `CostAnalyserService`,
|
|
42
|
+
which starts one tenant-scoped `meyi-cost-ai-analyser` Fargate task. The task
|
|
43
|
+
writes canonical Markdown and derived PDF artifacts to private S3. Keep ECS
|
|
44
|
+
task overrides tenant-scoped and never log the customer role external ID.
|
|
55
45
|
|
|
56
46
|
## Request and data flow
|
|
57
47
|
|
|
@@ -62,21 +52,17 @@ flowchart LR
|
|
|
62
52
|
Enabled --> Router[/api/v1/cost router]
|
|
63
53
|
Router --> Controller[Cost controller]
|
|
64
54
|
Controller --> Customer[Customer AWS context service]
|
|
65
|
-
|
|
66
|
-
Controller --> Source{Configured source}
|
|
67
|
-
Source -->|CUR| SaaS[SaaS Athena context service]
|
|
55
|
+
Controller --> SaaS[SaaS Athena context service]
|
|
68
56
|
SaaS --> Athena[Central tenant CUR partition]
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
Athena --> Controller
|
|
73
|
-
CE --> Controller
|
|
57
|
+
Athena --> Scheduler[Scheduled report worker]
|
|
58
|
+
Scheduler --> ECS[Cost analyser Fargate task]
|
|
59
|
+
ECS --> Artifacts[Private Markdown and PDF S3 objects]
|
|
74
60
|
```
|
|
75
61
|
|
|
76
62
|
The plugin currently owns these route groups under `${apiBaseUri}/cost`:
|
|
77
63
|
|
|
78
64
|
- `GET /accounts`, `/data-status`, `/overview`, `/filter-options`, `/reports`, `/tags`
|
|
79
|
-
- `GET`/`PUT /analysis/schedule`, `GET /analysis/reports`, and tenant-scoped PDF downloads
|
|
65
|
+
- `GET`/`PUT /analysis/schedule`, `GET /analysis/reports`, and tenant-scoped Markdown/PDF downloads
|
|
80
66
|
- `GET /budgets` and `POST /budgets`
|
|
81
67
|
- `DELETE /budgets/:id`
|
|
82
68
|
- `GET /budget-alert-dismissals` and `POST /budget-alert-dismissals`
|
package/README.md
CHANGED
|
@@ -258,12 +258,12 @@ SaaS Athena resources.
|
|
|
258
258
|
|
|
259
259
|
### AI analysis variables
|
|
260
260
|
|
|
261
|
-
AI
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
complete
|
|
266
|
-
|
|
261
|
+
AI reports are generated by a separate `meyi-cost-ai-analyser` Fargate task.
|
|
262
|
+
The backend only schedules jobs, launches and monitors the task, and serves
|
|
263
|
+
tenant-scoped artifacts. It does not invoke Bedrock or generate PDFs itself.
|
|
264
|
+
Daily reports cover the previous complete day, weekly reports cover the
|
|
265
|
+
previous seven complete days, and monthly reports cover the previous calendar
|
|
266
|
+
month in the configured timezone.
|
|
267
267
|
|
|
268
268
|
The authenticated report API is:
|
|
269
269
|
|
|
@@ -274,6 +274,7 @@ The authenticated report API is:
|
|
|
274
274
|
| `GET` | `/analysis/reports` | List tenant report history without PDF bytes. |
|
|
275
275
|
| `GET` | `/analysis/reports/:id` | Read one tenant report. |
|
|
276
276
|
| `GET` | `/analysis/reports/:id/pdf` | Download a completed report PDF. |
|
|
277
|
+
| `GET` | `/analysis/reports/:id/markdown` | Download the canonical Markdown report. |
|
|
277
278
|
|
|
278
279
|
Schedule and report rows live in `cost_ai_report_schedules` and
|
|
279
280
|
`cost_ai_reports`. The worker atomically claims due schedules with a lease, so
|
|
@@ -281,56 +282,30 @@ a restarted task can resume future work without relying on in-memory timers.
|
|
|
281
282
|
|
|
282
283
|
| Variable | Default | Required | Purpose |
|
|
283
284
|
| --- | --- | --- | --- |
|
|
284
|
-
| `COST_AI_ENABLED` | `false` | No | Enables the
|
|
285
|
-
| `
|
|
286
|
-
| `
|
|
287
|
-
| `
|
|
288
|
-
| `
|
|
289
|
-
| `
|
|
290
|
-
| `
|
|
285
|
+
| `COST_AI_ENABLED` | `false` | No | Enables the schedule worker and external analyser integration. |
|
|
286
|
+
| `COST_AI_ANALYSER_REGION` | `AWS_REGION` | When enabled | ECS and report-bucket region. |
|
|
287
|
+
| `COST_AI_ANALYSER_CLUSTER` | Empty | When enabled | ECS cluster ARN or name. |
|
|
288
|
+
| `COST_AI_ANALYSER_TASK_DEFINITION` | Empty | When enabled | Fargate task definition ARN. |
|
|
289
|
+
| `COST_AI_ANALYSER_CONTAINER_NAME` | `cost-ai-analyser` | No | Container override target. |
|
|
290
|
+
| `COST_AI_ANALYSER_SUBNETS` | Empty | When enabled | Comma-separated private subnet IDs. |
|
|
291
|
+
| `COST_AI_ANALYSER_SECURITY_GROUPS` | Empty | When enabled | Comma-separated task security groups. |
|
|
292
|
+
| `COST_AI_ANALYSER_REPORT_BUCKET` | Empty | When enabled | Private S3 Markdown/PDF artifact bucket. |
|
|
293
|
+
| `COST_AI_ANALYSER_BEDROCK_REGION` | Empty | When enabled | Bedrock region passed to the analyser task. |
|
|
294
|
+
| `COST_AI_ANALYSER_BEDROCK_MODEL_ID` | Empty | When enabled | Model passed only to the analyser task. |
|
|
295
|
+
| `COST_AI_ANALYSER_TARGET_REGIONS` | `AWS_REGION` | No | Customer regions inspected for resource detail. |
|
|
296
|
+
| `COST_AI_ANALYSER_TOP_N_SERVICES` | `10` | No | Maximum high-cost service agents run per report. |
|
|
297
|
+
| `COST_AI_ANALYSER_POLL_INTERVAL_MS` | `15000` | No | ECS task status polling interval. |
|
|
298
|
+
| `COST_AI_ANALYSER_TIMEOUT_MS` | `2700000` | No | Maximum external task duration. |
|
|
291
299
|
| `COST_AI_REPORT_POLL_INTERVAL_MS` | `60000` | No | How often the background worker claims due schedules; minimum 10 seconds. |
|
|
292
|
-
| `COST_AI_REPORT_LEASE_MS` | `
|
|
300
|
+
| `COST_AI_REPORT_LEASE_MS` | `3600000` | No | Claim lease used to recover from an interrupted worker. |
|
|
293
301
|
| `COST_AI_REPORT_BATCH_SIZE` | `5` | No | Maximum schedules claimed by one worker tick; clamped from 1 to 25. |
|
|
294
302
|
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
```js
|
|
302
|
-
createInsightCost({
|
|
303
|
-
app, db,
|
|
304
|
-
providerResolver: async (tenantId) => ({
|
|
305
|
-
provider: "anthropic", // bedrock | anthropic | openai
|
|
306
|
-
modelId: "claude-sonnet-4-5",
|
|
307
|
-
apiKey: "...", // required for anthropic and openai
|
|
308
|
-
region: "ap-south-2", // bedrock only
|
|
309
|
-
accessKeyId: "...", // bedrock only; omit to use ambient credentials
|
|
310
|
-
secretAccessKey: "...",
|
|
311
|
-
}),
|
|
312
|
-
})
|
|
313
|
-
```
|
|
314
|
-
|
|
315
|
-
Returning `null` falls back to the environment configuration, so a host that
|
|
316
|
-
does not pass a resolver behaves exactly as before. A resolver that throws is
|
|
317
|
-
logged and also falls back, so a credential-store outage degrades the feature
|
|
318
|
-
rather than failing the request.
|
|
319
|
-
|
|
320
|
-
Why this exists: Bedrock authenticates with SigV4 and is subject to
|
|
321
|
-
account-level Anthropic model access. The key-based providers are not, so a
|
|
322
|
-
deployment blocked on Bedrock model access can still run analysis. It also
|
|
323
|
-
lets a customer bring their own model account.
|
|
324
|
-
|
|
325
|
-
`GET /analysis/status` reports the provider that would serve the caller's
|
|
326
|
-
tenant, including a `source` of `tenant-configuration` or `environment`.
|
|
327
|
-
**Host applications are responsible for encrypting stored credentials.**
|
|
328
|
-
|
|
329
|
-
The runtime role needs `bedrock:InvokeModel` for the selected model or
|
|
330
|
-
inference profile. Analysis results are stored in the tenant-scoped
|
|
331
|
-
`cost_ai_analyses` table and scheduled PDF jobs are stored in
|
|
332
|
-
`cost_ai_reports`. Model or permission failures mark only that scheduled report
|
|
333
|
-
as failed and do not change CUR, standard reports, onboarding, or budgets.
|
|
303
|
+
`GET /analysis/status` reports `meyi-cost-ai-analyser` with an `ecs-task`
|
|
304
|
+
source. The analyser uses the shared ECS task role for Athena, S3, Bedrock, and
|
|
305
|
+
cross-account role assumption. New reports store task and private artifact
|
|
306
|
+
references in `cost_ai_reports`; existing database-backed PDF rows remain
|
|
307
|
+
downloadable for compatibility. Model or task failures mark only that report as
|
|
308
|
+
failed and do not change CUR, standard reports, or onboarding.
|
|
334
309
|
|
|
335
310
|
### CUR/Athena variables
|
|
336
311
|
|
package/index.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@meyicloud/meyi-cost-server",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.6.0",
|
|
4
4
|
"description": "Tenant-aware AWS CUR and Athena cost plugin server for MeyiConnect",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./index.js",
|
|
@@ -19,15 +19,14 @@
|
|
|
19
19
|
],
|
|
20
20
|
"dependencies": {
|
|
21
21
|
"@aws-sdk/client-athena": "^3.850.0",
|
|
22
|
-
"@aws-sdk/client-
|
|
22
|
+
"@aws-sdk/client-ecs": "^3.850.0",
|
|
23
23
|
"@aws-sdk/client-glue": "^3.850.0",
|
|
24
24
|
"@aws-sdk/client-s3": "^3.850.0",
|
|
25
25
|
"@aws-sdk/client-sts": "^3.850.0",
|
|
26
26
|
"@aws-sdk/credential-providers": "^3.850.0",
|
|
27
27
|
"drizzle-orm": "^0.44.7",
|
|
28
28
|
"express": "^4.21.1",
|
|
29
|
-
"luxon": "^3.7.2"
|
|
30
|
-
"pdfkit": "^0.17.2"
|
|
29
|
+
"luxon": "^3.7.2"
|
|
31
30
|
},
|
|
32
31
|
"peerDependencies": {
|
|
33
32
|
"pg": ">=8"
|
|
@@ -12,7 +12,7 @@ export class CostAnalysisController {
|
|
|
12
12
|
|
|
13
13
|
async latest(req, res, next) {
|
|
14
14
|
try {
|
|
15
|
-
return res.json({ analysis: await this.
|
|
15
|
+
return res.json({ analysis: await this.reportService.latest(req) });
|
|
16
16
|
} catch (error) { return next(error); }
|
|
17
17
|
}
|
|
18
18
|
|
|
@@ -45,4 +45,14 @@ export class CostAnalysisController {
|
|
|
45
45
|
return res.send(report.data);
|
|
46
46
|
} catch (error) { return next(error); }
|
|
47
47
|
}
|
|
48
|
+
|
|
49
|
+
async downloadMarkdown(req, res, next) {
|
|
50
|
+
try {
|
|
51
|
+
const report = await this.reportService.markdown(req, req.params.id);
|
|
52
|
+
res.setHeader("Content-Type", "text/markdown; charset=utf-8");
|
|
53
|
+
res.setHeader("Content-Disposition", `attachment; filename="${report.filename}"`);
|
|
54
|
+
res.setHeader("Content-Length", report.data.length);
|
|
55
|
+
return res.send(report.data);
|
|
56
|
+
} catch (error) { return next(error); }
|
|
57
|
+
}
|
|
48
58
|
}
|
package/src/plugin.js
CHANGED
|
@@ -11,13 +11,11 @@ import { BudgetController } from "./controllers/budget.controller.js";
|
|
|
11
11
|
import { CostAnalysisController } from "./controllers/cost-analysis.controller.js";
|
|
12
12
|
import { createCostRouter } from "./routes/index.js";
|
|
13
13
|
import { installCostBudgetSchema } from "./schema/cost-budget.schema.js";
|
|
14
|
-
import { installCostAnalysisSchema } from "./schema/cost-analysis.schema.js";
|
|
15
14
|
import { installCostAnalysisReportSchema } from "./schema/cost-analysis-report.schema.js";
|
|
16
|
-
import { CostAnalysisRepository } from "./repositories/cost-analysis.repository.js";
|
|
17
15
|
import { CostAnalysisReportRepository } from "./repositories/cost-analysis-report.repository.js";
|
|
18
|
-
import {
|
|
19
|
-
import { CostAnalysisService } from "./services/cost-analysis.service.js";
|
|
16
|
+
import { CostAnalyserService } from "./services/cost-analyser.service.js";
|
|
20
17
|
import { CostAnalysisReportService } from "./services/cost-analysis-report.service.js";
|
|
18
|
+
import { CostReportArtifactService } from "./services/cost-report-artifact.service.js";
|
|
21
19
|
import { CostAnalysisReportWorker } from "./workers/cost-analysis-report.worker.js";
|
|
22
20
|
import { installCurDiscoverySchema } from "./cur-discovery/schema.js";
|
|
23
21
|
import { CurDiscoveryRepository } from "./cur-discovery/cur-discovery.repository.js";
|
|
@@ -26,7 +24,7 @@ import { CurDiscoveryService } from "./cur-discovery/cur-discovery.service.js";
|
|
|
26
24
|
import { CurDiscoveryWorker } from "./cur-discovery/cur-discovery.worker.js";
|
|
27
25
|
import { safeSchema } from "./lib/cost-utils.js";
|
|
28
26
|
|
|
29
|
-
export function createInsightCost({ app, db, apiBaseUri = "/api/v1", logger = console
|
|
27
|
+
export function createInsightCost({ app, db, apiBaseUri = "/api/v1", logger = console } = {}) {
|
|
30
28
|
if (!db) throw new Error("db is required");
|
|
31
29
|
const schema = safeSchema(process.env.DB_SCHEMA || "meyiconnect");
|
|
32
30
|
const qSchema = `"${schema}"`;
|
|
@@ -35,7 +33,6 @@ export function createInsightCost({ app, db, apiBaseUri = "/api/v1", logger = co
|
|
|
35
33
|
const contextService = new CustomerAwsContextService({ repository: onboardingRepository, defaultTenant });
|
|
36
34
|
const budgetRepository = new BudgetRepository({ db, qSchema });
|
|
37
35
|
const budgetAlertRepository = new BudgetAlertRepository({ db, qSchema });
|
|
38
|
-
const costAnalysisRepository = new CostAnalysisRepository({ db, qSchema });
|
|
39
36
|
const costAnalysisReportRepository = new CostAnalysisReportRepository({ db, qSchema });
|
|
40
37
|
const budgetService = new BudgetService({ repository: budgetRepository });
|
|
41
38
|
const budgetAlertService = new BudgetAlertService({ repository: budgetAlertRepository });
|
|
@@ -54,20 +51,17 @@ export function createInsightCost({ app, db, apiBaseUri = "/api/v1", logger = co
|
|
|
54
51
|
const discoveryWorker = new CurDiscoveryWorker({ repository: discoveryRepository, service: discoveryService, logger });
|
|
55
52
|
const costController = new CostController({ contextService, curProvider, discoveryRepository, logger });
|
|
56
53
|
const budgetController = new BudgetController({ contextService, budgetService, budgetAlertService });
|
|
57
|
-
const
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
const
|
|
61
|
-
const
|
|
62
|
-
const costAnalysisReportWorker = new CostAnalysisReportWorker({ repository: costAnalysisReportRepository, analysisService: costAnalysisService, logger });
|
|
63
|
-
const costAnalysisController = new CostAnalysisController({ service: costAnalysisService, reportService: costAnalysisReportService });
|
|
54
|
+
const costAnalyserService = new CostAnalyserService({ contextService, athenaContextService, logger });
|
|
55
|
+
const artifactService = new CostReportArtifactService();
|
|
56
|
+
const costAnalysisReportService = new CostAnalysisReportService({ contextService, repository: costAnalysisReportRepository, artifactService });
|
|
57
|
+
const costAnalysisReportWorker = new CostAnalysisReportWorker({ repository: costAnalysisReportRepository, analyserService: costAnalyserService, logger });
|
|
58
|
+
const costAnalysisController = new CostAnalysisController({ service: costAnalyserService, reportService: costAnalysisReportService });
|
|
64
59
|
const router = createCostRouter({ costController, budgetController, costAnalysisController }, logger);
|
|
65
60
|
let mounted = false;
|
|
66
61
|
|
|
67
62
|
return {
|
|
68
63
|
async install() {
|
|
69
64
|
await installCostBudgetSchema(db, qSchema);
|
|
70
|
-
await installCostAnalysisSchema(db, qSchema);
|
|
71
65
|
await installCostAnalysisReportSchema(db, qSchema);
|
|
72
66
|
await installCurDiscoverySchema(db, qSchema);
|
|
73
67
|
logger.log?.("[Cost] Database migration verified");
|
|
@@ -24,9 +24,10 @@ export class AwsOnboardingRepository {
|
|
|
24
24
|
|
|
25
25
|
const connection = rows(await this.db.execute(sql`
|
|
26
26
|
SELECT
|
|
27
|
-
connection_id,
|
|
28
|
-
role_arn,
|
|
29
|
-
external_id
|
|
27
|
+
connection_id,
|
|
28
|
+
role_arn,
|
|
29
|
+
external_id,
|
|
30
|
+
management_account_id
|
|
30
31
|
FROM ${sql.raw(this.connectionsTable)}
|
|
31
32
|
WHERE tenant_id::text = ${tenant}
|
|
32
33
|
AND status = 'CONNECTED'
|
|
@@ -72,7 +73,8 @@ export class AwsOnboardingRepository {
|
|
|
72
73
|
status: item.status,
|
|
73
74
|
})),
|
|
74
75
|
meta: {
|
|
75
|
-
payerRoleArn: connection.role_arn,
|
|
76
|
+
payerRoleArn: connection.role_arn,
|
|
77
|
+
managementAccountId: connection.management_account_id,
|
|
76
78
|
externalId: connection.external_id,
|
|
77
79
|
connectionId: connection.connection_id,
|
|
78
80
|
curExportArn: curConfig.export_arn,
|
|
@@ -71,6 +71,22 @@ export class CostAnalysisReportRepository {
|
|
|
71
71
|
return rows(await this.db.execute(sql`UPDATE ${sql.raw(this.reports)} SET status = 'RUNNING', attempt_count = attempt_count + 1, started_at = now(), error = NULL, updated_at = now() WHERE id = ${id} AND status = 'PENDING' RETURNING *`))[0] || null;
|
|
72
72
|
}
|
|
73
73
|
|
|
74
|
+
async setTaskArn(id, taskArn) {
|
|
75
|
+
await this.db.execute(sql`UPDATE ${sql.raw(this.reports)} SET ecs_task_arn = ${taskArn}, updated_at = now() WHERE id = ${id}`);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
async completeExternal(id, result) {
|
|
79
|
+
await this.db.execute(sql`
|
|
80
|
+
UPDATE ${sql.raw(this.reports)} SET status = 'COMPLETED', ecs_task_arn = ${result.taskArn},
|
|
81
|
+
model_id = ${result.modelId}, data_source = ${result.dataSource}, facts = ${JSON.stringify(result.facts || {})}::jsonb,
|
|
82
|
+
result = ${JSON.stringify(result.summary || {})}::jsonb, artifact_bucket = ${result.artifactBucket},
|
|
83
|
+
markdown_key = ${result.markdownKey}, markdown_size = ${Number(result.markdownSize || 0)},
|
|
84
|
+
pdf_key = ${result.pdfKey}, pdf_size = ${Number(result.pdfSize || 0)}, pdf_data = NULL,
|
|
85
|
+
completed_at = now(), updated_at = now()
|
|
86
|
+
WHERE id = ${id}
|
|
87
|
+
`);
|
|
88
|
+
}
|
|
89
|
+
|
|
74
90
|
async complete(id, analysis, pdf) {
|
|
75
91
|
await this.db.execute(sql`
|
|
76
92
|
UPDATE ${sql.raw(this.reports)} SET status = 'COMPLETED', model_id = ${analysis.modelId}, data_source = ${analysis.dataSource || null},
|
|
@@ -82,13 +98,14 @@ export class CostAnalysisReportRepository {
|
|
|
82
98
|
}
|
|
83
99
|
|
|
84
100
|
async fail(id, error) {
|
|
85
|
-
await this.db.execute(sql`UPDATE ${sql.raw(this.reports)} SET status = 'FAILED', error = ${String(error?.message || error).slice(0, 2000)}, completed_at = now(), updated_at = now() WHERE id = ${id}`);
|
|
101
|
+
await this.db.execute(sql`UPDATE ${sql.raw(this.reports)} SET status = 'FAILED', ecs_task_arn = COALESCE(${error?.taskArn || null}, ecs_task_arn), error = ${String(error?.message || error).slice(0, 2000)}, completed_at = now(), updated_at = now() WHERE id = ${id}`);
|
|
86
102
|
}
|
|
87
103
|
|
|
88
104
|
async list(tenantId, limit = 100) {
|
|
89
105
|
return rows(await this.db.execute(sql`
|
|
90
106
|
SELECT id, schedule_frequency, scheduled_for, period_start, period_end, status,
|
|
91
|
-
attempt_count, model_id, data_source, facts, result,
|
|
107
|
+
attempt_count, model_id, data_source, facts, result, artifact_bucket,
|
|
108
|
+
markdown_key, markdown_size, pdf_key, pdf_size, error,
|
|
92
109
|
started_at, completed_at, created_at
|
|
93
110
|
FROM ${sql.raw(this.reports)} WHERE tenant_id = ${tenantId}
|
|
94
111
|
ORDER BY scheduled_for DESC LIMIT ${Math.min(Math.max(Number(limit) || 100, 1), 250)}
|
|
@@ -101,7 +118,8 @@ export class CostAnalysisReportRepository {
|
|
|
101
118
|
}
|
|
102
119
|
return rows(await this.db.execute(sql`
|
|
103
120
|
SELECT id, schedule_frequency, scheduled_for, period_start, period_end, status,
|
|
104
|
-
attempt_count, model_id, data_source, facts, result,
|
|
121
|
+
attempt_count, model_id, data_source, facts, result, artifact_bucket,
|
|
122
|
+
markdown_key, markdown_size, pdf_key, pdf_size, error,
|
|
105
123
|
started_at, completed_at, created_at
|
|
106
124
|
FROM ${sql.raw(this.reports)} WHERE tenant_id = ${tenantId} AND id = ${id} LIMIT 1
|
|
107
125
|
`))[0] || null;
|
package/src/routes/index.js
CHANGED
|
@@ -16,6 +16,7 @@ export function createCostRouter({ costController, budgetController, costAnalysi
|
|
|
16
16
|
router.get("/analysis/reports", costAnalysisController.reports.bind(costAnalysisController));
|
|
17
17
|
router.get("/analysis/reports/:id", costAnalysisController.report.bind(costAnalysisController));
|
|
18
18
|
router.get("/analysis/reports/:id/pdf", costAnalysisController.downloadPdf.bind(costAnalysisController));
|
|
19
|
+
router.get("/analysis/reports/:id/markdown", costAnalysisController.downloadMarkdown.bind(costAnalysisController));
|
|
19
20
|
router.get("/budgets", budgetController.list.bind(budgetController));
|
|
20
21
|
router.post("/budgets", budgetController.create.bind(budgetController));
|
|
21
22
|
router.delete("/budgets/:id", budgetController.delete.bind(budgetController));
|
|
@@ -31,6 +31,11 @@ export async function installCostAnalysisReportSchema(db, qSchema) {
|
|
|
31
31
|
result jsonb,
|
|
32
32
|
pdf_data bytea,
|
|
33
33
|
pdf_size integer,
|
|
34
|
+
ecs_task_arn text,
|
|
35
|
+
artifact_bucket text,
|
|
36
|
+
markdown_key text,
|
|
37
|
+
markdown_size integer,
|
|
38
|
+
pdf_key text,
|
|
34
39
|
error text,
|
|
35
40
|
started_at timestamptz,
|
|
36
41
|
completed_at timestamptz,
|
|
@@ -40,4 +45,9 @@ export async function installCostAnalysisReportSchema(db, qSchema) {
|
|
|
40
45
|
)`));
|
|
41
46
|
await db.execute(sql.raw(`CREATE INDEX IF NOT EXISTS cost_ai_reports_tenant_created_idx ON ${qSchema}.cost_ai_reports (tenant_id, created_at DESC)`));
|
|
42
47
|
await db.execute(sql.raw(`CREATE INDEX IF NOT EXISTS cost_ai_reports_status_idx ON ${qSchema}.cost_ai_reports (status, created_at)`));
|
|
48
|
+
await db.execute(sql.raw(`ALTER TABLE ${qSchema}.cost_ai_reports ADD COLUMN IF NOT EXISTS ecs_task_arn text`));
|
|
49
|
+
await db.execute(sql.raw(`ALTER TABLE ${qSchema}.cost_ai_reports ADD COLUMN IF NOT EXISTS artifact_bucket text`));
|
|
50
|
+
await db.execute(sql.raw(`ALTER TABLE ${qSchema}.cost_ai_reports ADD COLUMN IF NOT EXISTS markdown_key text`));
|
|
51
|
+
await db.execute(sql.raw(`ALTER TABLE ${qSchema}.cost_ai_reports ADD COLUMN IF NOT EXISTS markdown_size integer`));
|
|
52
|
+
await db.execute(sql.raw(`ALTER TABLE ${qSchema}.cost_ai_reports ADD COLUMN IF NOT EXISTS pdf_key text`));
|
|
43
53
|
}
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import {
|
|
2
|
+
DescribeTasksCommand,
|
|
3
|
+
ECSClient,
|
|
4
|
+
RunTaskCommand,
|
|
5
|
+
StopTaskCommand,
|
|
6
|
+
} from "@aws-sdk/client-ecs";
|
|
7
|
+
import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3";
|
|
8
|
+
import { truthy } from "../lib/cost-utils.js";
|
|
9
|
+
|
|
10
|
+
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
11
|
+
const list = (value) => String(value || "").split(",").map((item) => item.trim()).filter(Boolean);
|
|
12
|
+
|
|
13
|
+
function required(value, name) {
|
|
14
|
+
const text = String(value || "").trim();
|
|
15
|
+
if (!text) throw Object.assign(new Error(`${name} is required for external cost analysis`), { name: "CostAnalyserConfigError", statusCode: 503 });
|
|
16
|
+
return text;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function environment(values) {
|
|
20
|
+
return Object.entries(values).filter(([, value]) => value !== undefined && value !== null && value !== "").map(([name, value]) => ({ name, value: String(value) }));
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
async function bodyText(body) {
|
|
24
|
+
if (!body) return "";
|
|
25
|
+
if (typeof body.transformToString === "function") return body.transformToString();
|
|
26
|
+
const chunks = [];
|
|
27
|
+
for await (const chunk of body) chunks.push(Buffer.from(chunk));
|
|
28
|
+
return Buffer.concat(chunks).toString("utf8");
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export class CostAnalyserService {
|
|
32
|
+
constructor({ contextService, athenaContextService, logger = console, env = process.env, ecsClient = null, s3Client = null } = {}) {
|
|
33
|
+
this.contextService = contextService;
|
|
34
|
+
this.athenaContextService = athenaContextService;
|
|
35
|
+
this.logger = logger;
|
|
36
|
+
this.enabled = truthy(env.COST_AI_ENABLED);
|
|
37
|
+
this.region = String(env.COST_AI_ANALYSER_REGION || env.AWS_REGION || env.AWS_DEFAULT_REGION || "us-east-1");
|
|
38
|
+
this.cluster = String(env.COST_AI_ANALYSER_CLUSTER || "").trim();
|
|
39
|
+
this.taskDefinition = String(env.COST_AI_ANALYSER_TASK_DEFINITION || "").trim();
|
|
40
|
+
this.containerName = String(env.COST_AI_ANALYSER_CONTAINER_NAME || "cost-ai-analyser").trim();
|
|
41
|
+
this.subnets = list(env.COST_AI_ANALYSER_SUBNETS);
|
|
42
|
+
this.securityGroups = list(env.COST_AI_ANALYSER_SECURITY_GROUPS);
|
|
43
|
+
this.assignPublicIp = truthy(env.COST_AI_ANALYSER_ASSIGN_PUBLIC_IP) ? "ENABLED" : "DISABLED";
|
|
44
|
+
this.reportBucket = String(env.COST_AI_ANALYSER_REPORT_BUCKET || "").trim();
|
|
45
|
+
this.targetRegions = String(env.COST_AI_ANALYSER_TARGET_REGIONS || env.AWS_REGION || "us-east-1").trim();
|
|
46
|
+
this.bedrockRegion = String(env.COST_AI_ANALYSER_BEDROCK_REGION || "").trim();
|
|
47
|
+
this.bedrockModelId = String(env.COST_AI_ANALYSER_BEDROCK_MODEL_ID || "").trim();
|
|
48
|
+
this.topServices = String(env.COST_AI_ANALYSER_TOP_N_SERVICES || "10").trim();
|
|
49
|
+
this.pollMs = Math.max(Number(env.COST_AI_ANALYSER_POLL_INTERVAL_MS || 15_000), 2_000);
|
|
50
|
+
this.timeoutMs = Math.max(Number(env.COST_AI_ANALYSER_TIMEOUT_MS || 45 * 60_000), 60_000);
|
|
51
|
+
this.ecs = ecsClient || new ECSClient({ region: this.region });
|
|
52
|
+
this.s3 = s3Client || new S3Client({ region: this.region });
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
status() {
|
|
56
|
+
const configured = Boolean(this.cluster && this.taskDefinition && this.subnets.length && this.securityGroups.length && this.reportBucket && this.bedrockModelId);
|
|
57
|
+
return {
|
|
58
|
+
enabled: this.enabled && configured,
|
|
59
|
+
provider: "meyi-cost-ai-analyser",
|
|
60
|
+
modelId: configured ? this.bedrockModelId : null,
|
|
61
|
+
region: configured ? this.bedrockRegion : null,
|
|
62
|
+
source: "ecs-task",
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
validate() {
|
|
67
|
+
if (!this.enabled) throw Object.assign(new Error("AI cost analysis is not enabled for this deployment"), { name: "CostAiDisabledError", statusCode: 503 });
|
|
68
|
+
required(this.cluster, "COST_AI_ANALYSER_CLUSTER");
|
|
69
|
+
required(this.taskDefinition, "COST_AI_ANALYSER_TASK_DEFINITION");
|
|
70
|
+
if (!this.subnets.length) required("", "COST_AI_ANALYSER_SUBNETS");
|
|
71
|
+
if (!this.securityGroups.length) required("", "COST_AI_ANALYSER_SECURITY_GROUPS");
|
|
72
|
+
required(this.reportBucket, "COST_AI_ANALYSER_REPORT_BUCKET");
|
|
73
|
+
required(this.bedrockModelId, "COST_AI_ANALYSER_BEDROCK_MODEL_ID");
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
async readJson(key) {
|
|
77
|
+
const response = await this.s3.send(new GetObjectCommand({ Bucket: this.reportBucket, Key: key }));
|
|
78
|
+
return JSON.parse(await bodyText(response.Body));
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async execute({ tenantId, reportId, range, frequency, onStarted = null }) {
|
|
82
|
+
this.validate();
|
|
83
|
+
const customer = await this.contextService.resolve({ headers: {}, user: { tenant_id: tenantId, tenantId } });
|
|
84
|
+
const athena = this.athenaContextService.resolve(customer).config;
|
|
85
|
+
if (!athena.enabled) throw Object.assign(new Error("CUR discovery has not produced a queryable Athena table for this tenant"), { name: "CurDataNotReadyError", statusCode: 503 });
|
|
86
|
+
const targetRoleArn = required(customer.meta?.payerRoleArn, "customer TARGET_ROLE_ARN");
|
|
87
|
+
const targetAccountId = required(customer.meta?.managementAccountId || customer.accounts?.[0]?.id, "customer management account ID");
|
|
88
|
+
const prefix = `tenants/${tenantId}/reports/${reportId}`;
|
|
89
|
+
const values = {
|
|
90
|
+
REPORT_JOB_ID: reportId,
|
|
91
|
+
REPORT_TENANT_ID: tenantId,
|
|
92
|
+
REPORT_KEY_PREFIX: prefix,
|
|
93
|
+
REPORT_START_DATE: range.Start,
|
|
94
|
+
REPORT_END_DATE: range.End,
|
|
95
|
+
REPORT_FREQUENCY: frequency,
|
|
96
|
+
REPORT_BUCKET: this.reportBucket,
|
|
97
|
+
REPORT_BUCKET_REGION: this.region,
|
|
98
|
+
TARGET_ACCOUNT_ID: targetAccountId,
|
|
99
|
+
TARGET_ROLE_ARN: targetRoleArn,
|
|
100
|
+
TARGET_ROLE_EXTERNAL_ID: customer.meta?.externalId,
|
|
101
|
+
TARGET_REGIONS: this.targetRegions,
|
|
102
|
+
COST_CUR_DATABASE: athena.database,
|
|
103
|
+
COST_CUR_TABLE: athena.table,
|
|
104
|
+
COST_CUR_WORKGROUP: athena.workgroup,
|
|
105
|
+
COST_CUR_OUTPUT_LOCATION: athena.outputLocation,
|
|
106
|
+
COST_CUR_REGION: athena.region,
|
|
107
|
+
COST_CUR_TENANT_COLUMN: athena.tenantColumn,
|
|
108
|
+
COST_CUR_TENANT_PARTITION: athena.tenantPartition || tenantId,
|
|
109
|
+
BEDROCK_REGION: this.bedrockRegion,
|
|
110
|
+
BEDROCK_MODEL_ID: this.bedrockModelId,
|
|
111
|
+
TOP_N_SERVICES: this.topServices,
|
|
112
|
+
};
|
|
113
|
+
const launched = await this.ecs.send(new RunTaskCommand({
|
|
114
|
+
cluster: this.cluster,
|
|
115
|
+
taskDefinition: this.taskDefinition,
|
|
116
|
+
launchType: "FARGATE",
|
|
117
|
+
count: 1,
|
|
118
|
+
enableExecuteCommand: false,
|
|
119
|
+
networkConfiguration: { awsvpcConfiguration: { subnets: this.subnets, securityGroups: this.securityGroups, assignPublicIp: this.assignPublicIp } },
|
|
120
|
+
overrides: { containerOverrides: [{ name: this.containerName, environment: environment(values) }] },
|
|
121
|
+
startedBy: `meyi-cost-${reportId}`.slice(0, 36),
|
|
122
|
+
}));
|
|
123
|
+
if (launched.failures?.length || !launched.tasks?.[0]?.taskArn) {
|
|
124
|
+
const reason = launched.failures?.map((item) => item.reason || item.detail).filter(Boolean).join("; ") || "ECS did not return a task ARN";
|
|
125
|
+
throw Object.assign(new Error(`Unable to start cost analyser: ${reason}`), { name: "CostAnalyserLaunchError", statusCode: 502 });
|
|
126
|
+
}
|
|
127
|
+
const taskArn = launched.tasks[0].taskArn;
|
|
128
|
+
await onStarted?.(taskArn);
|
|
129
|
+
const deadline = Date.now() + this.timeoutMs;
|
|
130
|
+
while (Date.now() < deadline) {
|
|
131
|
+
const response = await this.ecs.send(new DescribeTasksCommand({ cluster: this.cluster, tasks: [taskArn] }));
|
|
132
|
+
const task = response.tasks?.[0];
|
|
133
|
+
if (task?.lastStatus === "STOPPED") {
|
|
134
|
+
const container = task.containers?.find((item) => item.name === this.containerName) || task.containers?.[0];
|
|
135
|
+
let status;
|
|
136
|
+
try { status = await this.readJson(`${prefix}/status.json`); } catch { status = null; }
|
|
137
|
+
if (Number(container?.exitCode ?? 1) !== 0 || status?.status !== "COMPLETED") {
|
|
138
|
+
throw Object.assign(new Error(status?.error || container?.reason || task.stoppedReason || "Cost analyser task failed"), { name: "CostAnalyserTaskError", statusCode: 502, taskArn });
|
|
139
|
+
}
|
|
140
|
+
return {
|
|
141
|
+
taskArn,
|
|
142
|
+
artifactBucket: this.reportBucket,
|
|
143
|
+
markdownKey: status.markdownKey || `${prefix}/report.md`,
|
|
144
|
+
pdfKey: status.pdfKey || `${prefix}/report.pdf`,
|
|
145
|
+
markdownSize: Number(status.markdownSize || 0),
|
|
146
|
+
pdfSize: Number(status.pdfSize || 0),
|
|
147
|
+
modelId: this.bedrockModelId,
|
|
148
|
+
dataSource: "AWS CUR",
|
|
149
|
+
facts: { currentTotal: Number(status.summary?.totalMonthlyCost || 0), currency: "USD" },
|
|
150
|
+
summary: status.summary || {},
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
await sleep(this.pollMs);
|
|
154
|
+
}
|
|
155
|
+
await this.ecs.send(new StopTaskCommand({ cluster: this.cluster, task: taskArn, reason: "Meyi cost analysis timeout" }));
|
|
156
|
+
throw Object.assign(new Error("Cost analyser task timed out"), { name: "CostAnalyserTimeoutError", statusCode: 504, taskArn });
|
|
157
|
+
}
|
|
158
|
+
}
|
|
@@ -11,17 +11,17 @@ function reportDto(row) {
|
|
|
11
11
|
period: { Start: String(row.period_start).slice(0, 10), End: String(row.period_end).slice(0, 10) },
|
|
12
12
|
status: row.status, attempts: Number(row.attempt_count || 0), modelId: row.model_id,
|
|
13
13
|
dataSource: row.data_source, facts: row.facts || null, ...(row.result || {}),
|
|
14
|
-
pdfReady: row.status === "COMPLETED" && Number(row.pdf_size || 0) > 0,
|
|
14
|
+
pdfReady: row.status === "COMPLETED" && Boolean(row.pdf_key || row.pdf_data || Number(row.pdf_size || 0) > 0),
|
|
15
15
|
pdfSize: Number(row.pdf_size || 0), error: row.error, startedAt: row.started_at,
|
|
16
16
|
completedAt: row.completed_at, createdAt: row.created_at,
|
|
17
17
|
};
|
|
18
18
|
}
|
|
19
19
|
|
|
20
20
|
export class CostAnalysisReportService {
|
|
21
|
-
constructor({ contextService, repository,
|
|
21
|
+
constructor({ contextService, repository, artifactService }) {
|
|
22
22
|
this.contextService = contextService;
|
|
23
23
|
this.repository = repository;
|
|
24
|
-
this.
|
|
24
|
+
this.artifactService = artifactService;
|
|
25
25
|
}
|
|
26
26
|
|
|
27
27
|
tenant(req) { return this.contextService.tenantId(req); }
|
|
@@ -37,6 +37,11 @@ export class CostAnalysisReportService {
|
|
|
37
37
|
|
|
38
38
|
async list(req, limit) { return (await this.repository.list(this.tenant(req), limit)).map(reportDto); }
|
|
39
39
|
|
|
40
|
+
async latest(req) {
|
|
41
|
+
const reports = await this.list(req, 1);
|
|
42
|
+
return reports[0] || null;
|
|
43
|
+
}
|
|
44
|
+
|
|
40
45
|
async get(req, id) {
|
|
41
46
|
const row = await this.repository.find(this.tenant(req), id);
|
|
42
47
|
if (!row) throw Object.assign(new Error("AI cost report was not found"), { statusCode: 404 });
|
|
@@ -46,7 +51,18 @@ export class CostAnalysisReportService {
|
|
|
46
51
|
async pdf(req, id) {
|
|
47
52
|
const row = await this.repository.find(this.tenant(req), id, true);
|
|
48
53
|
if (!row) throw Object.assign(new Error("AI cost report was not found"), { statusCode: 404 });
|
|
49
|
-
if (row.status !== "COMPLETED"
|
|
50
|
-
|
|
54
|
+
if (row.status !== "COMPLETED") throw Object.assign(new Error("The PDF is not ready"), { statusCode: 409 });
|
|
55
|
+
const data = row.pdf_key ? await this.artifactService.get(row.artifact_bucket, row.pdf_key) : row.pdf_data ? Buffer.from(row.pdf_data) : null;
|
|
56
|
+
if (!data?.length) throw Object.assign(new Error("The PDF artifact is unavailable"), { statusCode: 409 });
|
|
57
|
+
return { data, filename: `meyi-cost-analysis-${String(row.period_start).slice(0, 10)}-${id.slice(0, 8)}.pdf` };
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
async markdown(req, id) {
|
|
61
|
+
const row = await this.repository.find(this.tenant(req), id, true);
|
|
62
|
+
if (!row) throw Object.assign(new Error("AI cost report was not found"), { statusCode: 404 });
|
|
63
|
+
if (row.status !== "COMPLETED" || !row.markdown_key) throw Object.assign(new Error("The Markdown report is not ready"), { statusCode: 409 });
|
|
64
|
+
const data = await this.artifactService.get(row.artifact_bucket, row.markdown_key);
|
|
65
|
+
if (!data?.length) throw Object.assign(new Error("The Markdown artifact is unavailable"), { statusCode: 409 });
|
|
66
|
+
return { data, filename: `meyi-cost-analysis-${String(row.period_start).slice(0, 10)}-${id.slice(0, 8)}.md` };
|
|
51
67
|
}
|
|
52
68
|
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3";
|
|
2
|
+
|
|
3
|
+
async function bytes(body) {
|
|
4
|
+
if (!body) return Buffer.alloc(0);
|
|
5
|
+
if (typeof body.transformToByteArray === "function") return Buffer.from(await body.transformToByteArray());
|
|
6
|
+
const chunks = [];
|
|
7
|
+
for await (const chunk of body) chunks.push(Buffer.from(chunk));
|
|
8
|
+
return Buffer.concat(chunks);
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export class CostReportArtifactService {
|
|
12
|
+
constructor({ client = null, region = process.env.COST_AI_ANALYSER_REGION || process.env.AWS_REGION } = {}) {
|
|
13
|
+
this.client = client || new S3Client({ region });
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
async get(bucket, key) {
|
|
17
|
+
if (!bucket || !key) return null;
|
|
18
|
+
const response = await this.client.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
|
|
19
|
+
return bytes(response.Body);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|