@kici-dev/orchestrator 0.1.15 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app.d.ts +25 -0
- package/dist/approvals/apply-decision.d.ts +60 -0
- package/dist/approvals/approval-resolver.d.ts +66 -0
- package/dist/approvals/step-approval-bridge.d.ts +72 -0
- package/dist/approvals/team-membership-lookup.d.ts +13 -0
- package/dist/cache/user-cache.d.ts +1 -1
- package/dist/cli/api-client.d.ts +24 -0
- package/dist/cli/commands/firecracker/index.d.ts +11 -0
- package/dist/cli/commands/firecracker/provision.d.ts +13 -0
- package/dist/cli/commands/firecracker/teardown.d.ts +3 -0
- package/dist/cli/commands/firecracker/verify.d.ts +3 -0
- package/dist/cli/commands/scaler.d.ts +18 -0
- package/dist/cli/kici-admin.d.ts +10 -1
- package/dist/cli/service/image-digests.d.ts +21 -0
- package/dist/cli.js +3692 -411
- package/dist/cluster/peer-client.d.ts +23 -1
- package/dist/cluster/peer-handler.d.ts +9 -1
- package/dist/db/migrations/033_org_settings_approval.d.ts +19 -0
- package/dist/db/migrations/034_held_runs_generalize.d.ts +24 -0
- package/dist/db/types.d.ts +52 -1
- package/dist/diagnostics/bundle-writer.d.ts +1 -17
- package/dist/diagnostics/checks/firecracker-network.d.ts +13 -0
- package/dist/diagnostics/checks/index.d.ts +2 -1
- package/dist/diagnostics/fleet-collector.d.ts +52 -0
- package/dist/diagnostics/fleet-constants.d.ts +8 -0
- package/dist/diagnostics/fleet-selection.d.ts +15 -0
- package/dist/diagnostics/fleet-topology.d.ts +47 -0
- package/dist/diagnostics/fleet-wiring.d.ts +60 -0
- package/dist/environments/held-runs.d.ts +69 -1
- package/dist/firecracker/host-network.d.ts +83 -0
- package/dist/firecracker/persist.d.ts +14 -0
- package/dist/index.js +24 -6
- package/dist/orchestrator-core.d.ts +9 -1
- package/dist/pipeline/processor.d.ts +22 -0
- package/dist/routes/fleet.d.ts +20 -0
- package/dist/scaler/config.d.ts +4 -2
- package/dist/scaler/disk-guard.d.ts +27 -0
- package/dist/scaler/firecracker-backend.d.ts +18 -0
- package/dist/scaler/manager.d.ts +6 -0
- package/dist/scaler/nftables.d.ts +4 -0
- package/dist/scaler/reap-orphans.d.ts +27 -0
- package/dist/scaler/types.d.ts +2 -0
- package/dist/server.js +3205 -1214
- package/dist/stale-detector/stale-run-detector.d.ts +18 -0
- package/dist/standalone.js +2585 -1062
- package/dist/storage/filesystem.d.ts +2 -1
- package/dist/storage/s3.d.ts +2 -1
- package/dist/storage/types.d.ts +7 -0
- package/dist/ws/agent-handler.d.ts +32 -0
- package/dist/ws/dashboard-env-handler.d.ts +35 -0
- package/dist/ws/fleet-agent-collector.d.ts +23 -0
- package/installer-image-digests.json +7 -0
- package/package.json +6 -5
- package/sbom.spdx.json +62 -57
package/dist/cli.js
CHANGED
|
@@ -1,15 +1,17 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { createRequire } from "node:module";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
import * as path$1 from "node:path";
|
|
5
|
+
import path, { join, resolve } from "node:path";
|
|
3
6
|
import { Command } from "commander";
|
|
4
|
-
import { BaseColdStore, ChunkLru, chunkObjectKey, clearDispatchQueueDirect, computeChunkId, computeMigrationsHash, createDb, createDbRole, createEnvironmentTemplateDirect, createLogger, createPool, createReadOnlyDbUser, decodeChunk, deleteEnvironmentDirect, dropAndCreateDatabase, emitKiciEventDirect, encodeKeySegment, ensureDatabase, formatBytes, formatUptime, isSchemaCurrent, listEnvironmentsDirect, listExecutionRunsDirect, listQueueDirect, listRegistrationsDirect, maskDatabaseUrl, parseDatabaseUrl, parseManifest, prunePeerCredentialsDirect, purgeScopedSecretsDirect, purgeSecretBackendsDirect, purgeStaleExecutionDirect, purgeStaleSourcesDirect, registerWorkflowManualDirect, resetRaftStateDirect, seedEnvironmentBindingDirect, seedEnvironmentDirect, setEnvironmentPolicyDirect, setEnvironmentSecretDirect, sha256, showEnvironmentDirect, showExecutionRunDirect, showQueueEntryDirect, showRegistrationDirect, storeMigrationContentHash, tablePrefix, toErrorMessage } from "@kici-dev/shared";
|
|
5
7
|
import * as fs$1 from "node:fs";
|
|
6
|
-
import fs, { createWriteStream, promises, readFileSync } from "node:fs";
|
|
8
|
+
import fs, { closeSync, createReadStream, createWriteStream, existsSync, openSync, promises, readFileSync, statSync, unwatchFile, watchFile, writeFileSync } from "node:fs";
|
|
9
|
+
import { BaseColdStore, ChunkLru, addLogsToArchive, chunkObjectKey, clearDispatchQueueDirect, computeChunkId, computeMigrationsHash, createDb, createDbRole, createEnvironmentTemplateDirect, createLogger, createPool, createReadOnlyDbUser, decodeChunk, deleteEnvironmentDirect, dropAndCreateDatabase, emitKiciEventDirect, encodeKeySegment, ensureDatabase, formatBytes, formatUptime, isSchemaCurrent, listEnvironmentsDirect, listExecutionRunsDirect, listQueueDirect, listRegistrationsDirect, maskDatabaseUrl, parseDatabaseUrl, parseManifest, prunePeerCredentialsDirect, purgeScopedSecretsDirect, purgeSecretBackendsDirect, purgeStaleExecutionDirect, purgeStaleSourcesDirect, registerWorkflowManualDirect, resetRaftStateDirect, seedEnvironmentBindingDirect, seedEnvironmentDirect, setEnvironmentPolicyDirect, setEnvironmentSecretDirect, sha256, showEnvironmentDirect, showExecutionRunDirect, showQueueEntryDirect, showRegistrationDirect, storeMigrationContentHash, tablePrefix, toErrorMessage } from "@kici-dev/shared";
|
|
7
10
|
import crypto, { createDecipheriv, createHash, hkdfSync, randomBytes, randomUUID } from "node:crypto";
|
|
8
11
|
import { createInterface } from "node:readline";
|
|
9
|
-
import { readFile, writeFile } from "node:fs/promises";
|
|
10
|
-
import { stringify } from "yaml";
|
|
12
|
+
import { access, chmod, constants, copyFile, link, mkdir, open, readFile, readdir, rm, writeFile } from "node:fs/promises";
|
|
13
|
+
import { parse, stringify } from "yaml";
|
|
11
14
|
import { z } from "zod";
|
|
12
|
-
import "node:os";
|
|
13
15
|
import os from "node:os";
|
|
14
16
|
import { getDatabaseCollationDrift, refreshDatabaseCollationVersion, reindexDatabaseConcurrently } from "@kici-dev/shared/db-collation";
|
|
15
17
|
import { sql } from "kysely";
|
|
@@ -17,17 +19,17 @@ import { Migrator } from "kysely/migration";
|
|
|
17
19
|
import "jsonpath-plus";
|
|
18
20
|
import { execFile, execFileSync, execSync, spawn } from "node:child_process";
|
|
19
21
|
import { promisify } from "node:util";
|
|
20
|
-
import * as path$1 from "node:path";
|
|
21
|
-
import path from "node:path";
|
|
22
22
|
import archiver from "archiver";
|
|
23
|
+
import { confirm, input, password, select } from "@inquirer/prompts";
|
|
23
24
|
import JSZip from "jszip";
|
|
24
|
-
import { fileURLToPath } from "node:url";
|
|
25
25
|
import { pipeline } from "node:stream/promises";
|
|
26
|
-
import { confirm, input, password, select } from "@inquirer/prompts";
|
|
27
26
|
import "pg";
|
|
28
27
|
import { DASHBOARD_WRITE_OPERATIONS, DASHBOARD_WRITE_OPERATIONS_BY_NAME, DashboardWriteCategory, DashboardWriteSensitivity } from "@kici-dev/engine/protocol/dashboard-write-operations";
|
|
29
28
|
import { CLUSTER_NAME_FORMAT_MESSAGE } from "@kici-dev/engine/protocol/cluster-name";
|
|
30
|
-
import { TERMINAL_JOB_STATES, TERMINAL_RUN_STATES, WS_MAX_PAYLOAD_BYTES, accessLogWarmSqlCase, getAccessLogColdDays, getSecretAuditLogColdDays, minAccessLogWarmDays, minSecretAuditLogWarmDays, secretAuditLogWarmSqlCase } from "@kici-dev/engine";
|
|
29
|
+
import { KICI_AGENT_ENV_PREFIX, KNOWN_ROLES, ScalerBackendType, ScalerEventType, TERMINAL_JOB_STATES, TERMINAL_RUN_STATES, WS_MAX_PAYLOAD_BYTES, accessLogWarmSqlCase, getAccessLogColdDays, getSecretAuditLogColdDays, minAccessLogWarmDays, minSecretAuditLogWarmDays, scalerAgentLabels, secretAuditLogWarmSqlCase } from "@kici-dev/engine";
|
|
30
|
+
import http from "node:http";
|
|
31
|
+
import { PassThrough } from "node:stream";
|
|
32
|
+
import Docker from "dockerode";
|
|
31
33
|
var __defProp = Object.defineProperty;
|
|
32
34
|
var __esmMin = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
33
35
|
var __exportAll = (all, no_symbols) => {
|
|
@@ -42,6 +44,13 @@ var __exportAll = (all, no_symbols) => {
|
|
|
42
44
|
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
43
45
|
//#endregion
|
|
44
46
|
//#region src/cli/api-client.ts
|
|
47
|
+
/**
|
|
48
|
+
* HTTP client wrapper for the kici-admin CLI.
|
|
49
|
+
*
|
|
50
|
+
* Communicates with the orchestrator admin API via Bearer token authentication.
|
|
51
|
+
* All methods are thin wrappers around fetch() that handle JSON serialization,
|
|
52
|
+
* error formatting, and URL construction.
|
|
53
|
+
*/
|
|
45
54
|
var AdminApiClient = class {
|
|
46
55
|
constructor(baseUrl, token) {
|
|
47
56
|
this.baseUrl = baseUrl;
|
|
@@ -116,6 +125,27 @@ var AdminApiClient = class {
|
|
|
116
125
|
if (!res.ok) throw new Error(`HTTP ${res.status}: ${await res.text()}`);
|
|
117
126
|
return res.text();
|
|
118
127
|
}
|
|
128
|
+
/** Enumerate the cluster topology for `debug-bundle --fleet --list` / `--pick`. */
|
|
129
|
+
async getFleetTopology() {
|
|
130
|
+
return this.get("/admin/fleet-topology");
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Drive the fleet fan-out and write the assembled ZIP to `outPath`. The
|
|
134
|
+
* response is an octet-stream, so it is read as bytes rather than parsed JSON.
|
|
135
|
+
*/
|
|
136
|
+
async downloadFleetBundle(body, outPath) {
|
|
137
|
+
const res = await fetch(`${this.baseUrl}/admin/fleet-bundle`, {
|
|
138
|
+
method: "POST",
|
|
139
|
+
headers: {
|
|
140
|
+
Authorization: `Bearer ${this.token}`,
|
|
141
|
+
"Content-Type": "application/json"
|
|
142
|
+
},
|
|
143
|
+
body: JSON.stringify(body)
|
|
144
|
+
});
|
|
145
|
+
if (!res.ok) throw new Error(`HTTP ${res.status}: ${await res.text()}`);
|
|
146
|
+
const buf = Buffer.from(await res.arrayBuffer());
|
|
147
|
+
fs$1.writeFileSync(outPath, buf);
|
|
148
|
+
}
|
|
119
149
|
async listScopes(orgId) {
|
|
120
150
|
return this.request("GET", `/api/v1/admin/secrets/scopes?orgId=${encodeURIComponent(orgId)}`);
|
|
121
151
|
}
|
|
@@ -1518,8 +1548,8 @@ var init_client = __esmMin((() => {
|
|
|
1518
1548
|
//#region src/db/migrations/001_initial.ts
|
|
1519
1549
|
init_client();
|
|
1520
1550
|
var _001_initial_exports = /* @__PURE__ */ __exportAll({
|
|
1521
|
-
down: () => down$
|
|
1522
|
-
up: () => up$
|
|
1551
|
+
down: () => down$33,
|
|
1552
|
+
up: () => up$33
|
|
1523
1553
|
});
|
|
1524
1554
|
/**
|
|
1525
1555
|
* Squashed initial migration -- creates the complete Orchestrator database schema.
|
|
@@ -2211,7 +2241,7 @@ const DDL_STATEMENTS = [
|
|
|
2211
2241
|
`ALTER TABLE ONLY public.held_runs
|
|
2212
2242
|
ADD CONSTRAINT held_runs_environment_id_fkey FOREIGN KEY (environment_id) REFERENCES public.environments(id);`
|
|
2213
2243
|
];
|
|
2214
|
-
async function up$
|
|
2244
|
+
async function up$33(db) {
|
|
2215
2245
|
for (const stmt of DDL_STATEMENTS) await sql.raw(stmt).execute(db);
|
|
2216
2246
|
await sql`
|
|
2217
2247
|
INSERT INTO cluster_meta (key, value)
|
|
@@ -2233,7 +2263,7 @@ async function up$31(db) {
|
|
|
2233
2263
|
* Rollback drops everything created above. Uses CASCADE on table drops to cut
|
|
2234
2264
|
* through the FK graph without relying on exact topological order.
|
|
2235
2265
|
*/
|
|
2236
|
-
async function down$
|
|
2266
|
+
async function down$33(db) {
|
|
2237
2267
|
for (const [trig, tbl] of [["source_secrets_change_trigger", "scoped_secrets"], ["sources_change_trigger", "sources"]]) await sql.raw(`DROP TRIGGER IF EXISTS ${trig} ON public.${tbl}`).execute(db);
|
|
2238
2268
|
for (const table of [
|
|
2239
2269
|
"workflow_registrations",
|
|
@@ -2280,8 +2310,8 @@ async function down$31(db) {
|
|
|
2280
2310
|
//#endregion
|
|
2281
2311
|
//#region src/db/migrations/002_config_versions_key_version.ts
|
|
2282
2312
|
var _002_config_versions_key_version_exports = /* @__PURE__ */ __exportAll({
|
|
2283
|
-
down: () => down$
|
|
2284
|
-
up: () => up$
|
|
2313
|
+
down: () => down$32,
|
|
2314
|
+
up: () => up$32
|
|
2285
2315
|
});
|
|
2286
2316
|
/**
|
|
2287
2317
|
* Add key_version column to config_versions so that sensitive-field encryption
|
|
@@ -2295,13 +2325,13 @@ var _002_config_versions_key_version_exports = /* @__PURE__ */ __exportAll({
|
|
|
2295
2325
|
* No index is needed: rotation does a full-table scan; reads are by
|
|
2296
2326
|
* `version` primary key and never filter on `key_version`.
|
|
2297
2327
|
*/
|
|
2298
|
-
async function up$
|
|
2328
|
+
async function up$32(db) {
|
|
2299
2329
|
await sql`
|
|
2300
2330
|
ALTER TABLE public.config_versions
|
|
2301
2331
|
ADD COLUMN key_version integer NOT NULL DEFAULT 1
|
|
2302
2332
|
`.execute(db);
|
|
2303
2333
|
}
|
|
2304
|
-
async function down$
|
|
2334
|
+
async function down$32(db) {
|
|
2305
2335
|
await sql`
|
|
2306
2336
|
ALTER TABLE public.config_versions
|
|
2307
2337
|
DROP COLUMN key_version
|
|
@@ -2310,8 +2340,8 @@ async function down$30(db) {
|
|
|
2310
2340
|
//#endregion
|
|
2311
2341
|
//#region src/db/migrations/003_access_log.ts
|
|
2312
2342
|
var _003_access_log_exports = /* @__PURE__ */ __exportAll({
|
|
2313
|
-
down: () => down$
|
|
2314
|
-
up: () => up$
|
|
2343
|
+
down: () => down$31,
|
|
2344
|
+
up: () => up$31
|
|
2315
2345
|
});
|
|
2316
2346
|
/**
|
|
2317
2347
|
* Access log: one row per read or orchestrator-admin mutation attributable
|
|
@@ -2335,7 +2365,7 @@ var _003_access_log_exports = /* @__PURE__ */ __exportAll({
|
|
|
2335
2365
|
* Retention is TTL-based via expires_at; packages/orchestrator/src/queue/
|
|
2336
2366
|
* cleanup.ts picks up the prune pass.
|
|
2337
2367
|
*/
|
|
2338
|
-
async function up$
|
|
2368
|
+
async function up$31(db) {
|
|
2339
2369
|
await sql`
|
|
2340
2370
|
CREATE TABLE public.access_log (
|
|
2341
2371
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
@@ -2373,28 +2403,28 @@ async function up$29(db) {
|
|
|
2373
2403
|
ON public.access_log (actor_type, actor_id, created_at DESC)
|
|
2374
2404
|
`.execute(db);
|
|
2375
2405
|
}
|
|
2376
|
-
async function down$
|
|
2406
|
+
async function down$31(db) {
|
|
2377
2407
|
await sql`DROP TABLE IF EXISTS public.access_log`.execute(db);
|
|
2378
2408
|
}
|
|
2379
2409
|
//#endregion
|
|
2380
2410
|
//#region src/db/migrations/004_rename_bundle_to_source.ts
|
|
2381
2411
|
var _004_rename_bundle_to_source_exports = /* @__PURE__ */ __exportAll({
|
|
2382
|
-
down: () => down$
|
|
2383
|
-
up: () => up$
|
|
2412
|
+
down: () => down$30,
|
|
2413
|
+
up: () => up$30
|
|
2384
2414
|
});
|
|
2385
|
-
async function up$
|
|
2415
|
+
async function up$30(db) {
|
|
2386
2416
|
await db.schema.alterTable("dispatch_queue").renameColumn("bundle_url", "source_tar_url").execute();
|
|
2387
2417
|
await db.schema.alterTable("dispatch_queue").renameColumn("bundle_hash", "source_tar_hash").execute();
|
|
2388
2418
|
}
|
|
2389
|
-
async function down$
|
|
2419
|
+
async function down$30(db) {
|
|
2390
2420
|
await db.schema.alterTable("dispatch_queue").renameColumn("source_tar_url", "bundle_url").execute();
|
|
2391
2421
|
await db.schema.alterTable("dispatch_queue").renameColumn("source_tar_hash", "bundle_hash").execute();
|
|
2392
2422
|
}
|
|
2393
2423
|
//#endregion
|
|
2394
2424
|
//#region src/db/migrations/005_cold_store_chunk_counter.ts
|
|
2395
2425
|
var _005_cold_store_chunk_counter_exports = /* @__PURE__ */ __exportAll({
|
|
2396
|
-
down: () => down$
|
|
2397
|
-
up: () => up$
|
|
2426
|
+
down: () => down$29,
|
|
2427
|
+
up: () => up$29
|
|
2398
2428
|
});
|
|
2399
2429
|
/**
|
|
2400
2430
|
* Cold-store chunk counter table.
|
|
@@ -2411,7 +2441,7 @@ var _005_cold_store_chunk_counter_exports = /* @__PURE__ */ __exportAll({
|
|
|
2411
2441
|
*
|
|
2412
2442
|
* sections 5 and 8.
|
|
2413
2443
|
*/
|
|
2414
|
-
async function up$
|
|
2444
|
+
async function up$29(db) {
|
|
2415
2445
|
await sql`
|
|
2416
2446
|
CREATE TABLE public.cold_store_chunk_counts (
|
|
2417
2447
|
db TEXT NOT NULL,
|
|
@@ -2429,14 +2459,14 @@ async function up$27(db) {
|
|
|
2429
2459
|
ON public.cold_store_chunk_counts (db, table_name)
|
|
2430
2460
|
`.execute(db);
|
|
2431
2461
|
}
|
|
2432
|
-
async function down$
|
|
2462
|
+
async function down$29(db) {
|
|
2433
2463
|
await sql`DROP TABLE IF EXISTS public.cold_store_chunk_counts`.execute(db);
|
|
2434
2464
|
}
|
|
2435
2465
|
//#endregion
|
|
2436
2466
|
//#region src/db/migrations/006_runs_jobs_steps_archived_at.ts
|
|
2437
2467
|
var _006_runs_jobs_steps_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
2438
|
-
down: () => down$
|
|
2439
|
-
up: () => up$
|
|
2468
|
+
down: () => down$28,
|
|
2469
|
+
up: () => up$28
|
|
2440
2470
|
});
|
|
2441
2471
|
/**
|
|
2442
2472
|
* `execution_runs` / `execution_jobs` / `execution_steps` cold-store
|
|
@@ -2472,7 +2502,7 @@ var _006_runs_jobs_steps_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
|
2472
2502
|
* - `idx_execution_jobs_routing_key_created (routing_key, created_at)`
|
|
2473
2503
|
* - `idx_execution_steps_routing_key_created (routing_key, created_at)`
|
|
2474
2504
|
*/
|
|
2475
|
-
async function up$
|
|
2505
|
+
async function up$28(db) {
|
|
2476
2506
|
await sql`
|
|
2477
2507
|
ALTER TABLE public.execution_runs
|
|
2478
2508
|
ADD COLUMN archived_at TIMESTAMPTZ NULL,
|
|
@@ -2517,7 +2547,7 @@ async function up$26(db) {
|
|
|
2517
2547
|
ON public.execution_steps (routing_key, created_at)
|
|
2518
2548
|
`.execute(db);
|
|
2519
2549
|
}
|
|
2520
|
-
async function down$
|
|
2550
|
+
async function down$28(db) {
|
|
2521
2551
|
await sql`DROP INDEX IF EXISTS public.idx_execution_steps_routing_key_created`.execute(db);
|
|
2522
2552
|
await sql`
|
|
2523
2553
|
ALTER TABLE public.execution_steps
|
|
@@ -2542,8 +2572,8 @@ async function down$26(db) {
|
|
|
2542
2572
|
//#endregion
|
|
2543
2573
|
//#region src/db/migrations/007_audit_logs_archived_at.ts
|
|
2544
2574
|
var _007_audit_logs_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
2545
|
-
down: () => down$
|
|
2546
|
-
up: () => up$
|
|
2575
|
+
down: () => down$27,
|
|
2576
|
+
up: () => up$27
|
|
2547
2577
|
});
|
|
2548
2578
|
/**
|
|
2549
2579
|
* `secret_audit_log` and `access_log` cold-store schema additions, plus
|
|
@@ -2583,7 +2613,7 @@ var _007_audit_logs_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
|
2583
2613
|
* `down()` would not have meaningful retention bounds. Acceptable for
|
|
2584
2614
|
* staging.
|
|
2585
2615
|
*/
|
|
2586
|
-
async function up$
|
|
2616
|
+
async function up$27(db) {
|
|
2587
2617
|
await sql`
|
|
2588
2618
|
ALTER TABLE public.secret_audit_log
|
|
2589
2619
|
ADD COLUMN archived_at TIMESTAMPTZ NULL,
|
|
@@ -2604,7 +2634,7 @@ async function up$25(db) {
|
|
|
2604
2634
|
DROP COLUMN IF EXISTS expires_at
|
|
2605
2635
|
`.execute(db);
|
|
2606
2636
|
}
|
|
2607
|
-
async function down$
|
|
2637
|
+
async function down$27(db) {
|
|
2608
2638
|
await sql`
|
|
2609
2639
|
ALTER TABLE public.access_log
|
|
2610
2640
|
ADD COLUMN expires_at TIMESTAMPTZ NOT NULL DEFAULT (now() + INTERVAL '90 days')
|
|
@@ -2632,8 +2662,8 @@ async function down$25(db) {
|
|
|
2632
2662
|
//#endregion
|
|
2633
2663
|
//#region src/db/migrations/008_event_log_archived_at.ts
|
|
2634
2664
|
var _008_event_log_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
2635
|
-
down: () => down$
|
|
2636
|
-
up: () => up$
|
|
2665
|
+
down: () => down$26,
|
|
2666
|
+
up: () => up$26
|
|
2637
2667
|
});
|
|
2638
2668
|
/**
|
|
2639
2669
|
* `event_log` cold-store schema additions plus removal of the
|
|
@@ -2671,7 +2701,7 @@ var _008_event_log_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
|
2671
2701
|
* — best effort; rows inserted between `up()` and a hypothetical
|
|
2672
2702
|
* `down()` would not have meaningful retention bounds.
|
|
2673
2703
|
*/
|
|
2674
|
-
async function up$
|
|
2704
|
+
async function up$26(db) {
|
|
2675
2705
|
await sql`
|
|
2676
2706
|
ALTER TABLE public.event_log
|
|
2677
2707
|
ADD COLUMN archived_at TIMESTAMPTZ NULL,
|
|
@@ -2687,7 +2717,7 @@ async function up$24(db) {
|
|
|
2687
2717
|
DROP COLUMN IF EXISTS expires_at
|
|
2688
2718
|
`.execute(db);
|
|
2689
2719
|
}
|
|
2690
|
-
async function down$
|
|
2720
|
+
async function down$26(db) {
|
|
2691
2721
|
await sql`
|
|
2692
2722
|
ALTER TABLE public.event_log
|
|
2693
2723
|
ADD COLUMN expires_at TIMESTAMPTZ NOT NULL DEFAULT (now() + INTERVAL '30 days')
|
|
@@ -2706,8 +2736,8 @@ async function down$24(db) {
|
|
|
2706
2736
|
//#endregion
|
|
2707
2737
|
//#region src/db/migrations/009_access_log_trigram.ts
|
|
2708
2738
|
var _009_access_log_trigram_exports = /* @__PURE__ */ __exportAll({
|
|
2709
|
-
down: () => down$
|
|
2710
|
-
up: () => up$
|
|
2739
|
+
down: () => down$25,
|
|
2740
|
+
up: () => up$25
|
|
2711
2741
|
});
|
|
2712
2742
|
/**
|
|
2713
2743
|
* Trigram (pg_trgm) index on access_log.error_message for the federated
|
|
@@ -2720,7 +2750,7 @@ var _009_access_log_trigram_exports = /* @__PURE__ */ __exportAll({
|
|
|
2720
2750
|
* EXISTS` are both safe to re-run. No CONCURRENTLY because Kysely runs
|
|
2721
2751
|
* migrations inside a transaction; the lock is brief on a sampled table.
|
|
2722
2752
|
*/
|
|
2723
|
-
async function up$
|
|
2753
|
+
async function up$25(db) {
|
|
2724
2754
|
await sql`CREATE EXTENSION IF NOT EXISTS pg_trgm`.execute(db);
|
|
2725
2755
|
await sql`
|
|
2726
2756
|
CREATE INDEX IF NOT EXISTS access_log_error_message_trgm_idx
|
|
@@ -2729,14 +2759,14 @@ async function up$23(db) {
|
|
|
2729
2759
|
WHERE error_message IS NOT NULL
|
|
2730
2760
|
`.execute(db);
|
|
2731
2761
|
}
|
|
2732
|
-
async function down$
|
|
2762
|
+
async function down$25(db) {
|
|
2733
2763
|
await sql`DROP INDEX IF EXISTS public.access_log_error_message_trgm_idx`.execute(db);
|
|
2734
2764
|
}
|
|
2735
2765
|
//#endregion
|
|
2736
2766
|
//#region src/db/migrations/010_cold_store_chunks.ts
|
|
2737
2767
|
var _010_cold_store_chunks_exports = /* @__PURE__ */ __exportAll({
|
|
2738
|
-
down: () => down$
|
|
2739
|
-
up: () => up$
|
|
2768
|
+
down: () => down$24,
|
|
2769
|
+
up: () => up$24
|
|
2740
2770
|
});
|
|
2741
2771
|
/**
|
|
2742
2772
|
* Cold-store chunk index — Phase 2 (cold-store purge).
|
|
@@ -2768,7 +2798,7 @@ var _010_cold_store_chunks_exports = /* @__PURE__ */ __exportAll({
|
|
|
2768
2798
|
* forever. Adapters that don't opt into per-bucket archival via
|
|
2769
2799
|
* `coldTtlDays` don't insert here either.
|
|
2770
2800
|
*/
|
|
2771
|
-
async function up$
|
|
2801
|
+
async function up$24(db) {
|
|
2772
2802
|
await sql`
|
|
2773
2803
|
CREATE TABLE public.cold_store_chunks (
|
|
2774
2804
|
db TEXT NOT NULL,
|
|
@@ -2795,14 +2825,14 @@ async function up$22(db) {
|
|
|
2795
2825
|
ON public.cold_store_chunks (db, table_name, tenant_id, archived_at DESC)
|
|
2796
2826
|
`.execute(db);
|
|
2797
2827
|
}
|
|
2798
|
-
async function down$
|
|
2828
|
+
async function down$24(db) {
|
|
2799
2829
|
await sql`DROP TABLE IF EXISTS public.cold_store_chunks`.execute(db);
|
|
2800
2830
|
}
|
|
2801
2831
|
//#endregion
|
|
2802
2832
|
//#region src/db/migrations/011_drop_source_secrets_notify.ts
|
|
2803
2833
|
var _011_drop_source_secrets_notify_exports = /* @__PURE__ */ __exportAll({
|
|
2804
|
-
down: () => down$
|
|
2805
|
-
up: () => up$
|
|
2834
|
+
down: () => down$23,
|
|
2835
|
+
up: () => up$23
|
|
2806
2836
|
});
|
|
2807
2837
|
/**
|
|
2808
2838
|
* Drop the `source_secrets_change_trigger` and the
|
|
@@ -2821,11 +2851,11 @@ var _011_drop_source_secrets_notify_exports = /* @__PURE__ */ __exportAll({
|
|
|
2821
2851
|
* in `001_initial.ts`. They wake up no consumer until the `WebhookSecretManager`
|
|
2822
2852
|
* is restored, so this migration is safe to roll back.
|
|
2823
2853
|
*/
|
|
2824
|
-
async function up$
|
|
2854
|
+
async function up$23(db) {
|
|
2825
2855
|
await sql`DROP TRIGGER IF EXISTS source_secrets_change_trigger ON public.scoped_secrets`.execute(db);
|
|
2826
2856
|
await sql`DROP FUNCTION IF EXISTS public.notify_source_secrets_change() CASCADE`.execute(db);
|
|
2827
2857
|
}
|
|
2828
|
-
async function down$
|
|
2858
|
+
async function down$23(db) {
|
|
2829
2859
|
await sql`
|
|
2830
2860
|
CREATE OR REPLACE FUNCTION public.notify_source_secrets_change() RETURNS trigger
|
|
2831
2861
|
LANGUAGE plpgsql
|
|
@@ -2864,8 +2894,8 @@ async function down$21(db) {
|
|
|
2864
2894
|
//#endregion
|
|
2865
2895
|
//#region src/db/migrations/012_peer_credentials_active_uniq.ts
|
|
2866
2896
|
var _012_peer_credentials_active_uniq_exports = /* @__PURE__ */ __exportAll({
|
|
2867
|
-
down: () => down$
|
|
2868
|
-
up: () => up$
|
|
2897
|
+
down: () => down$22,
|
|
2898
|
+
up: () => up$22
|
|
2869
2899
|
});
|
|
2870
2900
|
/**
|
|
2871
2901
|
* Add a partial unique index on `peer_credentials (instance_id) WHERE
|
|
@@ -2891,7 +2921,7 @@ var _012_peer_credentials_active_uniq_exports = /* @__PURE__ */ __exportAll({
|
|
|
2891
2921
|
* `down()` only drops the index; it does NOT undo the dedupe (there's no
|
|
2892
2922
|
* safe way to recreate revoked rows, and the dedupe is monotonic).
|
|
2893
2923
|
*/
|
|
2894
|
-
async function up$
|
|
2924
|
+
async function up$22(db) {
|
|
2895
2925
|
await sql`
|
|
2896
2926
|
UPDATE public.peer_credentials
|
|
2897
2927
|
SET revoked_at = NOW()
|
|
@@ -2909,14 +2939,14 @@ async function up$20(db) {
|
|
|
2909
2939
|
WHERE revoked_at IS NULL
|
|
2910
2940
|
`.execute(db);
|
|
2911
2941
|
}
|
|
2912
|
-
async function down$
|
|
2942
|
+
async function down$22(db) {
|
|
2913
2943
|
await sql`DROP INDEX IF EXISTS public.peer_credentials_active_uniq`.execute(db);
|
|
2914
2944
|
}
|
|
2915
2945
|
//#endregion
|
|
2916
2946
|
//#region src/db/migrations/013_execution_log_bytes.ts
|
|
2917
2947
|
var _013_execution_log_bytes_exports = /* @__PURE__ */ __exportAll({
|
|
2918
|
-
down: () => down$
|
|
2919
|
-
up: () => up$
|
|
2948
|
+
down: () => down$21,
|
|
2949
|
+
up: () => up$21
|
|
2920
2950
|
});
|
|
2921
2951
|
/**
|
|
2922
2952
|
* Add `log_bytes BIGINT NOT NULL DEFAULT 0` columns to `execution_runs` and
|
|
@@ -2935,7 +2965,7 @@ var _013_execution_log_bytes_exports = /* @__PURE__ */ __exportAll({
|
|
|
2935
2965
|
*
|
|
2936
2966
|
* Idempotent (`ADD COLUMN IF NOT EXISTS`).
|
|
2937
2967
|
*/
|
|
2938
|
-
async function up$
|
|
2968
|
+
async function up$21(db) {
|
|
2939
2969
|
await sql`
|
|
2940
2970
|
ALTER TABLE public.execution_runs
|
|
2941
2971
|
ADD COLUMN IF NOT EXISTS log_bytes BIGINT NOT NULL DEFAULT 0
|
|
@@ -2945,15 +2975,15 @@ async function up$19(db) {
|
|
|
2945
2975
|
ADD COLUMN IF NOT EXISTS log_bytes BIGINT NOT NULL DEFAULT 0
|
|
2946
2976
|
`.execute(db);
|
|
2947
2977
|
}
|
|
2948
|
-
async function down$
|
|
2978
|
+
async function down$21(db) {
|
|
2949
2979
|
await sql`ALTER TABLE public.execution_runs DROP COLUMN IF EXISTS log_bytes`.execute(db);
|
|
2950
2980
|
await sql`ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS log_bytes`.execute(db);
|
|
2951
2981
|
}
|
|
2952
2982
|
//#endregion
|
|
2953
2983
|
//#region src/db/migrations/014_kici_events_lease_retry.ts
|
|
2954
2984
|
var _014_kici_events_lease_retry_exports = /* @__PURE__ */ __exportAll({
|
|
2955
|
-
down: () => down$
|
|
2956
|
-
up: () => up$
|
|
2985
|
+
down: () => down$20,
|
|
2986
|
+
up: () => up$20
|
|
2957
2987
|
});
|
|
2958
2988
|
/**
|
|
2959
2989
|
* Add lease + retry + DLQ columns to `kici_events` so the EventRouter can
|
|
@@ -2987,7 +3017,7 @@ var _014_kici_events_lease_retry_exports = /* @__PURE__ */ __exportAll({
|
|
|
2987
3017
|
*
|
|
2988
3018
|
* Idempotent (`ADD COLUMN IF NOT EXISTS` + `CREATE INDEX IF NOT EXISTS`).
|
|
2989
3019
|
*/
|
|
2990
|
-
async function up$
|
|
3020
|
+
async function up$20(db) {
|
|
2991
3021
|
await sql`
|
|
2992
3022
|
ALTER TABLE public.kici_events
|
|
2993
3023
|
ADD COLUMN IF NOT EXISTS claimed_at TIMESTAMPTZ,
|
|
@@ -3018,7 +3048,7 @@ async function up$18(db) {
|
|
|
3018
3048
|
WHERE dlq_at IS NOT NULL
|
|
3019
3049
|
`.execute(db);
|
|
3020
3050
|
}
|
|
3021
|
-
async function down$
|
|
3051
|
+
async function down$20(db) {
|
|
3022
3052
|
await sql`DROP INDEX IF EXISTS public.idx_kici_events_dlq`.execute(db);
|
|
3023
3053
|
await sql`DROP INDEX IF EXISTS public.idx_kici_events_lease_expired`.execute(db);
|
|
3024
3054
|
await sql`DROP INDEX IF EXISTS public.idx_kici_events_retry_due`.execute(db);
|
|
@@ -3036,8 +3066,8 @@ async function down$18(db) {
|
|
|
3036
3066
|
//#endregion
|
|
3037
3067
|
//#region src/db/migrations/015_org_settings_customer_scoped.ts
|
|
3038
3068
|
var _015_org_settings_customer_scoped_exports = /* @__PURE__ */ __exportAll({
|
|
3039
|
-
down: () => down$
|
|
3040
|
-
up: () => up$
|
|
3069
|
+
down: () => down$19,
|
|
3070
|
+
up: () => up$19
|
|
3041
3071
|
});
|
|
3042
3072
|
/**
|
|
3043
3073
|
* Org-scope `org_settings` and qualify each glob entry by source.
|
|
@@ -3065,7 +3095,7 @@ var _015_org_settings_customer_scoped_exports = /* @__PURE__ */ __exportAll({
|
|
|
3065
3095
|
* Idempotent: a re-run on an already-migrated DB sees `customer_id` exists
|
|
3066
3096
|
* and the list columns are already jsonb, so it is a no-op.
|
|
3067
3097
|
*/
|
|
3068
|
-
async function up$
|
|
3098
|
+
async function up$19(db) {
|
|
3069
3099
|
if ((await sql`
|
|
3070
3100
|
SELECT EXISTS (
|
|
3071
3101
|
SELECT 1 FROM information_schema.columns
|
|
@@ -3190,7 +3220,7 @@ async function up$17(db) {
|
|
|
3190
3220
|
await sql`DROP TABLE _org_settings_merged`.execute(db);
|
|
3191
3221
|
await sql`DROP TABLE _org_settings_stage`.execute(db);
|
|
3192
3222
|
}
|
|
3193
|
-
async function down$
|
|
3223
|
+
async function down$19(db) {
|
|
3194
3224
|
if (!(await sql`
|
|
3195
3225
|
SELECT EXISTS (
|
|
3196
3226
|
SELECT 1 FROM information_schema.columns
|
|
@@ -3215,8 +3245,8 @@ async function down$17(db) {
|
|
|
3215
3245
|
//#endregion
|
|
3216
3246
|
//#region src/db/migrations/016_org_settings_allow_http_npm.ts
|
|
3217
3247
|
var _016_org_settings_allow_http_npm_exports = /* @__PURE__ */ __exportAll({
|
|
3218
|
-
down: () => down$
|
|
3219
|
-
up: () => up$
|
|
3248
|
+
down: () => down$18,
|
|
3249
|
+
up: () => up$18
|
|
3220
3250
|
});
|
|
3221
3251
|
/**
|
|
3222
3252
|
* Add `org_settings.allow_http_npm_registries boolean NOT NULL DEFAULT false`.
|
|
@@ -3229,7 +3259,7 @@ var _016_org_settings_allow_http_npm_exports = /* @__PURE__ */ __exportAll({
|
|
|
3229
3259
|
*
|
|
3230
3260
|
* Idempotent: a re-run on a DB that already has the column is a no-op.
|
|
3231
3261
|
*/
|
|
3232
|
-
async function up$
|
|
3262
|
+
async function up$18(db) {
|
|
3233
3263
|
if ((await sql`
|
|
3234
3264
|
SELECT EXISTS (
|
|
3235
3265
|
SELECT 1 FROM information_schema.columns
|
|
@@ -3243,7 +3273,7 @@ async function up$16(db) {
|
|
|
3243
3273
|
ADD COLUMN allow_http_npm_registries boolean NOT NULL DEFAULT false
|
|
3244
3274
|
`.execute(db);
|
|
3245
3275
|
}
|
|
3246
|
-
async function down$
|
|
3276
|
+
async function down$18(db) {
|
|
3247
3277
|
await sql`
|
|
3248
3278
|
ALTER TABLE public.org_settings DROP COLUMN IF EXISTS allow_http_npm_registries
|
|
3249
3279
|
`.execute(db);
|
|
@@ -3251,8 +3281,8 @@ async function down$16(db) {
|
|
|
3251
3281
|
//#endregion
|
|
3252
3282
|
//#region src/db/migrations/017_org_id_widen.ts
|
|
3253
3283
|
var _017_org_id_widen_exports = /* @__PURE__ */ __exportAll({
|
|
3254
|
-
down: () => down$
|
|
3255
|
-
up: () => up$
|
|
3284
|
+
down: () => down$17,
|
|
3285
|
+
up: () => up$17
|
|
3256
3286
|
});
|
|
3257
3287
|
/**
|
|
3258
3288
|
* Widen every orchestrator-side `org_id varchar(12)` column to
|
|
@@ -3292,17 +3322,17 @@ const ORG_ID_TABLES$1 = [
|
|
|
3292
3322
|
"held_runs",
|
|
3293
3323
|
"scoped_secrets"
|
|
3294
3324
|
];
|
|
3295
|
-
async function up$
|
|
3325
|
+
async function up$17(db) {
|
|
3296
3326
|
for (const table of ORG_ID_TABLES$1) await sql.raw(`ALTER TABLE public.${table} ALTER COLUMN org_id TYPE varchar(16)`).execute(db);
|
|
3297
3327
|
}
|
|
3298
|
-
async function down$
|
|
3328
|
+
async function down$17(db) {
|
|
3299
3329
|
for (const table of ORG_ID_TABLES$1) await sql.raw(`ALTER TABLE public.${table} ALTER COLUMN org_id TYPE varchar(12)`).execute(db);
|
|
3300
3330
|
}
|
|
3301
3331
|
//#endregion
|
|
3302
3332
|
//#region src/db/migrations/018_org_id_prefix_backfill.ts
|
|
3303
3333
|
var _018_org_id_prefix_backfill_exports = /* @__PURE__ */ __exportAll({
|
|
3304
|
-
down: () => down$
|
|
3305
|
-
up: () => up$
|
|
3334
|
+
down: () => down$16,
|
|
3335
|
+
up: () => up$16
|
|
3306
3336
|
});
|
|
3307
3337
|
/**
|
|
3308
3338
|
* Prefix every orchestrator-side tenant string with `org_` to align
|
|
@@ -3347,19 +3377,19 @@ const CUSTOMER_ID_TABLES = [
|
|
|
3347
3377
|
"workflow_registrations",
|
|
3348
3378
|
"org_settings"
|
|
3349
3379
|
];
|
|
3350
|
-
async function up$
|
|
3380
|
+
async function up$16(db) {
|
|
3351
3381
|
for (const table of ORG_ID_TABLES) await sql.raw(`UPDATE public.${table} SET org_id = 'org_' || org_id WHERE org_id <> 'kici-admin' AND org_id NOT LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
|
|
3352
3382
|
for (const table of CUSTOMER_ID_TABLES) await sql.raw(`UPDATE public.${table} SET customer_id = 'org_' || customer_id WHERE customer_id <> 'kici-admin' AND customer_id NOT LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
|
|
3353
3383
|
}
|
|
3354
|
-
async function down$
|
|
3384
|
+
async function down$16(db) {
|
|
3355
3385
|
for (const table of CUSTOMER_ID_TABLES) await sql.raw(`UPDATE public.${table} SET customer_id = substring(customer_id from 5) WHERE customer_id LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
|
|
3356
3386
|
for (const table of ORG_ID_TABLES) await sql.raw(`UPDATE public.${table} SET org_id = substring(org_id from 5) WHERE org_id LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
|
|
3357
3387
|
}
|
|
3358
3388
|
//#endregion
|
|
3359
3389
|
//#region src/db/migrations/019_generic_sources_change_notify.ts
|
|
3360
3390
|
var _019_generic_sources_change_notify_exports = /* @__PURE__ */ __exportAll({
|
|
3361
|
-
down: () => down$
|
|
3362
|
-
up: () => up$
|
|
3391
|
+
down: () => down$15,
|
|
3392
|
+
up: () => up$15
|
|
3363
3393
|
});
|
|
3364
3394
|
/**
|
|
3365
3395
|
* Add a Postgres trigger on `generic_webhook_sources` that emits
|
|
@@ -3382,7 +3412,7 @@ var _019_generic_sources_change_notify_exports = /* @__PURE__ */ __exportAll({
|
|
|
3382
3412
|
* (`notify_sources_change()` + `sources_change_trigger`, defined in
|
|
3383
3413
|
* `001_initial.ts`).
|
|
3384
3414
|
*/
|
|
3385
|
-
async function up$
|
|
3415
|
+
async function up$15(db) {
|
|
3386
3416
|
await sql`
|
|
3387
3417
|
CREATE FUNCTION public.notify_generic_sources_change() RETURNS trigger
|
|
3388
3418
|
LANGUAGE plpgsql
|
|
@@ -3403,15 +3433,15 @@ async function up$13(db) {
|
|
|
3403
3433
|
FOR EACH ROW EXECUTE FUNCTION public.notify_generic_sources_change()
|
|
3404
3434
|
`.execute(db);
|
|
3405
3435
|
}
|
|
3406
|
-
async function down$
|
|
3436
|
+
async function down$15(db) {
|
|
3407
3437
|
await sql`DROP TRIGGER IF EXISTS generic_sources_change_trigger ON public.generic_webhook_sources`.execute(db);
|
|
3408
3438
|
await sql`DROP FUNCTION IF EXISTS public.notify_generic_sources_change()`.execute(db);
|
|
3409
3439
|
}
|
|
3410
3440
|
//#endregion
|
|
3411
3441
|
//#region src/db/migrations/020_org_settings_dashboard_write_policy.ts
|
|
3412
3442
|
var _020_org_settings_dashboard_write_policy_exports = /* @__PURE__ */ __exportAll({
|
|
3413
|
-
down: () => down$
|
|
3414
|
-
up: () => up$
|
|
3443
|
+
down: () => down$14,
|
|
3444
|
+
up: () => up$14
|
|
3415
3445
|
});
|
|
3416
3446
|
/**
|
|
3417
3447
|
* Add `org_settings.dashboard_write_policy jsonb NOT NULL DEFAULT '{}'`.
|
|
@@ -3426,7 +3456,7 @@ var _020_org_settings_dashboard_write_policy_exports = /* @__PURE__ */ __exportA
|
|
|
3426
3456
|
*
|
|
3427
3457
|
* Idempotent: a re-run on a DB that already has the column is a no-op.
|
|
3428
3458
|
*/
|
|
3429
|
-
async function up$
|
|
3459
|
+
async function up$14(db) {
|
|
3430
3460
|
if ((await sql`
|
|
3431
3461
|
SELECT EXISTS (
|
|
3432
3462
|
SELECT 1 FROM information_schema.columns
|
|
@@ -3440,7 +3470,7 @@ async function up$12(db) {
|
|
|
3440
3470
|
ADD COLUMN dashboard_write_policy jsonb NOT NULL DEFAULT '{}'::jsonb
|
|
3441
3471
|
`.execute(db);
|
|
3442
3472
|
}
|
|
3443
|
-
async function down$
|
|
3473
|
+
async function down$14(db) {
|
|
3444
3474
|
await sql`
|
|
3445
3475
|
ALTER TABLE public.org_settings DROP COLUMN IF EXISTS dashboard_write_policy
|
|
3446
3476
|
`.execute(db);
|
|
@@ -3448,8 +3478,8 @@ async function down$12(db) {
|
|
|
3448
3478
|
//#endregion
|
|
3449
3479
|
//#region src/db/migrations/021_check_run_tracking.ts
|
|
3450
3480
|
var _021_check_run_tracking_exports = /* @__PURE__ */ __exportAll({
|
|
3451
|
-
down: () => down$
|
|
3452
|
-
up: () => up$
|
|
3481
|
+
down: () => down$13,
|
|
3482
|
+
up: () => up$13
|
|
3453
3483
|
});
|
|
3454
3484
|
/**
|
|
3455
3485
|
* Add `check_run_tracking` table for HA-safe check-run state persistence.
|
|
@@ -3473,7 +3503,7 @@ var _021_check_run_tracking_exports = /* @__PURE__ */ __exportAll({
|
|
|
3473
3503
|
*
|
|
3474
3504
|
* Idempotent: a re-run on a DB that already has the table is a no-op.
|
|
3475
3505
|
*/
|
|
3476
|
-
async function up$
|
|
3506
|
+
async function up$13(db) {
|
|
3477
3507
|
if ((await sql`
|
|
3478
3508
|
SELECT EXISTS (
|
|
3479
3509
|
SELECT 1 FROM information_schema.tables
|
|
@@ -3504,14 +3534,14 @@ async function up$11(db) {
|
|
|
3504
3534
|
WHERE run_id IS NOT NULL
|
|
3505
3535
|
`.execute(db);
|
|
3506
3536
|
}
|
|
3507
|
-
async function down$
|
|
3537
|
+
async function down$13(db) {
|
|
3508
3538
|
await sql`DROP TABLE IF EXISTS public.check_run_tracking`.execute(db);
|
|
3509
3539
|
}
|
|
3510
3540
|
//#endregion
|
|
3511
3541
|
//#region src/db/migrations/022_scaler_manager_state.ts
|
|
3512
3542
|
var _022_scaler_manager_state_exports = /* @__PURE__ */ __exportAll({
|
|
3513
|
-
down: () => down$
|
|
3514
|
-
up: () => up$
|
|
3543
|
+
down: () => down$12,
|
|
3544
|
+
up: () => up$12
|
|
3515
3545
|
});
|
|
3516
3546
|
/**
|
|
3517
3547
|
* Add three tables persisting `ScalerManager` per-coord state:
|
|
@@ -3538,7 +3568,7 @@ var _022_scaler_manager_state_exports = /* @__PURE__ */ __exportAll({
|
|
|
3538
3568
|
* Idempotent: a re-run on a DB that already has any of these tables
|
|
3539
3569
|
* leaves the existing one alone.
|
|
3540
3570
|
*/
|
|
3541
|
-
async function up$
|
|
3571
|
+
async function up$12(db) {
|
|
3542
3572
|
const tableExists = async (name) => {
|
|
3543
3573
|
return (await sql`
|
|
3544
3574
|
SELECT EXISTS (
|
|
@@ -3588,7 +3618,7 @@ async function up$10(db) {
|
|
|
3588
3618
|
`.execute(db);
|
|
3589
3619
|
}
|
|
3590
3620
|
}
|
|
3591
|
-
async function down$
|
|
3621
|
+
async function down$12(db) {
|
|
3592
3622
|
await sql`DROP TABLE IF EXISTS public.scaler_reservations`.execute(db);
|
|
3593
3623
|
await sql`DROP TABLE IF EXISTS public.scaler_agent_jobs`.execute(db);
|
|
3594
3624
|
await sql`DROP TABLE IF EXISTS public.scaler_spawning_agents`.execute(db);
|
|
@@ -3596,8 +3626,8 @@ async function down$10(db) {
|
|
|
3596
3626
|
//#endregion
|
|
3597
3627
|
//#region src/db/migrations/023_dispatch_queue_recovery_deadline.ts
|
|
3598
3628
|
var _023_dispatch_queue_recovery_deadline_exports = /* @__PURE__ */ __exportAll({
|
|
3599
|
-
down: () => down$
|
|
3600
|
-
up: () => up$
|
|
3629
|
+
down: () => down$11,
|
|
3630
|
+
up: () => up$11
|
|
3601
3631
|
});
|
|
3602
3632
|
/**
|
|
3603
3633
|
* Add `dispatch_queue.recovery_deadline TIMESTAMPTZ` and
|
|
@@ -3621,7 +3651,7 @@ var _023_dispatch_queue_recovery_deadline_exports = /* @__PURE__ */ __exportAll(
|
|
|
3621
3651
|
* Idempotent: re-running on a DB that already has either column is a
|
|
3622
3652
|
* no-op.
|
|
3623
3653
|
*/
|
|
3624
|
-
async function up$
|
|
3654
|
+
async function up$11(db) {
|
|
3625
3655
|
const colExists = async (name) => {
|
|
3626
3656
|
return (await sql`
|
|
3627
3657
|
SELECT EXISTS (
|
|
@@ -3646,7 +3676,7 @@ async function up$9(db) {
|
|
|
3646
3676
|
WHERE recovery_deadline IS NOT NULL
|
|
3647
3677
|
`.execute(db);
|
|
3648
3678
|
}
|
|
3649
|
-
async function down$
|
|
3679
|
+
async function down$11(db) {
|
|
3650
3680
|
await sql`DROP INDEX IF EXISTS public.idx_dispatch_queue_recovery_deadline`.execute(db);
|
|
3651
3681
|
await sql`
|
|
3652
3682
|
ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS recovery_agent_id
|
|
@@ -3658,8 +3688,8 @@ async function down$9(db) {
|
|
|
3658
3688
|
//#endregion
|
|
3659
3689
|
//#region src/db/migrations/024_dispatch_queue_provisioning_error.ts
|
|
3660
3690
|
var _024_dispatch_queue_provisioning_error_exports = /* @__PURE__ */ __exportAll({
|
|
3661
|
-
down: () => down$
|
|
3662
|
-
up: () => up$
|
|
3691
|
+
down: () => down$10,
|
|
3692
|
+
up: () => up$10
|
|
3663
3693
|
});
|
|
3664
3694
|
/**
|
|
3665
3695
|
* Add `dispatch_queue.last_provisioning_error TEXT` recording the most
|
|
@@ -3675,7 +3705,7 @@ var _024_dispatch_queue_provisioning_error_exports = /* @__PURE__ */ __exportAll
|
|
|
3675
3705
|
*
|
|
3676
3706
|
* Idempotent: re-running on a DB that already has the column is a no-op.
|
|
3677
3707
|
*/
|
|
3678
|
-
async function up$
|
|
3708
|
+
async function up$10(db) {
|
|
3679
3709
|
const colExists = async (name) => {
|
|
3680
3710
|
return (await sql`
|
|
3681
3711
|
SELECT EXISTS (
|
|
@@ -3691,7 +3721,7 @@ async function up$8(db) {
|
|
|
3691
3721
|
ADD COLUMN last_provisioning_error TEXT
|
|
3692
3722
|
`.execute(db);
|
|
3693
3723
|
}
|
|
3694
|
-
async function down$
|
|
3724
|
+
async function down$10(db) {
|
|
3695
3725
|
await sql`
|
|
3696
3726
|
ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS last_provisioning_error
|
|
3697
3727
|
`.execute(db);
|
|
@@ -3699,8 +3729,8 @@ async function down$8(db) {
|
|
|
3699
3729
|
//#endregion
|
|
3700
3730
|
//#region src/db/migrations/025_init_failure.ts
|
|
3701
3731
|
var _025_init_failure_exports = /* @__PURE__ */ __exportAll({
|
|
3702
|
-
down: () => down$
|
|
3703
|
-
up: () => up$
|
|
3732
|
+
down: () => down$9,
|
|
3733
|
+
up: () => up$9
|
|
3704
3734
|
});
|
|
3705
3735
|
/**
|
|
3706
3736
|
* Add `init_failure jsonb` columns to `execution_runs` and `execution_jobs`.
|
|
@@ -3714,7 +3744,7 @@ var _025_init_failure_exports = /* @__PURE__ */ __exportAll({
|
|
|
3714
3744
|
* Idempotent: re-running on a DB that already has either column is a no-op
|
|
3715
3745
|
* for that column.
|
|
3716
3746
|
*/
|
|
3717
|
-
async function up$
|
|
3747
|
+
async function up$9(db) {
|
|
3718
3748
|
const colExists = async (table, name) => {
|
|
3719
3749
|
return (await sql`
|
|
3720
3750
|
SELECT EXISTS (
|
|
@@ -3734,7 +3764,7 @@ async function up$7(db) {
|
|
|
3734
3764
|
ADD COLUMN init_failure JSONB DEFAULT NULL
|
|
3735
3765
|
`.execute(db);
|
|
3736
3766
|
}
|
|
3737
|
-
async function down$
|
|
3767
|
+
async function down$9(db) {
|
|
3738
3768
|
await sql`
|
|
3739
3769
|
ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS init_failure
|
|
3740
3770
|
`.execute(db);
|
|
@@ -3745,8 +3775,8 @@ async function down$7(db) {
|
|
|
3745
3775
|
//#endregion
|
|
3746
3776
|
//#region src/db/migrations/026_event_log_lockfile_corrupt.ts
|
|
3747
3777
|
var _026_event_log_lockfile_corrupt_exports = /* @__PURE__ */ __exportAll({
|
|
3748
|
-
down: () => down$
|
|
3749
|
-
up: () => up$
|
|
3778
|
+
down: () => down$8,
|
|
3779
|
+
up: () => up$8
|
|
3750
3780
|
});
|
|
3751
3781
|
/**
|
|
3752
3782
|
* Extend the event_log.status CHECK constraint with 'lockfile_corrupt' so the
|
|
@@ -3755,7 +3785,7 @@ var _026_event_log_lockfile_corrupt_exports = /* @__PURE__ */ __exportAll({
|
|
|
3755
3785
|
*
|
|
3756
3786
|
* Idempotent: the DROP ... IF EXISTS / re-ADD pair re-runs cleanly.
|
|
3757
3787
|
*/
|
|
3758
|
-
async function up$
|
|
3788
|
+
async function up$8(db) {
|
|
3759
3789
|
await sql`ALTER TABLE event_log DROP CONSTRAINT IF EXISTS event_log_status_check`.execute(db);
|
|
3760
3790
|
await sql`
|
|
3761
3791
|
ALTER TABLE event_log ADD CONSTRAINT event_log_status_check
|
|
@@ -3765,7 +3795,7 @@ async function up$6(db) {
|
|
|
3765
3795
|
])))
|
|
3766
3796
|
`.execute(db);
|
|
3767
3797
|
}
|
|
3768
|
-
async function down$
|
|
3798
|
+
async function down$8(db) {
|
|
3769
3799
|
await sql`ALTER TABLE event_log DROP CONSTRAINT IF EXISTS event_log_status_check`.execute(db);
|
|
3770
3800
|
await sql`
|
|
3771
3801
|
ALTER TABLE event_log ADD CONSTRAINT event_log_status_check
|
|
@@ -3778,8 +3808,8 @@ async function down$6(db) {
|
|
|
3778
3808
|
//#endregion
|
|
3779
3809
|
//#region src/db/migrations/027_workflow_timeout.ts
|
|
3780
3810
|
var _027_workflow_timeout_exports = /* @__PURE__ */ __exportAll({
|
|
3781
|
-
down: () => down$
|
|
3782
|
-
up: () => up$
|
|
3811
|
+
down: () => down$7,
|
|
3812
|
+
up: () => up$7
|
|
3783
3813
|
});
|
|
3784
3814
|
/**
|
|
3785
3815
|
* Add `workflow_timeout_ms integer` to `execution_runs`.
|
|
@@ -3797,13 +3827,13 @@ var _027_workflow_timeout_exports = /* @__PURE__ */ __exportAll({
|
|
|
3797
3827
|
*
|
|
3798
3828
|
* Idempotent: re-running on a DB that already has the column is a no-op.
|
|
3799
3829
|
*/
|
|
3800
|
-
async function up$
|
|
3830
|
+
async function up$7(db) {
|
|
3801
3831
|
await sql`
|
|
3802
3832
|
ALTER TABLE public.execution_runs
|
|
3803
3833
|
ADD COLUMN IF NOT EXISTS workflow_timeout_ms INTEGER DEFAULT NULL
|
|
3804
3834
|
`.execute(db);
|
|
3805
3835
|
}
|
|
3806
|
-
async function down$
|
|
3836
|
+
async function down$7(db) {
|
|
3807
3837
|
await sql`
|
|
3808
3838
|
ALTER TABLE public.execution_runs DROP COLUMN IF EXISTS workflow_timeout_ms
|
|
3809
3839
|
`.execute(db);
|
|
@@ -3811,8 +3841,8 @@ async function down$5(db) {
|
|
|
3811
3841
|
//#endregion
|
|
3812
3842
|
//#region src/db/migrations/028_org_settings_user_cache.ts
|
|
3813
3843
|
var _028_org_settings_user_cache_exports = /* @__PURE__ */ __exportAll({
|
|
3814
|
-
down: () => down$
|
|
3815
|
-
up: () => up$
|
|
3844
|
+
down: () => down$6,
|
|
3845
|
+
up: () => up$6
|
|
3816
3846
|
});
|
|
3817
3847
|
/**
|
|
3818
3848
|
* Add `org_settings.user_cache_quota_bytes bigint` and
|
|
@@ -3841,7 +3871,7 @@ async function columnExists(db, column) {
|
|
|
3841
3871
|
) AS exists
|
|
3842
3872
|
`.execute(db)).rows[0]?.exists ?? false;
|
|
3843
3873
|
}
|
|
3844
|
-
async function up$
|
|
3874
|
+
async function up$6(db) {
|
|
3845
3875
|
if (!await columnExists(db, "user_cache_quota_bytes")) await sql`
|
|
3846
3876
|
ALTER TABLE public.org_settings
|
|
3847
3877
|
ADD COLUMN user_cache_quota_bytes bigint
|
|
@@ -3851,7 +3881,7 @@ async function up$4(db) {
|
|
|
3851
3881
|
ADD COLUMN user_cache_ttl_ms bigint
|
|
3852
3882
|
`.execute(db);
|
|
3853
3883
|
}
|
|
3854
|
-
async function down$
|
|
3884
|
+
async function down$6(db) {
|
|
3855
3885
|
await sql`
|
|
3856
3886
|
ALTER TABLE public.org_settings DROP COLUMN IF EXISTS user_cache_quota_bytes
|
|
3857
3887
|
`.execute(db);
|
|
@@ -3862,8 +3892,8 @@ async function down$4(db) {
|
|
|
3862
3892
|
//#endregion
|
|
3863
3893
|
//#region src/db/migrations/029_dispatch_queue_attempts.ts
|
|
3864
3894
|
var _029_dispatch_queue_attempts_exports = /* @__PURE__ */ __exportAll({
|
|
3865
|
-
down: () => down$
|
|
3866
|
-
up: () => up$
|
|
3895
|
+
down: () => down$5,
|
|
3896
|
+
up: () => up$5
|
|
3867
3897
|
});
|
|
3868
3898
|
/**
|
|
3869
3899
|
* Add `dispatch_queue.dispatch_attempts INT NOT NULL DEFAULT 0`.
|
|
@@ -3877,7 +3907,7 @@ var _029_dispatch_queue_attempts_exports = /* @__PURE__ */ __exportAll({
|
|
|
3877
3907
|
*
|
|
3878
3908
|
* Idempotent: re-running on a DB that already has the column is a no-op.
|
|
3879
3909
|
*/
|
|
3880
|
-
async function up$
|
|
3910
|
+
async function up$5(db) {
|
|
3881
3911
|
if (!((await sql`
|
|
3882
3912
|
SELECT EXISTS (
|
|
3883
3913
|
SELECT 1 FROM information_schema.columns
|
|
@@ -3890,7 +3920,7 @@ async function up$3(db) {
|
|
|
3890
3920
|
ADD COLUMN dispatch_attempts INT NOT NULL DEFAULT 0
|
|
3891
3921
|
`.execute(db);
|
|
3892
3922
|
}
|
|
3893
|
-
async function down$
|
|
3923
|
+
async function down$5(db) {
|
|
3894
3924
|
await sql`
|
|
3895
3925
|
ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS dispatch_attempts
|
|
3896
3926
|
`.execute(db);
|
|
@@ -3898,8 +3928,8 @@ async function down$3(db) {
|
|
|
3898
3928
|
//#endregion
|
|
3899
3929
|
//#region src/db/migrations/030_held_runs_env_set_null.ts
|
|
3900
3930
|
var _030_held_runs_env_set_null_exports = /* @__PURE__ */ __exportAll({
|
|
3901
|
-
down: () => down$
|
|
3902
|
-
up: () => up$
|
|
3931
|
+
down: () => down$4,
|
|
3932
|
+
up: () => up$4
|
|
3903
3933
|
});
|
|
3904
3934
|
/**
|
|
3905
3935
|
* held_runs.environment_id becomes nullable with ON DELETE SET NULL so
|
|
@@ -3910,14 +3940,14 @@ var _030_held_runs_env_set_null_exports = /* @__PURE__ */ __exportAll({
|
|
|
3910
3940
|
* Idempotent: dropping the NOT NULL and the constraint are both no-ops on a
|
|
3911
3941
|
* re-run, and the constraint is re-created with the SET NULL action.
|
|
3912
3942
|
*/
|
|
3913
|
-
async function up$
|
|
3943
|
+
async function up$4(db) {
|
|
3914
3944
|
await sql`ALTER TABLE public.held_runs ALTER COLUMN environment_id DROP NOT NULL`.execute(db);
|
|
3915
3945
|
await sql`ALTER TABLE public.held_runs DROP CONSTRAINT IF EXISTS held_runs_environment_id_fkey`.execute(db);
|
|
3916
3946
|
await sql`ALTER TABLE public.held_runs
|
|
3917
3947
|
ADD CONSTRAINT held_runs_environment_id_fkey
|
|
3918
3948
|
FOREIGN KEY (environment_id) REFERENCES public.environments(id) ON DELETE SET NULL`.execute(db);
|
|
3919
3949
|
}
|
|
3920
|
-
async function down$
|
|
3950
|
+
async function down$4(db) {
|
|
3921
3951
|
await sql`ALTER TABLE public.held_runs DROP CONSTRAINT IF EXISTS held_runs_environment_id_fkey`.execute(db);
|
|
3922
3952
|
await sql`ALTER TABLE public.held_runs
|
|
3923
3953
|
ADD CONSTRAINT held_runs_environment_id_fkey
|
|
@@ -3927,8 +3957,8 @@ async function down$2(db) {
|
|
|
3927
3957
|
//#endregion
|
|
3928
3958
|
//#region src/db/migrations/031_dispatch_queue_ack_deadline.ts
|
|
3929
3959
|
var _031_dispatch_queue_ack_deadline_exports = /* @__PURE__ */ __exportAll({
|
|
3930
|
-
down: () => down$
|
|
3931
|
-
up: () => up$
|
|
3960
|
+
down: () => down$3,
|
|
3961
|
+
up: () => up$3
|
|
3932
3962
|
});
|
|
3933
3963
|
/**
|
|
3934
3964
|
* Add `dispatch_queue.ack_deadline TIMESTAMPTZ` and
|
|
@@ -3945,7 +3975,7 @@ var _031_dispatch_queue_ack_deadline_exports = /* @__PURE__ */ __exportAll({
|
|
|
3945
3975
|
*
|
|
3946
3976
|
* Idempotent: re-running on a DB that already has either column is a no-op.
|
|
3947
3977
|
*/
|
|
3948
|
-
async function up$
|
|
3978
|
+
async function up$3(db) {
|
|
3949
3979
|
const colExists = async (name) => {
|
|
3950
3980
|
return (await sql`
|
|
3951
3981
|
SELECT EXISTS (
|
|
@@ -3970,7 +4000,7 @@ async function up$1(db) {
|
|
|
3970
4000
|
WHERE ack_deadline IS NOT NULL
|
|
3971
4001
|
`.execute(db);
|
|
3972
4002
|
}
|
|
3973
|
-
async function down$
|
|
4003
|
+
async function down$3(db) {
|
|
3974
4004
|
await sql`DROP INDEX IF EXISTS public.idx_dispatch_queue_ack_deadline`.execute(db);
|
|
3975
4005
|
await sql`ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS ack_agent_id`.execute(db);
|
|
3976
4006
|
await sql`ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS ack_deadline`.execute(db);
|
|
@@ -3978,8 +4008,8 @@ async function down$1(db) {
|
|
|
3978
4008
|
//#endregion
|
|
3979
4009
|
//#region src/db/migrations/032_org_settings_dispatch_ack_timeout.ts
|
|
3980
4010
|
var _032_org_settings_dispatch_ack_timeout_exports = /* @__PURE__ */ __exportAll({
|
|
3981
|
-
down: () => down,
|
|
3982
|
-
up: () => up
|
|
4011
|
+
down: () => down$2,
|
|
4012
|
+
up: () => up$2
|
|
3983
4013
|
});
|
|
3984
4014
|
/**
|
|
3985
4015
|
* Add `org_settings.dispatch_ack_timeout_ms BIGINT` (nullable).
|
|
@@ -3991,7 +4021,7 @@ var _032_org_settings_dispatch_ack_timeout_exports = /* @__PURE__ */ __exportAll
|
|
|
3991
4021
|
*
|
|
3992
4022
|
* Idempotent: a re-run on a DB that already has the column is a no-op.
|
|
3993
4023
|
*/
|
|
3994
|
-
async function up(db) {
|
|
4024
|
+
async function up$2(db) {
|
|
3995
4025
|
if ((await sql`
|
|
3996
4026
|
SELECT EXISTS (
|
|
3997
4027
|
SELECT 1 FROM information_schema.columns
|
|
@@ -4005,12 +4035,127 @@ async function up(db) {
|
|
|
4005
4035
|
ADD COLUMN dispatch_ack_timeout_ms BIGINT
|
|
4006
4036
|
`.execute(db);
|
|
4007
4037
|
}
|
|
4008
|
-
async function down(db) {
|
|
4038
|
+
async function down$2(db) {
|
|
4009
4039
|
await sql`
|
|
4010
4040
|
ALTER TABLE public.org_settings DROP COLUMN IF EXISTS dispatch_ack_timeout_ms
|
|
4011
4041
|
`.execute(db);
|
|
4012
4042
|
}
|
|
4013
4043
|
//#endregion
|
|
4044
|
+
//#region src/db/migrations/033_org_settings_approval.ts
|
|
4045
|
+
var _033_org_settings_approval_exports = /* @__PURE__ */ __exportAll({
|
|
4046
|
+
down: () => down$1,
|
|
4047
|
+
up: () => up$1
|
|
4048
|
+
});
|
|
4049
|
+
/**
|
|
4050
|
+
* Add the two approval-policy columns to `org_settings`:
|
|
4051
|
+
*
|
|
4052
|
+
* - `approval_expiry_seconds INTEGER NOT NULL DEFAULT 86400` — how long a held
|
|
4053
|
+
* approval element waits before it expires (and its run/job/step is
|
|
4054
|
+
* rejected). One day by default. An SDK `requireApproval` `timeout` overrides
|
|
4055
|
+
* this per element; otherwise this per-org value applies.
|
|
4056
|
+
* - `allow_self_approval BOOLEAN NOT NULL DEFAULT true` — whether the user who
|
|
4057
|
+
* triggered a run may also approve its held elements. Operators turn it off
|
|
4058
|
+
* to enforce four-eyes review.
|
|
4059
|
+
*
|
|
4060
|
+
* Both are cluster-configurable per org via `kici-admin org-settings approval`
|
|
4061
|
+
* and the orchestrator admin route. Idempotent: a re-run on a DB that already
|
|
4062
|
+
* has the columns is a no-op (each column is guarded independently).
|
|
4063
|
+
*/
|
|
4064
|
+
async function up$1(db) {
|
|
4065
|
+
const colExists = async (column) => {
|
|
4066
|
+
return (await sql`
|
|
4067
|
+
SELECT EXISTS (
|
|
4068
|
+
SELECT 1 FROM information_schema.columns
|
|
4069
|
+
WHERE table_schema = 'public'
|
|
4070
|
+
AND table_name = 'org_settings'
|
|
4071
|
+
AND column_name = ${column}
|
|
4072
|
+
) AS exists
|
|
4073
|
+
`.execute(db)).rows[0]?.exists ?? false;
|
|
4074
|
+
};
|
|
4075
|
+
if (!await colExists("approval_expiry_seconds")) await sql`
|
|
4076
|
+
ALTER TABLE public.org_settings
|
|
4077
|
+
ADD COLUMN approval_expiry_seconds INTEGER NOT NULL DEFAULT 86400
|
|
4078
|
+
`.execute(db);
|
|
4079
|
+
if (!await colExists("allow_self_approval")) await sql`
|
|
4080
|
+
ALTER TABLE public.org_settings
|
|
4081
|
+
ADD COLUMN allow_self_approval BOOLEAN NOT NULL DEFAULT true
|
|
4082
|
+
`.execute(db);
|
|
4083
|
+
}
|
|
4084
|
+
async function down$1(db) {
|
|
4085
|
+
await sql`
|
|
4086
|
+
ALTER TABLE public.org_settings DROP COLUMN IF EXISTS approval_expiry_seconds
|
|
4087
|
+
`.execute(db);
|
|
4088
|
+
await sql`
|
|
4089
|
+
ALTER TABLE public.org_settings DROP COLUMN IF EXISTS allow_self_approval
|
|
4090
|
+
`.execute(db);
|
|
4091
|
+
}
|
|
4092
|
+
//#endregion
|
|
4093
|
+
//#region src/db/migrations/034_held_runs_generalize.ts
|
|
4094
|
+
var _034_held_runs_generalize_exports = /* @__PURE__ */ __exportAll({
|
|
4095
|
+
down: () => down,
|
|
4096
|
+
up: () => up
|
|
4097
|
+
});
|
|
4098
|
+
/**
|
|
4099
|
+
* Generalize `held_runs` from an environment-only hold into the unified
|
|
4100
|
+
* "held element" model that backs per-element approvals, and add the
|
|
4101
|
+
* `held_run_approvals` table that records each approver's decision.
|
|
4102
|
+
*
|
|
4103
|
+
* New `held_runs` columns (all idempotent, column-exists guarded):
|
|
4104
|
+
* - `hold_scope text NOT NULL DEFAULT 'job'` — 'workflow' | 'job' | 'step'
|
|
4105
|
+
* (engine `HoldScope`). Existing rows held a single job, so they default to
|
|
4106
|
+
* 'job'.
|
|
4107
|
+
* - `step_index integer` — nullable; set only for step-scoped holds.
|
|
4108
|
+
* - `trigger_source text NOT NULL DEFAULT 'environment'` — 'environment' |
|
|
4109
|
+
* 'explicit' (engine `TriggerSource`). Existing holds came from environment
|
|
4110
|
+
* protection, so they default to 'environment'.
|
|
4111
|
+
* - `approval_requirement jsonb` — the normalized `ApprovalRequirement`
|
|
4112
|
+
* (clauses + expiresAt + reason) the hold must satisfy. Nullable for legacy
|
|
4113
|
+
* rows that predate the approval model.
|
|
4114
|
+
*
|
|
4115
|
+
* New `held_run_approvals` table: one row per approver decision, FK to
|
|
4116
|
+
* `held_runs.id` (uuid) with ON DELETE CASCADE.
|
|
4117
|
+
*/
|
|
4118
|
+
async function up(db) {
|
|
4119
|
+
const colExists = async (column) => {
|
|
4120
|
+
return (await sql`
|
|
4121
|
+
SELECT EXISTS (
|
|
4122
|
+
SELECT 1 FROM information_schema.columns
|
|
4123
|
+
WHERE table_schema = 'public'
|
|
4124
|
+
AND table_name = 'held_runs'
|
|
4125
|
+
AND column_name = ${column}
|
|
4126
|
+
) AS exists
|
|
4127
|
+
`.execute(db)).rows[0]?.exists ?? false;
|
|
4128
|
+
};
|
|
4129
|
+
if (!await colExists("hold_scope")) await sql`ALTER TABLE public.held_runs ADD COLUMN hold_scope text NOT NULL DEFAULT 'job'`.execute(db);
|
|
4130
|
+
if (!await colExists("step_index")) await sql`ALTER TABLE public.held_runs ADD COLUMN step_index integer`.execute(db);
|
|
4131
|
+
if (!await colExists("trigger_source")) await sql`ALTER TABLE public.held_runs ADD COLUMN trigger_source text NOT NULL DEFAULT 'environment'`.execute(db);
|
|
4132
|
+
if (!await colExists("approval_requirement")) await sql`ALTER TABLE public.held_runs ADD COLUMN approval_requirement jsonb`.execute(db);
|
|
4133
|
+
await sql`
|
|
4134
|
+
CREATE TABLE IF NOT EXISTS public.held_run_approvals (
|
|
4135
|
+
id uuid DEFAULT gen_random_uuid() NOT NULL,
|
|
4136
|
+
held_run_id uuid NOT NULL,
|
|
4137
|
+
approver_user_id text NOT NULL,
|
|
4138
|
+
decision text NOT NULL,
|
|
4139
|
+
clauses_satisfied jsonb,
|
|
4140
|
+
created_at timestamp with time zone DEFAULT now() NOT NULL,
|
|
4141
|
+
CONSTRAINT held_run_approvals_pkey PRIMARY KEY (id),
|
|
4142
|
+
CONSTRAINT held_run_approvals_held_run_id_fkey
|
|
4143
|
+
FOREIGN KEY (held_run_id) REFERENCES public.held_runs(id) ON DELETE CASCADE
|
|
4144
|
+
)
|
|
4145
|
+
`.execute(db);
|
|
4146
|
+
await sql`
|
|
4147
|
+
CREATE INDEX IF NOT EXISTS held_run_approvals_held_run_id_idx
|
|
4148
|
+
ON public.held_run_approvals USING btree (held_run_id)
|
|
4149
|
+
`.execute(db);
|
|
4150
|
+
}
|
|
4151
|
+
async function down(db) {
|
|
4152
|
+
await sql`DROP TABLE IF EXISTS public.held_run_approvals`.execute(db);
|
|
4153
|
+
await sql`ALTER TABLE public.held_runs DROP COLUMN IF EXISTS approval_requirement`.execute(db);
|
|
4154
|
+
await sql`ALTER TABLE public.held_runs DROP COLUMN IF EXISTS trigger_source`.execute(db);
|
|
4155
|
+
await sql`ALTER TABLE public.held_runs DROP COLUMN IF EXISTS step_index`.execute(db);
|
|
4156
|
+
await sql`ALTER TABLE public.held_runs DROP COLUMN IF EXISTS hold_scope`.execute(db);
|
|
4157
|
+
}
|
|
4158
|
+
//#endregion
|
|
4014
4159
|
//#region src/db/migration-provider.ts
|
|
4015
4160
|
function createMigrationProvider() {
|
|
4016
4161
|
return { async getMigrations() {
|
|
@@ -4046,7 +4191,9 @@ function createMigrationProvider() {
|
|
|
4046
4191
|
"029_dispatch_queue_attempts": _029_dispatch_queue_attempts_exports,
|
|
4047
4192
|
"030_held_runs_env_set_null": _030_held_runs_env_set_null_exports,
|
|
4048
4193
|
"031_dispatch_queue_ack_deadline": _031_dispatch_queue_ack_deadline_exports,
|
|
4049
|
-
"032_org_settings_dispatch_ack_timeout": _032_org_settings_dispatch_ack_timeout_exports
|
|
4194
|
+
"032_org_settings_dispatch_ack_timeout": _032_org_settings_dispatch_ack_timeout_exports,
|
|
4195
|
+
"033_org_settings_approval": _033_org_settings_approval_exports,
|
|
4196
|
+
"034_held_runs_generalize": _034_held_runs_generalize_exports
|
|
4050
4197
|
};
|
|
4051
4198
|
} };
|
|
4052
4199
|
}
|
|
@@ -4058,7 +4205,7 @@ function createMigrationProvider() {
|
|
|
4058
4205
|
* Uses pg_advisory_lock to ensure only one orchestrator instance runs
|
|
4059
4206
|
* migrations at a time, preventing conflicts in multi-instance deployments.
|
|
4060
4207
|
*/
|
|
4061
|
-
const logger$
|
|
4208
|
+
const logger$10 = createLogger({ prefix: "migrate" });
|
|
4062
4209
|
const ADVISORY_LOCK_KEY = 543210001;
|
|
4063
4210
|
/**
|
|
4064
4211
|
* Run all pending migrations with an advisory lock for concurrency safety.
|
|
@@ -4070,21 +4217,21 @@ const ADVISORY_LOCK_KEY = 543210001;
|
|
|
4070
4217
|
async function runMigrations(opts) {
|
|
4071
4218
|
const client = await opts.pool.connect();
|
|
4072
4219
|
try {
|
|
4073
|
-
logger$
|
|
4220
|
+
logger$10.info("Acquiring migration lock...");
|
|
4074
4221
|
await client.query("SELECT pg_advisory_lock($1)", [ADVISORY_LOCK_KEY]);
|
|
4075
|
-
logger$
|
|
4222
|
+
logger$10.info("Migration lock acquired");
|
|
4076
4223
|
const migrator = new Migrator({
|
|
4077
4224
|
db: opts.db,
|
|
4078
4225
|
provider: createMigrationProvider()
|
|
4079
4226
|
});
|
|
4080
4227
|
const start = Date.now();
|
|
4081
4228
|
const { results, error } = await migrator.migrateToLatest();
|
|
4082
|
-
for (const result of results ?? []) if (result.status === "Success") logger$
|
|
4083
|
-
else if (result.status === "Error") logger$
|
|
4229
|
+
for (const result of results ?? []) if (result.status === "Success") logger$10.info(`${result.migrationName} ... OK (${Date.now() - start}ms)`);
|
|
4230
|
+
else if (result.status === "Error") logger$10.error(`${result.migrationName} ... FAILED`);
|
|
4084
4231
|
if (error) throw error;
|
|
4085
4232
|
const applied = (results ?? []).filter((r) => r.status === "Success");
|
|
4086
|
-
if (applied.length === 0) logger$
|
|
4087
|
-
else logger$
|
|
4233
|
+
if (applied.length === 0) logger$10.info("Database schema is up to date");
|
|
4234
|
+
else logger$10.info(`Applied ${applied.length} migration(s)`);
|
|
4088
4235
|
const hash = await computeMigrationsHash(createMigrationProvider());
|
|
4089
4236
|
await storeMigrationContentHash(opts.pool, hash);
|
|
4090
4237
|
return results ?? [];
|
|
@@ -5989,142 +6136,6 @@ function registerDiagnoseCommand(program, getClient) {
|
|
|
5989
6136
|
}
|
|
5990
6137
|
});
|
|
5991
6138
|
}
|
|
5992
|
-
promisify(execFile);
|
|
5993
|
-
//#endregion
|
|
5994
|
-
//#region src/diagnostics/bundle-writer.ts
|
|
5995
|
-
/**
|
|
5996
|
-
* Debug bundle writer.
|
|
5997
|
-
*
|
|
5998
|
-
* Creates a sanitized ZIP diagnostic bundle containing config (secrets redacted),
|
|
5999
|
-
* recent logs, system info, metrics, cluster state, and execution data.
|
|
6000
|
-
* This is the primary support tool -- operators run debug-bundle and share
|
|
6001
|
-
* the ZIP for troubleshooting.
|
|
6002
|
-
*/
|
|
6003
|
-
/** Maximum total log bytes to include in bundle (50MB). */
|
|
6004
|
-
const MAX_LOG_BYTES = 50 * 1024 * 1024;
|
|
6005
|
-
/**
|
|
6006
|
-
* Config field names that are safe to include unredacted.
|
|
6007
|
-
* Everything else gets replaced with "****".
|
|
6008
|
-
*/
|
|
6009
|
-
const SAFE_CONFIG_KEYS = new Set([
|
|
6010
|
-
"mode",
|
|
6011
|
-
"host",
|
|
6012
|
-
"port",
|
|
6013
|
-
"logLevel",
|
|
6014
|
-
"region",
|
|
6015
|
-
"environment",
|
|
6016
|
-
"name",
|
|
6017
|
-
"label",
|
|
6018
|
-
"labels",
|
|
6019
|
-
"enabled",
|
|
6020
|
-
"disabled",
|
|
6021
|
-
"timeout",
|
|
6022
|
-
"interval",
|
|
6023
|
-
"maxRetries",
|
|
6024
|
-
"retries",
|
|
6025
|
-
"workers",
|
|
6026
|
-
"concurrency",
|
|
6027
|
-
"maxConcurrency",
|
|
6028
|
-
"batchSize",
|
|
6029
|
-
"bufferSize",
|
|
6030
|
-
"warmPool",
|
|
6031
|
-
"cooldown",
|
|
6032
|
-
"type",
|
|
6033
|
-
"provider",
|
|
6034
|
-
"scaler",
|
|
6035
|
-
"driver",
|
|
6036
|
-
"backend",
|
|
6037
|
-
"protocol",
|
|
6038
|
-
"scheme",
|
|
6039
|
-
"path",
|
|
6040
|
-
"basePath",
|
|
6041
|
-
"metricsPath",
|
|
6042
|
-
"healthPath",
|
|
6043
|
-
"logFormat",
|
|
6044
|
-
"logFile",
|
|
6045
|
-
"logDir",
|
|
6046
|
-
"dataDir",
|
|
6047
|
-
"version",
|
|
6048
|
-
"debug",
|
|
6049
|
-
"verbose",
|
|
6050
|
-
"quiet",
|
|
6051
|
-
"tls",
|
|
6052
|
-
"cors",
|
|
6053
|
-
"rateLimiting",
|
|
6054
|
-
"maxConnections",
|
|
6055
|
-
"poolSize",
|
|
6056
|
-
"minPool",
|
|
6057
|
-
"maxPool",
|
|
6058
|
-
"idleTimeout",
|
|
6059
|
-
"connectTimeout",
|
|
6060
|
-
"requestTimeout",
|
|
6061
|
-
"shutdownTimeout",
|
|
6062
|
-
"gracefulShutdown"
|
|
6063
|
-
]);
|
|
6064
|
-
/**
|
|
6065
|
-
* Redact config values using allowlist approach.
|
|
6066
|
-
* Only known-safe fields are preserved; everything else becomes "****".
|
|
6067
|
-
*/
|
|
6068
|
-
function redactConfig(obj, parentKey) {
|
|
6069
|
-
if (obj === null || obj === void 0) return obj;
|
|
6070
|
-
if (Array.isArray(obj)) return obj.map((item) => redactConfig(item, parentKey));
|
|
6071
|
-
if (typeof obj === "object") {
|
|
6072
|
-
const result = {};
|
|
6073
|
-
for (const [key, value] of Object.entries(obj)) result[key] = redactConfig(value, key);
|
|
6074
|
-
return result;
|
|
6075
|
-
}
|
|
6076
|
-
if (typeof obj === "string" && parentKey && !SAFE_CONFIG_KEYS.has(parentKey)) return "****";
|
|
6077
|
-
if (typeof obj === "number" || typeof obj === "boolean") return obj;
|
|
6078
|
-
if (typeof obj === "string" && parentKey && SAFE_CONFIG_KEYS.has(parentKey)) return obj;
|
|
6079
|
-
if (typeof obj === "string") return "****";
|
|
6080
|
-
return obj;
|
|
6081
|
-
}
|
|
6082
|
-
/**
|
|
6083
|
-
* Add log files from logDir to the archive, respecting MAX_LOG_BYTES cap
|
|
6084
|
-
* and the logWindow time filter. Matches any `*.log` file in the directory,
|
|
6085
|
-
* so the per-instance filename pattern produced by
|
|
6086
|
-
* `buildLogFilename()` is picked up without additional configuration.
|
|
6087
|
-
*
|
|
6088
|
-
* Exported for reuse by the `kici-admin debug-bundle` CLI command, which
|
|
6089
|
-
* runs outside the orchestrator process but still needs to include the
|
|
6090
|
-
* same log files in its locally-assembled bundle.
|
|
6091
|
-
*/
|
|
6092
|
-
async function addLogsToArchive(archive, logDir, logWindowHours) {
|
|
6093
|
-
const cutoff = Date.now() - logWindowHours * 60 * 60 * 1e3;
|
|
6094
|
-
const entries = fs$1.readdirSync(logDir).filter((f) => {
|
|
6095
|
-
if (!f.endsWith(".log")) return false;
|
|
6096
|
-
return fs$1.statSync(path$1.join(logDir, f)).mtimeMs >= cutoff;
|
|
6097
|
-
});
|
|
6098
|
-
entries.sort((a, b) => {
|
|
6099
|
-
const aStat = fs$1.statSync(path$1.join(logDir, a));
|
|
6100
|
-
return fs$1.statSync(path$1.join(logDir, b)).mtimeMs - aStat.mtimeMs;
|
|
6101
|
-
});
|
|
6102
|
-
let totalBytes = 0;
|
|
6103
|
-
let totalLines = 0;
|
|
6104
|
-
let errors = 0;
|
|
6105
|
-
let warnings = 0;
|
|
6106
|
-
for (const entry of entries) {
|
|
6107
|
-
const filePath = path$1.join(logDir, entry);
|
|
6108
|
-
const stat = fs$1.statSync(filePath);
|
|
6109
|
-
if (totalBytes + stat.size > MAX_LOG_BYTES) break;
|
|
6110
|
-
const content = fs$1.readFileSync(filePath, "utf-8");
|
|
6111
|
-
archive.append(content, { name: `logs/${entry}` });
|
|
6112
|
-
totalBytes += stat.size;
|
|
6113
|
-
const lines = content.split("\n");
|
|
6114
|
-
totalLines += lines.length;
|
|
6115
|
-
for (const line of lines) {
|
|
6116
|
-
if (/\berror\b/i.test(line)) errors++;
|
|
6117
|
-
if (/\bwarn(ing)?\b/i.test(line)) warnings++;
|
|
6118
|
-
}
|
|
6119
|
-
}
|
|
6120
|
-
const summary = {
|
|
6121
|
-
totalLines,
|
|
6122
|
-
errors,
|
|
6123
|
-
warnings,
|
|
6124
|
-
totalBytes
|
|
6125
|
-
};
|
|
6126
|
-
archive.append(JSON.stringify(summary, null, 2), { name: "logs/summary.json" });
|
|
6127
|
-
}
|
|
6128
6139
|
//#endregion
|
|
6129
6140
|
//#region src/cli/commands/debug-bundle.ts
|
|
6130
6141
|
/**
|
|
@@ -6140,9 +6151,13 @@ async function addLogsToArchive(archive, logDir, logWindowHours) {
|
|
|
6140
6151
|
* Register the debug-bundle command on the CLI program.
|
|
6141
6152
|
*/
|
|
6142
6153
|
function registerDebugBundleCommand(program, getClient) {
|
|
6143
|
-
program.command("debug-bundle").description("Generate a diagnostic debug bundle ZIP for troubleshooting").option("-o, --output <path>", "Output ZIP file path", `kici-debug-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19)}.zip`).option("--log-dir <path>", "Directory with rotated *.log files to include (defaults to $KICI_LOG_DIR)").option("--log-window <hours>", "Hours of log history to include in the bundle", (value) => parseInt(value, 10), 4).action(async (opts) => {
|
|
6154
|
+
program.command("debug-bundle").description("Generate a diagnostic debug bundle ZIP for troubleshooting").option("-o, --output <path>", "Output ZIP file path", `kici-debug-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19)}.zip`).option("--log-dir <path>", "Directory with rotated *.log files to include (defaults to $KICI_LOG_DIR)").option("--log-window <hours>", "Hours of log history to include in the bundle", (value) => parseInt(value, 10), 4).option("--fleet", "Collect logs from every node in the cluster (server-side fan-out)").option("--list", "With --fleet: print the fleet topology and exit (no collection)").option("--json", "With --fleet --list: emit the topology as JSON").option("--pick [selectors]", "With --fleet: comma-separated selectors (id/host*/label:k=v); bare flag opens an interactive picker on a TTY").option("--fleet-timeout <seconds>", "Per-node deadline for --fleet", (value) => parseInt(value, 10), 60).action(async (opts) => {
|
|
6144
6155
|
try {
|
|
6145
6156
|
const client = getClient();
|
|
6157
|
+
if (opts.fleet) {
|
|
6158
|
+
await runFleetBundle(client, opts);
|
|
6159
|
+
return;
|
|
6160
|
+
}
|
|
6146
6161
|
const outputPath = path$1.resolve(opts.output);
|
|
6147
6162
|
console.log("Collecting diagnostic data...");
|
|
6148
6163
|
const [diagnoseResult, configResult, systemInfo, clusterHealth, metrics] = await Promise.allSettled([
|
|
@@ -6195,6 +6210,57 @@ async function safeRequest(fn) {
|
|
|
6195
6210
|
return null;
|
|
6196
6211
|
}
|
|
6197
6212
|
}
|
|
6213
|
+
/**
|
|
6214
|
+
* Fleet mode: enumerate (`--list`), pick nodes (`--pick`), or collect the whole
|
|
6215
|
+
* cluster into one nested ZIP via the orchestrator's server-side fan-out.
|
|
6216
|
+
*/
|
|
6217
|
+
async function runFleetBundle(client, opts) {
|
|
6218
|
+
if (opts.list) {
|
|
6219
|
+
const topology = await client.getFleetTopology();
|
|
6220
|
+
if (opts.json) console.log(JSON.stringify(topology, null, 2));
|
|
6221
|
+
else printTopologyTable(topology);
|
|
6222
|
+
return;
|
|
6223
|
+
}
|
|
6224
|
+
const selectors = await resolveFleetSelectors(client, opts);
|
|
6225
|
+
const outputPath = path$1.resolve(opts.output);
|
|
6226
|
+
console.log("Collecting fleet bundle (this fans out to every selected node)...");
|
|
6227
|
+
await client.downloadFleetBundle({
|
|
6228
|
+
selectors,
|
|
6229
|
+
logWindowHours: opts.logWindow,
|
|
6230
|
+
timeoutSeconds: opts.fleetTimeout
|
|
6231
|
+
}, outputPath);
|
|
6232
|
+
const stat = fs$1.statSync(outputPath);
|
|
6233
|
+
console.log(`Fleet bundle written to ${outputPath} (${(stat.size / 1024).toFixed(1)} KB)`);
|
|
6234
|
+
}
|
|
6235
|
+
/**
|
|
6236
|
+
* Resolve the `--pick` selectors: a bare `--pick` on a TTY opens an interactive
|
|
6237
|
+
* checkbox over the enumerated topology; a value is split on commas; no flag
|
|
6238
|
+
* means "everything".
|
|
6239
|
+
*/
|
|
6240
|
+
async function resolveFleetSelectors(client, opts) {
|
|
6241
|
+
if (opts.pick === void 0) return [];
|
|
6242
|
+
if (typeof opts.pick === "string") return opts.pick.split(",").map((s) => s.trim()).filter(Boolean);
|
|
6243
|
+
if (!process.stdout.isTTY) throw new Error("--pick with no value requires a TTY; pass comma-separated selectors instead");
|
|
6244
|
+
const { checkbox } = await import("@inquirer/prompts");
|
|
6245
|
+
return checkbox({
|
|
6246
|
+
message: "Select nodes to collect",
|
|
6247
|
+
choices: (await client.getFleetTopology()).nodes.map((n) => ({
|
|
6248
|
+
name: `${n.kind === "orchestrator" ? `[${n.role ?? "orchestrator"}]` : " agent"} ${n.id}${n.hostname ? ` @ ${n.hostname}` : ""}`,
|
|
6249
|
+
value: n.id
|
|
6250
|
+
}))
|
|
6251
|
+
});
|
|
6252
|
+
}
|
|
6253
|
+
/** Print the fleet topology as an indented tree (orchestrators with their agents). */
|
|
6254
|
+
function printTopologyTable(topology) {
|
|
6255
|
+
const orchestrators = topology.nodes.filter((n) => n.kind === "orchestrator");
|
|
6256
|
+
console.log(`Fleet topology (${topology.nodes.length} node(s)):`);
|
|
6257
|
+
for (const o of orchestrators) {
|
|
6258
|
+
const root = o.parentId === null ? " (collector)" : "";
|
|
6259
|
+
console.log(` [${o.role ?? "orchestrator"}] ${o.id}${o.hostname ? ` @ ${o.hostname}` : ""}${root}`);
|
|
6260
|
+
const agents = topology.nodes.filter((n) => n.kind === "agent" && n.parentId === o.id);
|
|
6261
|
+
for (const a of agents) console.log(` agent ${a.id}`);
|
|
6262
|
+
}
|
|
6263
|
+
}
|
|
6198
6264
|
//#endregion
|
|
6199
6265
|
//#region src/diagnostics/bundle-reader.ts
|
|
6200
6266
|
/**
|
|
@@ -7675,6 +7741,56 @@ var init_windows = __esmMin((() => {
|
|
|
7675
7741
|
};
|
|
7676
7742
|
}));
|
|
7677
7743
|
//#endregion
|
|
7744
|
+
//#region src/cli/service/image-digests.ts
|
|
7745
|
+
/**
|
|
7746
|
+
* Resolves the customer installer's image reference from the digest record
|
|
7747
|
+
* shipped with this package (installer-image-digests.json at the package root).
|
|
7748
|
+
*
|
|
7749
|
+
* Emits a manifest-list-digest-pinned ref so a `docker`/`podman pull` verifies
|
|
7750
|
+
* the image hash — a registry that serves substituted bits fails the pull. When
|
|
7751
|
+
* the record is missing (a dev tree that never ran a release) or lacks the
|
|
7752
|
+
* image, falls back to the moving `:latest` tag and warns.
|
|
7753
|
+
*
|
|
7754
|
+
* The record resolves relative to this module's directory and works for both
|
|
7755
|
+
* layouts: the source tree (`src/cli/service/` → the JSON is three dirs up at
|
|
7756
|
+
* the package root) and the single-file bundle (`dist/cli.js` → one dir up).
|
|
7757
|
+
*/
|
|
7758
|
+
/**
|
|
7759
|
+
* Climb from a starting directory looking for installer-image-digests.json.
|
|
7760
|
+
* Works for the source layout (src/cli/service → 3 up) and the single-file
|
|
7761
|
+
* bundle layout (dist → 1 up).
|
|
7762
|
+
*/
|
|
7763
|
+
function findRecordFile(startDir) {
|
|
7764
|
+
let dir = startDir;
|
|
7765
|
+
for (let i = 0; i < 6; i++) {
|
|
7766
|
+
const candidate = path.join(dir, RECORD_FILENAME);
|
|
7767
|
+
if (existsSync(candidate)) return candidate;
|
|
7768
|
+
const parent = path.dirname(dir);
|
|
7769
|
+
if (parent === dir) break;
|
|
7770
|
+
dir = parent;
|
|
7771
|
+
}
|
|
7772
|
+
return null;
|
|
7773
|
+
}
|
|
7774
|
+
/**
|
|
7775
|
+
* Resolve `quay.io/kici-dev/<name>:<version>@sha256:<digest>` for an installer
|
|
7776
|
+
* image. Falls back to `:latest` (with a warning) when no recorded digest exists.
|
|
7777
|
+
*/
|
|
7778
|
+
function resolveImageRef(name, opts = {}) {
|
|
7779
|
+
const file = opts.filePath ?? findRecordFile(import.meta.dirname);
|
|
7780
|
+
if (file && existsSync(file)) try {
|
|
7781
|
+
const rec = JSON.parse(readFileSync(file, "utf8"));
|
|
7782
|
+
const digest = rec.images?.[name];
|
|
7783
|
+
if (digest && rec.version) return `${QUAY_PREFIX}/${name}:${rec.version}@${digest}`;
|
|
7784
|
+
} catch {}
|
|
7785
|
+
console.warn(`[kici] no recorded manifest-list digest for ${name}; pinning the mutable :latest tag. Reinstall from a released kici-admin to get a digest-pinned image.`);
|
|
7786
|
+
return `${QUAY_PREFIX}/${name}:latest`;
|
|
7787
|
+
}
|
|
7788
|
+
var QUAY_PREFIX, RECORD_FILENAME;
|
|
7789
|
+
var init_image_digests = __esmMin((() => {
|
|
7790
|
+
QUAY_PREFIX = "quay.io/kici-dev";
|
|
7791
|
+
RECORD_FILENAME = "installer-image-digests.json";
|
|
7792
|
+
}));
|
|
7793
|
+
//#endregion
|
|
7678
7794
|
//#region src/cli/service/compose.ts
|
|
7679
7795
|
/**
|
|
7680
7796
|
* Docker/Podman Compose service manager.
|
|
@@ -7691,7 +7807,7 @@ var compose_exports = /* @__PURE__ */ __exportAll({ ComposeServiceManager: () =>
|
|
|
7691
7807
|
*
|
|
7692
7808
|
* @throws When neither runtime is found.
|
|
7693
7809
|
*/
|
|
7694
|
-
function detectRuntime() {
|
|
7810
|
+
function detectRuntime$1() {
|
|
7695
7811
|
try {
|
|
7696
7812
|
execSync("podman compose version", { stdio: "pipe" });
|
|
7697
7813
|
return "podman";
|
|
@@ -7734,10 +7850,13 @@ function getRestartMode(config) {
|
|
|
7734
7850
|
* The generated file includes the service definition with env_file,
|
|
7735
7851
|
* restart policy, and volume mounts.
|
|
7736
7852
|
*
|
|
7737
|
-
* The `image:` line
|
|
7738
|
-
*
|
|
7739
|
-
*
|
|
7740
|
-
*
|
|
7853
|
+
* The `image:` line is pinned by manifest-list digest via `resolveImageRef`
|
|
7854
|
+
* (`quay.io/kici-dev/<service-name>:<version>@sha256:<digest>`), falling back
|
|
7855
|
+
* to the mutable `:latest` tag only when no recorded digest ships with this
|
|
7856
|
+
* package. Source of truth for the publish targets:
|
|
7857
|
+
* packages/ci/src/publish-images.ts. See
|
|
7858
|
+
* .claude/rules/container-publish-registry.md — Quay is the sole registry for
|
|
7859
|
+
* public KiCI images we publish.
|
|
7741
7860
|
*/
|
|
7742
7861
|
function generateComposeYaml(config) {
|
|
7743
7862
|
const restartMode = getRestartMode(config);
|
|
@@ -7747,7 +7866,7 @@ function generateComposeYaml(config) {
|
|
|
7747
7866
|
"",
|
|
7748
7867
|
"services:",
|
|
7749
7868
|
` ${config.name}:`,
|
|
7750
|
-
` image:
|
|
7869
|
+
` image: ${resolveImageRef(config.name)}`,
|
|
7751
7870
|
` container_name: ${config.name}`,
|
|
7752
7871
|
` restart: ${restartMode}`,
|
|
7753
7872
|
" env_file:",
|
|
@@ -7764,11 +7883,12 @@ function generateComposeYaml(config) {
|
|
|
7764
7883
|
}
|
|
7765
7884
|
var ComposeServiceManager;
|
|
7766
7885
|
var init_compose = __esmMin((() => {
|
|
7886
|
+
init_image_digests();
|
|
7767
7887
|
ComposeServiceManager = class {
|
|
7768
7888
|
runtime = null;
|
|
7769
7889
|
/** Get the runtime binary (`podman` or `docker`), detecting it if not already done. */
|
|
7770
7890
|
getRuntime() {
|
|
7771
|
-
if (!this.runtime) this.runtime = detectRuntime();
|
|
7891
|
+
if (!this.runtime) this.runtime = detectRuntime$1();
|
|
7772
7892
|
return this.runtime;
|
|
7773
7893
|
}
|
|
7774
7894
|
/** Run a compose command for the given service config. */
|
|
@@ -9981,6 +10101,8 @@ function formatSettings(s, format) {
|
|
|
9981
10101
|
lines.push(`User-cache quota: ${s.userCacheQuotaBytes === null ? "(cluster default)" : `${s.userCacheQuotaBytes} bytes`}`);
|
|
9982
10102
|
lines.push(`User-cache TTL: ${s.userCacheTtlMs === null ? "(cluster default)" : `${s.userCacheTtlMs} ms`}`);
|
|
9983
10103
|
lines.push(`Dispatch ack timeout: ${s.dispatchAckTimeoutMs === null ? "(cluster default)" : `${s.dispatchAckTimeoutMs} ms`}`);
|
|
10104
|
+
lines.push(`Approval expiry: ${s.approvalExpirySeconds} s`);
|
|
10105
|
+
lines.push(`Allow self-approval: ${s.allowSelfApproval}`);
|
|
9984
10106
|
if (s.createdAt) lines.push(`Created at: ${s.createdAt}`);
|
|
9985
10107
|
if (s.updatedAt) lines.push(`Updated at: ${s.updatedAt}`);
|
|
9986
10108
|
return lines.join("\n");
|
|
@@ -10057,6 +10179,64 @@ function registerOrgSettingsCommands(program, getClient) {
|
|
|
10057
10179
|
});
|
|
10058
10180
|
registerUserCacheCommands(orgSettings, getClient);
|
|
10059
10181
|
registerDispatchAckCommands(orgSettings, getClient);
|
|
10182
|
+
registerApprovalCommands(orgSettings, getClient);
|
|
10183
|
+
}
|
|
10184
|
+
/**
|
|
10185
|
+
* `kici-admin org-settings approval <show|set-expiry|set-self-approval>`.
|
|
10186
|
+
*
|
|
10187
|
+
* The per-org held-approval policy: how long a held element waits before it
|
|
10188
|
+
* expires (`approval_expiry_seconds`, default 86400) and whether a run's
|
|
10189
|
+
* triggerer may approve its own held elements (`allow_self_approval`, default
|
|
10190
|
+
* true). Both have NOT NULL DB defaults, so there is no "reset to cluster
|
|
10191
|
+
* default" — set replaces the current value.
|
|
10192
|
+
*/
|
|
10193
|
+
function registerApprovalCommands(orgSettings, getClient) {
|
|
10194
|
+
const ap = orgSettings.command("approval").description("Manage the per-org held-approval expiry + self-approval policy");
|
|
10195
|
+
ap.command("show").description("Print the current per-org approval policy").option("--customer-id <id>", "Customer / org id (alias: --org)").option("--org <id>", "Alias for --customer-id").option("--format <format>", "Output format: json|table", "table").action(async (opts) => {
|
|
10196
|
+
const customerId = resolveCustomerId(opts);
|
|
10197
|
+
try {
|
|
10198
|
+
const settings = await fetchSettings(getClient(), customerId);
|
|
10199
|
+
console.log(formatSettings(settings, opts.format));
|
|
10200
|
+
} catch (err) {
|
|
10201
|
+
console.error(`Error: ${toErrorMessage(err)}`);
|
|
10202
|
+
process.exit(1);
|
|
10203
|
+
}
|
|
10204
|
+
});
|
|
10205
|
+
ap.command("set-expiry <seconds>").description("Set the per-org held-approval expiry (integer seconds, >= 1)").option("--customer-id <id>", "Customer / org id (alias: --org)").option("--org <id>", "Alias for --customer-id").option("--format <format>", "Output format: json|table", "table").action(async (value, opts) => {
|
|
10206
|
+
const n = Number(value);
|
|
10207
|
+
if (!Number.isInteger(n) || n < 1) {
|
|
10208
|
+
console.error("Error: value must be an integer >= 1 (seconds)");
|
|
10209
|
+
process.exit(1);
|
|
10210
|
+
}
|
|
10211
|
+
const customerId = resolveCustomerId(opts);
|
|
10212
|
+
try {
|
|
10213
|
+
const updated = await patchSettings(getClient(), {
|
|
10214
|
+
customerId,
|
|
10215
|
+
approvalExpirySeconds: n
|
|
10216
|
+
});
|
|
10217
|
+
console.log(formatSettings(updated, opts.format));
|
|
10218
|
+
} catch (err) {
|
|
10219
|
+
console.error(`Error: ${toErrorMessage(err)}`);
|
|
10220
|
+
process.exit(1);
|
|
10221
|
+
}
|
|
10222
|
+
});
|
|
10223
|
+
ap.command("set-self-approval <value>").description("Allow or forbid a run triggerer approving its own held elements (true|false)").option("--customer-id <id>", "Customer / org id (alias: --org)").option("--org <id>", "Alias for --customer-id").option("--format <format>", "Output format: json|table", "table").action(async (value, opts) => {
|
|
10224
|
+
if (value !== "true" && value !== "false") {
|
|
10225
|
+
console.error("Error: value must be \"true\" or \"false\"");
|
|
10226
|
+
process.exit(1);
|
|
10227
|
+
}
|
|
10228
|
+
const customerId = resolveCustomerId(opts);
|
|
10229
|
+
try {
|
|
10230
|
+
const updated = await patchSettings(getClient(), {
|
|
10231
|
+
customerId,
|
|
10232
|
+
allowSelfApproval: value === "true"
|
|
10233
|
+
});
|
|
10234
|
+
console.log(formatSettings(updated, opts.format));
|
|
10235
|
+
} catch (err) {
|
|
10236
|
+
console.error(`Error: ${toErrorMessage(err)}`);
|
|
10237
|
+
process.exit(1);
|
|
10238
|
+
}
|
|
10239
|
+
});
|
|
10060
10240
|
}
|
|
10061
10241
|
/**
|
|
10062
10242
|
* `kici-admin org-settings dispatch-ack <show|set|reset>`.
|
|
@@ -12648,9 +12828,9 @@ async function build(deps) {
|
|
|
12648
12828
|
sizeOf: (v) => v.byteLength
|
|
12649
12829
|
}),
|
|
12650
12830
|
log: (level, msg, extra) => {
|
|
12651
|
-
if (level === "info") logger$
|
|
12652
|
-
else if (level === "warn") logger$
|
|
12653
|
-
else logger$
|
|
12831
|
+
if (level === "info") logger$6.info(msg, extra);
|
|
12832
|
+
else if (level === "warn") logger$6.warn(msg, extra);
|
|
12833
|
+
else logger$6.error(msg, extra);
|
|
12654
12834
|
}
|
|
12655
12835
|
}),
|
|
12656
12836
|
kdb,
|
|
@@ -13092,11 +13272,11 @@ async function peekChunk(opts) {
|
|
|
13092
13272
|
await b.close();
|
|
13093
13273
|
}
|
|
13094
13274
|
}
|
|
13095
|
-
var logger$
|
|
13275
|
+
var logger$6;
|
|
13096
13276
|
var init_cold_store_impl = __esmMin((() => {
|
|
13097
13277
|
init_client();
|
|
13098
13278
|
init_orchestrator_cold_store();
|
|
13099
|
-
logger$
|
|
13279
|
+
logger$6 = createLogger({ prefix: "kici-admin-cold-store" });
|
|
13100
13280
|
}));
|
|
13101
13281
|
//#endregion
|
|
13102
13282
|
//#region src/cli/commands/cold-store.ts
|
|
@@ -13272,91 +13452,3177 @@ function registerColdStoreCommands(program, _getClient) {
|
|
|
13272
13452
|
});
|
|
13273
13453
|
}
|
|
13274
13454
|
//#endregion
|
|
13275
|
-
//#region src/
|
|
13455
|
+
//#region src/scaler/nftables.ts
|
|
13276
13456
|
/**
|
|
13277
|
-
*
|
|
13457
|
+
* Programmatic nftables rule management for agent network isolation.
|
|
13278
13458
|
*
|
|
13279
|
-
*
|
|
13280
|
-
*
|
|
13281
|
-
* 2. Send join.request with the join token
|
|
13282
|
-
* 3. Receive join.response with encrypted config bundle
|
|
13283
|
-
* 4. Decrypt bundle using token-derived AES-256-GCM key
|
|
13284
|
-
* 5. Write decrypted config as local YAML file
|
|
13459
|
+
* Manages RFC1918 + cloud metadata blocking rules per-interface (Firecracker/container)
|
|
13460
|
+
* or per-UID (bare-metal). All operations use `nft` CLI via child_process.execFile.
|
|
13285
13461
|
*
|
|
13286
|
-
*
|
|
13287
|
-
*
|
|
13288
|
-
*
|
|
13462
|
+
* Table layout:
|
|
13463
|
+
* table ip kici {
|
|
13464
|
+
* chain forward { type filter hook forward priority 0; policy accept; }
|
|
13465
|
+
* chain output { type filter hook output priority 0; policy accept; }
|
|
13466
|
+
* }
|
|
13289
13467
|
*/
|
|
13290
|
-
const
|
|
13468
|
+
const execFile$1 = promisify(execFile);
|
|
13469
|
+
const logger$5 = createLogger({ prefix: "nftables" });
|
|
13470
|
+
/** Timeout for nft commands in milliseconds. */
|
|
13471
|
+
const NFT_TIMEOUT_MS = 1e4;
|
|
13472
|
+
/** RFC1918 private address ranges. */
|
|
13473
|
+
const RFC1918_RANGES = [
|
|
13474
|
+
"10.0.0.0/8",
|
|
13475
|
+
"172.16.0.0/12",
|
|
13476
|
+
"192.168.0.0/16"
|
|
13477
|
+
];
|
|
13478
|
+
/** Cloud metadata service range (AWS/GCP/Azure link-local). */
|
|
13479
|
+
const METADATA_RANGE = "169.254.0.0/16";
|
|
13291
13480
|
/**
|
|
13292
|
-
*
|
|
13481
|
+
* Execute an nft command with timeout.
|
|
13482
|
+
* @returns stdout from the command
|
|
13483
|
+
* @throws Error on non-zero exit or timeout
|
|
13293
13484
|
*/
|
|
13294
|
-
function
|
|
13295
|
-
|
|
13485
|
+
async function nft(opts, ...args) {
|
|
13486
|
+
const { stdout } = opts.requireSudo === true ? await execFile$1("sudo", [
|
|
13487
|
+
"-n",
|
|
13488
|
+
"nft",
|
|
13489
|
+
...args
|
|
13490
|
+
], { timeout: NFT_TIMEOUT_MS }) : await execFile$1("nft", args, { timeout: NFT_TIMEOUT_MS });
|
|
13491
|
+
return stdout;
|
|
13296
13492
|
}
|
|
13297
13493
|
/**
|
|
13298
|
-
*
|
|
13299
|
-
*
|
|
13494
|
+
* Validate that nftables is available and the process has NET_ADMIN capability.
|
|
13495
|
+
* Attempts `nft list tables` -- if it fails:
|
|
13496
|
+
* - ENOENT: nft binary not installed
|
|
13497
|
+
* - EPERM: nft binary present but NET_ADMIN capability missing
|
|
13498
|
+
* Throws with a clear error message in both cases.
|
|
13300
13499
|
*/
|
|
13301
|
-
function
|
|
13302
|
-
|
|
13303
|
-
|
|
13304
|
-
|
|
13305
|
-
|
|
13500
|
+
async function validateNftablesAvailability(opts = {}) {
|
|
13501
|
+
try {
|
|
13502
|
+
await nft(opts, "list", "tables");
|
|
13503
|
+
} catch (err) {
|
|
13504
|
+
const message = toErrorMessage(err);
|
|
13505
|
+
if (message.includes("ENOENT") || message.includes("not found")) throw new Error("nftables binary not found at /usr/sbin/nft. The orchestrator container image must include nftables (apk add nftables). Network isolation for agents cannot be established without nftables.");
|
|
13506
|
+
if (message.includes("EPERM") || message.includes("Operation not permitted")) throw new Error("nftables operation denied -- missing NET_ADMIN capability. Start the orchestrator container with --cap-add=NET_ADMIN. Network isolation for agents requires this capability.");
|
|
13507
|
+
throw new Error(`nftables validation failed: ${message}`);
|
|
13508
|
+
}
|
|
13306
13509
|
}
|
|
13307
13510
|
/**
|
|
13308
|
-
*
|
|
13511
|
+
* Ensure the `kici` nftables table and required chains exist.
|
|
13512
|
+
* Idempotent -- safe to call multiple times.
|
|
13309
13513
|
*/
|
|
13310
|
-
async function
|
|
13311
|
-
|
|
13514
|
+
async function ensureKiciTable(opts = {}) {
|
|
13515
|
+
try {
|
|
13516
|
+
await nft(opts, "list", "table", "ip", "kici");
|
|
13517
|
+
logger$5.debug("kici nftables table already exists");
|
|
13518
|
+
return;
|
|
13519
|
+
} catch {}
|
|
13520
|
+
logger$5.info("Creating kici nftables table with forward and output chains");
|
|
13521
|
+
await nft(opts, "-f", "/dev/stdin").catch(() => {});
|
|
13522
|
+
await nft(opts, "add", "table", "ip", "kici");
|
|
13523
|
+
try {
|
|
13524
|
+
await nft(opts, "add", "chain", "ip", "kici", "forward", "{ type filter hook forward priority 0; policy accept; }");
|
|
13525
|
+
} catch {
|
|
13526
|
+
logger$5.debug("kici forward chain already exists");
|
|
13527
|
+
}
|
|
13528
|
+
try {
|
|
13529
|
+
await nft(opts, "add", "chain", "ip", "kici", "output", "{ type filter hook output priority 0; policy accept; }");
|
|
13530
|
+
} catch {
|
|
13531
|
+
logger$5.debug("kici output chain already exists");
|
|
13532
|
+
}
|
|
13533
|
+
logger$5.info("kici nftables table and chains created");
|
|
13312
13534
|
}
|
|
13313
|
-
|
|
13314
|
-
|
|
13315
|
-
|
|
13316
|
-
|
|
13317
|
-
|
|
13535
|
+
/**
|
|
13536
|
+
* Build the per-identifier isolation rule operations.
|
|
13537
|
+
*
|
|
13538
|
+
* Placement matters because nftables is first-match-wins and the `kici`
|
|
13539
|
+
* forward chain is shared: it accumulates rules for every live agent plus
|
|
13540
|
+
* any host-level baseline (e.g. wildcard `iifname "kici-*"` RFC1918 drops
|
|
13541
|
+
* installed at host bootstrap). Accepts therefore go in via `insert`
|
|
13542
|
+
* (chain head) so they beat any pre-existing drop that also matches the
|
|
13543
|
+
* traffic; an appended accept after a wildcard drop is dead code. Drops go
|
|
13544
|
+
* in via `add` (tail) — they only need to beat the chain's accept policy.
|
|
13545
|
+
*
|
|
13546
|
+
* Effective per-identifier order: gateway + allowlist accepts (head),
|
|
13547
|
+
* RFC1918 / metadata / denyAll drops (tail).
|
|
13548
|
+
*/
|
|
13549
|
+
function buildIsolationRuleOps(matchClause, gatewayIp, networkPolicy) {
|
|
13550
|
+
const ops = [];
|
|
13551
|
+
ops.push({
|
|
13552
|
+
verb: "insert",
|
|
13553
|
+
tokens: [
|
|
13554
|
+
...matchClause,
|
|
13555
|
+
"ip",
|
|
13556
|
+
"daddr",
|
|
13557
|
+
gatewayIp,
|
|
13558
|
+
"accept"
|
|
13559
|
+
]
|
|
13560
|
+
});
|
|
13561
|
+
if (networkPolicy?.allowlist) for (const cidr of networkPolicy.allowlist) ops.push({
|
|
13562
|
+
verb: "insert",
|
|
13563
|
+
tokens: [
|
|
13564
|
+
...matchClause,
|
|
13565
|
+
"ip",
|
|
13566
|
+
"daddr",
|
|
13567
|
+
cidr,
|
|
13568
|
+
"accept"
|
|
13569
|
+
]
|
|
13570
|
+
});
|
|
13571
|
+
for (const range of RFC1918_RANGES) ops.push({
|
|
13572
|
+
verb: "add",
|
|
13573
|
+
tokens: [
|
|
13574
|
+
...matchClause,
|
|
13575
|
+
"ip",
|
|
13576
|
+
"daddr",
|
|
13577
|
+
range,
|
|
13578
|
+
"drop"
|
|
13579
|
+
]
|
|
13580
|
+
});
|
|
13581
|
+
ops.push({
|
|
13582
|
+
verb: "add",
|
|
13583
|
+
tokens: [
|
|
13584
|
+
...matchClause,
|
|
13585
|
+
"ip",
|
|
13586
|
+
"daddr",
|
|
13587
|
+
METADATA_RANGE,
|
|
13588
|
+
"drop"
|
|
13589
|
+
]
|
|
13590
|
+
});
|
|
13591
|
+
if (networkPolicy?.denyAll) ops.push({
|
|
13592
|
+
verb: "add",
|
|
13593
|
+
tokens: [...matchClause, "drop"]
|
|
13594
|
+
});
|
|
13595
|
+
return ops;
|
|
13596
|
+
}
|
|
13597
|
+
/**
|
|
13598
|
+
* Add network isolation rules for a specific network interface.
|
|
13599
|
+
* Used by Firecracker and container backends.
|
|
13600
|
+
*
|
|
13601
|
+
* Rule placement (see buildIsolationRuleOps):
|
|
13602
|
+
* 1. Accept gateway traffic (inserted at chain head)
|
|
13603
|
+
* 2. Accept allowlisted CIDRs (inserted at chain head)
|
|
13604
|
+
* 3. Drop RFC1918 ranges (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16)
|
|
13605
|
+
* 4. Drop cloud metadata (169.254.0.0/16)
|
|
13606
|
+
* 5. Drop all remaining traffic (only if denyAll is true)
|
|
13607
|
+
*
|
|
13608
|
+
* @param identifier - Network interface name or source IP to match (e.g., "veth-abc123" or "172.30.0.5")
|
|
13609
|
+
* @param gatewayIp - Gateway IP that must remain accessible (e.g., "10.0.0.1")
|
|
13610
|
+
* @param networkPolicy - Optional policy with allowlist and denyAll settings
|
|
13611
|
+
* @param matchMode - How to match traffic: 'iifname' for interface name (default), 'saddr' for source IP
|
|
13612
|
+
*/
|
|
13613
|
+
async function addIsolationRules(identifier, gatewayIp, networkPolicy, matchMode = "iifname", opts = {}) {
|
|
13614
|
+
const matchLabel = matchMode === "iifname" ? `interface ${identifier}` : `source IP ${identifier}`;
|
|
13615
|
+
logger$5.info(`Adding isolation rules for ${matchLabel} (gateway: ${gatewayIp})`);
|
|
13616
|
+
const matchClause = matchMode === "iifname" ? ["iifname", identifier] : [
|
|
13617
|
+
"ip",
|
|
13618
|
+
"saddr",
|
|
13619
|
+
identifier
|
|
13620
|
+
];
|
|
13621
|
+
if (networkPolicy?.allowlist) for (const cidr of networkPolicy.allowlist) logger$5.debug(`Allowlisting ${cidr} for ${matchLabel}`);
|
|
13622
|
+
if (networkPolicy?.denyAll) logger$5.info(`Blocking all outbound traffic for ${matchLabel} (denyAll)`);
|
|
13623
|
+
for (const op of buildIsolationRuleOps(matchClause, gatewayIp, networkPolicy)) await nft(opts, op.verb, "rule", "ip", "kici", "forward", ...op.tokens);
|
|
13624
|
+
logger$5.info(`Isolation rules applied for ${matchLabel}`);
|
|
13625
|
+
}
|
|
13626
|
+
/**
|
|
13627
|
+
* Remove all isolation rules matching a specific interface from the kici forward chain.
|
|
13628
|
+
* Called during agent cleanup/destruction.
|
|
13629
|
+
*
|
|
13630
|
+
* Lists all rules with handles, finds those matching the interface name, and deletes them.
|
|
13631
|
+
* Best-effort: logs errors but does not throw (cleanup should not block destruction).
|
|
13632
|
+
*
|
|
13633
|
+
* @param interfaceName - Network interface whose rules should be removed
|
|
13634
|
+
*/
|
|
13635
|
+
async function removeIsolationRules(interfaceName, opts = {}) {
|
|
13636
|
+
logger$5.info(`Removing isolation rules for interface ${interfaceName}`);
|
|
13637
|
+
try {
|
|
13638
|
+
const handles = parseRuleHandles(await nft(opts, "-a", "list", "chain", "ip", "kici", "forward"), interfaceName);
|
|
13639
|
+
if (handles.length === 0) {
|
|
13640
|
+
logger$5.debug(`No rules found for interface ${interfaceName}`);
|
|
13641
|
+
return;
|
|
13642
|
+
}
|
|
13643
|
+
for (const handle of handles.sort((a, b) => b - a)) try {
|
|
13644
|
+
await nft(opts, "delete", "rule", "ip", "kici", "forward", "handle", String(handle));
|
|
13645
|
+
} catch (err) {
|
|
13646
|
+
logger$5.warn(`Failed to delete rule handle ${handle}: ${err}`);
|
|
13647
|
+
}
|
|
13648
|
+
logger$5.info(`Removed ${handles.length} rules for interface ${interfaceName}`);
|
|
13649
|
+
} catch (err) {
|
|
13650
|
+
logger$5.warn(`Failed to list/remove rules for ${interfaceName}: ${err}`);
|
|
13318
13651
|
}
|
|
13319
|
-
|
|
13320
|
-
|
|
13321
|
-
|
|
13322
|
-
|
|
13323
|
-
|
|
13324
|
-
|
|
13325
|
-
|
|
13326
|
-
|
|
13327
|
-
|
|
13328
|
-
|
|
13329
|
-
|
|
13330
|
-
|
|
13331
|
-
|
|
13332
|
-
|
|
13333
|
-
|
|
13334
|
-
if (!
|
|
13335
|
-
|
|
13336
|
-
const
|
|
13337
|
-
|
|
13338
|
-
logger.info("Join successful, writing config...", { clusterId: bundle.clusterId });
|
|
13339
|
-
const localConfig = buildLocalConfig(bundle);
|
|
13340
|
-
const configPath = this.options.configPath ?? "./kici-orchestrator.yaml";
|
|
13341
|
-
await writeConfigFile(configPath, localConfig);
|
|
13342
|
-
logger.info(`Config written to ${configPath}`);
|
|
13343
|
-
logger.info("Start the orchestrator with: kici-admin orchestrator start");
|
|
13652
|
+
}
|
|
13653
|
+
/**
|
|
13654
|
+
* Parse nft rule listing output and extract handles for rules matching a given identifier.
|
|
13655
|
+
* Handles lines like: ` iifname "veth-abc" ip daddr 10.0.0.0/8 drop # handle 42`
|
|
13656
|
+
*
|
|
13657
|
+
* @param nftOutput - Raw output from `nft -a list chain ip kici <chain>`
|
|
13658
|
+
* @param identifier - String to search for in each rule line (interface name or UID)
|
|
13659
|
+
* @returns Array of numeric rule handles
|
|
13660
|
+
*/
|
|
13661
|
+
function parseRuleHandles(nftOutput, identifier) {
|
|
13662
|
+
const handles = [];
|
|
13663
|
+
const lines = nftOutput.split("\n");
|
|
13664
|
+
const escaped = identifier.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
13665
|
+
const boundaryRe = new RegExp(`(^|[\\s"])${escaped}([\\s"]|$)`);
|
|
13666
|
+
for (const line of lines) {
|
|
13667
|
+
if (!line.includes("# handle")) continue;
|
|
13668
|
+
if (!boundaryRe.test(line)) continue;
|
|
13669
|
+
const match = line.match(/# handle (\d+)/);
|
|
13670
|
+
if (match) handles.push(parseInt(match[1], 10));
|
|
13344
13671
|
}
|
|
13345
|
-
|
|
13346
|
-
|
|
13347
|
-
|
|
13348
|
-
|
|
13349
|
-
|
|
13350
|
-
|
|
13351
|
-
|
|
13352
|
-
|
|
13353
|
-
|
|
13354
|
-
|
|
13355
|
-
|
|
13356
|
-
|
|
13357
|
-
|
|
13358
|
-
|
|
13359
|
-
|
|
13672
|
+
return handles;
|
|
13673
|
+
}
|
|
13674
|
+
//#endregion
|
|
13675
|
+
//#region src/firecracker/host-network.ts
|
|
13676
|
+
/**
|
|
13677
|
+
* Firecracker host-network provisioning.
|
|
13678
|
+
*
|
|
13679
|
+
* Creates the per-coordinator bridge (kici-brN), assigns its gateway IP,
|
|
13680
|
+
* marks kici-* interfaces unmanaged by NetworkManager, enables IP forwarding,
|
|
13681
|
+
* and builds a disjoint, source-scoped nftables table for NAT + egress
|
|
13682
|
+
* isolation. One pure command-builder drives live provisioning, the rendered
|
|
13683
|
+
* boot script, and (read-only) verification.
|
|
13684
|
+
*
|
|
13685
|
+
* This is HOST setup, distinct from the runtime per-VM isolation in
|
|
13686
|
+
* scaler/nftables.ts (added at spawn / removed at destroy). The two share the
|
|
13687
|
+
* nft table name but have separate lifecycles; this module never touches the
|
|
13688
|
+
* per-VM rules.
|
|
13689
|
+
*/
|
|
13690
|
+
const execFileP$1 = promisify(execFile);
|
|
13691
|
+
const logger$4 = createLogger({ prefix: "fc-host-network" });
|
|
13692
|
+
/** Privileged binaries that need `sudo -n` on a non-root orchestrator host. */
|
|
13693
|
+
const PRIVILEGED_BINS = new Set([
|
|
13694
|
+
"ip",
|
|
13695
|
+
"nft",
|
|
13696
|
+
"sysctl"
|
|
13697
|
+
]);
|
|
13698
|
+
/** Timeout for a single host-network provisioning command. */
|
|
13699
|
+
const COMMAND_TIMEOUT_MS = 3e4;
|
|
13700
|
+
/**
|
|
13701
|
+
* Derive the network address (CIDR) from a gateway CIDR by masking host bits.
|
|
13702
|
+
* '10.0.0.1/24' -> '10.0.0.0/24'.
|
|
13703
|
+
*/
|
|
13704
|
+
function cidrToNetwork(cidr) {
|
|
13705
|
+
const m = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})\/(\d{1,2})$/.exec(cidr);
|
|
13706
|
+
if (!m) throw new Error(`Invalid CIDR "${cidr}" (expected a.b.c.d/prefix)`);
|
|
13707
|
+
const octets = [
|
|
13708
|
+
m[1],
|
|
13709
|
+
m[2],
|
|
13710
|
+
m[3],
|
|
13711
|
+
m[4]
|
|
13712
|
+
].map((o) => Number(o));
|
|
13713
|
+
for (const o of octets) if (o > 255) throw new Error(`Invalid octet in CIDR "${cidr}" (each octet must be 0-255)`);
|
|
13714
|
+
const prefix = Number(m[5]);
|
|
13715
|
+
if (prefix > 32) throw new Error(`Invalid prefix /${prefix} in CIDR "${cidr}" (max /32)`);
|
|
13716
|
+
const net = ((octets[0] << 24 | octets[1] << 16 | octets[2] << 8 | octets[3]) >>> 0 & (prefix === 0 ? 0 : 4294967295 << 32 - prefix >>> 0)) >>> 0;
|
|
13717
|
+
return `${[
|
|
13718
|
+
net >>> 24 & 255,
|
|
13719
|
+
net >>> 16 & 255,
|
|
13720
|
+
net >>> 8 & 255,
|
|
13721
|
+
net & 255
|
|
13722
|
+
].join(".")}/${prefix}`;
|
|
13723
|
+
}
|
|
13724
|
+
/** Bridge-name prefix used as the iifname match (matches the bridge + its TAP members). */
|
|
13725
|
+
function bridgePrefix(bridgeName) {
|
|
13726
|
+
return `${bridgeName.split("-")[0]}-*`;
|
|
13727
|
+
}
|
|
13728
|
+
/**
|
|
13729
|
+
* Build the ordered command list that provisions one Firecracker host bridge.
|
|
13730
|
+
* Pure — performs no I/O. `provisionBridge` executes these; `renderBootScript`
|
|
13731
|
+
* serializes them.
|
|
13732
|
+
*
|
|
13733
|
+
* The nft `delete table`/`add table` here only ever touches `cfg.table`, so a
|
|
13734
|
+
* coord-B provision never wipes coord A's table (and vice versa). Every
|
|
13735
|
+
* forward/postrouting/MSS rule is source-scoped to the bridge subnet so two
|
|
13736
|
+
* tables on the shared hooks do not cross-drop each other's traffic.
|
|
13737
|
+
*/
|
|
13738
|
+
function buildBridgeCommands(cfg) {
|
|
13739
|
+
const iface = cfg.hostIface;
|
|
13740
|
+
if (!iface) throw new Error("buildBridgeCommands requires a resolved hostIface (call resolveHostIface first)");
|
|
13741
|
+
const subnet = cidrToNetwork(cfg.bridgeCidr);
|
|
13742
|
+
const gatewayIp = cfg.bridgeCidr.split("/")[0];
|
|
13743
|
+
const prefix = bridgePrefix(cfg.bridgeName);
|
|
13744
|
+
const t = cfg.table;
|
|
13745
|
+
const nft = (...args) => ({
|
|
13746
|
+
bin: "nft",
|
|
13747
|
+
args
|
|
13748
|
+
});
|
|
13749
|
+
const saddr = [
|
|
13750
|
+
"ip",
|
|
13751
|
+
"saddr",
|
|
13752
|
+
subnet
|
|
13753
|
+
];
|
|
13754
|
+
return [
|
|
13755
|
+
{
|
|
13756
|
+
bin: "ip",
|
|
13757
|
+
args: [
|
|
13758
|
+
"link",
|
|
13759
|
+
"add",
|
|
13760
|
+
"name",
|
|
13761
|
+
cfg.bridgeName,
|
|
13762
|
+
"type",
|
|
13763
|
+
"bridge"
|
|
13764
|
+
]
|
|
13765
|
+
},
|
|
13766
|
+
{
|
|
13767
|
+
bin: "ip",
|
|
13768
|
+
args: [
|
|
13769
|
+
"addr",
|
|
13770
|
+
"add",
|
|
13771
|
+
cfg.bridgeCidr,
|
|
13772
|
+
"dev",
|
|
13773
|
+
cfg.bridgeName
|
|
13774
|
+
]
|
|
13775
|
+
},
|
|
13776
|
+
{
|
|
13777
|
+
bin: "ip",
|
|
13778
|
+
args: [
|
|
13779
|
+
"link",
|
|
13780
|
+
"set",
|
|
13781
|
+
cfg.bridgeName,
|
|
13782
|
+
"up"
|
|
13783
|
+
]
|
|
13784
|
+
},
|
|
13785
|
+
{
|
|
13786
|
+
bin: "sysctl",
|
|
13787
|
+
args: ["-w", "net.ipv4.ip_forward=1"]
|
|
13788
|
+
},
|
|
13789
|
+
nft("delete", "table", "ip", t),
|
|
13790
|
+
nft("add", "table", "ip", t),
|
|
13791
|
+
nft("add", "chain", t, "postrouting", "{ type nat hook postrouting priority srcnat; policy accept; }"),
|
|
13792
|
+
nft("add", "rule", t, "postrouting", ...saddr, "oifname", iface, "masquerade"),
|
|
13793
|
+
nft("add", "chain", t, "forward", "{ type filter hook forward priority filter; policy accept; }"),
|
|
13794
|
+
nft("add", "rule", t, "forward", ...saddr, "iifname", prefix, "ip", "daddr", gatewayIp, "accept"),
|
|
13795
|
+
...RFC1918_RANGES.map((r) => nft("add", "rule", t, "forward", ...saddr, "iifname", prefix, "ip", "daddr", r, "drop")),
|
|
13796
|
+
nft("add", "rule", t, "forward", ...saddr, "iifname", prefix, "ip", "daddr", METADATA_RANGE, "drop"),
|
|
13797
|
+
nft("add", "rule", t, "forward", ...saddr, "iifname", prefix, "oifname", iface, "accept"),
|
|
13798
|
+
nft("add", "rule", t, "forward", "iifname", iface, "ip", "daddr", subnet, "oifname", prefix, "ct", "state", "related,established", "accept"),
|
|
13799
|
+
nft("add", "rule", t, "forward", ...saddr, "tcp", "flags", "syn", "/", "syn,rst", "tcp", "option", "maxseg", "size", "set", "1460")
|
|
13800
|
+
];
|
|
13801
|
+
}
|
|
13802
|
+
const NM_CONF_PATH = "/etc/NetworkManager/conf.d/90-kici-unmanaged.conf";
|
|
13803
|
+
const NM_CONF_CONTENT = [
|
|
13804
|
+
"# Managed by kici-admin firecracker — do not hand-edit.",
|
|
13805
|
+
"#",
|
|
13806
|
+
"# KiCI's Firecracker scaler creates and destroys many kici-* interfaces",
|
|
13807
|
+
"# (TAP devices + bridges). NetworkManager must not manage them: it auto-adopts",
|
|
13808
|
+
"# them and can wedge the NM main thread at 100% CPU under churn.",
|
|
13809
|
+
"[keyfile]",
|
|
13810
|
+
"unmanaged-devices=interface-name:kici-*",
|
|
13811
|
+
""
|
|
13812
|
+
].join("\n");
|
|
13813
|
+
const defaultFileWriter = (path, content) => writeFile(path, content, "utf8");
|
|
13814
|
+
function defaultRunner(requireSudo) {
|
|
13815
|
+
return async ({ bin, args, stdin }) => {
|
|
13816
|
+
const child = requireSudo && PRIVILEGED_BINS.has(bin) ? execFileP$1("sudo", [
|
|
13817
|
+
"-n",
|
|
13818
|
+
bin,
|
|
13819
|
+
...args
|
|
13820
|
+
], { timeout: COMMAND_TIMEOUT_MS }) : execFileP$1(bin, args, { timeout: COMMAND_TIMEOUT_MS });
|
|
13821
|
+
if (stdin && child.child.stdin) child.child.stdin.end(stdin);
|
|
13822
|
+
const { stdout } = await child;
|
|
13823
|
+
return { stdout: stdout.toString() };
|
|
13824
|
+
};
|
|
13825
|
+
}
|
|
13826
|
+
/** Resolve the default-route egress interface. */
|
|
13827
|
+
async function resolveHostIface(opts = {}) {
|
|
13828
|
+
const { stdout } = await (opts.runner ?? defaultRunner(opts.requireSudo ?? false))({
|
|
13829
|
+
bin: "ip",
|
|
13830
|
+
args: [
|
|
13831
|
+
"-j",
|
|
13832
|
+
"route",
|
|
13833
|
+
"show",
|
|
13834
|
+
"default"
|
|
13835
|
+
]
|
|
13836
|
+
});
|
|
13837
|
+
const dev = JSON.parse(stdout).find((r) => r.dev)?.dev;
|
|
13838
|
+
if (!dev) throw new Error("could not detect the default-route interface for NAT egress");
|
|
13839
|
+
return dev;
|
|
13840
|
+
}
|
|
13841
|
+
/** Provision (or heal) one Firecracker host bridge. Throws on any failure. */
|
|
13842
|
+
async function provisionBridge(cfg, opts = {}) {
|
|
13843
|
+
const runner = opts.runner ?? defaultRunner(opts.requireSudo ?? false);
|
|
13844
|
+
const iface = cfg.hostIface ?? await resolveHostIface(opts);
|
|
13845
|
+
const resolved = {
|
|
13846
|
+
...cfg,
|
|
13847
|
+
hostIface: iface
|
|
13848
|
+
};
|
|
13849
|
+
const writeNmConf = opts.writeNmConf ?? defaultFileWriter;
|
|
13850
|
+
try {
|
|
13851
|
+
await writeNmConf(NM_CONF_PATH, NM_CONF_CONTENT);
|
|
13852
|
+
} catch (err) {
|
|
13853
|
+
if (!toErrorMessage(err).includes("ENOENT")) throw new Error(`failed to write ${NM_CONF_PATH}: ${toErrorMessage(err)}`);
|
|
13854
|
+
}
|
|
13855
|
+
for (const spec of buildBridgeCommands(resolved)) {
|
|
13856
|
+
const line = [spec.bin, ...spec.args].join(" ");
|
|
13857
|
+
try {
|
|
13858
|
+
await runner(spec);
|
|
13859
|
+
} catch (err) {
|
|
13860
|
+
const msg = toErrorMessage(err);
|
|
13861
|
+
const benignDelete = line.startsWith("nft delete table") && /No such file|does not exist/i.test(msg);
|
|
13862
|
+
const benignExists = /File exists|already a member|already assigned|exists/i.test(msg) && (line.includes("link add") || line.includes("addr add"));
|
|
13863
|
+
if (benignDelete || benignExists) {
|
|
13864
|
+
logger$4.debug(`idempotent skip: ${line} (${msg})`);
|
|
13865
|
+
continue;
|
|
13866
|
+
}
|
|
13867
|
+
throw new Error(`FC host-network command failed: ${line}: ${msg}`);
|
|
13868
|
+
}
|
|
13869
|
+
}
|
|
13870
|
+
logger$4.info(`provisioned bridge ${cfg.bridgeName} (${cfg.bridgeCidr}, table ${cfg.table}, egress ${iface})`);
|
|
13871
|
+
}
|
|
13872
|
+
/** Read-only health probe for one bridge. Never throws on a missing resource. */
|
|
13873
|
+
async function verifyBridge(cfg, opts = {}) {
|
|
13874
|
+
const runner = opts.runner ?? defaultRunner(opts.requireSudo ?? false);
|
|
13875
|
+
const want = cfg.bridgeCidr.split("/")[0];
|
|
13876
|
+
const wantPrefix = Number(cfg.bridgeCidr.split("/")[1]);
|
|
13877
|
+
let bridgeExists = false;
|
|
13878
|
+
let bridgeUp = false;
|
|
13879
|
+
let addrPresent = false;
|
|
13880
|
+
let tablePresent = false;
|
|
13881
|
+
const misses = [];
|
|
13882
|
+
try {
|
|
13883
|
+
const { stdout } = await runner({
|
|
13884
|
+
bin: "ip",
|
|
13885
|
+
args: [
|
|
13886
|
+
"-j",
|
|
13887
|
+
"link",
|
|
13888
|
+
"show",
|
|
13889
|
+
cfg.bridgeName
|
|
13890
|
+
]
|
|
13891
|
+
});
|
|
13892
|
+
const link = JSON.parse(stdout);
|
|
13893
|
+
bridgeExists = link.length > 0;
|
|
13894
|
+
const adminUp = (link[0]?.flags ?? []).includes("UP");
|
|
13895
|
+
const operUp = link[0]?.operstate === "UP" || link[0]?.operstate === "UNKNOWN";
|
|
13896
|
+
bridgeUp = adminUp || operUp;
|
|
13897
|
+
if (!bridgeUp) misses.push(`${cfg.bridgeName} not up`);
|
|
13898
|
+
} catch {
|
|
13899
|
+
misses.push(`${cfg.bridgeName} does not exist`);
|
|
13900
|
+
}
|
|
13901
|
+
try {
|
|
13902
|
+
const { stdout } = await runner({
|
|
13903
|
+
bin: "ip",
|
|
13904
|
+
args: [
|
|
13905
|
+
"-j",
|
|
13906
|
+
"addr",
|
|
13907
|
+
"show",
|
|
13908
|
+
cfg.bridgeName
|
|
13909
|
+
]
|
|
13910
|
+
});
|
|
13911
|
+
addrPresent = (JSON.parse(stdout)[0]?.addr_info ?? []).some((a) => a.local === want && a.prefixlen === wantPrefix);
|
|
13912
|
+
if (!addrPresent) misses.push(`${cfg.bridgeCidr} not assigned to ${cfg.bridgeName}`);
|
|
13913
|
+
} catch {
|
|
13914
|
+
misses.push(`could not read addrs on ${cfg.bridgeName}`);
|
|
13915
|
+
}
|
|
13916
|
+
try {
|
|
13917
|
+
await runner({
|
|
13918
|
+
bin: "nft",
|
|
13919
|
+
args: [
|
|
13920
|
+
"list",
|
|
13921
|
+
"table",
|
|
13922
|
+
"ip",
|
|
13923
|
+
cfg.table
|
|
13924
|
+
]
|
|
13925
|
+
});
|
|
13926
|
+
tablePresent = true;
|
|
13927
|
+
} catch {
|
|
13928
|
+
misses.push(`nft table ip ${cfg.table} missing`);
|
|
13929
|
+
}
|
|
13930
|
+
const healthy = bridgeExists && bridgeUp && addrPresent && tablePresent;
|
|
13931
|
+
return {
|
|
13932
|
+
bridgeName: cfg.bridgeName,
|
|
13933
|
+
bridgeExists,
|
|
13934
|
+
bridgeUp,
|
|
13935
|
+
addrPresent,
|
|
13936
|
+
tablePresent,
|
|
13937
|
+
healthy,
|
|
13938
|
+
detail: healthy ? "healthy" : misses.join("; ")
|
|
13939
|
+
};
|
|
13940
|
+
}
|
|
13941
|
+
/** Remove the bridge + its nft table. Leaves the host-scoped NM conf in place. */
|
|
13942
|
+
async function teardownBridge(cfg, opts = {}) {
|
|
13943
|
+
const runner = opts.runner ?? defaultRunner(opts.requireSudo ?? false);
|
|
13944
|
+
for (const spec of [{
|
|
13945
|
+
bin: "nft",
|
|
13946
|
+
args: [
|
|
13947
|
+
"delete",
|
|
13948
|
+
"table",
|
|
13949
|
+
"ip",
|
|
13950
|
+
cfg.table
|
|
13951
|
+
]
|
|
13952
|
+
}, {
|
|
13953
|
+
bin: "ip",
|
|
13954
|
+
args: [
|
|
13955
|
+
"link",
|
|
13956
|
+
"del",
|
|
13957
|
+
cfg.bridgeName
|
|
13958
|
+
]
|
|
13959
|
+
}]) try {
|
|
13960
|
+
await runner(spec);
|
|
13961
|
+
} catch (err) {
|
|
13962
|
+
logger$4.debug(`teardown skip ${spec.bin}: ${toErrorMessage(err)}`);
|
|
13963
|
+
}
|
|
13964
|
+
}
|
|
13965
|
+
const HOST_IFACE_SENTINEL = "__HOST_IFACE__";
|
|
13966
|
+
/** Serialize the provisioning command list into a dependency-free boot script. */
|
|
13967
|
+
function renderBootScript(cfg) {
|
|
13968
|
+
const lines = buildBridgeCommands({
|
|
13969
|
+
...cfg,
|
|
13970
|
+
hostIface: HOST_IFACE_SENTINEL
|
|
13971
|
+
}).map((s) => {
|
|
13972
|
+
const benign = s.bin === "nft" && s.args[0] === "delete" && s.args[1] === "table";
|
|
13973
|
+
const argv = [s.bin, ...s.args].map((tok) => tok === HOST_IFACE_SENTINEL ? "\"$HOST_IFACE\"" : shellQuote(tok)).join(" ");
|
|
13974
|
+
return benign ? `${argv} 2>/dev/null || true` : argv;
|
|
13975
|
+
});
|
|
13976
|
+
return [
|
|
13977
|
+
"#!/usr/bin/env bash",
|
|
13978
|
+
`# Generated by kici-admin firecracker --persist for bridge ${cfg.bridgeName}. Do not hand-edit.`,
|
|
13979
|
+
"set -euo pipefail",
|
|
13980
|
+
`printf '%s\\n' "${NM_CONF_CONTENT.replace(/\n/g, "\\n").replace(/"/g, "\\\"")}" > ${NM_CONF_PATH} || true`,
|
|
13981
|
+
"HOST_IFACE=$(ip -j route show default | sed -n 's/.*\"dev\":\"\\([^\"]*\\)\".*/\\1/p' | head -1)",
|
|
13982
|
+
"if [ -z \"$HOST_IFACE\" ]; then echo \"no default-route iface\" >&2; exit 1; fi",
|
|
13983
|
+
...lines,
|
|
13984
|
+
""
|
|
13985
|
+
].join("\n");
|
|
13986
|
+
}
|
|
13987
|
+
/**
|
|
13988
|
+
* Minimal POSIX-safe single-quote shell escaping for boot-script tokens.
|
|
13989
|
+
*
|
|
13990
|
+
* `*` is deliberately NOT in the bare-token allow-list: an `nft` arg like
|
|
13991
|
+
* `kici-*` (the iifname match) must reach nft literally, but an unquoted
|
|
13992
|
+
* `kici-*` would be glob-expanded by bash against the CWD. Tokens carrying any
|
|
13993
|
+
* shell metacharacter (incl. `*`, spaces, `{`/`;` from chain definitions) are
|
|
13994
|
+
* single-quoted so they pass through verbatim.
|
|
13995
|
+
*/
|
|
13996
|
+
function shellQuote(tok) {
|
|
13997
|
+
if (/^[A-Za-z0-9_./:=,+-]+$/.test(tok)) return tok;
|
|
13998
|
+
return `'${tok.replace(/'/g, `'\\''`)}'`;
|
|
13999
|
+
}
|
|
14000
|
+
//#endregion
|
|
14001
|
+
//#region src/firecracker/persist.ts
|
|
14002
|
+
/**
|
|
14003
|
+
* Boot-persistence for Firecracker host bridges: install a dependency-free
|
|
14004
|
+
* provisioning script + a per-bridge systemd oneshot unit so the bridge is
|
|
14005
|
+
* recreated on reboot (the kernel bridge/nft state is non-persistent).
|
|
14006
|
+
*/
|
|
14007
|
+
const execFileP = promisify(execFile);
|
|
14008
|
+
const logger$3 = createLogger({ prefix: "fc-persist" });
|
|
14009
|
+
const BOOT_SCRIPT_DIR = "/usr/local/lib/kici-firecracker";
|
|
14010
|
+
const SYSTEMD_DIR = "/etc/systemd/system";
|
|
14011
|
+
function persistUnitName(bridgeName) {
|
|
14012
|
+
return `kici-fc-net-${bridgeName}.service`;
|
|
14013
|
+
}
|
|
14014
|
+
function renderPersistUnit(bridgeName) {
|
|
14015
|
+
return [
|
|
14016
|
+
"[Unit]",
|
|
14017
|
+
`Description=KiCI Firecracker host bridge ${bridgeName}`,
|
|
14018
|
+
"After=network-online.target",
|
|
14019
|
+
"Wants=network-online.target",
|
|
14020
|
+
"",
|
|
14021
|
+
"[Service]",
|
|
14022
|
+
"Type=oneshot",
|
|
14023
|
+
"RemainAfterExit=yes",
|
|
14024
|
+
`ExecStart=${BOOT_SCRIPT_DIR}/provision-${bridgeName}.sh`,
|
|
14025
|
+
"",
|
|
14026
|
+
"[Install]",
|
|
14027
|
+
"WantedBy=multi-user.target",
|
|
14028
|
+
""
|
|
14029
|
+
].join("\n");
|
|
14030
|
+
}
|
|
14031
|
+
/**
|
|
14032
|
+
* Install the boot script + unit and `systemctl enable` it (enable only — the
|
|
14033
|
+
* deploy/CLI provisions the bridge live in the same run, so the unit fires on
|
|
14034
|
+
* the NEXT boot rather than re-churning nft now).
|
|
14035
|
+
*/
|
|
14036
|
+
async function installPersist(cfg, opts = {}) {
|
|
14037
|
+
const sudo = opts.requireSudo ?? false;
|
|
14038
|
+
const run = (bin, args) => sudo ? execFileP("sudo", [
|
|
14039
|
+
"-n",
|
|
14040
|
+
bin,
|
|
14041
|
+
...args
|
|
14042
|
+
], { timeout: 3e4 }) : execFileP(bin, args, { timeout: 3e4 });
|
|
14043
|
+
const scriptPath = `${BOOT_SCRIPT_DIR}/provision-${cfg.bridgeName}.sh`;
|
|
14044
|
+
const unitPath = `${SYSTEMD_DIR}/${persistUnitName(cfg.bridgeName)}`;
|
|
14045
|
+
await mkdir(BOOT_SCRIPT_DIR, { recursive: true });
|
|
14046
|
+
await writeFile(scriptPath, renderBootScript(cfg), "utf8");
|
|
14047
|
+
await chmod(scriptPath, 493);
|
|
14048
|
+
await writeFile(unitPath, renderPersistUnit(cfg.bridgeName), "utf8");
|
|
14049
|
+
await run("systemctl", ["daemon-reload"]);
|
|
14050
|
+
await run("systemctl", ["enable", persistUnitName(cfg.bridgeName)]);
|
|
14051
|
+
logger$3.info(`installed + enabled ${persistUnitName(cfg.bridgeName)} (boot script ${scriptPath})`);
|
|
14052
|
+
}
|
|
14053
|
+
//#endregion
|
|
14054
|
+
//#region src/cli/commands/firecracker/provision.ts
|
|
14055
|
+
function optionsToConfig(opts) {
|
|
14056
|
+
if (!opts.cidr) throw new Error("--cidr <gateway/prefix> is required");
|
|
14057
|
+
return {
|
|
14058
|
+
bridgeName: opts.bridge,
|
|
14059
|
+
bridgeCidr: opts.cidr,
|
|
14060
|
+
table: opts.table ?? "kici",
|
|
14061
|
+
hostIface: opts.hostIface
|
|
14062
|
+
};
|
|
14063
|
+
}
|
|
14064
|
+
function registerProvision(group) {
|
|
14065
|
+
group.command("provision").description("Create/heal a Firecracker host bridge (NAT + egress isolation)").requiredOption("--bridge <name>", "bridge interface name (e.g. kici-br0)").requiredOption("--cidr <cidr>", "gateway IP + prefix (e.g. 10.0.0.1/24)").option("--table <name>", "nft table name", "kici").option("--host-iface <iface>", "NAT egress interface (auto-detected if omitted)").option("--persist", "install a systemd oneshot so the bridge survives reboot").option("--sudo", "wrap privileged commands with sudo -n (non-root host)").action(async (opts) => {
|
|
14066
|
+
try {
|
|
14067
|
+
const cfg = optionsToConfig(opts);
|
|
14068
|
+
await provisionBridge(cfg, { requireSudo: opts.sudo });
|
|
14069
|
+
if (opts.persist) await installPersist(cfg, { requireSudo: opts.sudo });
|
|
14070
|
+
console.log(`Firecracker bridge ${cfg.bridgeName} provisioned${opts.persist ? " + persisted" : ""}.`);
|
|
14071
|
+
} catch (err) {
|
|
14072
|
+
console.error(`firecracker provision failed: ${toErrorMessage(err)}`);
|
|
14073
|
+
process.exit(1);
|
|
14074
|
+
}
|
|
14075
|
+
});
|
|
14076
|
+
}
|
|
14077
|
+
//#endregion
|
|
14078
|
+
//#region src/cli/commands/firecracker/verify.ts
|
|
14079
|
+
function registerVerify(group) {
|
|
14080
|
+
group.command("verify").description("Check a Firecracker host bridge is up with its addr + nft table").requiredOption("--bridge <name>", "bridge interface name").requiredOption("--cidr <cidr>", "gateway IP + prefix").option("--table <name>", "nft table name", "kici").option("--sudo", "wrap privileged commands with sudo -n").action(async (opts) => {
|
|
14081
|
+
const cfg = optionsToConfig(opts);
|
|
14082
|
+
const h = await verifyBridge(cfg, { requireSudo: opts.sudo });
|
|
14083
|
+
if (h.healthy) {
|
|
14084
|
+
console.log(`OK: ${cfg.bridgeName} healthy (${cfg.bridgeCidr}, table ${cfg.table})`);
|
|
14085
|
+
return;
|
|
14086
|
+
}
|
|
14087
|
+
console.error(`UNHEALTHY: ${cfg.bridgeName}: ${h.detail}`);
|
|
14088
|
+
process.exit(1);
|
|
14089
|
+
});
|
|
14090
|
+
}
|
|
14091
|
+
//#endregion
|
|
14092
|
+
//#region src/cli/commands/firecracker/teardown.ts
|
|
14093
|
+
function registerTeardown(group) {
|
|
14094
|
+
group.command("teardown").description("Remove a Firecracker host bridge + its nft table (leaves NM conf in place)").requiredOption("--bridge <name>", "bridge interface name").option("--cidr <cidr>", "gateway IP + prefix (unused but accepted for symmetry)", "0.0.0.0/0").option("--table <name>", "nft table name", "kici").option("--sudo", "wrap privileged commands with sudo -n").action(async (opts) => {
|
|
14095
|
+
await teardownBridge(optionsToConfig(opts), { requireSudo: opts.sudo });
|
|
14096
|
+
console.log(`Firecracker bridge ${opts.bridge} torn down.`);
|
|
14097
|
+
});
|
|
14098
|
+
}
|
|
14099
|
+
//#endregion
|
|
14100
|
+
//#region src/cli/commands/firecracker/index.ts
|
|
14101
|
+
function registerFirecrackerCommands(program) {
|
|
14102
|
+
const group = program.command("firecracker").description("Provision and verify Firecracker host networking");
|
|
14103
|
+
registerProvision(group);
|
|
14104
|
+
registerVerify(group);
|
|
14105
|
+
registerTeardown(group);
|
|
14106
|
+
}
|
|
14107
|
+
//#endregion
|
|
14108
|
+
//#region src/config/loader.ts
|
|
14109
|
+
/**
|
|
14110
|
+
* YAML configuration file loader for the orchestrator.
|
|
14111
|
+
*
|
|
14112
|
+
* Reads the local YAML config file from a configurable path with fallback:
|
|
14113
|
+
* 1. Explicit configPath argument
|
|
14114
|
+
* 2. KICI_CONFIG environment variable
|
|
14115
|
+
* 3. /etc/kici/orchestrator.yaml default
|
|
14116
|
+
*
|
|
14117
|
+
* When no YAML file is found and no explicit path was given,
|
|
14118
|
+
* returns an empty config (YAML is optional -- env-only mode).
|
|
14119
|
+
*/
|
|
14120
|
+
init_schema();
|
|
14121
|
+
/** Default config file path when no explicit path or env var is set */
|
|
14122
|
+
const DEFAULT_CONFIG_PATH = "/etc/kici/orchestrator.yaml";
|
|
14123
|
+
/**
|
|
14124
|
+
* Load and validate the local orchestrator YAML configuration.
|
|
14125
|
+
*
|
|
14126
|
+
* @param configPath - Explicit path to the config file (highest precedence)
|
|
14127
|
+
* @returns Validated LocalConfig
|
|
14128
|
+
* @throws Error if explicit path doesn't exist, or if validation fails
|
|
14129
|
+
*/
|
|
14130
|
+
async function loadLocalConfig(configPath) {
|
|
14131
|
+
const resolvedPath = configPath ?? process.env.KICI_CONFIG ?? DEFAULT_CONFIG_PATH;
|
|
14132
|
+
const isExplicit = configPath !== void 0 || process.env.KICI_CONFIG !== void 0;
|
|
14133
|
+
let rawYaml;
|
|
14134
|
+
try {
|
|
14135
|
+
rawYaml = await readFile(resolvedPath, "utf-8");
|
|
14136
|
+
} catch (err) {
|
|
14137
|
+
if (!isExplicit && isNodeError(err) && err.code === "ENOENT") return { database: { url: "" } };
|
|
14138
|
+
throw new Error(`Failed to read config file at ${resolvedPath}: ${toErrorMessage(err)}`);
|
|
14139
|
+
}
|
|
14140
|
+
const parsed = parse(rawYaml);
|
|
14141
|
+
if (!parsed || typeof parsed !== "object") {
|
|
14142
|
+
if (isExplicit) throw new Error(`Config file at ${resolvedPath} is empty or not a valid YAML object`);
|
|
14143
|
+
return { database: { url: "" } };
|
|
14144
|
+
}
|
|
14145
|
+
const result = localConfigSchema.safeParse(parsed);
|
|
14146
|
+
if (!result.success) {
|
|
14147
|
+
const errors = result.error.issues.map((e) => ` - ${e.path.join(".")}: ${e.message}`).join("\n");
|
|
14148
|
+
throw new Error(`Config validation failed for ${resolvedPath}:\n${errors}`);
|
|
14149
|
+
}
|
|
14150
|
+
return result.data;
|
|
14151
|
+
}
|
|
14152
|
+
/**
|
|
14153
|
+
* Type guard for Node.js errors with a `code` property.
|
|
14154
|
+
*/
|
|
14155
|
+
function isNodeError(err) {
|
|
14156
|
+
return err instanceof Error && "code" in err;
|
|
14157
|
+
}
|
|
14158
|
+
//#endregion
|
|
14159
|
+
//#region src/scaler/config.ts
|
|
14160
|
+
/**
|
|
14161
|
+
* YAML configuration loading and Zod validation for the scaler module.
|
|
14162
|
+
*
|
|
14163
|
+
* Handles:
|
|
14164
|
+
* - Main YAML config file parsing
|
|
14165
|
+
* - scalers.d/ directory merging (alphabetically sorted)
|
|
14166
|
+
* - Zod schema validation with type-specific refinements
|
|
14167
|
+
* - Memory string parsing (e.g., "2g" -> bytes)
|
|
14168
|
+
*/
|
|
14169
|
+
/**
|
|
14170
|
+
* Zod schema for a single resource spec (cpus + memory).
|
|
14171
|
+
*
|
|
14172
|
+
* The flat shape used to be the public scaler config form (`{ cpus, memory }`).
|
|
14173
|
+
* It's still accepted at the label-set / defaults level via `resourceRequestSchema`
|
|
14174
|
+
* below, where it's normalized to the nested `{ requests, limits }` form.
|
|
14175
|
+
*/
|
|
14176
|
+
const resourceSpecSchema = z.object({
|
|
14177
|
+
memory: z.string().optional(),
|
|
14178
|
+
cpus: z.number().positive().max(256).optional()
|
|
14179
|
+
}).strict();
|
|
14180
|
+
/**
|
|
14181
|
+
* Zod schema for nested-form `ResourceRequest` (`{ requests, limits }`).
|
|
14182
|
+
*/
|
|
14183
|
+
const nestedResourceRequestSchema = z.object({
|
|
14184
|
+
requests: resourceSpecSchema.optional(),
|
|
14185
|
+
limits: resourceSpecSchema.optional()
|
|
14186
|
+
}).strict();
|
|
14187
|
+
/**
|
|
14188
|
+
* Resource request schema accepted at the label-set / defaults level.
|
|
14189
|
+
*
|
|
14190
|
+
* Two input forms:
|
|
14191
|
+
* - Nested (preferred): `{ requests?: {cpus, memory}, limits?: {cpus, memory} }`.
|
|
14192
|
+
* - Flat shorthand: `{ cpus, memory }` -- treated as `limits`, `requests` mirrored from it.
|
|
14193
|
+
*
|
|
14194
|
+
* Always normalizes to the nested form so downstream code sees one shape.
|
|
14195
|
+
*/
|
|
14196
|
+
const resourceRequestSchema = z.union([nestedResourceRequestSchema, resourceSpecSchema]).transform((value) => {
|
|
14197
|
+
if ("cpus" in value || "memory" in value) {
|
|
14198
|
+
const flat = value;
|
|
14199
|
+
if (flat.cpus === void 0 && flat.memory === void 0) return {};
|
|
14200
|
+
return {
|
|
14201
|
+
requests: { ...flat },
|
|
14202
|
+
limits: { ...flat }
|
|
14203
|
+
};
|
|
14204
|
+
}
|
|
14205
|
+
return value;
|
|
14206
|
+
});
|
|
14207
|
+
/**
|
|
14208
|
+
* Zod schema for an aggregate resource cap (per-scaler / per-orchestrator / per-machine).
|
|
14209
|
+
*
|
|
14210
|
+
* `maxMemory` is parsed at config-load to bytes, exposed downstream as
|
|
14211
|
+
* `maxMemoryBytes` so the scaler doesn't re-parse on every spawn check.
|
|
14212
|
+
*/
|
|
14213
|
+
const resourceCapSchema = z.object({
|
|
14214
|
+
maxCpu: z.number().positive().optional(),
|
|
14215
|
+
maxMemory: z.string().optional()
|
|
14216
|
+
}).strict().transform((value, ctx) => {
|
|
14217
|
+
let maxMemoryBytes;
|
|
14218
|
+
if (value.maxMemory !== void 0) try {
|
|
14219
|
+
maxMemoryBytes = parseMemoryString(value.maxMemory);
|
|
14220
|
+
} catch (err) {
|
|
14221
|
+
ctx.addIssue({
|
|
14222
|
+
code: z.ZodIssueCode.custom,
|
|
14223
|
+
message: err instanceof Error ? err.message : String(err),
|
|
14224
|
+
path: ["maxMemory"]
|
|
14225
|
+
});
|
|
14226
|
+
return z.NEVER;
|
|
14227
|
+
}
|
|
14228
|
+
return {
|
|
14229
|
+
maxCpu: value.maxCpu,
|
|
14230
|
+
maxMemoryBytes
|
|
14231
|
+
};
|
|
14232
|
+
});
|
|
14233
|
+
/**
|
|
14234
|
+
* Zod schema for warm pool configuration.
|
|
14235
|
+
*/
|
|
14236
|
+
const warmPoolSchema = z.object({
|
|
14237
|
+
enabled: z.boolean().default(false),
|
|
14238
|
+
size: z.number().int().nonnegative().default(0),
|
|
14239
|
+
idleTimeoutSeconds: z.number().default(300)
|
|
14240
|
+
}).optional();
|
|
14241
|
+
/**
|
|
14242
|
+
* Zod schema for network policy configuration on a label set.
|
|
14243
|
+
* Controls RFC1918 and internet access for spawned agents.
|
|
14244
|
+
*/
|
|
14245
|
+
const networkPolicySchema = z.object({
|
|
14246
|
+
/** Allow traffic to these CIDR ranges (exceptions to default RFC1918 block) */
|
|
14247
|
+
allowlist: z.array(z.string()).optional(),
|
|
14248
|
+
/** Block all outbound traffic except allowlisted ranges */
|
|
14249
|
+
denyAll: z.boolean().default(false)
|
|
14250
|
+
}).optional();
|
|
14251
|
+
/**
|
|
14252
|
+
* Zod schema for a label-set configuration entry.
|
|
14253
|
+
*/
|
|
14254
|
+
const labelSetConfigSchema = z.object({
|
|
14255
|
+
labels: z.array(z.string()).min(1).refine((labels) => !labels.some((l) => l.toLowerCase().startsWith("kici:")), { message: "Labels with 'kici:' prefix are reserved and cannot be used in scaler labelSets" }),
|
|
14256
|
+
image: z.string().optional(),
|
|
14257
|
+
imagePullPolicy: z.enum([
|
|
14258
|
+
"Always",
|
|
14259
|
+
"IfNotPresent",
|
|
14260
|
+
"Never"
|
|
14261
|
+
]).default("Always"),
|
|
14262
|
+
binaryPath: z.string().optional(),
|
|
14263
|
+
/**
|
|
14264
|
+
* Per-label-set resource request and limit. Accepts either the nested
|
|
14265
|
+
* form `{ requests, limits }` or the legacy flat `{ cpus, memory }`
|
|
14266
|
+
* (treated as `limits`, with `requests` mirrored). Always normalized to nested.
|
|
14267
|
+
*/
|
|
14268
|
+
resources: resourceRequestSchema.optional(),
|
|
14269
|
+
volumes: z.array(z.string()).optional(),
|
|
14270
|
+
containerSocket: z.boolean().default(false),
|
|
14271
|
+
env: z.record(z.string(), z.string()).optional(),
|
|
14272
|
+
backpressureMode: z.enum(["pause", "drop"]).optional(),
|
|
14273
|
+
networkPolicy: networkPolicySchema,
|
|
14274
|
+
rootfsPath: z.string().optional(),
|
|
14275
|
+
kernelPath: z.string().optional(),
|
|
14276
|
+
vcpuCount: z.number().int().positive().optional(),
|
|
14277
|
+
memSizeMib: z.number().int().positive().optional(),
|
|
14278
|
+
overlayDriveSizeMib: z.number().int().positive().optional()
|
|
14279
|
+
}).strict();
|
|
14280
|
+
/**
|
|
14281
|
+
* Zod schema for a single scaler entry.
|
|
14282
|
+
* Includes type-specific refinements:
|
|
14283
|
+
* - Docker scalers require 'image' on every label set
|
|
14284
|
+
* - Bare-metal scalers require 'binaryPath' on every label set
|
|
14285
|
+
*/
|
|
14286
|
+
const scalerEntrySchema = z.object({
|
|
14287
|
+
name: z.string(),
|
|
14288
|
+
type: ScalerBackendType.exclude(["kubernetes"]),
|
|
14289
|
+
maxAgents: z.number().int().positive(),
|
|
14290
|
+
labelSets: z.array(labelSetConfigSchema).min(1),
|
|
14291
|
+
host: z.string().optional(),
|
|
14292
|
+
socketPath: z.string().optional(),
|
|
14293
|
+
runtime: z.enum([
|
|
14294
|
+
"docker",
|
|
14295
|
+
"podman",
|
|
14296
|
+
"auto"
|
|
14297
|
+
]).default("auto"),
|
|
14298
|
+
orchestratorUrl: z.string().optional(),
|
|
14299
|
+
extraHosts: z.array(z.string()).optional(),
|
|
14300
|
+
networkIsolation: z.boolean().default(true),
|
|
14301
|
+
warmPool: warmPoolSchema,
|
|
14302
|
+
/**
|
|
14303
|
+
* Labels a job MUST declare in `runsOn` to allocate on this scaler.
|
|
14304
|
+
*
|
|
14305
|
+
* Mirrors the Kubernetes "taints" concept: if a scaler has
|
|
14306
|
+
* `mandatoryLabels: ['gpu']`, a job's `runsOn` must include `gpu` for
|
|
14307
|
+
* the scaler to be considered. Generic jobs that don't list every
|
|
14308
|
+
* mandatory label are blocked from this scaler entirely, even if
|
|
14309
|
+
* their other labels are a subset of one of the scaler's labelSets.
|
|
14310
|
+
*
|
|
14311
|
+
* The `kici:` prefix is reserved for auto-injected system labels
|
|
14312
|
+
* (kici:role:*, kici:os:*, kici:arch:*, kici:agent:*, kici:scaler:*)
|
|
14313
|
+
* and cannot be used here — same as `labelSets[].labels`.
|
|
14314
|
+
*
|
|
14315
|
+
* Default: `[]` (no gating; behavior is unchanged from before this feature).
|
|
14316
|
+
*/
|
|
14317
|
+
mandatoryLabels: z.array(z.string()).default([]).refine((labels) => !labels.some((l) => l.toLowerCase().startsWith("kici:")), { message: "Labels with 'kici:' prefix are reserved and cannot be used in mandatoryLabels" }),
|
|
14318
|
+
roles: z.array(z.enum([...KNOWN_ROLES, "all"])).optional().transform((roles) => {
|
|
14319
|
+
if (!roles) return void 0;
|
|
14320
|
+
if (roles.includes("all")) return void 0;
|
|
14321
|
+
if (roles.length === 0) return [];
|
|
14322
|
+
return roles.filter((r) => r !== "all");
|
|
14323
|
+
}),
|
|
14324
|
+
resourceCap: resourceCapSchema.optional(),
|
|
14325
|
+
machinePool: z.string().optional(),
|
|
14326
|
+
enforceCgroups: z.boolean().default(false),
|
|
14327
|
+
firecrackerPath: z.string().optional(),
|
|
14328
|
+
jailerPath: z.string().optional(),
|
|
14329
|
+
kernelPath: z.string().optional(),
|
|
14330
|
+
chrootBaseDir: z.string().default("/srv/jailer"),
|
|
14331
|
+
uid: z.number().int().optional(),
|
|
14332
|
+
gid: z.number().int().optional(),
|
|
14333
|
+
vcpuCount: z.number().int().positive().default(2),
|
|
14334
|
+
memSizeMib: z.number().int().positive().default(512),
|
|
14335
|
+
/**
|
|
14336
|
+
* When the orchestrator runs as a non-root user (e.g. user-mode systemd on
|
|
14337
|
+
* an edge worker), set this to true so privileged commands (`ip`, `chown`)
|
|
14338
|
+
* are wrapped with `sudo -n`. Operators must have a NOPASSWD sudoers entry
|
|
14339
|
+
* for those binaries. On hosts where the orchestrator is already root,
|
|
14340
|
+
* leave unset.
|
|
14341
|
+
*/
|
|
14342
|
+
requireSudo: z.boolean().default(false)
|
|
14343
|
+
}).strict().superRefine((data, ctx) => {
|
|
14344
|
+
if (data.type === ScalerBackendType.enum.container) data.labelSets.forEach((ls, i) => {
|
|
14345
|
+
if (!ls.image) ctx.addIssue({
|
|
14346
|
+
code: z.ZodIssueCode.custom,
|
|
14347
|
+
message: `Container scaler "${data.name}" label set [${i}] requires an 'image' field`,
|
|
14348
|
+
path: [
|
|
14349
|
+
"labelSets",
|
|
14350
|
+
i,
|
|
14351
|
+
"image"
|
|
14352
|
+
]
|
|
14353
|
+
});
|
|
14354
|
+
});
|
|
14355
|
+
if (data.type === ScalerBackendType.enum["bare-metal"]) data.labelSets.forEach((ls, i) => {
|
|
14356
|
+
if (!ls.binaryPath) ctx.addIssue({
|
|
14357
|
+
code: z.ZodIssueCode.custom,
|
|
14358
|
+
message: `Bare-metal scaler "${data.name}" label set [${i}] requires a 'binaryPath' field`,
|
|
14359
|
+
path: [
|
|
14360
|
+
"labelSets",
|
|
14361
|
+
i,
|
|
14362
|
+
"binaryPath"
|
|
14363
|
+
]
|
|
14364
|
+
});
|
|
14365
|
+
});
|
|
14366
|
+
if (data.mandatoryLabels.length > 0) {
|
|
14367
|
+
const mandatoryLower = data.mandatoryLabels.map((l) => l.toLowerCase());
|
|
14368
|
+
data.labelSets.forEach((ls, i) => {
|
|
14369
|
+
const labelsLower = new Set(ls.labels.map((l) => l.toLowerCase()));
|
|
14370
|
+
for (const required of mandatoryLower) if (!labelsLower.has(required)) ctx.addIssue({
|
|
14371
|
+
code: z.ZodIssueCode.custom,
|
|
14372
|
+
message: `Scaler "${data.name}" mandatoryLabel "${required}" is missing from labelSets[${i}].labels — every mandatory label must appear in every labelSet, otherwise jobs can route through this labelSet and bypass the gate`,
|
|
14373
|
+
path: [
|
|
14374
|
+
"labelSets",
|
|
14375
|
+
i,
|
|
14376
|
+
"labels"
|
|
14377
|
+
]
|
|
14378
|
+
});
|
|
14379
|
+
});
|
|
14380
|
+
}
|
|
14381
|
+
if (data.type === ScalerBackendType.enum.firecracker) {
|
|
14382
|
+
if (!data.firecrackerPath) ctx.addIssue({
|
|
14383
|
+
code: z.ZodIssueCode.custom,
|
|
14384
|
+
message: `Firecracker scaler "${data.name}" requires a 'firecrackerPath' field`,
|
|
14385
|
+
path: ["firecrackerPath"]
|
|
14386
|
+
});
|
|
14387
|
+
if (!data.jailerPath) ctx.addIssue({
|
|
14388
|
+
code: z.ZodIssueCode.custom,
|
|
14389
|
+
message: `Firecracker scaler "${data.name}" requires a 'jailerPath' field`,
|
|
14390
|
+
path: ["jailerPath"]
|
|
14391
|
+
});
|
|
14392
|
+
if (!data.kernelPath) ctx.addIssue({
|
|
14393
|
+
code: z.ZodIssueCode.custom,
|
|
14394
|
+
message: `Firecracker scaler "${data.name}" requires a 'kernelPath' field`,
|
|
14395
|
+
path: ["kernelPath"]
|
|
14396
|
+
});
|
|
14397
|
+
if (data.uid === void 0) ctx.addIssue({
|
|
14398
|
+
code: z.ZodIssueCode.custom,
|
|
14399
|
+
message: `Firecracker scaler "${data.name}" requires a 'uid' field`,
|
|
14400
|
+
path: ["uid"]
|
|
14401
|
+
});
|
|
14402
|
+
if (data.gid === void 0) ctx.addIssue({
|
|
14403
|
+
code: z.ZodIssueCode.custom,
|
|
14404
|
+
message: `Firecracker scaler "${data.name}" requires a 'gid' field`,
|
|
14405
|
+
path: ["gid"]
|
|
14406
|
+
});
|
|
14407
|
+
data.labelSets.forEach((ls, i) => {
|
|
14408
|
+
if (!ls.rootfsPath) ctx.addIssue({
|
|
14409
|
+
code: z.ZodIssueCode.custom,
|
|
14410
|
+
message: `Firecracker scaler "${data.name}" label set [${i}] requires a 'rootfsPath' field`,
|
|
14411
|
+
path: [
|
|
14412
|
+
"labelSets",
|
|
14413
|
+
i,
|
|
14414
|
+
"rootfsPath"
|
|
14415
|
+
]
|
|
14416
|
+
});
|
|
14417
|
+
});
|
|
14418
|
+
}
|
|
14419
|
+
});
|
|
14420
|
+
/**
|
|
14421
|
+
* Zod schema for Firecracker network configuration.
|
|
14422
|
+
* Global across all Firecracker scalers on the orchestrator.
|
|
14423
|
+
*/
|
|
14424
|
+
const firecrackerNetworkSchema = z.object({
|
|
14425
|
+
cidr: z.string().default("10.0.0.0/24"),
|
|
14426
|
+
bridgeName: z.string().default("kici-br0"),
|
|
14427
|
+
gateway: z.string().default("10.0.0.1"),
|
|
14428
|
+
netmask: z.string().default("255.255.255.0"),
|
|
14429
|
+
/** nft table name for this coordinator's host bridge (disjoint per bridge). */
|
|
14430
|
+
table: z.string().default("kici")
|
|
14431
|
+
});
|
|
14432
|
+
/**
|
|
14433
|
+
* Zod schema for the complete scaler configuration file.
|
|
14434
|
+
* Validates version, global limits, defaults, and scaler entries.
|
|
14435
|
+
*/
|
|
14436
|
+
const scalerFileSchema = z.object({
|
|
14437
|
+
version: z.literal(1),
|
|
14438
|
+
globalMaxAgents: z.number().int().positive().default(50),
|
|
14439
|
+
defaults: z.object({ resources: resourceRequestSchema.optional() }).optional(),
|
|
14440
|
+
/** Cap on summed `requests` across every agent this orchestrator is running. */
|
|
14441
|
+
globalResourceCap: resourceCapSchema.optional(),
|
|
14442
|
+
/** Named pools for cross-orchestrator coordination on the same host. */
|
|
14443
|
+
machinePools: z.array(z.object({
|
|
14444
|
+
name: z.string().min(1),
|
|
14445
|
+
cap: resourceCapSchema
|
|
14446
|
+
}).strict()).optional(),
|
|
14447
|
+
scalers: z.array(scalerEntrySchema),
|
|
14448
|
+
firecracker: firecrackerNetworkSchema.optional()
|
|
14449
|
+
}).superRefine((data, ctx) => {
|
|
14450
|
+
const poolNames = new Set((data.machinePools ?? []).map((p) => p.name));
|
|
14451
|
+
data.scalers.forEach((entry, i) => {
|
|
14452
|
+
if (entry.machinePool !== void 0 && !poolNames.has(entry.machinePool)) ctx.addIssue({
|
|
14453
|
+
code: z.ZodIssueCode.custom,
|
|
14454
|
+
message: `Scaler "${entry.name}" references unknown machinePool "${entry.machinePool}". Define it under top-level machinePools[].`,
|
|
14455
|
+
path: [
|
|
14456
|
+
"scalers",
|
|
14457
|
+
i,
|
|
14458
|
+
"machinePool"
|
|
14459
|
+
]
|
|
14460
|
+
});
|
|
14461
|
+
});
|
|
14462
|
+
const seen = /* @__PURE__ */ new Set();
|
|
14463
|
+
(data.machinePools ?? []).forEach((p, i) => {
|
|
14464
|
+
if (seen.has(p.name)) ctx.addIssue({
|
|
14465
|
+
code: z.ZodIssueCode.custom,
|
|
14466
|
+
message: `Duplicate machinePool name "${p.name}".`,
|
|
14467
|
+
path: [
|
|
14468
|
+
"machinePools",
|
|
14469
|
+
i,
|
|
14470
|
+
"name"
|
|
14471
|
+
]
|
|
14472
|
+
});
|
|
14473
|
+
seen.add(p.name);
|
|
14474
|
+
});
|
|
14475
|
+
});
|
|
14476
|
+
/**
|
|
14477
|
+
* Parse a memory string with unit suffix to bytes.
|
|
14478
|
+
*
|
|
14479
|
+
* Supported formats:
|
|
14480
|
+
* - "512m" or "512M" -> 536870912 (512 * 1024^2)
|
|
14481
|
+
* - "2g" or "2G" -> 2147483648 (2 * 1024^3)
|
|
14482
|
+
* - "1024k" or "1024K" -> 1048576 (1024 * 1024)
|
|
14483
|
+
*
|
|
14484
|
+
* @throws Error if the format is invalid
|
|
14485
|
+
*/
|
|
14486
|
+
function parseMemoryString(memory) {
|
|
14487
|
+
const match = memory.match(/^(\d+(?:\.\d+)?)\s*([kmgKMG])$/);
|
|
14488
|
+
if (!match) throw new Error(`Invalid memory format: "${memory}". Expected format like "512m", "2g", or "1024k".`);
|
|
14489
|
+
const value = parseFloat(match[1]);
|
|
14490
|
+
const unit = match[2].toLowerCase();
|
|
14491
|
+
switch (unit) {
|
|
14492
|
+
case "k": return Math.floor(value * 1024);
|
|
14493
|
+
case "m": return Math.floor(value * 1024 * 1024);
|
|
14494
|
+
case "g": return Math.floor(value * 1024 * 1024 * 1024);
|
|
14495
|
+
default: throw new Error(`Unknown memory unit: "${unit}"`);
|
|
14496
|
+
}
|
|
14497
|
+
}
|
|
14498
|
+
/**
|
|
14499
|
+
* Load and validate the scaler configuration from YAML files.
|
|
14500
|
+
*
|
|
14501
|
+
* 1. Reads the main config file
|
|
14502
|
+
* 2. If scalersDirPath provided, reads all *.yaml and *.yml files
|
|
14503
|
+
* from the directory (sorted alphabetically) and merges their
|
|
14504
|
+
* scalers arrays into the main config
|
|
14505
|
+
* 3. Validates the merged config through Zod schema
|
|
14506
|
+
*
|
|
14507
|
+
* @param configPath - Path to the main scaler YAML config file
|
|
14508
|
+
* @param scalersDirPath - Optional path to scalers.d/ directory for additional scaler definitions
|
|
14509
|
+
* @returns Validated ScalerConfig
|
|
14510
|
+
* @throws ZodError if validation fails
|
|
14511
|
+
*/
|
|
14512
|
+
async function loadScalerConfig(configPath, scalersDirPath) {
|
|
14513
|
+
const mainConfig = parse(await readFile(configPath, "utf-8"));
|
|
14514
|
+
if (scalersDirPath) {
|
|
14515
|
+
const yamlFiles = (await readdir(scalersDirPath)).filter((f) => f.endsWith(".yaml") || f.endsWith(".yml")).sort();
|
|
14516
|
+
for (const file of yamlFiles) {
|
|
14517
|
+
const parsed = parse(await readFile(join(scalersDirPath, file), "utf-8"));
|
|
14518
|
+
if (parsed?.scalers && Array.isArray(parsed.scalers)) mainConfig.scalers = [...mainConfig.scalers ?? [], ...parsed.scalers];
|
|
14519
|
+
}
|
|
14520
|
+
}
|
|
14521
|
+
return scalerFileSchema.parse(mainConfig);
|
|
14522
|
+
}
|
|
14523
|
+
//#endregion
|
|
14524
|
+
//#region src/scaler/label-matcher.ts
|
|
14525
|
+
/**
|
|
14526
|
+
* Label-set matching and overlap detection for the scaler module.
|
|
14527
|
+
*
|
|
14528
|
+
* Label sets are normalized (sorted, deduplicated, lowercased) for
|
|
14529
|
+
* deterministic comparison.
|
|
14530
|
+
*
|
|
14531
|
+
* findBackendForLabels uses subset matching: a job requiring ['macos']
|
|
14532
|
+
* matches a scaler with ['macos', 'darwin', 'bare-metal']. This is
|
|
14533
|
+
* consistent with peer routing (findPeersWithCapacity) and allows
|
|
14534
|
+
* workflows to target broad labels like 'macos' without listing every
|
|
14535
|
+
* label the scaler defines.
|
|
14536
|
+
*/
|
|
14537
|
+
/**
|
|
14538
|
+
* Normalize a label set for comparison.
|
|
14539
|
+
* Sorts alphabetically, deduplicates, and lowercases all labels.
|
|
14540
|
+
* Produces a deterministic string key for comparison.
|
|
14541
|
+
*
|
|
14542
|
+
* @example
|
|
14543
|
+
* normalizeLabelSet(['Docker', 'linux', 'linux']) // => "docker,linux"
|
|
14544
|
+
* normalizeLabelSet([]) // => ""
|
|
14545
|
+
*/
|
|
14546
|
+
function normalizeLabelSet(labels) {
|
|
14547
|
+
return [...new Set(labels.map((l) => l.toLowerCase()))].sort().join(",");
|
|
14548
|
+
}
|
|
14549
|
+
//#endregion
|
|
14550
|
+
//#region src/scaler/firecracker-api.ts
|
|
14551
|
+
/**
|
|
14552
|
+
* Firecracker REST API client over Unix domain sockets.
|
|
14553
|
+
*
|
|
14554
|
+
* Provides a thin wrapper around Node.js built-in `http` module for communicating
|
|
14555
|
+
* with Firecracker's REST API. Uses the `socketPath` option for HTTP-over-Unix-socket.
|
|
14556
|
+
*
|
|
14557
|
+
* Only used for post-boot operations:
|
|
14558
|
+
* - MMDS data injection (passing agent config to the guest VM)
|
|
14559
|
+
* - Graceful shutdown via SendCtrlAltDel (x86_64 only)
|
|
14560
|
+
* - Socket readiness polling after jailer invocation
|
|
14561
|
+
*
|
|
14562
|
+
* Pre-boot VM configuration uses `--config-file` JSON (atomic, simpler).
|
|
14563
|
+
*
|
|
14564
|
+
* @see https://github.com/firecracker-microvm/firecracker/blob/main/docs/getting-started.md
|
|
14565
|
+
* @see https://github.com/firecracker-microvm/firecracker/blob/main/docs/mmds/mmds-user-guide.md
|
|
14566
|
+
*/
|
|
14567
|
+
/**
|
|
14568
|
+
* Error thrown when the Firecracker API returns a non-2xx status code
|
|
14569
|
+
* or a network-level error occurs.
|
|
14570
|
+
*/
|
|
14571
|
+
var FirecrackerApiError = class extends Error {
|
|
14572
|
+
/** HTTP status code from the Firecracker API (undefined for network errors) */
|
|
14573
|
+
statusCode;
|
|
14574
|
+
/** The API path that was requested */
|
|
14575
|
+
path;
|
|
14576
|
+
/** The response body from the Firecracker API */
|
|
14577
|
+
responseBody;
|
|
14578
|
+
constructor(message, path, statusCode, responseBody) {
|
|
14579
|
+
super(message);
|
|
14580
|
+
this.name = "FirecrackerApiError";
|
|
14581
|
+
this.path = path;
|
|
14582
|
+
this.statusCode = statusCode;
|
|
14583
|
+
this.responseBody = responseBody ?? "";
|
|
14584
|
+
}
|
|
14585
|
+
};
|
|
14586
|
+
/**
|
|
14587
|
+
* Thin REST client for the Firecracker API over Unix domain sockets.
|
|
14588
|
+
*
|
|
14589
|
+
* No external dependencies -- uses Node.js built-in `http` module with
|
|
14590
|
+
* the `socketPath` option for HTTP-over-Unix-socket communication.
|
|
14591
|
+
*/
|
|
14592
|
+
var FirecrackerApi = class {
|
|
14593
|
+
constructor(socketPath) {
|
|
14594
|
+
this.socketPath = socketPath;
|
|
14595
|
+
}
|
|
14596
|
+
/**
|
|
14597
|
+
* Send an HTTP request to the Firecracker API.
|
|
14598
|
+
*
|
|
14599
|
+
* @param method - HTTP method (GET, PUT, PATCH)
|
|
14600
|
+
* @param path - API path (e.g., '/mmds', '/actions', '/')
|
|
14601
|
+
* @param body - Optional JSON body for PUT/PATCH requests
|
|
14602
|
+
* @returns Response with statusCode and body
|
|
14603
|
+
* @throws FirecrackerApiError on non-2xx status or network error
|
|
14604
|
+
*/
|
|
14605
|
+
request(method, path, body) {
|
|
14606
|
+
return new Promise((resolve, reject) => {
|
|
14607
|
+
const headers = {};
|
|
14608
|
+
let data;
|
|
14609
|
+
if (body !== void 0) {
|
|
14610
|
+
data = JSON.stringify(body);
|
|
14611
|
+
headers["Content-Type"] = "application/json";
|
|
14612
|
+
headers["Content-Length"] = Buffer.byteLength(data);
|
|
14613
|
+
}
|
|
14614
|
+
const req = http.request({
|
|
14615
|
+
socketPath: this.socketPath,
|
|
14616
|
+
path,
|
|
14617
|
+
method,
|
|
14618
|
+
headers
|
|
14619
|
+
}, (res) => {
|
|
14620
|
+
let responseBody = "";
|
|
14621
|
+
res.on("data", (chunk) => {
|
|
14622
|
+
responseBody += chunk;
|
|
14623
|
+
});
|
|
14624
|
+
res.on("end", () => {
|
|
14625
|
+
const statusCode = res.statusCode ?? 0;
|
|
14626
|
+
if (statusCode >= 200 && statusCode < 300) resolve({
|
|
14627
|
+
statusCode,
|
|
14628
|
+
body: responseBody
|
|
14629
|
+
});
|
|
14630
|
+
else reject(new FirecrackerApiError(`Firecracker API ${method} ${path}: ${statusCode} ${responseBody}`, path, statusCode, responseBody));
|
|
14631
|
+
});
|
|
14632
|
+
});
|
|
14633
|
+
req.on("error", (err) => {
|
|
14634
|
+
reject(new FirecrackerApiError(`Firecracker API ${method} ${path}: ${err.message}`, path, void 0, ""));
|
|
14635
|
+
});
|
|
14636
|
+
if (data !== void 0) req.write(data);
|
|
14637
|
+
req.end();
|
|
14638
|
+
});
|
|
14639
|
+
}
|
|
14640
|
+
/**
|
|
14641
|
+
* Inject MMDS (Micro VM Metadata Service) data into a running VM.
|
|
14642
|
+
*
|
|
14643
|
+
* Uses MMDS v2 data injection endpoint. The data should follow the structure:
|
|
14644
|
+
* `{ latest: { 'meta-data': { ... } } }`
|
|
14645
|
+
*
|
|
14646
|
+
* The guest VM reads this data from http://169.254.169.254/latest/meta-data/
|
|
14647
|
+
* using a token-based session (MMDS v2).
|
|
14648
|
+
*
|
|
14649
|
+
* @param data - Metadata object to inject
|
|
14650
|
+
* @see https://github.com/firecracker-microvm/firecracker/blob/main/docs/mmds/mmds-user-guide.md
|
|
14651
|
+
*/
|
|
14652
|
+
async putMmds(data) {
|
|
14653
|
+
await this.request("PUT", "/mmds", data);
|
|
14654
|
+
}
|
|
14655
|
+
/**
|
|
14656
|
+
* Clear MMDS data after agent has received config via WS.
|
|
14657
|
+
* Replaces the MMDS data with an empty meta-data object.
|
|
14658
|
+
* Called after receiving config.ack from the agent.
|
|
14659
|
+
*/
|
|
14660
|
+
async clearMmds() {
|
|
14661
|
+
await this.request("PUT", "/mmds", { latest: { "meta-data": {} } });
|
|
14662
|
+
}
|
|
14663
|
+
/**
|
|
14664
|
+
* Send a SendCtrlAltDel action for graceful x86_64 shutdown.
|
|
14665
|
+
*
|
|
14666
|
+
* This simulates pressing Ctrl+Alt+Del on the virtual keyboard,
|
|
14667
|
+
* which triggers a clean shutdown in most Linux distributions.
|
|
14668
|
+
*
|
|
14669
|
+
* Note: Only works on x86_64. On arm64, the i8042 keyboard controller
|
|
14670
|
+
* is not available, so this action is not supported. Use process kill
|
|
14671
|
+
* as fallback on arm64.
|
|
14672
|
+
*
|
|
14673
|
+
* @see https://github.com/firecracker-microvm/firecracker/blob/main/docs/getting-started.md
|
|
14674
|
+
*/
|
|
14675
|
+
async sendCtrlAltDel() {
|
|
14676
|
+
await this.request("PUT", "/actions", { action_type: "SendCtrlAltDel" });
|
|
14677
|
+
}
|
|
14678
|
+
/**
|
|
14679
|
+
* Get instance info from the Firecracker API root endpoint.
|
|
14680
|
+
*
|
|
14681
|
+
* Used to check if the API socket is reachable and the VM is running.
|
|
14682
|
+
* Returns the parsed JSON response with instance information.
|
|
14683
|
+
*/
|
|
14684
|
+
async getInstanceInfo() {
|
|
14685
|
+
const response = await this.request("GET", "/");
|
|
14686
|
+
return JSON.parse(response.body);
|
|
14687
|
+
}
|
|
14688
|
+
/**
|
|
14689
|
+
* Poll the Firecracker API socket until it becomes reachable or timeout expires.
|
|
14690
|
+
*
|
|
14691
|
+
* Used after jailer invocation to wait for Firecracker to create the API socket
|
|
14692
|
+
* and become ready to accept requests.
|
|
14693
|
+
*
|
|
14694
|
+
* @param timeoutMs - Maximum time to wait in milliseconds (default: 5000)
|
|
14695
|
+
* @param intervalMs - Polling interval in milliseconds (default: 100)
|
|
14696
|
+
* @returns true if socket became reachable, false on timeout
|
|
14697
|
+
*/
|
|
14698
|
+
async waitForSocket(timeoutMs = 5e3, intervalMs = 100) {
|
|
14699
|
+
const deadline = Date.now() + timeoutMs;
|
|
14700
|
+
while (Date.now() < deadline) try {
|
|
14701
|
+
await this.getInstanceInfo();
|
|
14702
|
+
return true;
|
|
14703
|
+
} catch {
|
|
14704
|
+
await new Promise((resolve) => setTimeout(resolve, intervalMs));
|
|
14705
|
+
}
|
|
14706
|
+
return false;
|
|
14707
|
+
}
|
|
14708
|
+
};
|
|
14709
|
+
//#endregion
|
|
14710
|
+
//#region src/scaler/file-tail.ts
|
|
14711
|
+
/**
|
|
14712
|
+
* File tailing utility for append-only log files.
|
|
14713
|
+
*
|
|
14714
|
+
* Provides an async generator that yields complete lines as they are appended
|
|
14715
|
+
* to a file, using `fs.watch()` for change notifications and partial line
|
|
14716
|
+
* buffering for split writes.
|
|
14717
|
+
*
|
|
14718
|
+
* Used by the Firecracker backend to tail serial console and VMM log files
|
|
14719
|
+
* from the jailer chroot directory.
|
|
14720
|
+
*/
|
|
14721
|
+
/**
|
|
14722
|
+
* Tail an append-only file, yielding complete lines as they are written.
|
|
14723
|
+
*
|
|
14724
|
+
* Pre-creates the file if it does not exist to avoid watch errors.
|
|
14725
|
+
* Tracks byte offset and partial (incomplete) trailing lines across reads.
|
|
14726
|
+
* Stops when the provided AbortSignal is triggered.
|
|
14727
|
+
*
|
|
14728
|
+
* @param filePath - Path to the file to tail
|
|
14729
|
+
* @param signal - AbortSignal to stop tailing
|
|
14730
|
+
* @yields Complete lines (non-empty) as they are appended to the file
|
|
14731
|
+
*/
|
|
14732
|
+
async function* tailFile(filePath, signal) {
|
|
14733
|
+
await writeFile(filePath, "", { flag: "a" });
|
|
14734
|
+
let offset = 0;
|
|
14735
|
+
let partial = "";
|
|
14736
|
+
/**
|
|
14737
|
+
* Read new data from the file starting at the current offset.
|
|
14738
|
+
* Returns complete lines and buffers any incomplete trailing line.
|
|
14739
|
+
*/
|
|
14740
|
+
function readNewData() {
|
|
14741
|
+
return new Promise((resolve, reject) => {
|
|
14742
|
+
let fileSize;
|
|
14743
|
+
try {
|
|
14744
|
+
fileSize = statSync(filePath).size;
|
|
14745
|
+
} catch {
|
|
14746
|
+
resolve([]);
|
|
14747
|
+
return;
|
|
14748
|
+
}
|
|
14749
|
+
if (fileSize <= offset) {
|
|
14750
|
+
resolve([]);
|
|
14751
|
+
return;
|
|
14752
|
+
}
|
|
14753
|
+
const chunks = [];
|
|
14754
|
+
const stream = createReadStream(filePath, {
|
|
14755
|
+
start: offset,
|
|
14756
|
+
encoding: "utf-8"
|
|
14757
|
+
});
|
|
14758
|
+
stream.on("data", (chunk) => {
|
|
14759
|
+
chunks.push(typeof chunk === "string" ? chunk : chunk.toString("utf-8"));
|
|
14760
|
+
});
|
|
14761
|
+
stream.on("end", () => {
|
|
14762
|
+
const parts = chunks.join("").split("\n");
|
|
14763
|
+
const incompletePart = parts.pop();
|
|
14764
|
+
if (parts.length > 0) {
|
|
14765
|
+
parts[0] = partial + parts[0];
|
|
14766
|
+
partial = incompletePart;
|
|
14767
|
+
} else {
|
|
14768
|
+
partial += incompletePart;
|
|
14769
|
+
resolve([]);
|
|
14770
|
+
try {
|
|
14771
|
+
offset = statSync(filePath).size;
|
|
14772
|
+
} catch {}
|
|
14773
|
+
return;
|
|
14774
|
+
}
|
|
14775
|
+
const lines = parts.filter((l) => l.length > 0);
|
|
14776
|
+
try {
|
|
14777
|
+
offset = statSync(filePath).size;
|
|
14778
|
+
} catch {}
|
|
14779
|
+
resolve(lines);
|
|
14780
|
+
});
|
|
14781
|
+
stream.on("error", (err) => {
|
|
14782
|
+
reject(err);
|
|
14783
|
+
});
|
|
14784
|
+
});
|
|
14785
|
+
}
|
|
14786
|
+
const lineQueue = [];
|
|
14787
|
+
let resolveWaiter = null;
|
|
14788
|
+
let stopped = false;
|
|
14789
|
+
function onFileChange() {
|
|
14790
|
+
if (stopped) return;
|
|
14791
|
+
readNewData().then((lines) => {
|
|
14792
|
+
if (lines.length > 0) {
|
|
14793
|
+
lineQueue.push(...lines);
|
|
14794
|
+
if (resolveWaiter) {
|
|
14795
|
+
const r = resolveWaiter;
|
|
14796
|
+
resolveWaiter = null;
|
|
14797
|
+
r();
|
|
14798
|
+
}
|
|
14799
|
+
}
|
|
14800
|
+
}, () => {});
|
|
14801
|
+
}
|
|
14802
|
+
watchFile(filePath, { interval: 100 }, onFileChange);
|
|
14803
|
+
onFileChange();
|
|
14804
|
+
const onAbort = () => {
|
|
14805
|
+
stopped = true;
|
|
14806
|
+
unwatchFile(filePath, onFileChange);
|
|
14807
|
+
if (resolveWaiter) {
|
|
14808
|
+
const r = resolveWaiter;
|
|
14809
|
+
resolveWaiter = null;
|
|
14810
|
+
r();
|
|
14811
|
+
}
|
|
14812
|
+
};
|
|
14813
|
+
if (signal.aborted) {
|
|
14814
|
+
stopped = true;
|
|
14815
|
+
unwatchFile(filePath, onFileChange);
|
|
14816
|
+
} else signal.addEventListener("abort", onAbort, { once: true });
|
|
14817
|
+
try {
|
|
14818
|
+
while (!stopped) if (lineQueue.length > 0) yield lineQueue.shift();
|
|
14819
|
+
else await new Promise((resolve) => {
|
|
14820
|
+
resolveWaiter = resolve;
|
|
14821
|
+
if (stopped) {
|
|
14822
|
+
resolveWaiter = null;
|
|
14823
|
+
resolve();
|
|
14824
|
+
}
|
|
14825
|
+
});
|
|
14826
|
+
if (partial.length > 0) {
|
|
14827
|
+
yield partial;
|
|
14828
|
+
partial = "";
|
|
14829
|
+
}
|
|
14830
|
+
} finally {
|
|
14831
|
+
stopped = true;
|
|
14832
|
+
unwatchFile(filePath, onFileChange);
|
|
14833
|
+
signal.removeEventListener("abort", onAbort);
|
|
14834
|
+
}
|
|
14835
|
+
}
|
|
14836
|
+
//#endregion
|
|
14837
|
+
//#region src/scaler/log-forwarder.ts
|
|
14838
|
+
/**
|
|
14839
|
+
* Enrich and forward a single log line to an output stream.
|
|
14840
|
+
*
|
|
14841
|
+
* Standalone function for use in the WS agent.log path (agent-handler).
|
|
14842
|
+
*
|
|
14843
|
+
* @param line - Raw log line from the agent
|
|
14844
|
+
* @param agentId - The agent that produced the line
|
|
14845
|
+
* @param output - Writable stream to write enriched JSON to
|
|
14846
|
+
* @param context - Optional trace context fields
|
|
14847
|
+
* @param logsSource - Optional log source identifier (e.g. 'docker', 'podman', 'bare-metal', 'firecracker-serial')
|
|
14848
|
+
*/
|
|
14849
|
+
function forwardLine(line, agentId, output, context, logsSource) {
|
|
14850
|
+
let parsed;
|
|
14851
|
+
try {
|
|
14852
|
+
parsed = JSON.parse(line);
|
|
14853
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) parsed = {
|
|
14854
|
+
level: "info",
|
|
14855
|
+
message: line
|
|
14856
|
+
};
|
|
14857
|
+
} catch {
|
|
14858
|
+
parsed = {
|
|
14859
|
+
level: "info",
|
|
14860
|
+
message: line
|
|
14861
|
+
};
|
|
14862
|
+
}
|
|
14863
|
+
parsed.service = "agent";
|
|
14864
|
+
parsed.agentId = agentId;
|
|
14865
|
+
if (context?.runId) parsed.runId = context.runId;
|
|
14866
|
+
if (context?.requestId) parsed.requestId = context.requestId;
|
|
14867
|
+
if (context?.jobId) parsed.jobId = context.jobId;
|
|
14868
|
+
if (logsSource) parsed.logsSource = logsSource;
|
|
14869
|
+
output.write(JSON.stringify(parsed) + "\n");
|
|
14870
|
+
}
|
|
14871
|
+
//#endregion
|
|
14872
|
+
//#region src/scaler/firecracker-backend.ts
|
|
14873
|
+
/**
|
|
14874
|
+
* Firecracker scaler backend implementation.
|
|
14875
|
+
*
|
|
14876
|
+
* Manages ephemeral agent microVMs using the Firecracker VMM via the jailer.
|
|
14877
|
+
* Handles full VM lifecycle: prepare chroot, hardlink rootfs+kernel, create overlay drive,
|
|
14878
|
+
* invoke jailer, inject MMDS metadata, and destroy with cleanup.
|
|
14879
|
+
*
|
|
14880
|
+
* Log capture: Firecracker backend handles log forwarding internally.
|
|
14881
|
+
* The jailer is spawned as a non-daemonized detached child with stdout redirected to a
|
|
14882
|
+
* serial console log file and VMM logs captured via --log-path. Both files are tailed
|
|
14883
|
+
* using fs.watchFile() and forwarded with distinct logsSource tags ('firecracker-serial',
|
|
14884
|
+
* 'firecracker-vmm'). The serial-console path is best-effort only: Firecracker's Rust
|
|
14885
|
+
* BufWriter around stdout + jailer's uid drop make sparse userspace writes unreliable,
|
|
14886
|
+
* so it's treated as defense-in-depth. The canonical agent log path is WS agent.log
|
|
14887
|
+
* (attached unconditionally by the agent; see packages/agent/src/server.ts step 6).
|
|
14888
|
+
*
|
|
14889
|
+
* Follows the Docker backend pattern closely for consistency.
|
|
14890
|
+
*/
|
|
14891
|
+
/** Hardlink src to dest; fall back to copy if on different filesystems (EXDEV). */
|
|
14892
|
+
async function linkOrCopy(src, dest) {
|
|
14893
|
+
try {
|
|
14894
|
+
await link(src, dest);
|
|
14895
|
+
} catch (err) {
|
|
14896
|
+
if (err.code === "EXDEV") await copyFile(src, dest);
|
|
14897
|
+
else throw err;
|
|
14898
|
+
}
|
|
14899
|
+
}
|
|
14900
|
+
const execFileAsync = promisify(execFile);
|
|
14901
|
+
const logger$2 = createLogger({ prefix: "firecracker-backend" });
|
|
14902
|
+
/** Log file names within the jailer chroot directory */
|
|
14903
|
+
const SERIAL_LOG_FILE = "serial-console.log";
|
|
14904
|
+
const VMM_LOG_FILE = "vmm.log";
|
|
14905
|
+
/**
|
|
14906
|
+
* Maximum total bytes allowed for forwarded env vars in a single VM's MMDS payload.
|
|
14907
|
+
*
|
|
14908
|
+
* Firecracker's default MMDS data store cap is ~51 KiB shared across every metadata
|
|
14909
|
+
* field for that VM. The other static fields (URL, agent ID, labels, token, gateway,
|
|
14910
|
+
* backpressure) total well under 1 KiB, so 32 KiB is a generous-but-safe budget for
|
|
14911
|
+
* operator-defined env vars. Vars exceeding the remaining budget are skipped with a
|
|
14912
|
+
* warning rather than silently triggering an opaque MMDS PUT failure mid-spawn.
|
|
14913
|
+
*/
|
|
14914
|
+
const MMDS_FORWARDED_ENV_BUDGET_BYTES = 32 * 1024;
|
|
14915
|
+
/** POSIX shell-safe env var name pattern. Reject anything else to avoid MMDS path injection. */
|
|
14916
|
+
const POSIX_ENV_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
14917
|
+
/**
|
|
14918
|
+
* Interface-name pattern for per-VM TAP devices.
|
|
14919
|
+
* Matches `kici-${vmId.slice(0, 8)}` — 8 lowercase hex chars.
|
|
14920
|
+
*/
|
|
14921
|
+
const VM_TAP_PATTERN = /^kici-[0-9a-f]{8}$/;
|
|
14922
|
+
/**
|
|
14923
|
+
* Infrastructure interfaces that share the `kici-` prefix but must NEVER be
|
|
14924
|
+
* deleted by the orphan sweep. Bridges (kici-br0/kici-br1) carry all VM
|
|
14925
|
+
* traffic; kici-m01 is the staging metadata interface. Losing any of these
|
|
14926
|
+
* kills all running VMs.
|
|
14927
|
+
*/
|
|
14928
|
+
const PROTECTED_INTERFACES = [
|
|
14929
|
+
"kici-br0",
|
|
14930
|
+
"kici-br1",
|
|
14931
|
+
"kici-m01"
|
|
14932
|
+
];
|
|
14933
|
+
/** Convert a dotted IPv4 netmask ('255.255.255.0') to a CIDR prefix length (24). */
|
|
14934
|
+
function netmaskToPrefix(netmask) {
|
|
14935
|
+
return netmask.split(".").map((o) => Number(o).toString(2).padStart(8, "0")).join("").split("1").length - 1;
|
|
14936
|
+
}
|
|
14937
|
+
var FirecrackerScalerBackend = class FirecrackerScalerBackend {
|
|
14938
|
+
type = ScalerBackendType.enum.firecracker;
|
|
14939
|
+
spawnsOnLocalHost = true;
|
|
14940
|
+
maxAgents;
|
|
14941
|
+
logsSource = "firecracker-serial";
|
|
14942
|
+
/** AbortControllers for file tailing per managed VM (keyed by agent ID) */
|
|
14943
|
+
tailAbortControllers = /* @__PURE__ */ new Map();
|
|
14944
|
+
_labelSets;
|
|
14945
|
+
name;
|
|
14946
|
+
ipAllocator;
|
|
14947
|
+
firecrackerPath;
|
|
14948
|
+
jailerPath;
|
|
14949
|
+
kernelPath;
|
|
14950
|
+
chrootBaseDir;
|
|
14951
|
+
uid;
|
|
14952
|
+
gid;
|
|
14953
|
+
vcpuCount;
|
|
14954
|
+
memSizeMib;
|
|
14955
|
+
bridgeName;
|
|
14956
|
+
cidr;
|
|
14957
|
+
gateway;
|
|
14958
|
+
netmask;
|
|
14959
|
+
table;
|
|
14960
|
+
tokenStore;
|
|
14961
|
+
tokenTtlMs;
|
|
14962
|
+
roles;
|
|
14963
|
+
requireSudo;
|
|
14964
|
+
/** Tracks all managed VM agents by ManagedAgent.id */
|
|
14965
|
+
agents = /* @__PURE__ */ new Map();
|
|
14966
|
+
/** Periodic orphan sweep timer (null when stopped). */
|
|
14967
|
+
orphanSweepTimer = null;
|
|
14968
|
+
/** Guard against re-entrant sweeps if a single run takes longer than the interval. */
|
|
14969
|
+
orphanSweepInFlight = false;
|
|
14970
|
+
constructor(options) {
|
|
14971
|
+
this.name = options.name;
|
|
14972
|
+
this._labelSets = options.labelSets;
|
|
14973
|
+
this.maxAgents = options.maxAgents;
|
|
14974
|
+
this.ipAllocator = options.ipAllocator;
|
|
14975
|
+
this.firecrackerPath = options.firecrackerPath;
|
|
14976
|
+
this.jailerPath = options.jailerPath;
|
|
14977
|
+
this.kernelPath = options.kernelPath;
|
|
14978
|
+
this.chrootBaseDir = options.chrootBaseDir ?? "/srv/jailer";
|
|
14979
|
+
this.uid = options.uid;
|
|
14980
|
+
this.gid = options.gid;
|
|
14981
|
+
this.vcpuCount = options.vcpuCount ?? 2;
|
|
14982
|
+
this.memSizeMib = options.memSizeMib ?? 512;
|
|
14983
|
+
this.bridgeName = options.bridgeName;
|
|
14984
|
+
this.cidr = options.cidr;
|
|
14985
|
+
this.gateway = options.gateway;
|
|
14986
|
+
this.netmask = options.netmask;
|
|
14987
|
+
this.table = options.table ?? "kici";
|
|
14988
|
+
this.tokenStore = options.tokenStore;
|
|
14989
|
+
this.tokenTtlMs = options.tokenTtlMs ?? 36e5;
|
|
14990
|
+
this.roles = options.roles;
|
|
14991
|
+
this.requireSudo = options.requireSudo ?? false;
|
|
14992
|
+
}
|
|
14993
|
+
/**
|
|
14994
|
+
* Declare required tools for a firecracker scaler entry.
|
|
14995
|
+
*/
|
|
14996
|
+
static getRequiredTools(entry) {
|
|
14997
|
+
const reqs = [];
|
|
14998
|
+
const name = entry.name;
|
|
14999
|
+
if (entry.firecrackerPath) reqs.push({
|
|
15000
|
+
type: "file-access",
|
|
15001
|
+
path: entry.firecrackerPath,
|
|
15002
|
+
mode: "executable",
|
|
15003
|
+
reason: `firecracker binary for scaler "${name}"`
|
|
15004
|
+
});
|
|
15005
|
+
if (entry.jailerPath) reqs.push({
|
|
15006
|
+
type: "file-access",
|
|
15007
|
+
path: entry.jailerPath,
|
|
15008
|
+
mode: "executable",
|
|
15009
|
+
reason: `jailer binary for scaler "${name}"`
|
|
15010
|
+
});
|
|
15011
|
+
if (entry.kernelPath) reqs.push({
|
|
15012
|
+
type: "file-access",
|
|
15013
|
+
path: entry.kernelPath,
|
|
15014
|
+
mode: "readable",
|
|
15015
|
+
reason: `kernel image for scaler "${name}"`
|
|
15016
|
+
});
|
|
15017
|
+
for (const ls of entry.labelSets) if (ls.rootfsPath) reqs.push({
|
|
15018
|
+
type: "file-access",
|
|
15019
|
+
path: ls.rootfsPath,
|
|
15020
|
+
mode: "readable",
|
|
15021
|
+
reason: `rootfs image for scaler "${name}" label set [${ls.labels.join(",")}]`
|
|
15022
|
+
});
|
|
15023
|
+
reqs.push({
|
|
15024
|
+
type: "path-binary",
|
|
15025
|
+
name: "ip",
|
|
15026
|
+
reason: `required by firecracker scaler "${name}" for TAP device management`
|
|
15027
|
+
});
|
|
15028
|
+
reqs.push({
|
|
15029
|
+
type: "path-binary",
|
|
15030
|
+
name: "mkfs.ext4",
|
|
15031
|
+
reason: `required by firecracker scaler "${name}" for overlay drive creation`
|
|
15032
|
+
});
|
|
15033
|
+
return reqs;
|
|
15034
|
+
}
|
|
15035
|
+
get labelSets() {
|
|
15036
|
+
return this._labelSets;
|
|
15037
|
+
}
|
|
15038
|
+
getActiveCount() {
|
|
15039
|
+
return this.agents.size;
|
|
15040
|
+
}
|
|
15041
|
+
async spawn(labelSet, agentId, orchestratorUrl, onEvent, effectiveLimits) {
|
|
15042
|
+
const emit = (eventType, detail) => {
|
|
15043
|
+
onEvent?.({
|
|
15044
|
+
agentId,
|
|
15045
|
+
eventType,
|
|
15046
|
+
detail,
|
|
15047
|
+
timestampMs: Date.now()
|
|
15048
|
+
});
|
|
15049
|
+
};
|
|
15050
|
+
const normalizedTarget = normalizeLabelSet(labelSet);
|
|
15051
|
+
const matchedLabelSet = this._labelSets.find((ls) => normalizeLabelSet(ls.labels) === normalizedTarget);
|
|
15052
|
+
if (!matchedLabelSet) throw new Error(`Label set [${labelSet.join(", ")}] not supported by Firecracker backend "${this.name}"`);
|
|
15053
|
+
if (this.getActiveCount() >= this.maxAgents) throw new Error(`Firecracker backend "${this.name}" at capacity (${this.maxAgents}/${this.maxAgents})`);
|
|
15054
|
+
const managed = {
|
|
15055
|
+
id: agentId,
|
|
15056
|
+
labelSet,
|
|
15057
|
+
backendRef: "",
|
|
15058
|
+
spawnedAt: Date.now(),
|
|
15059
|
+
state: "spawning",
|
|
15060
|
+
ip: "",
|
|
15061
|
+
tapDevice: ""
|
|
15062
|
+
};
|
|
15063
|
+
this.agents.set(managed.id, managed);
|
|
15064
|
+
let alloc;
|
|
15065
|
+
try {
|
|
15066
|
+
emit(ScalerEventType.enum["scaler.provisioning"], "preparing rootfs and kernel");
|
|
15067
|
+
alloc = await this.ipAllocator.allocate(agentId, this.name);
|
|
15068
|
+
managed.ip = alloc.ip;
|
|
15069
|
+
managed.tapDevice = alloc.tapDevice;
|
|
15070
|
+
emit(ScalerEventType.enum["scaler.network"], `configuring TAP device ${alloc.tapDevice}`);
|
|
15071
|
+
await this.execAsync("ip", [
|
|
15072
|
+
"tuntap",
|
|
15073
|
+
"add",
|
|
15074
|
+
alloc.tapDevice,
|
|
15075
|
+
"mode",
|
|
15076
|
+
"tap"
|
|
15077
|
+
]);
|
|
15078
|
+
await this.execAsync("ip", [
|
|
15079
|
+
"link",
|
|
15080
|
+
"set",
|
|
15081
|
+
alloc.tapDevice,
|
|
15082
|
+
"master",
|
|
15083
|
+
this.bridgeName
|
|
15084
|
+
]);
|
|
15085
|
+
await this.execAsync("ip", [
|
|
15086
|
+
"link",
|
|
15087
|
+
"set",
|
|
15088
|
+
alloc.tapDevice,
|
|
15089
|
+
"up"
|
|
15090
|
+
]);
|
|
15091
|
+
emit(ScalerEventType.enum["scaler.network"], `allocating IP ${alloc.ip}`);
|
|
15092
|
+
const nftOpts = { requireSudo: this.requireSudo };
|
|
15093
|
+
await ensureKiciTable(nftOpts);
|
|
15094
|
+
await addIsolationRules(alloc.ip, this.gateway, matchedLabelSet.networkPolicy, "saddr", nftOpts);
|
|
15095
|
+
const chrootDir = this.getChrootDir(agentId);
|
|
15096
|
+
await mkdir(chrootDir, { recursive: true });
|
|
15097
|
+
await linkOrCopy(matchedLabelSet.rootfsPath, join(chrootDir, "rootfs.ext4"));
|
|
15098
|
+
await linkOrCopy(matchedLabelSet.kernelPath ?? this.kernelPath, join(chrootDir, "kernel"));
|
|
15099
|
+
const overlayPath = join(chrootDir, "overlay.ext4");
|
|
15100
|
+
const overlayMib = matchedLabelSet.overlayDriveSizeMib ?? 2048;
|
|
15101
|
+
await this.createOverlayDrive(overlayPath, overlayMib);
|
|
15102
|
+
const config = this.buildVmConfig(alloc, matchedLabelSet, effectiveLimits);
|
|
15103
|
+
await writeFile(join(chrootDir, "config.json"), JSON.stringify(config, null, 2));
|
|
15104
|
+
const serialLogPath = join(chrootDir, SERIAL_LOG_FILE);
|
|
15105
|
+
const vmmLogPath = join(chrootDir, VMM_LOG_FILE);
|
|
15106
|
+
writeFileSync(serialLogPath, "");
|
|
15107
|
+
writeFileSync(vmmLogPath, "");
|
|
15108
|
+
const stdoutFd = openSync(serialLogPath, "w");
|
|
15109
|
+
const stderrFd = openSync(serialLogPath, "a");
|
|
15110
|
+
const vmDir = join(this.chrootBaseDir, "firecracker", agentId);
|
|
15111
|
+
await this.execAsync("chown", [
|
|
15112
|
+
"-R",
|
|
15113
|
+
`${this.uid}:${this.gid}`,
|
|
15114
|
+
vmDir
|
|
15115
|
+
]);
|
|
15116
|
+
const orchUid = process.getuid?.();
|
|
15117
|
+
const orchGid = process.getgid?.();
|
|
15118
|
+
if (orchUid !== void 0 && orchUid !== 0) await this.execAsync("chown", [
|
|
15119
|
+
`${orchUid}:${orchGid}`,
|
|
15120
|
+
join(chrootDir, "rootfs.ext4"),
|
|
15121
|
+
join(chrootDir, "kernel")
|
|
15122
|
+
]);
|
|
15123
|
+
emit(ScalerEventType.enum["scaler.provisioning"], "booting microVM");
|
|
15124
|
+
const child = spawn(this.jailerPath, [
|
|
15125
|
+
"--id",
|
|
15126
|
+
agentId,
|
|
15127
|
+
"--exec-file",
|
|
15128
|
+
this.firecrackerPath,
|
|
15129
|
+
"--uid",
|
|
15130
|
+
String(this.uid),
|
|
15131
|
+
"--gid",
|
|
15132
|
+
String(this.gid),
|
|
15133
|
+
"--chroot-base-dir",
|
|
15134
|
+
this.chrootBaseDir,
|
|
15135
|
+
"--new-pid-ns",
|
|
15136
|
+
"--",
|
|
15137
|
+
"--config-file",
|
|
15138
|
+
"/config.json",
|
|
15139
|
+
"--log-path",
|
|
15140
|
+
"/vmm.log",
|
|
15141
|
+
"--level",
|
|
15142
|
+
"Warning"
|
|
15143
|
+
], {
|
|
15144
|
+
detached: true,
|
|
15145
|
+
stdio: [
|
|
15146
|
+
"ignore",
|
|
15147
|
+
stdoutFd,
|
|
15148
|
+
stderrFd
|
|
15149
|
+
]
|
|
15150
|
+
});
|
|
15151
|
+
child.unref();
|
|
15152
|
+
closeSync(stdoutFd);
|
|
15153
|
+
closeSync(stderrFd);
|
|
15154
|
+
this.startLogTailing(agentId, serialLogPath, vmmLogPath, child);
|
|
15155
|
+
const api = new FirecrackerApi(this.getSocketPath(agentId));
|
|
15156
|
+
let chmodLoosenInterval;
|
|
15157
|
+
if (this.requireSudo) {
|
|
15158
|
+
const chrootHostPath = join(join(this.chrootBaseDir, "firecracker", agentId), "root");
|
|
15159
|
+
chmodLoosenInterval = setInterval(() => {
|
|
15160
|
+
this.execAsync("chmod", [
|
|
15161
|
+
"-R",
|
|
15162
|
+
"go+rX",
|
|
15163
|
+
chrootHostPath
|
|
15164
|
+
]).catch(() => {});
|
|
15165
|
+
}, 100);
|
|
15166
|
+
}
|
|
15167
|
+
let ready;
|
|
15168
|
+
try {
|
|
15169
|
+
ready = await api.waitForSocket(5e3);
|
|
15170
|
+
} finally {
|
|
15171
|
+
if (chmodLoosenInterval) clearInterval(chmodLoosenInterval);
|
|
15172
|
+
}
|
|
15173
|
+
if (!ready) throw new Error(`Firecracker API socket not ready within 5s for agent ${agentId}`);
|
|
15174
|
+
const fullLabels = scalerAgentLabels(labelSet, this.type, this.name, this.roles);
|
|
15175
|
+
let agentToken;
|
|
15176
|
+
if (this.tokenStore) agentToken = await this.tokenStore.createEphemeral(agentId, fullLabels, this.tokenTtlMs);
|
|
15177
|
+
const acceptedEnv = this.buildForwardedEnv(matchedLabelSet, agentId);
|
|
15178
|
+
await api.putMmds({ latest: { "meta-data": {
|
|
15179
|
+
"kici-orchestrator-url": orchestratorUrl,
|
|
15180
|
+
"kici-agent-id": agentId,
|
|
15181
|
+
"kici-labels": fullLabels.join(","),
|
|
15182
|
+
"kici-scaler-managed": "1",
|
|
15183
|
+
"kici-gateway-ip": this.gateway,
|
|
15184
|
+
...agentToken ? { "kici-agent-token": agentToken } : {},
|
|
15185
|
+
...matchedLabelSet.backpressureMode ? { "kici-backpressure-mode": matchedLabelSet.backpressureMode } : {},
|
|
15186
|
+
...Object.keys(acceptedEnv).length > 0 ? { "kici-env": acceptedEnv } : {}
|
|
15187
|
+
} } });
|
|
15188
|
+
emit(ScalerEventType.enum["scaler.ready"], "microVM booted");
|
|
15189
|
+
managed.state = "running";
|
|
15190
|
+
managed.backendRef = agentId;
|
|
15191
|
+
emit(ScalerEventType.enum["agent.connecting"], "waiting for agent WS registration from VM");
|
|
15192
|
+
return managed;
|
|
15193
|
+
} catch (err) {
|
|
15194
|
+
emit(ScalerEventType.enum["scaler.failed"], err instanceof Error ? err.message : String(err));
|
|
15195
|
+
await this.cleanupFailedSpawn(agentId, alloc);
|
|
15196
|
+
this.agents.delete(managed.id);
|
|
15197
|
+
throw err;
|
|
15198
|
+
}
|
|
15199
|
+
}
|
|
15200
|
+
getScalerContext(agentId) {
|
|
15201
|
+
const managed = this.agents.get(agentId);
|
|
15202
|
+
if (!managed) return void 0;
|
|
15203
|
+
const normalizedTarget = normalizeLabelSet(managed.labelSet);
|
|
15204
|
+
const matchedLabelSet = this._labelSets.find((ls) => normalizeLabelSet(ls.labels) === normalizedTarget);
|
|
15205
|
+
return {
|
|
15206
|
+
backendType: "firecracker",
|
|
15207
|
+
scalerName: this.name,
|
|
15208
|
+
rootfsPath: matchedLabelSet?.rootfsPath,
|
|
15209
|
+
kernelPath: matchedLabelSet?.kernelPath ?? this.kernelPath,
|
|
15210
|
+
vcpuCount: matchedLabelSet?.vcpuCount ?? this.vcpuCount,
|
|
15211
|
+
memSizeMib: matchedLabelSet?.memSizeMib ?? this.memSizeMib,
|
|
15212
|
+
ip: managed.ip,
|
|
15213
|
+
bridgeName: this.bridgeName,
|
|
15214
|
+
gateway: this.gateway,
|
|
15215
|
+
netmask: this.netmask
|
|
15216
|
+
};
|
|
15217
|
+
}
|
|
15218
|
+
/**
|
|
15219
|
+
* Bridge config for host-network diagnostics (read-only snapshot).
|
|
15220
|
+
* `bridgeCidr` is the gateway IP with the network prefix (e.g. '10.0.0.1/24'),
|
|
15221
|
+
* which is exactly what `provisionBridge`/`verifyBridge` consume. The prefix
|
|
15222
|
+
* comes from the configured network `cidr`; if absent, it is derived from the
|
|
15223
|
+
* dotted `netmask`.
|
|
15224
|
+
*/
|
|
15225
|
+
getBridgeConfig() {
|
|
15226
|
+
const prefix = this.cidr ? this.cidr.split("/")[1] : netmaskToPrefix(this.netmask);
|
|
15227
|
+
return {
|
|
15228
|
+
bridgeName: this.bridgeName,
|
|
15229
|
+
bridgeCidr: `${this.gateway}/${prefix}`,
|
|
15230
|
+
table: this.table
|
|
15231
|
+
};
|
|
15232
|
+
}
|
|
15233
|
+
async destroy(managedId) {
|
|
15234
|
+
const managed = this.agents.get(managedId);
|
|
15235
|
+
if (!managed) return;
|
|
15236
|
+
managed.state = "destroying";
|
|
15237
|
+
const abortController = this.tailAbortControllers.get(managedId);
|
|
15238
|
+
if (abortController) {
|
|
15239
|
+
abortController.abort();
|
|
15240
|
+
this.tailAbortControllers.delete(managedId);
|
|
15241
|
+
}
|
|
15242
|
+
try {
|
|
15243
|
+
await new FirecrackerApi(this.getSocketPath(managedId)).sendCtrlAltDel();
|
|
15244
|
+
await this.waitForProcessExit(managedId, 5e3);
|
|
15245
|
+
} catch {}
|
|
15246
|
+
try {
|
|
15247
|
+
const pidStr = await readFile(join(this.getChrootDir(managedId), "firecracker.pid"), "utf-8");
|
|
15248
|
+
const pid = parseInt(pidStr.trim(), 10);
|
|
15249
|
+
if (!isNaN(pid)) try {
|
|
15250
|
+
process.kill(pid, "SIGKILL");
|
|
15251
|
+
} catch {}
|
|
15252
|
+
} catch {}
|
|
15253
|
+
if (managed.ip) try {
|
|
15254
|
+
await removeIsolationRules(managed.ip, { requireSudo: this.requireSudo });
|
|
15255
|
+
} catch {}
|
|
15256
|
+
if (managed.tapDevice) try {
|
|
15257
|
+
await this.execAsync("ip", [
|
|
15258
|
+
"link",
|
|
15259
|
+
"del",
|
|
15260
|
+
managed.tapDevice
|
|
15261
|
+
]);
|
|
15262
|
+
} catch {}
|
|
15263
|
+
await this.ipAllocator.release(managedId);
|
|
15264
|
+
try {
|
|
15265
|
+
await this.removeChrootDir(managedId);
|
|
15266
|
+
} catch {}
|
|
15267
|
+
this.agents.delete(managedId);
|
|
15268
|
+
}
|
|
15269
|
+
async shutdownAll() {
|
|
15270
|
+
this.stopPeriodicOrphanSweep();
|
|
15271
|
+
const ids = [...this.agents.keys()];
|
|
15272
|
+
await Promise.allSettled(ids.map((id) => this.destroy(id)));
|
|
15273
|
+
}
|
|
15274
|
+
/**
|
|
15275
|
+
* Clear MMDS data for an agent after receiving config.ack via WS.
|
|
15276
|
+
* Belt-and-suspenders: even though MMDS only contains orchestrator URL,
|
|
15277
|
+
* clearing it reduces the attack surface to zero post-startup.
|
|
15278
|
+
*
|
|
15279
|
+
* Note: Wiring from agent-handler is done in Plan 04.
|
|
15280
|
+
*
|
|
15281
|
+
* @param agentId - The agent ID whose VM MMDS should be cleared
|
|
15282
|
+
*/
|
|
15283
|
+
async clearAgentMmds(agentId) {
|
|
15284
|
+
try {
|
|
15285
|
+
await new FirecrackerApi(this.getSocketPath(agentId)).clearMmds();
|
|
15286
|
+
} catch (err) {}
|
|
15287
|
+
}
|
|
15288
|
+
reload(labelSets) {
|
|
15289
|
+
const errors = [];
|
|
15290
|
+
labelSets.forEach((ls, i) => {
|
|
15291
|
+
if (!ls.rootfsPath) errors.push(`Label set [${i}] requires a 'rootfsPath' field for Firecracker backend`);
|
|
15292
|
+
});
|
|
15293
|
+
if (errors.length > 0) return {
|
|
15294
|
+
valid: false,
|
|
15295
|
+
errors
|
|
15296
|
+
};
|
|
15297
|
+
this._labelSets = labelSets;
|
|
15298
|
+
return { valid: true };
|
|
15299
|
+
}
|
|
15300
|
+
/**
|
|
15301
|
+
* Clean up orphaned VMs.
|
|
15302
|
+
*
|
|
15303
|
+
* Safe to call both at startup AND periodically while the backend is active
|
|
15304
|
+
* (see startPeriodicOrphanSweep): each pass explicitly skips VMs that are
|
|
15305
|
+
* currently tracked in-memory (spawning / running / destroying), and Pass 3
|
|
15306
|
+
* re-reads DB allocations after listing host interfaces to close the spawn
|
|
15307
|
+
* race (allocate → ip tuntap add is not atomic).
|
|
15308
|
+
*
|
|
15309
|
+
* Three passes:
|
|
15310
|
+
* 1. DB allocations: check if VM process is still running, clean dead ones
|
|
15311
|
+
* 2. Filesystem: scan chroot dir for directories not in DB allocations
|
|
15312
|
+
* 3. Network: scan host interfaces for orphan TAP devices
|
|
15313
|
+
*
|
|
15314
|
+
* Returns the count of cleaned orphans.
|
|
15315
|
+
*/
|
|
15316
|
+
async cleanupOrphans() {
|
|
15317
|
+
let cleaned = 0;
|
|
15318
|
+
const trackedVmIds = new Set(this.agents.keys());
|
|
15319
|
+
const chrootParent = join(this.chrootBaseDir, "firecracker");
|
|
15320
|
+
let chrootEntries = [];
|
|
15321
|
+
try {
|
|
15322
|
+
chrootEntries = await readdir(chrootParent);
|
|
15323
|
+
} catch {}
|
|
15324
|
+
const liveVmIds = /* @__PURE__ */ new Set();
|
|
15325
|
+
for (const entry of chrootEntries) if (trackedVmIds.has(entry) || await this.isVmProcessAlive(entry)) liveVmIds.add(entry);
|
|
15326
|
+
const liveTapNames = new Set([...liveVmIds].map((id) => `kici-${id.slice(0, 8)}`));
|
|
15327
|
+
try {
|
|
15328
|
+
const allocations = await this.ipAllocator.getAllocations();
|
|
15329
|
+
const scalerAllocations = allocations.filter((a) => a.scaler_name === this.name);
|
|
15330
|
+
for (const alloc of scalerAllocations) {
|
|
15331
|
+
if (trackedVmIds.has(alloc.vm_id)) continue;
|
|
15332
|
+
if (!(liveVmIds.has(alloc.vm_id) || await this.isVmProcessAlive(alloc.vm_id))) {
|
|
15333
|
+
try {
|
|
15334
|
+
const chrootDir = this.getChrootDir(alloc.vm_id);
|
|
15335
|
+
await this.forwardRemainingLogs(alloc.vm_id, chrootDir);
|
|
15336
|
+
} catch {}
|
|
15337
|
+
try {
|
|
15338
|
+
await this.execAsync("ip", [
|
|
15339
|
+
"link",
|
|
15340
|
+
"del",
|
|
15341
|
+
alloc.tap_device
|
|
15342
|
+
]);
|
|
15343
|
+
} catch {}
|
|
15344
|
+
await this.ipAllocator.release(alloc.vm_id);
|
|
15345
|
+
try {
|
|
15346
|
+
await this.removeChrootDir(alloc.vm_id);
|
|
15347
|
+
} catch {}
|
|
15348
|
+
cleaned++;
|
|
15349
|
+
}
|
|
15350
|
+
}
|
|
15351
|
+
const allocatedVmIds = new Set(allocations.map((a) => a.vm_id));
|
|
15352
|
+
for (const entry of chrootEntries) {
|
|
15353
|
+
if (allocatedVmIds.has(entry)) continue;
|
|
15354
|
+
if (trackedVmIds.has(entry)) continue;
|
|
15355
|
+
if (liveVmIds.has(entry)) continue;
|
|
15356
|
+
try {
|
|
15357
|
+
const orphanChrootDir = join(chrootParent, entry, "root");
|
|
15358
|
+
await this.forwardRemainingLogs(entry, orphanChrootDir);
|
|
15359
|
+
} catch {}
|
|
15360
|
+
try {
|
|
15361
|
+
await this.removeChrootDir(entry);
|
|
15362
|
+
cleaned++;
|
|
15363
|
+
} catch {}
|
|
15364
|
+
}
|
|
15365
|
+
const trackedTapDevices = new Set([...this.agents.values()].map((a) => a.tapDevice).filter((tap) => tap.length > 0));
|
|
15366
|
+
const allocatedTapsBefore = new Set(allocations.map((a) => a.tap_device));
|
|
15367
|
+
try {
|
|
15368
|
+
const { stdout } = await this.execAsync("ip", ["-br", "link"]);
|
|
15369
|
+
const ifaceNames = stdout.split("\n").map((line) => line.split(/\s+/)[0]).filter((name) => !!name);
|
|
15370
|
+
let allocatedTapsAfter;
|
|
15371
|
+
try {
|
|
15372
|
+
const allocationsAfter = await this.ipAllocator.getAllocations();
|
|
15373
|
+
allocatedTapsAfter = new Set(allocationsAfter.map((a) => a.tap_device));
|
|
15374
|
+
} catch {
|
|
15375
|
+
allocatedTapsAfter = allocatedTapsBefore;
|
|
15376
|
+
}
|
|
15377
|
+
for (const name of ifaceNames) {
|
|
15378
|
+
if (!VM_TAP_PATTERN.test(name)) continue;
|
|
15379
|
+
if (PROTECTED_INTERFACES.includes(name)) continue;
|
|
15380
|
+
if (allocatedTapsBefore.has(name)) continue;
|
|
15381
|
+
if (allocatedTapsAfter.has(name)) continue;
|
|
15382
|
+
if (trackedTapDevices.has(name)) continue;
|
|
15383
|
+
if (liveTapNames.has(name)) continue;
|
|
15384
|
+
try {
|
|
15385
|
+
await this.execAsync("ip", [
|
|
15386
|
+
"link",
|
|
15387
|
+
"del",
|
|
15388
|
+
name
|
|
15389
|
+
]);
|
|
15390
|
+
cleaned++;
|
|
15391
|
+
} catch {}
|
|
15392
|
+
}
|
|
15393
|
+
} catch {}
|
|
15394
|
+
} catch {}
|
|
15395
|
+
return cleaned;
|
|
15396
|
+
}
|
|
15397
|
+
/**
|
|
15398
|
+
* Default interval for periodic orphan sweeps (15 minutes).
|
|
15399
|
+
*
|
|
15400
|
+
* Long-running orchestrators need in-process orphan sweeps because the
|
|
15401
|
+
* external kici-leak-sweep timer skips TAP cleanup while a staging
|
|
15402
|
+
* orchestrator is active (to avoid racing spawns). Without this, leaked
|
|
15403
|
+
* TAPs accumulate until the orchestrator restarts — which can be weeks.
|
|
15404
|
+
*/
|
|
15405
|
+
static ORPHAN_SWEEP_INTERVAL_MS = 900 * 1e3;
|
|
15406
|
+
/**
|
|
15407
|
+
* Start a periodic orphan sweep. Idempotent — calling twice has no effect
|
|
15408
|
+
* beyond the first call. Use stopPeriodicOrphanSweep() to stop.
|
|
15409
|
+
*
|
|
15410
|
+
* @param intervalMs override the default interval (primarily for tests)
|
|
15411
|
+
*/
|
|
15412
|
+
startPeriodicOrphanSweep(intervalMs = FirecrackerScalerBackend.ORPHAN_SWEEP_INTERVAL_MS) {
|
|
15413
|
+
if (this.orphanSweepTimer !== null) return;
|
|
15414
|
+
const runSweep = async () => {
|
|
15415
|
+
if (this.orphanSweepInFlight) return;
|
|
15416
|
+
this.orphanSweepInFlight = true;
|
|
15417
|
+
try {
|
|
15418
|
+
const cleaned = await this.cleanupOrphans();
|
|
15419
|
+
if (cleaned > 0) logger$2.info(`Periodic orphan sweep cleaned ${cleaned} resources`, { backend: this.name });
|
|
15420
|
+
} catch (err) {
|
|
15421
|
+
logger$2.warn("Periodic orphan sweep failed", {
|
|
15422
|
+
backend: this.name,
|
|
15423
|
+
error: toErrorMessage(err)
|
|
15424
|
+
});
|
|
15425
|
+
} finally {
|
|
15426
|
+
this.orphanSweepInFlight = false;
|
|
15427
|
+
}
|
|
15428
|
+
};
|
|
15429
|
+
this.orphanSweepTimer = setInterval(() => {
|
|
15430
|
+
runSweep().catch(() => {});
|
|
15431
|
+
}, intervalMs);
|
|
15432
|
+
this.orphanSweepTimer.unref?.();
|
|
15433
|
+
}
|
|
15434
|
+
/**
|
|
15435
|
+
* Stop the periodic orphan sweep started by startPeriodicOrphanSweep().
|
|
15436
|
+
* Safe to call when the sweep was never started.
|
|
15437
|
+
*/
|
|
15438
|
+
stopPeriodicOrphanSweep() {
|
|
15439
|
+
if (this.orphanSweepTimer !== null) {
|
|
15440
|
+
clearInterval(this.orphanSweepTimer);
|
|
15441
|
+
this.orphanSweepTimer = null;
|
|
15442
|
+
}
|
|
15443
|
+
}
|
|
15444
|
+
/**
|
|
15445
|
+
* Build the merged forwarded-env map for an agent's MMDS payload.
|
|
15446
|
+
*
|
|
15447
|
+
* Mirrors bare-metal/container precedence: KICI_AGENT_ENV_*-prefixed vars from the
|
|
15448
|
+
* orchestrator's process.env (prefix stripped) are seeded first, then scalers.yaml
|
|
15449
|
+
* `env:` overlays them (yaml wins on conflict).
|
|
15450
|
+
*
|
|
15451
|
+
* Two safety filters apply:
|
|
15452
|
+
* - Keys must match POSIX_ENV_NAME_PATTERN. Otherwise rejected to keep MMDS path
|
|
15453
|
+
* construction safe inside the VM and to prevent the guest from being asked to
|
|
15454
|
+
* `export` something the shell can't represent.
|
|
15455
|
+
* - The cumulative byte cost (key + value + 2-byte overhead per entry) must stay
|
|
15456
|
+
* under MMDS_FORWARDED_ENV_BUDGET_BYTES. Once exceeded, remaining entries are
|
|
15457
|
+
* skipped and warnings logged. This protects the ~51 KiB MMDS data store cap
|
|
15458
|
+
* from being blown by an accidentally-huge env value.
|
|
15459
|
+
*/
|
|
15460
|
+
buildForwardedEnv(matchedLabelSet, agentId) {
|
|
15461
|
+
const merged = {};
|
|
15462
|
+
for (const [k, v] of Object.entries(process.env)) {
|
|
15463
|
+
if (!k.startsWith(KICI_AGENT_ENV_PREFIX) || v === void 0) continue;
|
|
15464
|
+
const stripped = k.slice(KICI_AGENT_ENV_PREFIX.length);
|
|
15465
|
+
if (stripped.length === 0) continue;
|
|
15466
|
+
merged[stripped] = v;
|
|
15467
|
+
}
|
|
15468
|
+
Object.assign(merged, matchedLabelSet.env ?? {});
|
|
15469
|
+
const accepted = {};
|
|
15470
|
+
let usedBytes = 0;
|
|
15471
|
+
for (const [k, v] of Object.entries(merged)) {
|
|
15472
|
+
if (!POSIX_ENV_NAME_PATTERN.test(k)) {
|
|
15473
|
+
logger$2.warn(`Skipping forwarded env var "${k}" for agent ${agentId}: not a POSIX-safe identifier`);
|
|
15474
|
+
continue;
|
|
15475
|
+
}
|
|
15476
|
+
const cost = Buffer.byteLength(k) + Buffer.byteLength(v) + 2;
|
|
15477
|
+
if (usedBytes + cost > MMDS_FORWARDED_ENV_BUDGET_BYTES) {
|
|
15478
|
+
logger$2.warn(`Skipping forwarded env var "${k}" for agent ${agentId}: would exceed ${MMDS_FORWARDED_ENV_BUDGET_BYTES}-byte MMDS budget`);
|
|
15479
|
+
continue;
|
|
15480
|
+
}
|
|
15481
|
+
accepted[k] = v;
|
|
15482
|
+
usedBytes += cost;
|
|
15483
|
+
}
|
|
15484
|
+
return accepted;
|
|
15485
|
+
}
|
|
15486
|
+
/**
|
|
15487
|
+
* Build the Firecracker VM configuration JSON.
|
|
15488
|
+
*
|
|
15489
|
+
* Resolution order for vcpu_count and mem_size_mib:
|
|
15490
|
+
* 1. effectiveLimits (resolved by ScalerManager from job/scaler/global caps) — wins if set.
|
|
15491
|
+
* 2. label-set's vcpuCount/memSizeMib.
|
|
15492
|
+
* 3. backend-level default vcpuCount/memSizeMib.
|
|
15493
|
+
*
|
|
15494
|
+
* Firecracker requires integer vCPUs and integer MiB. We round CPUs UP via
|
|
15495
|
+
* Math.ceil to never under-provision (a 0.5-CPU request still gets a whole vCPU)
|
|
15496
|
+
* and floor-divide bytes -> MiB. A debug log is emitted when rounding actually
|
|
15497
|
+
* changes the value, so operators can spot frequent fractional requests.
|
|
15498
|
+
*/
|
|
15499
|
+
buildVmConfig(alloc, labelSet, effectiveLimits) {
|
|
15500
|
+
let vcpuCount = labelSet.vcpuCount ?? this.vcpuCount;
|
|
15501
|
+
let memSizeMib = labelSet.memSizeMib ?? this.memSizeMib;
|
|
15502
|
+
if (effectiveLimits) {
|
|
15503
|
+
if (typeof effectiveLimits.cpus === "number" && effectiveLimits.cpus > 0) {
|
|
15504
|
+
const rounded = Math.max(1, Math.ceil(effectiveLimits.cpus));
|
|
15505
|
+
if (rounded !== effectiveLimits.cpus) logger$2.debug(`Rounding effective CPU limit ${effectiveLimits.cpus} up to ${rounded} vCPUs for Firecracker (integer vCPU required)`);
|
|
15506
|
+
vcpuCount = rounded;
|
|
15507
|
+
}
|
|
15508
|
+
if (typeof effectiveLimits.memBytes === "number" && effectiveLimits.memBytes > 0) {
|
|
15509
|
+
const mib = Math.floor(effectiveLimits.memBytes / (1024 * 1024));
|
|
15510
|
+
if (mib > 0) {
|
|
15511
|
+
if (mib * 1024 * 1024 !== effectiveLimits.memBytes) logger$2.debug(`Truncating effective memory limit ${effectiveLimits.memBytes}B to ${mib} MiB for Firecracker (integer MiB required)`);
|
|
15512
|
+
memSizeMib = mib;
|
|
15513
|
+
}
|
|
15514
|
+
}
|
|
15515
|
+
}
|
|
15516
|
+
return {
|
|
15517
|
+
"boot-source": {
|
|
15518
|
+
kernel_image_path: "/kernel",
|
|
15519
|
+
boot_args: `console=ttyS0 reboot=k panic=1 random.trust_cpu=on init=/init ip=${alloc.ip}::${this.gateway}:${this.netmask}::eth0:off`
|
|
15520
|
+
},
|
|
15521
|
+
drives: [{
|
|
15522
|
+
drive_id: "rootfs",
|
|
15523
|
+
path_on_host: "/rootfs.ext4",
|
|
15524
|
+
is_root_device: true,
|
|
15525
|
+
is_read_only: true
|
|
15526
|
+
}, {
|
|
15527
|
+
drive_id: "overlay",
|
|
15528
|
+
path_on_host: "/overlay.ext4",
|
|
15529
|
+
is_root_device: false,
|
|
15530
|
+
is_read_only: false
|
|
15531
|
+
}],
|
|
15532
|
+
"machine-config": {
|
|
15533
|
+
vcpu_count: vcpuCount,
|
|
15534
|
+
mem_size_mib: memSizeMib,
|
|
15535
|
+
smt: false
|
|
15536
|
+
},
|
|
15537
|
+
"network-interfaces": [{
|
|
15538
|
+
iface_id: "eth0",
|
|
15539
|
+
guest_mac: alloc.mac,
|
|
15540
|
+
host_dev_name: alloc.tapDevice
|
|
15541
|
+
}],
|
|
15542
|
+
"mmds-config": {
|
|
15543
|
+
network_interfaces: ["eth0"],
|
|
15544
|
+
ipv4_address: "169.254.169.254"
|
|
15545
|
+
}
|
|
15546
|
+
};
|
|
15547
|
+
}
|
|
15548
|
+
/**
|
|
15549
|
+
* Create a sparse ext4 overlay drive for per-VM writable layer.
|
|
15550
|
+
* The file is sparse (allocates no disk blocks until written) and
|
|
15551
|
+
* pre-formatted as ext4 so the guest can mount it immediately.
|
|
15552
|
+
*/
|
|
15553
|
+
async createOverlayDrive(path, sizeMib) {
|
|
15554
|
+
const fd = await open(path, "w");
|
|
15555
|
+
await fd.truncate(sizeMib * 1024 * 1024);
|
|
15556
|
+
await fd.close();
|
|
15557
|
+
await this.execAsync("mkfs.ext4", ["-qF", path]);
|
|
15558
|
+
}
|
|
15559
|
+
/**
|
|
15560
|
+
* Promisified execFile wrapper with 30s timeout.
|
|
15561
|
+
*/
|
|
15562
|
+
async execAsync(cmd, args) {
|
|
15563
|
+
if (this.requireSudo && (cmd === "ip" || cmd === "chown" || cmd === "chmod")) return execFileAsync("sudo", [
|
|
15564
|
+
"-n",
|
|
15565
|
+
cmd,
|
|
15566
|
+
...args
|
|
15567
|
+
], { timeout: 3e4 });
|
|
15568
|
+
return execFileAsync(cmd, args, { timeout: 3e4 });
|
|
15569
|
+
}
|
|
15570
|
+
/**
|
|
15571
|
+
* Remove a VM's jailer chroot directory.
|
|
15572
|
+
*
|
|
15573
|
+
* On rootless nodes (requireSudo), the jailer chowns the chroot contents to
|
|
15574
|
+
* the jailer uid/gid, leaving the inner directories owned by that uid with
|
|
15575
|
+
* mode 0755 — so the orchestrator process (a different, non-root uid) has no
|
|
15576
|
+
* write permission on them and a plain `rm` fails with EACCES. destroy() and
|
|
15577
|
+
* both cleanupOrphans passes treat removal as best-effort and swallow that
|
|
15578
|
+
* failure, so each spawn would otherwise leak a multi-GiB chroot until the
|
|
15579
|
+
* data disk fills and the orchestrator crash-loops on ENOSPC (it can no
|
|
15580
|
+
* longer write its own files to start, so the in-process orphan sweep never
|
|
15581
|
+
* runs to recover). Reclaim ownership via the same sudo-wrapped `chown` path
|
|
15582
|
+
* used for `ip` before the `rm` runs as ourselves. On root nodes (requireSudo
|
|
15583
|
+
* false) the chown is skipped and `rm` works directly.
|
|
15584
|
+
*/
|
|
15585
|
+
async removeChrootDir(vmId) {
|
|
15586
|
+
const dir = join(this.chrootBaseDir, "firecracker", vmId);
|
|
15587
|
+
if (this.requireSudo) {
|
|
15588
|
+
const owner = `${process.getuid?.() ?? 0}:${process.getgid?.() ?? 0}`;
|
|
15589
|
+
try {
|
|
15590
|
+
await this.execAsync("chown", [
|
|
15591
|
+
"-R",
|
|
15592
|
+
owner,
|
|
15593
|
+
dir
|
|
15594
|
+
]);
|
|
15595
|
+
} catch {}
|
|
15596
|
+
}
|
|
15597
|
+
await rm(dir, {
|
|
15598
|
+
recursive: true,
|
|
15599
|
+
force: true
|
|
15600
|
+
});
|
|
15601
|
+
}
|
|
15602
|
+
/**
|
|
15603
|
+
* Get the API socket path for an agent's Firecracker VM.
|
|
15604
|
+
*/
|
|
15605
|
+
getSocketPath(agentId) {
|
|
15606
|
+
return join(this.chrootBaseDir, "firecracker", agentId, "root", "run", "firecracker.socket");
|
|
15607
|
+
}
|
|
15608
|
+
/**
|
|
15609
|
+
* Get the chroot directory path for an agent.
|
|
15610
|
+
*/
|
|
15611
|
+
getChrootDir(agentId) {
|
|
15612
|
+
return join(this.chrootBaseDir, "firecracker", agentId, "root");
|
|
15613
|
+
}
|
|
15614
|
+
/**
|
|
15615
|
+
* Check if a VM process is still alive by reading its PID file and sending signal 0.
|
|
15616
|
+
*/
|
|
15617
|
+
async isVmProcessAlive(vmId) {
|
|
15618
|
+
try {
|
|
15619
|
+
const pidStr = await readFile(join(this.getChrootDir(vmId), "firecracker.pid"), "utf-8");
|
|
15620
|
+
const pid = parseInt(pidStr.trim(), 10);
|
|
15621
|
+
if (isNaN(pid)) return false;
|
|
15622
|
+
process.kill(pid, 0);
|
|
15623
|
+
return true;
|
|
15624
|
+
} catch {
|
|
15625
|
+
return false;
|
|
15626
|
+
}
|
|
15627
|
+
}
|
|
15628
|
+
/**
|
|
15629
|
+
* Wait for a VM's jailer process to exit within a timeout.
|
|
15630
|
+
*/
|
|
15631
|
+
async waitForProcessExit(vmId, timeoutMs) {
|
|
15632
|
+
const deadline = Date.now() + timeoutMs;
|
|
15633
|
+
const interval = 200;
|
|
15634
|
+
while (Date.now() < deadline) {
|
|
15635
|
+
if (!await this.isVmProcessAlive(vmId)) return true;
|
|
15636
|
+
await new Promise((resolve) => setTimeout(resolve, interval));
|
|
15637
|
+
}
|
|
15638
|
+
return false;
|
|
15639
|
+
}
|
|
15640
|
+
/**
|
|
15641
|
+
* Clean up resources after a failed spawn attempt.
|
|
15642
|
+
*/
|
|
15643
|
+
async cleanupFailedSpawn(agentId, alloc) {
|
|
15644
|
+
if (alloc) {
|
|
15645
|
+
try {
|
|
15646
|
+
await removeIsolationRules(alloc.ip, { requireSudo: this.requireSudo });
|
|
15647
|
+
} catch {}
|
|
15648
|
+
try {
|
|
15649
|
+
await this.ipAllocator.release(agentId);
|
|
15650
|
+
} catch {}
|
|
15651
|
+
try {
|
|
15652
|
+
await this.execAsync("ip", [
|
|
15653
|
+
"link",
|
|
15654
|
+
"del",
|
|
15655
|
+
alloc.tapDevice
|
|
15656
|
+
]);
|
|
15657
|
+
} catch {}
|
|
15658
|
+
}
|
|
15659
|
+
try {
|
|
15660
|
+
await rm(join(this.chrootBaseDir, "firecracker", agentId), {
|
|
15661
|
+
recursive: true,
|
|
15662
|
+
force: true
|
|
15663
|
+
});
|
|
15664
|
+
} catch {}
|
|
15665
|
+
}
|
|
15666
|
+
/**
|
|
15667
|
+
* Start tailing serial console and VMM log files, forwarding lines directly
|
|
15668
|
+
* via forwardLine() with distinct logsSource tags.
|
|
15669
|
+
*
|
|
15670
|
+
* Also monitors the jailer child process for early exit (boot failure detection).
|
|
15671
|
+
* Forwarding is handled internally (not through ScalerManager/AgentLogForwarder)
|
|
15672
|
+
* because Firecracker has two distinct log sources per VM.
|
|
15673
|
+
*/
|
|
15674
|
+
startLogTailing(agentId, serialLogPath, vmmLogPath, child) {
|
|
15675
|
+
const abortController = new AbortController();
|
|
15676
|
+
this.tailAbortControllers.set(agentId, abortController);
|
|
15677
|
+
const signal = abortController.signal;
|
|
15678
|
+
const output = process.stdout;
|
|
15679
|
+
this.tailAndForward(serialLogPath, agentId, "firecracker-serial", signal, output);
|
|
15680
|
+
this.tailAndForward(vmmLogPath, agentId, "firecracker-vmm", signal, output);
|
|
15681
|
+
child.on("exit", (code, sig) => {
|
|
15682
|
+
setTimeout(() => {
|
|
15683
|
+
abortController.abort();
|
|
15684
|
+
this.tailAbortControllers.delete(agentId);
|
|
15685
|
+
if (code !== 0 && code !== null) {
|
|
15686
|
+
const failureType = this.detectFailureType(serialLogPath);
|
|
15687
|
+
forwardLine(JSON.stringify({
|
|
15688
|
+
level: "error",
|
|
15689
|
+
message: "VM boot failed",
|
|
15690
|
+
exitCode: code,
|
|
15691
|
+
signal: sig,
|
|
15692
|
+
failureType
|
|
15693
|
+
}), agentId, output, void 0, "firecracker-serial");
|
|
15694
|
+
}
|
|
15695
|
+
}, 500);
|
|
15696
|
+
});
|
|
15697
|
+
}
|
|
15698
|
+
/**
|
|
15699
|
+
* Tail a single log file and forward each line via forwardLine().
|
|
15700
|
+
* For serial console lines: non-JSON lines (kernel boot output) are wrapped
|
|
15701
|
+
* as debug-level JSON; failure patterns (kernel panic, init failure) are elevated to error.
|
|
15702
|
+
*/
|
|
15703
|
+
async tailAndForward(filePath, agentId, logsSource, signal, output) {
|
|
15704
|
+
try {
|
|
15705
|
+
for await (const line of tailFile(filePath, signal)) {
|
|
15706
|
+
if (logsSource === "firecracker-serial") {
|
|
15707
|
+
let isJson = false;
|
|
15708
|
+
try {
|
|
15709
|
+
JSON.parse(line);
|
|
15710
|
+
isJson = true;
|
|
15711
|
+
} catch {}
|
|
15712
|
+
if (!isJson) {
|
|
15713
|
+
const level = this.isFailurePattern(line) ? "error" : "debug";
|
|
15714
|
+
forwardLine(JSON.stringify({
|
|
15715
|
+
level,
|
|
15716
|
+
message: line
|
|
15717
|
+
}), agentId, output, void 0, logsSource);
|
|
15718
|
+
continue;
|
|
15719
|
+
}
|
|
15720
|
+
}
|
|
15721
|
+
forwardLine(line, agentId, output, void 0, logsSource);
|
|
15722
|
+
}
|
|
15723
|
+
} catch (err) {
|
|
15724
|
+
if (err.name === "AbortError") return;
|
|
15725
|
+
forwardLine(JSON.stringify({
|
|
15726
|
+
level: "warn",
|
|
15727
|
+
message: `Log tailing error for ${filePath}: ${err}`
|
|
15728
|
+
}), agentId, output, void 0, logsSource);
|
|
15729
|
+
}
|
|
15730
|
+
}
|
|
15731
|
+
/**
|
|
15732
|
+
* Check if a serial console line matches known boot failure patterns.
|
|
15733
|
+
*/
|
|
15734
|
+
isFailurePattern(line) {
|
|
15735
|
+
return [
|
|
15736
|
+
/Kernel panic/i,
|
|
15737
|
+
/Failed to execute \/init/i,
|
|
15738
|
+
/failed to start/i,
|
|
15739
|
+
/systemd\[1\]: Failed/i,
|
|
15740
|
+
/VFS: Unable to mount root fs/i,
|
|
15741
|
+
/not syncing/i
|
|
15742
|
+
].some((p) => p.test(line));
|
|
15743
|
+
}
|
|
15744
|
+
/**
|
|
15745
|
+
* Read the serial console log file and detect the type of boot failure.
|
|
15746
|
+
*/
|
|
15747
|
+
detectFailureType(serialLogPath) {
|
|
15748
|
+
try {
|
|
15749
|
+
const content = readFileSync(serialLogPath, "utf-8");
|
|
15750
|
+
if (/Kernel panic/i.test(content)) return "kernel-panic";
|
|
15751
|
+
if (/Failed to execute \/init/i.test(content) || /systemd\[1\]: Failed/i.test(content)) return "init-failure";
|
|
15752
|
+
if (/VFS: Unable to mount root fs/i.test(content)) return "rootfs-mount-failure";
|
|
15753
|
+
return "unknown";
|
|
15754
|
+
} catch {
|
|
15755
|
+
return "unknown";
|
|
15756
|
+
}
|
|
15757
|
+
}
|
|
15758
|
+
/**
|
|
15759
|
+
* Read and forward any remaining log file content for an orphaned/dead VM.
|
|
15760
|
+
* Called during cleanupOrphans() before deleting the chroot directory.
|
|
15761
|
+
*/
|
|
15762
|
+
async forwardRemainingLogs(vmId, chrootDir) {
|
|
15763
|
+
const output = process.stdout;
|
|
15764
|
+
for (const [fileName, logsSource] of [[SERIAL_LOG_FILE, "firecracker-serial"], [VMM_LOG_FILE, "firecracker-vmm"]]) try {
|
|
15765
|
+
const lines = (await readFile(join(chrootDir, fileName), "utf-8")).split("\n").filter((l) => l.length > 0);
|
|
15766
|
+
for (const line of lines) forwardLine(line, vmId, output, void 0, logsSource);
|
|
15767
|
+
} catch {}
|
|
15768
|
+
}
|
|
15769
|
+
};
|
|
15770
|
+
//#endregion
|
|
15771
|
+
//#region src/scaler/container-backend.ts
|
|
15772
|
+
/**
|
|
15773
|
+
* Container scaler backend implementation.
|
|
15774
|
+
*
|
|
15775
|
+
* Manages ephemeral agent containers using Docker or Podman via dockerode.
|
|
15776
|
+
* Supports auto-detection of container runtime socket.
|
|
15777
|
+
* Handles full container lifecycle: pull, create, start, stop, remove.
|
|
15778
|
+
* Supports container socket sharing, resource limits, and orphan cleanup.
|
|
15779
|
+
*/
|
|
15780
|
+
const logger$1 = createLogger({ prefix: "container-backend" });
|
|
15781
|
+
/** Name of the isolated bridge network for agent containers. */
|
|
15782
|
+
const ISOLATED_NETWORK_NAME = "kici-agent-net";
|
|
15783
|
+
/** Subnet for the isolated agent network. */
|
|
15784
|
+
const ISOLATED_NETWORK_SUBNET = "172.30.0.0/16";
|
|
15785
|
+
/** Gateway IP for the isolated agent network (host-side). */
|
|
15786
|
+
const ISOLATED_NETWORK_GATEWAY = "172.30.0.1";
|
|
15787
|
+
const PROBE_ORDER = [{
|
|
15788
|
+
path: "/var/run/docker.sock",
|
|
15789
|
+
runtime: "docker"
|
|
15790
|
+
}, {
|
|
15791
|
+
path: "/run/podman/podman.sock",
|
|
15792
|
+
runtime: "podman"
|
|
15793
|
+
}];
|
|
15794
|
+
/**
|
|
15795
|
+
* Probe known socket paths to detect which container runtime is available.
|
|
15796
|
+
* Returns the first accessible socket found, or null if none found.
|
|
15797
|
+
*/
|
|
15798
|
+
async function detectRuntime(runtimeHint) {
|
|
15799
|
+
const probes = [...PROBE_ORDER];
|
|
15800
|
+
const xdgDir = process.env.XDG_RUNTIME_DIR;
|
|
15801
|
+
if (xdgDir) probes.push({
|
|
15802
|
+
path: `${xdgDir}/podman/podman.sock`,
|
|
15803
|
+
runtime: "podman"
|
|
15804
|
+
});
|
|
15805
|
+
const filtered = runtimeHint && runtimeHint !== "auto" ? probes.filter((p) => p.runtime === runtimeHint) : probes;
|
|
15806
|
+
for (const probe of filtered) try {
|
|
15807
|
+
await access(probe.path, constants.R_OK | constants.W_OK);
|
|
15808
|
+
return {
|
|
15809
|
+
socketPath: probe.path,
|
|
15810
|
+
runtime: probe.runtime
|
|
15811
|
+
};
|
|
15812
|
+
} catch {
|
|
15813
|
+
continue;
|
|
15814
|
+
}
|
|
15815
|
+
return null;
|
|
15816
|
+
}
|
|
15817
|
+
/**
|
|
15818
|
+
* Create a LogCapture from a running container's stdout/stderr streams.
|
|
15819
|
+
* Uses dockerode's demuxStream to strip Docker's 8-byte multiplexed headers.
|
|
15820
|
+
*/
|
|
15821
|
+
async function createContainerLogCapture(docker, containerId) {
|
|
15822
|
+
const stream = await docker.getContainer(containerId).logs({
|
|
15823
|
+
follow: true,
|
|
15824
|
+
stdout: true,
|
|
15825
|
+
stderr: true,
|
|
15826
|
+
timestamps: false
|
|
15827
|
+
});
|
|
15828
|
+
const passthrough = new PassThrough();
|
|
15829
|
+
docker.modem.demuxStream(stream, passthrough, passthrough);
|
|
15830
|
+
const TAIL_MAX_LINES = 50;
|
|
15831
|
+
const tailBuf = [];
|
|
15832
|
+
const pushTail = (line) => {
|
|
15833
|
+
tailBuf.push(line);
|
|
15834
|
+
if (tailBuf.length > TAIL_MAX_LINES) tailBuf.shift();
|
|
15835
|
+
};
|
|
15836
|
+
const rl = createInterface({
|
|
15837
|
+
input: passthrough,
|
|
15838
|
+
crlfDelay: Infinity
|
|
15839
|
+
});
|
|
15840
|
+
rl.on("line", pushTail);
|
|
15841
|
+
return {
|
|
15842
|
+
async *lines() {
|
|
15843
|
+
for await (const line of rl) yield line;
|
|
15844
|
+
},
|
|
15845
|
+
tail() {
|
|
15846
|
+
return tailBuf.join("\n");
|
|
15847
|
+
},
|
|
15848
|
+
close() {
|
|
15849
|
+
stream.destroy?.();
|
|
15850
|
+
rl.close();
|
|
15851
|
+
}
|
|
15852
|
+
};
|
|
15853
|
+
}
|
|
15854
|
+
var ContainerScalerBackend = class ContainerScalerBackend {
|
|
15855
|
+
type = ScalerBackendType.enum.container;
|
|
15856
|
+
spawnsOnLocalHost;
|
|
15857
|
+
maxAgents;
|
|
15858
|
+
_labelSets;
|
|
15859
|
+
name;
|
|
15860
|
+
docker;
|
|
15861
|
+
defaultResources;
|
|
15862
|
+
/** The resolved socket path (for socket sharing bind mounts) */
|
|
15863
|
+
resolvedSocketPath;
|
|
15864
|
+
/** The detected or configured runtime type */
|
|
15865
|
+
detectedRuntime;
|
|
15866
|
+
/** Extra host:IP mappings for spawned containers */
|
|
15867
|
+
extraHosts;
|
|
15868
|
+
/** Whether nftables-based network isolation is enabled */
|
|
15869
|
+
networkIsolation;
|
|
15870
|
+
/** Token store for creating ephemeral agent auth tokens */
|
|
15871
|
+
tokenStore;
|
|
15872
|
+
/** TTL for ephemeral agent tokens in ms */
|
|
15873
|
+
tokenTtlMs;
|
|
15874
|
+
/** Agent roles for this scaler. undefined = all, [] = execution only. */
|
|
15875
|
+
roles;
|
|
15876
|
+
/** ID of the isolated bridge network (set after creation/discovery) */
|
|
15877
|
+
isolatedNetworkId = "";
|
|
15878
|
+
/** Host bridge interface name for the isolated network (used by nftables) */
|
|
15879
|
+
isolatedBridgeIface = "";
|
|
15880
|
+
/** Tracks all managed agent containers by ManagedAgent.id */
|
|
15881
|
+
agents = /* @__PURE__ */ new Map();
|
|
15882
|
+
/** Maps container ID to ManagedAgent.id for reverse lookup */
|
|
15883
|
+
containerToManaged = /* @__PURE__ */ new Map();
|
|
15884
|
+
/** LogCapture instances for each managed agent (keyed by ManagedAgent.id) */
|
|
15885
|
+
logCaptures = /* @__PURE__ */ new Map();
|
|
15886
|
+
/** Maps managedId → container IP on the isolated network (for per-container nftables cleanup) */
|
|
15887
|
+
containerIps = /* @__PURE__ */ new Map();
|
|
15888
|
+
constructor(options, socketPath, runtime) {
|
|
15889
|
+
this.name = options.name;
|
|
15890
|
+
this._labelSets = options.labelSets;
|
|
15891
|
+
this.maxAgents = options.maxAgents;
|
|
15892
|
+
this.defaultResources = options.defaultResources;
|
|
15893
|
+
this.extraHosts = options.extraHosts;
|
|
15894
|
+
this.networkIsolation = options.networkIsolation !== false;
|
|
15895
|
+
this.tokenStore = options.tokenStore;
|
|
15896
|
+
this.tokenTtlMs = options.tokenTtlMs ?? 36e5;
|
|
15897
|
+
this.roles = options.roles;
|
|
15898
|
+
this.resolvedSocketPath = socketPath;
|
|
15899
|
+
this.detectedRuntime = runtime;
|
|
15900
|
+
this.spawnsOnLocalHost = !options.host;
|
|
15901
|
+
if (options.host) {
|
|
15902
|
+
this.docker = new Docker({ host: options.host });
|
|
15903
|
+
this.resolvedSocketPath = "";
|
|
15904
|
+
this.detectedRuntime = options.runtime === "podman" ? "podman" : "docker";
|
|
15905
|
+
} else this.docker = new Docker({ socketPath });
|
|
15906
|
+
}
|
|
15907
|
+
/**
|
|
15908
|
+
* Ensure the isolated bridge network exists and nftables rules are applied.
|
|
15909
|
+
*
|
|
15910
|
+
* Creates `kici-agent-net` if it doesn't exist, inspects the network to
|
|
15911
|
+
* discover the host bridge interface name, then applies RFC1918 + metadata
|
|
15912
|
+
* blocking rules via nftables with a gateway exception.
|
|
15913
|
+
*
|
|
15914
|
+
* Idempotent: safe to call across orchestrator restarts.
|
|
15915
|
+
*/
|
|
15916
|
+
async ensureIsolatedNetwork() {
|
|
15917
|
+
const existing = (await this.docker.listNetworks({ filters: { name: [ISOLATED_NETWORK_NAME] } })).find((n) => n.Name === ISOLATED_NETWORK_NAME);
|
|
15918
|
+
let networkId;
|
|
15919
|
+
if (existing) {
|
|
15920
|
+
networkId = existing.Id;
|
|
15921
|
+
logger$1.info(`Isolated network ${ISOLATED_NETWORK_NAME} already exists (${networkId.slice(0, 12)})`);
|
|
15922
|
+
} else try {
|
|
15923
|
+
networkId = (await this.docker.createNetwork({
|
|
15924
|
+
Name: ISOLATED_NETWORK_NAME,
|
|
15925
|
+
Driver: "bridge",
|
|
15926
|
+
IPAM: { Config: [{
|
|
15927
|
+
Subnet: ISOLATED_NETWORK_SUBNET,
|
|
15928
|
+
Gateway: ISOLATED_NETWORK_GATEWAY
|
|
15929
|
+
}] },
|
|
15930
|
+
Labels: { "kici-managed": "true" }
|
|
15931
|
+
})).id;
|
|
15932
|
+
logger$1.info(`Created isolated network ${ISOLATED_NETWORK_NAME} (${networkId.slice(0, 12)})`);
|
|
15933
|
+
} catch (err) {
|
|
15934
|
+
if (err.statusCode === 409) {
|
|
15935
|
+
logger$1.info("Isolated network creation raced -- using existing network");
|
|
15936
|
+
const found = (await this.docker.listNetworks({ filters: { name: [ISOLATED_NETWORK_NAME] } })).find((n) => n.Name === ISOLATED_NETWORK_NAME);
|
|
15937
|
+
if (!found) throw new Error(`Failed to find isolated network ${ISOLATED_NETWORK_NAME} after 409 conflict`);
|
|
15938
|
+
networkId = found.Id;
|
|
15939
|
+
} else throw err;
|
|
15940
|
+
}
|
|
15941
|
+
this.isolatedNetworkId = networkId;
|
|
15942
|
+
const bridgeName = (await this.docker.getNetwork(networkId).inspect()).Options?.["com.docker.network.bridge.name"] ?? `br-${networkId.slice(0, 12)}`;
|
|
15943
|
+
this.isolatedBridgeIface = bridgeName;
|
|
15944
|
+
logger$1.info(`Isolated network bridge interface: ${bridgeName}`);
|
|
15945
|
+
if (this.networkIsolation) {
|
|
15946
|
+
await ensureKiciTable();
|
|
15947
|
+
logger$1.info("nftables kici table ready — per-container rules will be applied during spawn");
|
|
15948
|
+
} else logger$1.warn("Network isolation DISABLED for container backend — nftables rules will NOT be applied. Agent containers will have unrestricted network access.");
|
|
15949
|
+
}
|
|
15950
|
+
/**
|
|
15951
|
+
* Declare required tools for a container scaler entry.
|
|
15952
|
+
*
|
|
15953
|
+
* For the auto-detect case (no explicit socketPath / remote host) the
|
|
15954
|
+
* orchestrator must have a local container runtime — docker OR podman — on
|
|
15955
|
+
* PATH, otherwise the scaler cannot spawn agent containers. Declaring it
|
|
15956
|
+
* here lets the startup tool-validation gate fail fast with a clear error
|
|
15957
|
+
* instead of the first job hanging. When a socketPath or remote host is
|
|
15958
|
+
* configured the binary need not be on PATH (the runtime may be remote), so
|
|
15959
|
+
* reachability is validated later in create().
|
|
15960
|
+
*/
|
|
15961
|
+
static getRequiredTools(entry) {
|
|
15962
|
+
if (entry.host || entry.socketPath) return [];
|
|
15963
|
+
return [{
|
|
15964
|
+
type: "any-path-binary",
|
|
15965
|
+
names: ["docker", "podman"],
|
|
15966
|
+
reason: `container scaler "${entry.name}" needs a local container runtime to spawn agents. Install Docker or Podman, or set socketPath / host in scalers.yaml for a remote runtime.`
|
|
15967
|
+
}];
|
|
15968
|
+
}
|
|
15969
|
+
/**
|
|
15970
|
+
* Create a ContainerScalerBackend with auto-detected or configured socket.
|
|
15971
|
+
* Throws if no container runtime is found and no host is configured.
|
|
15972
|
+
*/
|
|
15973
|
+
static async create(options) {
|
|
15974
|
+
if (options.networkIsolation !== false) await validateNftablesAvailability();
|
|
15975
|
+
let backend;
|
|
15976
|
+
if (options.host) backend = new ContainerScalerBackend(options, "", options.runtime === "podman" ? "podman" : "docker");
|
|
15977
|
+
else if (options.socketPath) {
|
|
15978
|
+
const runtime = options.socketPath.includes("podman") ? "podman" : "docker";
|
|
15979
|
+
logger$1.info(`Using configured socket at ${options.socketPath}`, { runtime });
|
|
15980
|
+
backend = new ContainerScalerBackend(options, options.socketPath, runtime);
|
|
15981
|
+
} else {
|
|
15982
|
+
const detected = await detectRuntime(options.runtime);
|
|
15983
|
+
if (!detected) throw new Error("No container runtime found. Install Docker or Podman, or configure host.");
|
|
15984
|
+
logger$1.info(`Detected ${detected.runtime} at ${detected.socketPath}`);
|
|
15985
|
+
backend = new ContainerScalerBackend(options, detected.socketPath, detected.runtime);
|
|
15986
|
+
}
|
|
15987
|
+
if (options.networkIsolation !== false) await backend.ensureIsolatedNetwork();
|
|
15988
|
+
else logger$1.info("Network isolation disabled — skipping isolated network and nftables setup");
|
|
15989
|
+
return backend;
|
|
15990
|
+
}
|
|
15991
|
+
/** Log source identifier: 'docker' or 'podman' based on detected runtime. */
|
|
15992
|
+
get logsSource() {
|
|
15993
|
+
return this.detectedRuntime;
|
|
15994
|
+
}
|
|
15995
|
+
get labelSets() {
|
|
15996
|
+
return this._labelSets;
|
|
15997
|
+
}
|
|
15998
|
+
getActiveCount() {
|
|
15999
|
+
return this.agents.size;
|
|
16000
|
+
}
|
|
16001
|
+
async spawn(labelSet, agentId, orchestratorUrl, onEvent, effectiveLimits, spawnContext) {
|
|
16002
|
+
const emit = (eventType, detail) => {
|
|
16003
|
+
onEvent?.({
|
|
16004
|
+
agentId,
|
|
16005
|
+
eventType,
|
|
16006
|
+
detail,
|
|
16007
|
+
timestampMs: Date.now()
|
|
16008
|
+
});
|
|
16009
|
+
};
|
|
16010
|
+
const normalizedTarget = normalizeLabelSet(labelSet);
|
|
16011
|
+
const matchedLabelSet = this._labelSets.find((ls) => normalizeLabelSet(ls.labels) === normalizedTarget);
|
|
16012
|
+
if (!matchedLabelSet) throw new Error(`Label set [${labelSet.join(", ")}] not supported by container backend "${this.name}"`);
|
|
16013
|
+
if (this.getActiveCount() >= this.maxAgents) throw new Error(`Container backend "${this.name}" at capacity (${this.maxAgents}/${this.maxAgents})`);
|
|
16014
|
+
const managed = {
|
|
16015
|
+
id: agentId,
|
|
16016
|
+
labelSet,
|
|
16017
|
+
backendRef: "",
|
|
16018
|
+
spawnedAt: Date.now(),
|
|
16019
|
+
state: "spawning"
|
|
16020
|
+
};
|
|
16021
|
+
this.agents.set(managed.id, managed);
|
|
16022
|
+
try {
|
|
16023
|
+
const agentEnvForwarded = [];
|
|
16024
|
+
for (const [key, value] of Object.entries(process.env)) if (key.startsWith(KICI_AGENT_ENV_PREFIX) && value !== void 0) {
|
|
16025
|
+
const stripped = key.slice(KICI_AGENT_ENV_PREFIX.length);
|
|
16026
|
+
if (stripped.length > 0) agentEnvForwarded.push(`${stripped}=${value}`);
|
|
16027
|
+
}
|
|
16028
|
+
const fullLabels = scalerAgentLabels(labelSet, this.type, this.name, this.roles);
|
|
16029
|
+
let agentToken;
|
|
16030
|
+
if (this.tokenStore) agentToken = await this.tokenStore.createEphemeral(agentId, fullLabels, this.tokenTtlMs);
|
|
16031
|
+
const env = [
|
|
16032
|
+
`KICI_ORCHESTRATOR_URL=${orchestratorUrl}`,
|
|
16033
|
+
`KICI_AGENT_ID=${agentId}`,
|
|
16034
|
+
`KICI_LABELS=${fullLabels.join(",")}`,
|
|
16035
|
+
`KICI_SCALER_MANAGED=1`,
|
|
16036
|
+
`KICI_EXECUTION_MODE=bare-metal`,
|
|
16037
|
+
...agentToken ? [`KICI_AGENT_TOKEN=${agentToken}`] : [],
|
|
16038
|
+
...matchedLabelSet.backpressureMode ? [`KICI_BACKPRESSURE_MODE=${matchedLabelSet.backpressureMode}`] : [],
|
|
16039
|
+
...agentEnvForwarded,
|
|
16040
|
+
...Object.entries(matchedLabelSet.env ?? {}).map(([k, v]) => `${k}=${v}`)
|
|
16041
|
+
];
|
|
16042
|
+
const binds = [...matchedLabelSet.volumes ?? []];
|
|
16043
|
+
if (matchedLabelSet.containerSocket && this.resolvedSocketPath) binds.push(`${this.resolvedSocketPath}:${this.resolvedSocketPath}`);
|
|
16044
|
+
let memoryBytes;
|
|
16045
|
+
let nanoCpus;
|
|
16046
|
+
if (effectiveLimits && (effectiveLimits.cpus || effectiveLimits.memBytes)) {
|
|
16047
|
+
if (effectiveLimits.memBytes) memoryBytes = effectiveLimits.memBytes;
|
|
16048
|
+
if (effectiveLimits.cpus) nanoCpus = effectiveLimits.cpus * 1e9;
|
|
16049
|
+
} else {
|
|
16050
|
+
const limits = (matchedLabelSet.resources ?? this.defaultResources)?.limits;
|
|
16051
|
+
if (limits?.memory) memoryBytes = parseMemoryString(limits.memory);
|
|
16052
|
+
if (limits?.cpus) nanoCpus = limits.cpus * 1e9;
|
|
16053
|
+
}
|
|
16054
|
+
const pullPolicy = matchedLabelSet.imagePullPolicy ?? "Always";
|
|
16055
|
+
let shouldPull = pullPolicy === "Always";
|
|
16056
|
+
if (pullPolicy === "IfNotPresent") try {
|
|
16057
|
+
await this.docker.getImage(matchedLabelSet.image).inspect();
|
|
16058
|
+
shouldPull = false;
|
|
16059
|
+
} catch {
|
|
16060
|
+
shouldPull = true;
|
|
16061
|
+
}
|
|
16062
|
+
if (shouldPull) {
|
|
16063
|
+
emit(ScalerEventType.enum["scaler.provisioning"], `pulling image ${matchedLabelSet.image}`);
|
|
16064
|
+
const stream = await this.docker.pull(matchedLabelSet.image);
|
|
16065
|
+
await new Promise((resolve, reject) => {
|
|
16066
|
+
this.docker.modem.followProgress(stream, (err) => {
|
|
16067
|
+
if (err) reject(err);
|
|
16068
|
+
else resolve();
|
|
16069
|
+
});
|
|
16070
|
+
});
|
|
16071
|
+
}
|
|
16072
|
+
emit(ScalerEventType.enum["scaler.provisioning"], "creating container");
|
|
16073
|
+
const normalizedLabelSetStr = normalizeLabelSet(labelSet);
|
|
16074
|
+
const container = await this.docker.createContainer({
|
|
16075
|
+
Image: matchedLabelSet.image,
|
|
16076
|
+
Env: env,
|
|
16077
|
+
Labels: {
|
|
16078
|
+
"kici-managed": "true",
|
|
16079
|
+
"kici-scaler-name": this.name,
|
|
16080
|
+
"kici-agent-id": agentId,
|
|
16081
|
+
"kici-labels": normalizedLabelSetStr,
|
|
16082
|
+
...spawnContext?.boundJobId && { "kici-bound-job-id": spawnContext.boundJobId },
|
|
16083
|
+
...spawnContext?.runId && { "kici-run-id": spawnContext.runId }
|
|
16084
|
+
},
|
|
16085
|
+
HostConfig: {
|
|
16086
|
+
Memory: memoryBytes,
|
|
16087
|
+
NanoCpus: nanoCpus,
|
|
16088
|
+
Binds: binds.length > 0 ? binds : void 0,
|
|
16089
|
+
ExtraHosts: this.extraHosts?.length ? this.extraHosts : void 0,
|
|
16090
|
+
AutoRemove: false
|
|
16091
|
+
},
|
|
16092
|
+
...this.networkIsolation && { NetworkingConfig: { EndpointsConfig: { [ISOLATED_NETWORK_NAME]: {} } } }
|
|
16093
|
+
});
|
|
16094
|
+
if (this.networkIsolation) emit(ScalerEventType.enum["scaler.network"], "configuring network isolation");
|
|
16095
|
+
await container.start();
|
|
16096
|
+
emit(ScalerEventType.enum["scaler.ready"], "container started");
|
|
16097
|
+
if (this.networkIsolation) {
|
|
16098
|
+
const containerIp = (await this.docker.getContainer(container.id).inspect()).NetworkSettings?.Networks?.[ISOLATED_NETWORK_NAME]?.IPAddress;
|
|
16099
|
+
if (containerIp) {
|
|
16100
|
+
this.containerIps.set(managed.id, containerIp);
|
|
16101
|
+
await addIsolationRules(containerIp, ISOLATED_NETWORK_GATEWAY, matchedLabelSet.networkPolicy, "saddr");
|
|
16102
|
+
} else logger$1.warn("Could not determine container IP for nftables rules", { agentId });
|
|
16103
|
+
}
|
|
16104
|
+
try {
|
|
16105
|
+
const capture = await createContainerLogCapture(this.docker, container.id);
|
|
16106
|
+
this.logCaptures.set(managed.id, capture);
|
|
16107
|
+
} catch (err) {
|
|
16108
|
+
logger$1.warn("Failed to create log capture for container", {
|
|
16109
|
+
agentId,
|
|
16110
|
+
error: toErrorMessage(err)
|
|
16111
|
+
});
|
|
16112
|
+
}
|
|
16113
|
+
managed.state = "running";
|
|
16114
|
+
managed.backendRef = container.id;
|
|
16115
|
+
this.containerToManaged.set(container.id, managed.id);
|
|
16116
|
+
emit(ScalerEventType.enum["agent.connecting"], "waiting for agent WS registration");
|
|
16117
|
+
return managed;
|
|
16118
|
+
} catch (err) {
|
|
16119
|
+
const t = this.logCaptures.get(managed.id)?.tail() ?? "";
|
|
16120
|
+
const base = toErrorMessage(err);
|
|
16121
|
+
emit(ScalerEventType.enum["scaler.failed"], t ? `${base}\n--- captured output ---\n${t}` : base);
|
|
16122
|
+
const failedIp = this.containerIps.get(managed.id);
|
|
16123
|
+
if (failedIp) {
|
|
16124
|
+
try {
|
|
16125
|
+
await removeIsolationRules(failedIp);
|
|
16126
|
+
} catch {}
|
|
16127
|
+
this.containerIps.delete(managed.id);
|
|
16128
|
+
}
|
|
16129
|
+
this.agents.delete(managed.id);
|
|
16130
|
+
throw err;
|
|
16131
|
+
}
|
|
16132
|
+
}
|
|
16133
|
+
getScalerContext(agentId) {
|
|
16134
|
+
const managed = this.agents.get(agentId);
|
|
16135
|
+
if (!managed) return void 0;
|
|
16136
|
+
const normalizedTarget = normalizeLabelSet(managed.labelSet);
|
|
16137
|
+
const matchedLabelSet = this._labelSets.find((ls) => normalizeLabelSet(ls.labels) === normalizedTarget);
|
|
16138
|
+
return {
|
|
16139
|
+
backendType: "container",
|
|
16140
|
+
scalerName: this.name,
|
|
16141
|
+
image: matchedLabelSet?.image,
|
|
16142
|
+
imagePullPolicy: matchedLabelSet?.imagePullPolicy ?? "Always",
|
|
16143
|
+
runtime: this.detectedRuntime,
|
|
16144
|
+
resources: matchedLabelSet?.resources ?? this.defaultResources,
|
|
16145
|
+
networkIsolation: this.networkIsolation,
|
|
16146
|
+
volumes: matchedLabelSet?.volumes,
|
|
16147
|
+
extraHosts: this.extraHosts
|
|
16148
|
+
};
|
|
16149
|
+
}
|
|
16150
|
+
async destroy(managedId) {
|
|
16151
|
+
const managed = this.agents.get(managedId);
|
|
16152
|
+
if (!managed) return;
|
|
16153
|
+
managed.state = "destroying";
|
|
16154
|
+
const containerIp = this.containerIps.get(managedId);
|
|
16155
|
+
this.containerIps.delete(managedId);
|
|
16156
|
+
this.containerToManaged.delete(managed.backendRef);
|
|
16157
|
+
this.agents.delete(managedId);
|
|
16158
|
+
if (containerIp && this.networkIsolation) try {
|
|
16159
|
+
await removeIsolationRules(containerIp);
|
|
16160
|
+
} catch {}
|
|
16161
|
+
const capture = this.logCaptures.get(managedId);
|
|
16162
|
+
if (capture) {
|
|
16163
|
+
capture.close();
|
|
16164
|
+
this.logCaptures.delete(managedId);
|
|
16165
|
+
}
|
|
16166
|
+
try {
|
|
16167
|
+
const container = this.docker.getContainer(managed.backendRef);
|
|
16168
|
+
try {
|
|
16169
|
+
await container.stop({ t: 10 });
|
|
16170
|
+
} catch {}
|
|
16171
|
+
try {
|
|
16172
|
+
await container.remove({ force: true });
|
|
16173
|
+
} catch {}
|
|
16174
|
+
} catch {}
|
|
16175
|
+
}
|
|
16176
|
+
/**
|
|
16177
|
+
* Get the LogCapture for a managed agent (used by ScalerManager for log forwarding).
|
|
16178
|
+
*/
|
|
16179
|
+
getLogCapture(managedId) {
|
|
16180
|
+
return this.logCaptures.get(managedId);
|
|
16181
|
+
}
|
|
16182
|
+
async shutdownAll() {
|
|
16183
|
+
const ids = [...this.agents.keys()];
|
|
16184
|
+
await Promise.allSettled(ids.map((id) => this.destroy(id)));
|
|
16185
|
+
}
|
|
16186
|
+
reload(labelSets) {
|
|
16187
|
+
const errors = [];
|
|
16188
|
+
labelSets.forEach((ls, i) => {
|
|
16189
|
+
if (!ls.image) errors.push(`Label set [${i}] requires an 'image' field for container backend`);
|
|
16190
|
+
});
|
|
16191
|
+
if (errors.length > 0) return {
|
|
16192
|
+
valid: false,
|
|
16193
|
+
errors
|
|
16194
|
+
};
|
|
16195
|
+
this._labelSets = labelSets;
|
|
16196
|
+
return { valid: true };
|
|
16197
|
+
}
|
|
16198
|
+
/**
|
|
16199
|
+
* Clean up orphaned kici-managed containers on startup.
|
|
16200
|
+
* Finds containers with the `kici-managed=true` label and removes them.
|
|
16201
|
+
* Returns the count of cleaned containers.
|
|
16202
|
+
*/
|
|
16203
|
+
async cleanupOrphans() {
|
|
16204
|
+
const containers = await this.docker.listContainers({
|
|
16205
|
+
all: true,
|
|
16206
|
+
filters: { label: ["kici-managed=true"] }
|
|
16207
|
+
});
|
|
16208
|
+
let cleaned = 0;
|
|
16209
|
+
for (const info of containers) {
|
|
16210
|
+
const container = this.docker.getContainer(info.Id);
|
|
16211
|
+
try {
|
|
16212
|
+
await container.stop({ t: 5 });
|
|
16213
|
+
} catch {}
|
|
16214
|
+
try {
|
|
16215
|
+
await container.remove({ force: true });
|
|
16216
|
+
cleaned++;
|
|
16217
|
+
} catch {}
|
|
16218
|
+
}
|
|
16219
|
+
return cleaned;
|
|
16220
|
+
}
|
|
16221
|
+
};
|
|
16222
|
+
//#endregion
|
|
16223
|
+
//#region src/scaler/ip-allocator.ts
|
|
16224
|
+
/**
|
|
16225
|
+
* Parse an IPv4 address string to a 32-bit unsigned integer.
|
|
16226
|
+
*/
|
|
16227
|
+
function ipToNumber(ip) {
|
|
16228
|
+
const parts = ip.split(".");
|
|
16229
|
+
if (parts.length !== 4) throw new Error(`Invalid IP address: "${ip}"`);
|
|
16230
|
+
let result = 0;
|
|
16231
|
+
for (const part of parts) {
|
|
16232
|
+
const octet = parseInt(part, 10);
|
|
16233
|
+
if (isNaN(octet) || octet < 0 || octet > 255) throw new Error(`Invalid IP address: "${ip}"`);
|
|
16234
|
+
result = result << 8 | octet;
|
|
16235
|
+
}
|
|
16236
|
+
return result >>> 0;
|
|
16237
|
+
}
|
|
16238
|
+
/**
|
|
16239
|
+
* Convert a 32-bit unsigned integer back to an IPv4 address string.
|
|
16240
|
+
*/
|
|
16241
|
+
function numberToIp(n) {
|
|
16242
|
+
const unsigned = n >>> 0;
|
|
16243
|
+
return [
|
|
16244
|
+
unsigned >>> 24 & 255,
|
|
16245
|
+
unsigned >>> 16 & 255,
|
|
16246
|
+
unsigned >>> 8 & 255,
|
|
16247
|
+
unsigned & 255
|
|
16248
|
+
].join(".");
|
|
16249
|
+
}
|
|
16250
|
+
/**
|
|
16251
|
+
* Parse a CIDR notation string into a CidrRange.
|
|
16252
|
+
*
|
|
16253
|
+
* @example parseCidr('10.0.0.0/24')
|
|
16254
|
+
* // => { networkAddress: 167772160, prefixLength: 24, startIp: 167772161, endIp: 167772414 }
|
|
16255
|
+
*/
|
|
16256
|
+
function parseCidr(cidr) {
|
|
16257
|
+
const parts = cidr.split("/");
|
|
16258
|
+
if (parts.length !== 2) throw new Error(`Invalid CIDR notation: "${cidr}"`);
|
|
16259
|
+
const ip = parts[0];
|
|
16260
|
+
const prefixLength = parseInt(parts[1], 10);
|
|
16261
|
+
if (isNaN(prefixLength) || prefixLength < 0 || prefixLength > 32) throw new Error(`Invalid CIDR prefix length: ${parts[1]}`);
|
|
16262
|
+
const networkAddress = ipToNumber(ip);
|
|
16263
|
+
const broadcastAddress = (networkAddress | (1 << 32 - prefixLength) - 1) >>> 0;
|
|
16264
|
+
return {
|
|
16265
|
+
networkAddress,
|
|
16266
|
+
prefixLength,
|
|
16267
|
+
startIp: networkAddress + 1 >>> 0,
|
|
16268
|
+
endIp: broadcastAddress - 1 >>> 0
|
|
16269
|
+
};
|
|
16270
|
+
}
|
|
16271
|
+
/**
|
|
16272
|
+
* Generate a deterministic MAC address from an IP address.
|
|
16273
|
+
*
|
|
16274
|
+
* Uses the locally-administered unicast prefix 06:00:AC followed by
|
|
16275
|
+
* the last 3 octets of the IP address encoded as hex.
|
|
16276
|
+
*
|
|
16277
|
+
* @example generateMac('10.0.0.5') // => '06:00:AC:00:00:05'
|
|
16278
|
+
* @example generateMac('10.0.1.42') // => '06:00:AC:00:01:2A'
|
|
16279
|
+
*/
|
|
16280
|
+
function generateMac(ip) {
|
|
16281
|
+
const parts = ip.split(".");
|
|
16282
|
+
if (parts.length !== 4) throw new Error(`Invalid IP address for MAC generation: "${ip}"`);
|
|
16283
|
+
const hex = (n) => n.toString(16).toUpperCase().padStart(2, "0");
|
|
16284
|
+
return `06:00:AC:${hex(parseInt(parts[1], 10))}:${hex(parseInt(parts[2], 10))}:${hex(parseInt(parts[3], 10))}`;
|
|
16285
|
+
}
|
|
16286
|
+
/**
|
|
16287
|
+
* Generate a TAP device name from a VM ID.
|
|
16288
|
+
* Linux interface names are limited to 15 characters (IFNAMSIZ).
|
|
16289
|
+
* Format: kici-XXXXXXXX (5 + 8 = 13 chars, well within limit).
|
|
16290
|
+
*
|
|
16291
|
+
* Uses the LAST 8 characters of the vmId to avoid collisions.
|
|
16292
|
+
* VM IDs have format "scaler-firecracker-XXXXXXXX" where the unique
|
|
16293
|
+
* random suffix is at the end. Using slice(0, 8) would always produce
|
|
16294
|
+
* "scaler-f" for all Firecracker VMs.
|
|
16295
|
+
*/
|
|
16296
|
+
function generateTapName(vmId) {
|
|
16297
|
+
return `kici-${vmId.slice(-8)}`;
|
|
16298
|
+
}
|
|
16299
|
+
/**
|
|
16300
|
+
* Walk the CIDR range and call `pickIp` with each candidate IP (as a
|
|
16301
|
+
* dotted-quad string). The first IP for which `pickIp` returns a truthy
|
|
16302
|
+
* result is allocated and returned. Skips the gateway. Throws if the
|
|
16303
|
+
* range is exhausted. Shared by both `DbIpAllocator` and
|
|
16304
|
+
* `InMemoryIpAllocator` — keeps the "scan + skip gateway + skip
|
|
16305
|
+
* already-allocated" loop in one place.
|
|
16306
|
+
*/
|
|
16307
|
+
function findFirstFreeIp(range, gatewayNum, isAllocated) {
|
|
16308
|
+
for (let n = range.startIp; n <= range.endIp; n++) {
|
|
16309
|
+
const unsigned = n >>> 0;
|
|
16310
|
+
if (unsigned === gatewayNum) continue;
|
|
16311
|
+
const candidateIp = numberToIp(unsigned);
|
|
16312
|
+
if (isAllocated(candidateIp)) continue;
|
|
16313
|
+
return candidateIp;
|
|
16314
|
+
}
|
|
16315
|
+
throw new Error(`IP pool exhausted: no available addresses in ${numberToIp(range.startIp)}-${numberToIp(range.endIp)}`);
|
|
16316
|
+
}
|
|
16317
|
+
/**
|
|
16318
|
+
* In-memory CIDR IP allocator.
|
|
16319
|
+
*
|
|
16320
|
+
* Used by worker peers where the cluster Postgres is not reachable. State
|
|
16321
|
+
* lives in a `Map<vmId, IpAllocationRecord>` for the lifetime of the
|
|
16322
|
+
* orch process; on restart the allocator starts empty and the
|
|
16323
|
+
* firecracker-backend's existing `cleanupOrphans()` Pass-2/Pass-3 walks
|
|
16324
|
+
* the jailer chroot tree + host TAP devices (matching `VM_TAP_PATTERN`)
|
|
16325
|
+
* to reap any leftover state. No persistence and no chroot recovery
|
|
16326
|
+
* inside the allocator — the filesystem is the source of truth, and the
|
|
16327
|
+
* cleanup pass already consults it.
|
|
16328
|
+
*
|
|
16329
|
+
* Each orch has its own bridge with its own subnet, so per-process
|
|
16330
|
+
* state is by design. No locking needed: every method is synchronous on
|
|
16331
|
+
* the in-memory Map and only awaits to satisfy the async interface.
|
|
16332
|
+
*/
|
|
16333
|
+
var InMemoryIpAllocator = class {
|
|
16334
|
+
range;
|
|
16335
|
+
gateway;
|
|
16336
|
+
gatewayNum;
|
|
16337
|
+
netmask;
|
|
16338
|
+
byVmId = /* @__PURE__ */ new Map();
|
|
16339
|
+
allocatedIps = /* @__PURE__ */ new Set();
|
|
16340
|
+
constructor(opts) {
|
|
16341
|
+
this.range = parseCidr(opts.cidr);
|
|
16342
|
+
this.gateway = opts.gateway;
|
|
16343
|
+
this.gatewayNum = ipToNumber(opts.gateway);
|
|
16344
|
+
this.netmask = opts.netmask;
|
|
16345
|
+
}
|
|
16346
|
+
async allocate(vmId, scalerName) {
|
|
16347
|
+
const candidateIp = findFirstFreeIp(this.range, this.gatewayNum, (ip) => this.allocatedIps.has(ip));
|
|
16348
|
+
const mac = generateMac(candidateIp);
|
|
16349
|
+
const tapDevice = generateTapName(vmId);
|
|
16350
|
+
this.byVmId.set(vmId, {
|
|
16351
|
+
ip: candidateIp,
|
|
16352
|
+
vm_id: vmId,
|
|
16353
|
+
scaler_name: scalerName,
|
|
16354
|
+
tap_device: tapDevice,
|
|
16355
|
+
mac_address: mac
|
|
16356
|
+
});
|
|
16357
|
+
this.allocatedIps.add(candidateIp);
|
|
16358
|
+
return {
|
|
16359
|
+
ip: candidateIp,
|
|
16360
|
+
gateway: this.gateway,
|
|
16361
|
+
netmask: this.netmask,
|
|
16362
|
+
mac,
|
|
16363
|
+
tapDevice
|
|
16364
|
+
};
|
|
16365
|
+
}
|
|
16366
|
+
async release(vmId) {
|
|
16367
|
+
const existing = this.byVmId.get(vmId);
|
|
16368
|
+
if (existing) {
|
|
16369
|
+
this.allocatedIps.delete(existing.ip);
|
|
16370
|
+
this.byVmId.delete(vmId);
|
|
16371
|
+
}
|
|
16372
|
+
}
|
|
16373
|
+
async releaseByIp(ip) {
|
|
16374
|
+
for (const [vmId, record] of this.byVmId) if (record.ip === ip) {
|
|
16375
|
+
this.byVmId.delete(vmId);
|
|
16376
|
+
this.allocatedIps.delete(ip);
|
|
16377
|
+
return;
|
|
16378
|
+
}
|
|
16379
|
+
}
|
|
16380
|
+
async getAllocations() {
|
|
16381
|
+
return [...this.byVmId.values()];
|
|
16382
|
+
}
|
|
16383
|
+
async getAllocationForVm(vmId) {
|
|
16384
|
+
return this.byVmId.get(vmId) ?? null;
|
|
16385
|
+
}
|
|
16386
|
+
};
|
|
16387
|
+
//#endregion
|
|
16388
|
+
//#region src/scaler/reap-orphans.ts
|
|
16389
|
+
/**
|
|
16390
|
+
* Standalone orphan reaper.
|
|
16391
|
+
*
|
|
16392
|
+
* Reconstructs scaler backends from a loaded ScalerConfig and runs their
|
|
16393
|
+
* liveness-driven orphan cleanup WITHOUT a running orchestrator and (for
|
|
16394
|
+
* Firecracker) WITHOUT a database — an InMemoryIpAllocator stands in for the
|
|
16395
|
+
* coordinator's DbIpAllocator, exactly as worker mode does. Used by the
|
|
16396
|
+
* `kici-admin scaler reap-orphans` command and by the startup disk-space guard
|
|
16397
|
+
* to recover a node whose data disk is full and whose orchestrator therefore
|
|
16398
|
+
* cannot start.
|
|
16399
|
+
*
|
|
16400
|
+
* Writes nothing to disk: progress is the caller's concern (counts are
|
|
16401
|
+
* returned). removeChrootDir already chowns-before-rm on rootless (requireSudo)
|
|
16402
|
+
* nodes, so the reap frees disk owned by the jailer subuid.
|
|
16403
|
+
*/
|
|
16404
|
+
/**
|
|
16405
|
+
* Reap orphan Firecracker resources (dead-VM chroots + TAPs) for every
|
|
16406
|
+
* `firecracker` scaler in the config. Safe to run alongside a live
|
|
16407
|
+
* orchestrator: the reap is liveness-driven and only touches dead VMs.
|
|
16408
|
+
*/
|
|
16409
|
+
async function reapFirecrackerOrphans(scalerConfig) {
|
|
16410
|
+
const counts = {};
|
|
16411
|
+
const fcNet = scalerConfig.firecracker;
|
|
16412
|
+
const cidr = fcNet?.cidr ?? "10.0.0.0/24";
|
|
16413
|
+
const bridgeName = fcNet?.bridgeName ?? "kici-br0";
|
|
16414
|
+
const gateway = fcNet?.gateway ?? "10.0.0.1";
|
|
16415
|
+
const netmask = fcNet?.netmask ?? "255.255.255.0";
|
|
16416
|
+
const table = fcNet?.table ?? "kici";
|
|
16417
|
+
for (const s of scalerConfig.scalers) {
|
|
16418
|
+
if (s.type !== "firecracker") continue;
|
|
16419
|
+
const backend = new FirecrackerScalerBackend({
|
|
16420
|
+
name: s.name,
|
|
16421
|
+
labelSets: s.labelSets,
|
|
16422
|
+
maxAgents: s.maxAgents,
|
|
16423
|
+
ipAllocator: new InMemoryIpAllocator({
|
|
16424
|
+
cidr,
|
|
16425
|
+
gateway,
|
|
16426
|
+
netmask
|
|
16427
|
+
}),
|
|
16428
|
+
firecrackerPath: s.firecrackerPath,
|
|
16429
|
+
jailerPath: s.jailerPath,
|
|
16430
|
+
kernelPath: s.kernelPath,
|
|
16431
|
+
chrootBaseDir: s.chrootBaseDir,
|
|
16432
|
+
uid: s.uid,
|
|
16433
|
+
gid: s.gid,
|
|
16434
|
+
vcpuCount: s.vcpuCount,
|
|
16435
|
+
memSizeMib: s.memSizeMib,
|
|
16436
|
+
bridgeName,
|
|
16437
|
+
cidr,
|
|
16438
|
+
gateway,
|
|
16439
|
+
netmask,
|
|
16440
|
+
table,
|
|
16441
|
+
roles: s.roles,
|
|
16442
|
+
requireSudo: s.requireSudo
|
|
16443
|
+
});
|
|
16444
|
+
counts[s.name] = await backend.cleanupOrphans();
|
|
16445
|
+
}
|
|
16446
|
+
return counts;
|
|
16447
|
+
}
|
|
16448
|
+
/**
|
|
16449
|
+
* Reap orphan container resources for every `container` scaler. Unlike the FC
|
|
16450
|
+
* reap, container cleanup is unconditional (it removes every kici-managed
|
|
16451
|
+
* container), so callers MUST ensure the orchestrator is NOT running before
|
|
16452
|
+
* invoking this — otherwise live agents are killed. The CLI command enforces
|
|
16453
|
+
* this via the /health gate.
|
|
16454
|
+
*/
|
|
16455
|
+
async function reapContainerOrphans(scalerConfig) {
|
|
16456
|
+
const counts = {};
|
|
16457
|
+
for (const s of scalerConfig.scalers) {
|
|
16458
|
+
if (s.type !== "container") continue;
|
|
16459
|
+
const backend = await ContainerScalerBackend.create({
|
|
16460
|
+
name: s.name,
|
|
16461
|
+
labelSets: s.labelSets,
|
|
16462
|
+
maxAgents: s.maxAgents,
|
|
16463
|
+
host: s.host,
|
|
16464
|
+
socketPath: s.socketPath,
|
|
16465
|
+
runtime: s.runtime
|
|
16466
|
+
});
|
|
16467
|
+
counts[s.name] = await backend.cleanupOrphans();
|
|
16468
|
+
}
|
|
16469
|
+
return counts;
|
|
16470
|
+
}
|
|
16471
|
+
/**
|
|
16472
|
+
* Reap orphans for all supported backends. Firecracker is always reaped
|
|
16473
|
+
* (liveness-driven, safe). Containers are reaped only when includeContainers
|
|
16474
|
+
* is set (caller has confirmed the orchestrator is down).
|
|
16475
|
+
*/
|
|
16476
|
+
async function reapAllOrphans(opts) {
|
|
16477
|
+
const fc = await reapFirecrackerOrphans(opts.scalerConfig);
|
|
16478
|
+
if (!opts.includeContainers) return fc;
|
|
16479
|
+
const ct = await reapContainerOrphans(opts.scalerConfig);
|
|
16480
|
+
return {
|
|
16481
|
+
...fc,
|
|
16482
|
+
...ct
|
|
16483
|
+
};
|
|
16484
|
+
}
|
|
16485
|
+
//#endregion
|
|
16486
|
+
//#region src/cli/commands/scaler.ts
|
|
16487
|
+
/** Probe the local orchestrator /health endpoint. Healthy => it reaps itself. */
|
|
16488
|
+
async function isOrchestratorHealthy(port, basePath) {
|
|
16489
|
+
const url = `http://127.0.0.1:${port}${basePath === "/" ? "" : basePath.replace(/\/$/, "")}/health`;
|
|
16490
|
+
try {
|
|
16491
|
+
return (await fetch(url, { signal: AbortSignal.timeout(2e3) })).status === 200;
|
|
16492
|
+
} catch {
|
|
16493
|
+
return false;
|
|
16494
|
+
}
|
|
16495
|
+
}
|
|
16496
|
+
function registerScalerCommands(program) {
|
|
16497
|
+
program.command("scaler").description("Scaler maintenance (local, no orchestrator)").command("reap-orphans").description("Free leaked Firecracker/container resources without a running orchestrator").option("--config <path>", "Path to the orchestrator config (default: KICI_CONFIG or /etc/kici/orchestrator.yaml)").option("--force", "Reap even if the local orchestrator reports healthy", false).option("--json", "Emit machine-readable JSON counts", false).action(async (opts) => {
|
|
16498
|
+
try {
|
|
16499
|
+
const local = await loadLocalConfig(opts.config);
|
|
16500
|
+
const scalerConfigPath = local.scaler?.configPath ?? process.env.KICI_SCALER_CONFIG_PATH;
|
|
16501
|
+
const scalerConfigDir = local.scaler?.configDir ?? process.env.KICI_SCALER_CONFIG_DIR;
|
|
16502
|
+
if (!scalerConfigPath) {
|
|
16503
|
+
console.error("Error: no scaler config path found (set scaler.configPath in the orchestrator config, KICI_SCALER_CONFIG_PATH in the environment, or pass --config)");
|
|
16504
|
+
process.exit(1);
|
|
16505
|
+
}
|
|
16506
|
+
const scalerConfig = await loadScalerConfig(scalerConfigPath, scalerConfigDir);
|
|
16507
|
+
const port = local.server?.port ?? Number(process.env.KICI_PORT ?? "4000");
|
|
16508
|
+
const basePath = local.server?.basePath ?? process.env.KICI_BASE_PATH ?? "/";
|
|
16509
|
+
if ((opts.force ? false : await isOrchestratorHealthy(port, basePath)) && !opts.force) {
|
|
16510
|
+
const msg = "orchestrator is up and reaps orphans itself; pass --force to reap anyway";
|
|
16511
|
+
if (opts.json) console.log(JSON.stringify({
|
|
16512
|
+
skipped: true,
|
|
16513
|
+
reason: "orchestrator-healthy",
|
|
16514
|
+
counts: {}
|
|
16515
|
+
}));
|
|
16516
|
+
else console.log(msg);
|
|
16517
|
+
process.exit(0);
|
|
16518
|
+
}
|
|
16519
|
+
const counts = await reapAllOrphans({
|
|
16520
|
+
scalerConfig,
|
|
16521
|
+
includeContainers: true
|
|
16522
|
+
});
|
|
16523
|
+
const total = Object.values(counts).reduce((a, b) => a + b, 0);
|
|
16524
|
+
if (opts.json) console.log(JSON.stringify({
|
|
16525
|
+
skipped: false,
|
|
16526
|
+
counts,
|
|
16527
|
+
total
|
|
16528
|
+
}));
|
|
16529
|
+
else if (total === 0) console.log("No orphan resources found.");
|
|
16530
|
+
else {
|
|
16531
|
+
console.log(`Reaped ${total} orphan resource(s):`);
|
|
16532
|
+
for (const [name, n] of Object.entries(counts)) console.log(` ${name}: ${n}`);
|
|
16533
|
+
}
|
|
16534
|
+
} catch (err) {
|
|
16535
|
+
console.error(`Error: ${toErrorMessage(err)}`);
|
|
16536
|
+
process.exit(1);
|
|
16537
|
+
}
|
|
16538
|
+
});
|
|
16539
|
+
}
|
|
16540
|
+
//#endregion
|
|
16541
|
+
//#region src/cluster/join-client.ts
|
|
16542
|
+
/**
|
|
16543
|
+
* Join client for zero-knowledge cluster bootstrap.
|
|
16544
|
+
*
|
|
16545
|
+
* Provides the joiner-side logic for `kici-admin join`:
|
|
16546
|
+
* 1. Connect to Platform relay (WS) or direct peer (HTTP POST)
|
|
16547
|
+
* 2. Send join.request with the join token
|
|
16548
|
+
* 3. Receive join.response with encrypted config bundle
|
|
16549
|
+
* 4. Decrypt bundle using token-derived AES-256-GCM key
|
|
16550
|
+
* 5. Write decrypted config as local YAML file
|
|
16551
|
+
*
|
|
16552
|
+
* The token carries routing info (cleartext for Platform relay) and a secret
|
|
16553
|
+
* (used for HKDF key derivation). Only the joiner and the token creator
|
|
16554
|
+
* can derive the encryption key -- the Platform relay sees only ciphertext.
|
|
16555
|
+
*/
|
|
16556
|
+
const logger = createLogger({ prefix: "join-client" });
|
|
16557
|
+
/**
|
|
16558
|
+
* Decrypt a base64-encoded encrypted config bundle using a derived encryption key.
|
|
16559
|
+
*/
|
|
16560
|
+
function decryptAndParseBundle(encryptedB64, encryptionKey) {
|
|
16561
|
+
return decryptBundle(Buffer.from(encryptedB64, "base64"), encryptionKey);
|
|
16562
|
+
}
|
|
16563
|
+
/**
|
|
16564
|
+
* Build a JoinLocalConfig from a decrypted ConfigBundle.
|
|
16565
|
+
* Maps bundle fields to the structure expected by loadLocalConfig().
|
|
16566
|
+
*/
|
|
16567
|
+
function buildLocalConfig(bundle) {
|
|
16568
|
+
const config = { database: { url: bundle.databaseUrl } };
|
|
16569
|
+
if (bundle.storage) config.storage = bundle.storage;
|
|
16570
|
+
if (bundle.secretKey) config.secrets = { key: bundle.secretKey };
|
|
16571
|
+
return config;
|
|
16572
|
+
}
|
|
16573
|
+
/**
|
|
16574
|
+
* Write a local config object to a YAML file.
|
|
16575
|
+
*/
|
|
16576
|
+
async function writeConfigFile(path, config) {
|
|
16577
|
+
await writeFile(path, stringify(config), "utf-8");
|
|
16578
|
+
}
|
|
16579
|
+
var JoinClient = class {
|
|
16580
|
+
constructor(options) {
|
|
16581
|
+
this.options = options;
|
|
16582
|
+
if (!options.platformUrl && !options.peerUrl) throw new Error("Either --platform or --peer must be specified");
|
|
16583
|
+
if (options.platformUrl && options.peerUrl) throw new Error("--platform and --peer are mutually exclusive");
|
|
16584
|
+
}
|
|
16585
|
+
/**
|
|
16586
|
+
* Execute the join flow:
|
|
16587
|
+
* 1. Send join.request with token to Platform relay or direct peer
|
|
16588
|
+
* 2. Receive join.response with encrypted config bundle
|
|
16589
|
+
* 3. Decrypt bundle using token-derived key
|
|
16590
|
+
* 4. Write local config YAML
|
|
16591
|
+
*/
|
|
16592
|
+
async join() {
|
|
16593
|
+
const request = {
|
|
16594
|
+
type: "join.request",
|
|
16595
|
+
token: this.options.token
|
|
16596
|
+
};
|
|
16597
|
+
logger.info("Sending join request...");
|
|
16598
|
+
const response = this.options.platformUrl ? await this.joinViaPlatform(request) : await this.joinViaPeer(request);
|
|
16599
|
+
if (!response.success) throw new Error(`Join rejected: ${response.error ?? "unknown error"}`);
|
|
16600
|
+
if (!response.encryptedBundle) throw new Error("Join response missing encrypted bundle");
|
|
16601
|
+
const parsed = parseToken(this.options.token);
|
|
16602
|
+
const keys = deriveKeys(Buffer.from(parsed.secretHex, "hex"));
|
|
16603
|
+
const bundle = decryptAndParseBundle(response.encryptedBundle, keys.encryptionKey);
|
|
16604
|
+
logger.info("Join successful, writing config...", { clusterId: bundle.clusterId });
|
|
16605
|
+
const localConfig = buildLocalConfig(bundle);
|
|
16606
|
+
const configPath = this.options.configPath ?? "./kici-orchestrator.yaml";
|
|
16607
|
+
await writeConfigFile(configPath, localConfig);
|
|
16608
|
+
logger.info(`Config written to ${configPath}`);
|
|
16609
|
+
logger.info("Start the orchestrator with: kici-admin orchestrator start");
|
|
16610
|
+
}
|
|
16611
|
+
/**
|
|
16612
|
+
* Join via Platform relay: connect WS, authenticate, send join.request, receive join.response.
|
|
16613
|
+
*/
|
|
16614
|
+
async joinViaPlatform(request) {
|
|
16615
|
+
const url = this.options.platformUrl;
|
|
16616
|
+
const apiKey = this.options.apiKey;
|
|
16617
|
+
if (!apiKey) throw new Error("--api-key is required for Platform relay mode");
|
|
16618
|
+
const { default: WebSocket } = await import("ws");
|
|
16619
|
+
return new Promise((resolve, reject) => {
|
|
16620
|
+
const ws = new WebSocket(url, {
|
|
16621
|
+
maxPayload: WS_MAX_PAYLOAD_BYTES,
|
|
16622
|
+
perMessageDeflate: {
|
|
16623
|
+
concurrencyLimit: 10,
|
|
16624
|
+
threshold: 128
|
|
16625
|
+
}
|
|
13360
16626
|
});
|
|
13361
16627
|
let authenticated = false;
|
|
13362
16628
|
let resolved = false;
|
|
@@ -13452,54 +16718,69 @@ function registerJoinCommand(program) {
|
|
|
13452
16718
|
* --url / -u Orchestrator URL (env: KICI_ADMIN_URL, default: http://localhost:8080)
|
|
13453
16719
|
* --token / -t Admin API token (env: KICI_ADMIN_TOKEN, required)
|
|
13454
16720
|
*/
|
|
13455
|
-
const program = new Command();
|
|
13456
|
-
program.name("kici-admin").description("KiCI orchestrator admin CLI for managing config, secrets, and tokens").version("0.0.1", "-V, --cli-version").option("-u, --url <url>", "Orchestrator URL", process.env.KICI_ADMIN_URL ?? "http://localhost:8080").option("-t, --token <token>", "Admin API token", process.env.KICI_ADMIN_TOKEN);
|
|
13457
16721
|
/**
|
|
13458
|
-
*
|
|
13459
|
-
*
|
|
16722
|
+
* Build the kici-admin Commander program with every command group registered.
|
|
16723
|
+
* Exported so the surface registry can walk the real command tree without
|
|
16724
|
+
* parsing argv (no action runs during a tree walk, so `getClient` is
|
|
16725
|
+
* constructed but never invoked).
|
|
13460
16726
|
*/
|
|
13461
|
-
function
|
|
13462
|
-
const
|
|
13463
|
-
|
|
13464
|
-
|
|
13465
|
-
|
|
16727
|
+
function buildProgram() {
|
|
16728
|
+
const program = new Command();
|
|
16729
|
+
program.name("kici-admin").description("KiCI orchestrator admin CLI for managing config, secrets, and tokens").version("0.0.1", "-V, --cli-version").option("-u, --url <url>", "Orchestrator URL", process.env.KICI_ADMIN_URL ?? "http://localhost:8080").option("-t, --token <token>", "Admin API token", process.env.KICI_ADMIN_TOKEN);
|
|
16730
|
+
/**
|
|
16731
|
+
* Create an AdminApiClient from global options.
|
|
16732
|
+
* Called lazily by each command to allow --help without requiring --token.
|
|
16733
|
+
*/
|
|
16734
|
+
function getClient() {
|
|
16735
|
+
const opts = program.opts();
|
|
16736
|
+
if (!opts.token) {
|
|
16737
|
+
console.error("Error: --token or KICI_ADMIN_TOKEN is required");
|
|
16738
|
+
process.exit(1);
|
|
16739
|
+
}
|
|
16740
|
+
return new AdminApiClient(opts.url, opts.token);
|
|
13466
16741
|
}
|
|
13467
|
-
|
|
13468
|
-
|
|
13469
|
-
|
|
13470
|
-
|
|
13471
|
-
|
|
13472
|
-
|
|
13473
|
-
|
|
13474
|
-
|
|
13475
|
-
|
|
13476
|
-
|
|
13477
|
-
|
|
13478
|
-
|
|
13479
|
-
|
|
13480
|
-
|
|
13481
|
-
|
|
13482
|
-
|
|
13483
|
-
|
|
13484
|
-
|
|
13485
|
-
|
|
13486
|
-
|
|
13487
|
-
registerAgentServiceCommands(program);
|
|
13488
|
-
registerPeerCommands(program, getClient);
|
|
13489
|
-
registerOrgSettingsCommands(program, getClient);
|
|
13490
|
-
registerClusterNameCommands(program, getClient);
|
|
13491
|
-
registerMaintenanceCommands(program, getClient);
|
|
13492
|
-
registerEnvironmentCommands(program, getClient);
|
|
13493
|
-
registerVariableCommands(program, getClient);
|
|
13494
|
-
registerQueueCommands(program, getClient);
|
|
13495
|
-
registerExecutionCommands(program, getClient);
|
|
13496
|
-
registerRegistrationCommands(program, getClient);
|
|
13497
|
-
registerEventCommands(program, getClient);
|
|
13498
|
-
registerEventDlqCommands(program, getClient);
|
|
13499
|
-
registerColdStoreCommands(program, getClient);
|
|
13500
|
-
registerJoinCommand(program);
|
|
13501
|
-
program
|
|
13502
|
-
|
|
13503
|
-
|
|
16742
|
+
registerSecretCommands(program, getClient);
|
|
16743
|
+
registerRotateCommand(program, getClient);
|
|
16744
|
+
registerAuditCommands(program, getClient);
|
|
16745
|
+
registerTokenCommands(program, getClient);
|
|
16746
|
+
registerAgentCommands(program, getClient);
|
|
16747
|
+
registerApiKeyCommands(program, getClient);
|
|
16748
|
+
registerConfigCommands(program, getClient);
|
|
16749
|
+
registerDbCommands(program, getClient);
|
|
16750
|
+
registerSourceCommands(program, getClient);
|
|
16751
|
+
registerWorkflowCommands(program, getClient);
|
|
16752
|
+
registerRunsCommands(program, getClient);
|
|
16753
|
+
registerEventLogCommands(program, getClient);
|
|
16754
|
+
registerAccessLogCommands(program, getClient);
|
|
16755
|
+
registerBackendCommands(program, getClient);
|
|
16756
|
+
registerDiagnoseCommand(program, getClient);
|
|
16757
|
+
registerDebugBundleCommand(program, getClient);
|
|
16758
|
+
registerInspectBundleCommand(program);
|
|
16759
|
+
registerOrchestratorServiceCommands(program);
|
|
16760
|
+
registerFirecrackerCommands(program);
|
|
16761
|
+
registerScalerCommands(program);
|
|
16762
|
+
registerAgentServiceCommands(program);
|
|
16763
|
+
registerPeerCommands(program, getClient);
|
|
16764
|
+
registerOrgSettingsCommands(program, getClient);
|
|
16765
|
+
registerClusterNameCommands(program, getClient);
|
|
16766
|
+
registerMaintenanceCommands(program, getClient);
|
|
16767
|
+
registerEnvironmentCommands(program, getClient);
|
|
16768
|
+
registerVariableCommands(program, getClient);
|
|
16769
|
+
registerQueueCommands(program, getClient);
|
|
16770
|
+
registerExecutionCommands(program, getClient);
|
|
16771
|
+
registerRegistrationCommands(program, getClient);
|
|
16772
|
+
registerEventCommands(program, getClient);
|
|
16773
|
+
registerEventDlqCommands(program, getClient);
|
|
16774
|
+
registerColdStoreCommands(program, getClient);
|
|
16775
|
+
registerJoinCommand(program);
|
|
16776
|
+
return program;
|
|
16777
|
+
}
|
|
16778
|
+
/** Build the program and parse argv — the bin-shim entry point. */
|
|
16779
|
+
function runCli(argv = process.argv) {
|
|
16780
|
+
buildProgram().parse(argv);
|
|
16781
|
+
}
|
|
16782
|
+
if (resolve(process.argv[1] ?? "") === fileURLToPath(import.meta.url)) runCli();
|
|
16783
|
+
//#endregion
|
|
16784
|
+
export { buildProgram, runCli };
|
|
13504
16785
|
|
|
13505
16786
|
//# sourceMappingURL=cli.js.map
|