@stacksjs/ts-cloud 0.7.63 → 0.7.65
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/cli.js +1556 -1284
- package/dist/{chunk-0zbqbrqg.js → chunk-0z7ry8fm.js} +889 -89
- package/dist/{chunk-t092y37p.js → chunk-267f6qd0.js} +525 -145
- package/dist/{chunk-zvk0f89t.js → chunk-28vh1h91.js} +1 -1
- package/dist/{chunk-tq46tnxv.js → chunk-ecsyc3p0.js} +4 -1
- package/dist/{chunk-rxhqny6z.js → chunk-zcvdzvmz.js} +2 -2
- package/dist/deploy/dashboard-data-server.d.ts +27 -1
- package/dist/deploy/index.js +4 -4
- package/dist/deploy/telemetry-collection.d.ts +2 -0
- package/dist/drivers/hetzner/client.d.ts +84 -2
- package/dist/drivers/hetzner/host-optimization.d.ts +33 -0
- package/dist/drivers/hetzner/monitoring.d.ts +46 -0
- package/dist/drivers/hetzner/resize-remote.d.ts +38 -0
- package/dist/drivers/hetzner/resize-state.d.ts +24 -0
- package/dist/drivers/hetzner/resize.d.ts +86 -0
- package/dist/drivers/index.d.ts +8 -0
- package/dist/drivers/index.js +35 -3
- package/dist/drivers/shared/compute-deploy.d.ts +15 -0
- package/dist/drivers/shared/ufw.d.ts +4 -2
- package/dist/index.js +4 -4
- package/dist/security/pre-deploy-scanner.test.d.ts +1 -0
- package/dist/ui/access-denied.html +2 -2
- package/dist/ui/account/automation.html +4 -4
- package/dist/ui/account/security.html +2 -2
- package/dist/ui/applications/compose.html +4 -4
- package/dist/ui/applications/new.html +2 -2
- package/dist/ui/data/backups.html +4 -4
- package/dist/ui/data/services.html +4 -4
- package/dist/ui/data/volumes.html +4 -4
- package/dist/ui/index.html +12 -7
- package/dist/ui/integrations.html +2 -2
- package/dist/ui/operations/alerts.html +4 -4
- package/dist/ui/operations/configuration.html +4 -4
- package/dist/ui/operations/jobs.html +4 -4
- package/dist/ui/operations/maintenance.html +3 -3
- package/dist/ui/operations/observability.html +13 -7
- package/dist/ui/operations/previews.html +4 -4
- package/dist/ui/operations/queue.html +4 -4
- package/dist/ui/operations/regions.html +4 -4
- package/dist/ui/operations/releases.html +4 -4
- package/dist/ui/operations/workloads.html +4 -4
- package/dist/ui/security.html +2 -2
- package/dist/ui/server/actions.html +3 -3
- package/dist/ui/server/activity.html +2 -2
- package/dist/ui/server/capacity.html +4 -4
- package/dist/ui/server/database.html +4 -4
- package/dist/ui/server/deployments.html +4 -4
- package/dist/ui/server/firewall.html +3 -3
- package/dist/ui/server/fleet.html +4 -4
- package/dist/ui/server/logs.html +4 -4
- package/dist/ui/server/metrics.html +203 -3
- package/dist/ui/server/services.html +2 -2
- package/dist/ui/server/sites.html +102 -6
- package/dist/ui/server/ssh-keys.html +4 -4
- package/dist/ui/server/team.html +4 -4
- package/dist/ui/server/terminal.html +2 -2
- package/dist/ui/serverless/alarms.html +4 -4
- package/dist/ui/serverless/assets.html +2 -2
- package/dist/ui/serverless/cost.html +2 -2
- package/dist/ui/serverless/data.html +4 -4
- package/dist/ui/serverless/firewall.html +2 -2
- package/dist/ui/serverless/functions.html +4 -4
- package/dist/ui/serverless/logs.html +3 -3
- package/dist/ui/serverless/metrics.html +2 -2
- package/dist/ui/serverless/queues.html +4 -4
- package/dist/ui/serverless/secrets.html +4 -4
- package/dist/ui/serverless/traces.html +4 -4
- package/dist/ui/serverless.html +4 -4
- package/dist/ui-src/pages/index.stx +12 -10
- package/dist/ui-src/pages/operations/observability.stx +6 -4
- package/dist/ui-src/pages/server/metrics.stx +262 -106
- package/dist/ui-src/pages/server/sites.stx +98 -6
- package/package.json +3 -3
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
resolveProjectStackName,
|
|
9
9
|
resolveStatePath,
|
|
10
10
|
statePath
|
|
11
|
-
} from "./chunk-
|
|
11
|
+
} from "./chunk-ecsyc3p0.js";
|
|
12
12
|
import {
|
|
13
13
|
EC2Client,
|
|
14
14
|
SSMClient
|
|
@@ -346,15 +346,14 @@ function buildAutoUpdatesScript(enabled2 = true) {
|
|
|
346
346
|
return [];
|
|
347
347
|
return [
|
|
348
348
|
"export DEBIAN_FRONTEND=noninteractive",
|
|
349
|
-
|
|
350
|
-
|
|
349
|
+
`dpkg-query -W -f='\${Status}' unattended-upgrades 2>/dev/null | grep -q 'ok installed' || apt-get install -y unattended-upgrades`,
|
|
350
|
+
`cat > /etc/apt/apt.conf.d/20auto-upgrades <<'TS_CLOUD_AUTOUPD_EOF'`,
|
|
351
351
|
'APT::Periodic::Update-Package-Lists "1";',
|
|
352
352
|
'APT::Periodic::Unattended-Upgrade "1";',
|
|
353
353
|
'APT::Periodic::Download-Upgradeable-Packages "1";',
|
|
354
354
|
'APT::Periodic::AutocleanInterval "7";',
|
|
355
355
|
"TS_CLOUD_AUTOUPD_EOF",
|
|
356
|
-
"systemctl enable unattended-upgrades 2>/dev/null || true"
|
|
357
|
-
"systemctl restart unattended-upgrades 2>/dev/null || true"
|
|
356
|
+
"systemctl enable --now unattended-upgrades 2>/dev/null || true"
|
|
358
357
|
];
|
|
359
358
|
}
|
|
360
359
|
|
|
@@ -366,11 +365,14 @@ var DEFAULT_MEM_PERCENT = 90;
|
|
|
366
365
|
var DEFAULT_DISK_PERCENT = 90;
|
|
367
366
|
var SERVICE_PROBES = [
|
|
368
367
|
["nginx", 80],
|
|
368
|
+
["https", 443],
|
|
369
369
|
["phpFpm", 9074],
|
|
370
370
|
["mysql", 3306],
|
|
371
371
|
["postgres", 5432],
|
|
372
372
|
["redis", 6379],
|
|
373
|
-
["meilisearch", 7700]
|
|
373
|
+
["meilisearch", 7700],
|
|
374
|
+
["typesense", 8108],
|
|
375
|
+
["smtp", 25]
|
|
374
376
|
];
|
|
375
377
|
function resolveMonitoring(monitoring = true) {
|
|
376
378
|
const obj = typeof monitoring === "object" ? monitoring : {};
|
|
@@ -389,18 +391,20 @@ function buildMonitoringScript(monitoring = true) {
|
|
|
389
391
|
const probeLines = SERVICE_PROBES.map(([name, port]) => `SVC_${name.toUpperCase()}=$(probe ${port})`);
|
|
390
392
|
const servicesJson = SERVICE_PROBES.map(([name]) => `"${name}":"$SVC_${name.toUpperCase()}"`).join(",");
|
|
391
393
|
return [
|
|
394
|
+
"if ! command -v sadf >/dev/null 2>&1; then export DEBIAN_FRONTEND=noninteractive; apt-get update -qq; apt-get install -y -qq sysstat; fi",
|
|
395
|
+
"systemctl enable --now sysstat.service 2>/dev/null || true",
|
|
392
396
|
"mkdir -p /var/lib/ts-cloud",
|
|
393
|
-
|
|
397
|
+
`cat > /usr/local/bin/ts-cloud-metrics.sh <<'TS_CLOUD_METRICS_EOF'`,
|
|
394
398
|
"#!/bin/bash",
|
|
395
399
|
"set -uo pipefail",
|
|
396
|
-
|
|
400
|
+
`LOAD=$(cut -d' ' -f1 /proc/loadavg)`,
|
|
397
401
|
"CPUS=$(nproc)",
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
+
`MEM_TOTAL=$(free -m | awk '/^Mem:/{print $2}')`,
|
|
403
|
+
`MEM_USED=$(free -m | awk '/^Mem:/{print $3}')`,
|
|
404
|
+
`SWAP_TOTAL=$(free -m | awk '/^Swap:/{print $2}')`,
|
|
405
|
+
`SWAP_USED=$(free -m | awk '/^Swap:/{print $3}')`,
|
|
402
406
|
`DISK_PCT=$(df -P / | awk 'NR==2{gsub("%","",$5); print $5}')`,
|
|
403
|
-
|
|
407
|
+
`UPTIME_SEC=$(cut -d' ' -f1 /proc/uptime | cut -d. -f1)`,
|
|
404
408
|
`RX_BYTES=$(awk -F'[: ]+' 'NR>2 && $2!="lo"{rx+=$3} END{print rx+0}' /proc/net/dev)`,
|
|
405
409
|
`TX_BYTES=$(awk -F'[: ]+' 'NR>2 && $2!="lo"{tx+=$11} END{print tx+0}' /proc/net/dev)`,
|
|
406
410
|
"probe(){ (exec 3<>/dev/tcp/127.0.0.1/$1) 2>/dev/null && echo up || echo down; }",
|
|
@@ -427,7 +431,7 @@ function buildMonitoringScript(monitoring = true) {
|
|
|
427
431
|
"fi",
|
|
428
432
|
"TS_CLOUD_METRICS_EOF",
|
|
429
433
|
"chmod +x /usr/local/bin/ts-cloud-metrics.sh",
|
|
430
|
-
|
|
434
|
+
`cat > /etc/systemd/system/ts-cloud-metrics.service <<'TS_CLOUD_METRICS_SVC_EOF'`,
|
|
431
435
|
"[Unit]",
|
|
432
436
|
"Description=ts-cloud metrics collector",
|
|
433
437
|
"",
|
|
@@ -435,20 +439,22 @@ function buildMonitoringScript(monitoring = true) {
|
|
|
435
439
|
"Type=oneshot",
|
|
436
440
|
"ExecStart=/usr/local/bin/ts-cloud-metrics.sh",
|
|
437
441
|
"TS_CLOUD_METRICS_SVC_EOF",
|
|
438
|
-
|
|
442
|
+
`cat > /etc/systemd/system/ts-cloud-metrics.timer <<'TS_CLOUD_METRICS_TMR_EOF'`,
|
|
439
443
|
"[Unit]",
|
|
440
444
|
"Description=Run ts-cloud metrics collector every minute",
|
|
441
445
|
"",
|
|
442
446
|
"[Timer]",
|
|
443
|
-
"
|
|
444
|
-
"
|
|
447
|
+
"OnCalendar=*-*-* *:*:00",
|
|
448
|
+
"AccuracySec=1s",
|
|
449
|
+
"RandomizedDelaySec=5s",
|
|
450
|
+
"Persistent=true",
|
|
445
451
|
"",
|
|
446
452
|
"[Install]",
|
|
447
453
|
"WantedBy=timers.target",
|
|
448
454
|
"TS_CLOUD_METRICS_TMR_EOF",
|
|
449
455
|
"systemctl daemon-reload",
|
|
450
|
-
"systemctl enable ts-cloud-metrics.timer",
|
|
451
|
-
"systemctl start ts-cloud-metrics.
|
|
456
|
+
"systemctl enable --now ts-cloud-metrics.timer",
|
|
457
|
+
"systemctl start ts-cloud-metrics.service"
|
|
452
458
|
];
|
|
453
459
|
}
|
|
454
460
|
|
|
@@ -1170,6 +1176,7 @@ import { readdirSync, readFileSync } from 'node:fs'
|
|
|
1170
1176
|
const dir = ${JSON.stringify(sitesDir)}
|
|
1171
1177
|
const proxies = []
|
|
1172
1178
|
const seen = new Set()
|
|
1179
|
+
const owners = new Map()
|
|
1173
1180
|
const suffixes = new Set()
|
|
1174
1181
|
const guardHosts = new Set()
|
|
1175
1182
|
let email
|
|
@@ -1190,8 +1197,12 @@ for (const f of files) {
|
|
|
1190
1197
|
catch (err) { console.error('[rpx-assembler] SKIPPING malformed fragment ' + f + ' — its host(s) will 404 until fixed: ' + err); continue }
|
|
1191
1198
|
for (const p of frag.proxies ?? []) {
|
|
1192
1199
|
const key = p.id || (p.to + (p.path ?? ''))
|
|
1193
|
-
if (seen.has(key))
|
|
1200
|
+
if (seen.has(key)) {
|
|
1201
|
+
console.warn('[rpx-assembler] duplicate route ' + key + ' in ' + f + ' ignored; first declared by ' + owners.get(key))
|
|
1202
|
+
continue
|
|
1203
|
+
}
|
|
1194
1204
|
seen.add(key)
|
|
1205
|
+
owners.set(key, f)
|
|
1195
1206
|
proxies.push(p)
|
|
1196
1207
|
}
|
|
1197
1208
|
for (const s of frag.onDemandTls?.allowedSuffixes ?? []) suffixes.add(s)
|
|
@@ -1427,7 +1438,8 @@ function buildUfwScript(firewall = {}) {
|
|
|
1427
1438
|
const ports = [...new Set([...UFW_BASE_PORTS, ...firewall.allowedPorts || []])].filter((p) => Number.isInteger(p) && p >= 1 && p <= 65535).sort((a, b) => a - b);
|
|
1428
1439
|
const lines = [
|
|
1429
1440
|
"export DEBIAN_FRONTEND=noninteractive",
|
|
1430
|
-
"apt-get install -y ufw",
|
|
1441
|
+
"command -v ufw >/dev/null 2>&1 || apt-get install -y ufw",
|
|
1442
|
+
"ufw --force reset",
|
|
1431
1443
|
"ufw default deny incoming",
|
|
1432
1444
|
"ufw default allow outgoing",
|
|
1433
1445
|
"ufw allow OpenSSH"
|
|
@@ -2132,6 +2144,28 @@ function resolveHetznerSettings(config, overrides = {}) {
|
|
|
2132
2144
|
// src/drivers/hetzner/client.ts
|
|
2133
2145
|
var DEFAULT_API_URL = "https://api.hetzner.cloud/v1";
|
|
2134
2146
|
|
|
2147
|
+
class HetznerApiError extends Error {
|
|
2148
|
+
status;
|
|
2149
|
+
code;
|
|
2150
|
+
details;
|
|
2151
|
+
constructor(message, status, code, details) {
|
|
2152
|
+
super(message);
|
|
2153
|
+
this.status = status;
|
|
2154
|
+
this.code = code;
|
|
2155
|
+
this.details = details;
|
|
2156
|
+
this.name = "HetznerApiError";
|
|
2157
|
+
}
|
|
2158
|
+
}
|
|
2159
|
+
|
|
2160
|
+
class HetznerActionError extends Error {
|
|
2161
|
+
action;
|
|
2162
|
+
constructor(message, action) {
|
|
2163
|
+
super(message);
|
|
2164
|
+
this.action = action;
|
|
2165
|
+
this.name = "HetznerActionError";
|
|
2166
|
+
}
|
|
2167
|
+
}
|
|
2168
|
+
|
|
2135
2169
|
class HetznerClient {
|
|
2136
2170
|
name = "hetzner";
|
|
2137
2171
|
apiToken;
|
|
@@ -2165,7 +2199,7 @@ class HetznerClient {
|
|
|
2165
2199
|
if (!response.ok) {
|
|
2166
2200
|
const message = data.error?.message || response.statusText || "Hetzner API error";
|
|
2167
2201
|
const code = data.error?.code ? ` [${data.error.code}]` : "";
|
|
2168
|
-
throw new
|
|
2202
|
+
throw new HetznerApiError(`Hetzner API ${method} ${path} (${response.status})${code}: ${message}`, response.status, data.error?.code, data.error?.details);
|
|
2169
2203
|
}
|
|
2170
2204
|
return data;
|
|
2171
2205
|
}
|
|
@@ -2189,6 +2223,23 @@ class HetznerClient {
|
|
|
2189
2223
|
const data = await this.request("GET", `/servers/${id}`);
|
|
2190
2224
|
return data.server;
|
|
2191
2225
|
}
|
|
2226
|
+
async getServerMetrics(id, options) {
|
|
2227
|
+
const query = new URLSearchParams({
|
|
2228
|
+
type: options.types.join(","),
|
|
2229
|
+
start: options.start.toISOString(),
|
|
2230
|
+
end: options.end.toISOString(),
|
|
2231
|
+
step: String(Math.max(60, Math.floor(options.step ?? 60)))
|
|
2232
|
+
});
|
|
2233
|
+
const data = await this.request("GET", `/servers/${id}/metrics?${query}`);
|
|
2234
|
+
return data.metrics;
|
|
2235
|
+
}
|
|
2236
|
+
async listServerTypes(name) {
|
|
2237
|
+
const query = name ? `?name=${encodeURIComponent(name)}` : "";
|
|
2238
|
+
return this.requestAll(`/server_types${query}`, "server_types");
|
|
2239
|
+
}
|
|
2240
|
+
async getServerType(name) {
|
|
2241
|
+
return (await this.listServerTypes(name)).find((item) => item.name === name) ?? null;
|
|
2242
|
+
}
|
|
2192
2243
|
async createServer(options) {
|
|
2193
2244
|
const data = await this.request("POST", "/servers", {
|
|
2194
2245
|
name: options.name,
|
|
@@ -2300,16 +2351,35 @@ class HetznerClient {
|
|
|
2300
2351
|
});
|
|
2301
2352
|
return data.ssh_key;
|
|
2302
2353
|
}
|
|
2354
|
+
async getAction(actionId) {
|
|
2355
|
+
const data = await this.request("GET", `/actions/${actionId}`);
|
|
2356
|
+
return data.action;
|
|
2357
|
+
}
|
|
2358
|
+
async shutdownServer(serverId) {
|
|
2359
|
+
const data = await this.request("POST", `/servers/${serverId}/actions/shutdown`, {});
|
|
2360
|
+
return data.action;
|
|
2361
|
+
}
|
|
2362
|
+
async powerOnServer(serverId) {
|
|
2363
|
+
const data = await this.request("POST", `/servers/${serverId}/actions/poweron`, {});
|
|
2364
|
+
return data.action;
|
|
2365
|
+
}
|
|
2366
|
+
async changeServerType(serverId, serverType, upgradeDisk) {
|
|
2367
|
+
const data = await this.request("POST", `/servers/${serverId}/actions/change_type`, {
|
|
2368
|
+
server_type: serverType,
|
|
2369
|
+
upgrade_disk: upgradeDisk
|
|
2370
|
+
});
|
|
2371
|
+
return data.action;
|
|
2372
|
+
}
|
|
2303
2373
|
async waitForAction(actionId, options) {
|
|
2304
2374
|
const pollInterval = options?.pollIntervalMs ?? 2000;
|
|
2305
2375
|
const maxWait = options?.maxWaitMs ?? 300000;
|
|
2306
2376
|
const start = Date.now();
|
|
2307
2377
|
while (Date.now() - start < maxWait) {
|
|
2308
|
-
const
|
|
2309
|
-
if (
|
|
2310
|
-
return
|
|
2311
|
-
if (
|
|
2312
|
-
throw new
|
|
2378
|
+
const action = await this.getAction(actionId);
|
|
2379
|
+
if (action.status === "success")
|
|
2380
|
+
return action;
|
|
2381
|
+
if (action.status === "error") {
|
|
2382
|
+
throw new HetznerActionError(action.error?.message || "Hetzner action failed", action);
|
|
2313
2383
|
}
|
|
2314
2384
|
await new Promise((resolve) => setTimeout(resolve, pollInterval));
|
|
2315
2385
|
}
|
|
@@ -2327,6 +2397,18 @@ class HetznerClient {
|
|
|
2327
2397
|
}
|
|
2328
2398
|
throw new Error(`Timed out waiting for server ${serverId} to reach running state`);
|
|
2329
2399
|
}
|
|
2400
|
+
async waitForServerStatus(serverId, status, options) {
|
|
2401
|
+
const pollInterval = options?.pollIntervalMs ?? 3000;
|
|
2402
|
+
const maxWait = options?.maxWaitMs ?? 600000;
|
|
2403
|
+
const start = Date.now();
|
|
2404
|
+
while (Date.now() - start < maxWait) {
|
|
2405
|
+
const server = await this.getServer(serverId);
|
|
2406
|
+
if (server.status === status)
|
|
2407
|
+
return server;
|
|
2408
|
+
await new Promise((resolve) => setTimeout(resolve, pollInterval));
|
|
2409
|
+
}
|
|
2410
|
+
throw new Error(`Timed out waiting for server ${serverId} to reach ${status} state`);
|
|
2411
|
+
}
|
|
2330
2412
|
}
|
|
2331
2413
|
function resolveHetznerApiToken2(configToken, config) {
|
|
2332
2414
|
const token = resolveHetznerApiToken(configToken, config);
|
|
@@ -2499,7 +2581,7 @@ class HetznerDriver {
|
|
|
2499
2581
|
}
|
|
2500
2582
|
const labels = tsCloudLabels(slug, environment, "app");
|
|
2501
2583
|
const sites = config.sites || {};
|
|
2502
|
-
const sitePorts = this.collectUpstreamPorts(sites);
|
|
2584
|
+
const sitePorts = usesRpxProxy(compute) ? [] : this.collectUpstreamPorts(sites);
|
|
2503
2585
|
const firewallName = `${slug}-${environment}-app-fw`;
|
|
2504
2586
|
const { firewall } = await this.ensureFirewall(firewallName, labels, buildHetznerFirewallRules({
|
|
2505
2587
|
allowSsh: compute.allowSsh !== false,
|
|
@@ -3430,43 +3512,179 @@ class CloudDriverFactory {
|
|
|
3430
3512
|
}
|
|
3431
3513
|
}
|
|
3432
3514
|
var cloudDrivers = new CloudDriverFactory;
|
|
3433
|
-
// src/drivers/hetzner/
|
|
3434
|
-
|
|
3435
|
-
|
|
3436
|
-
|
|
3437
|
-
|
|
3438
|
-
|
|
3439
|
-
|
|
3440
|
-
|
|
3441
|
-
|
|
3442
|
-
|
|
3443
|
-
}
|
|
3444
|
-
|
|
3515
|
+
// src/drivers/hetzner/resize.ts
|
|
3516
|
+
function serverLocation(server) {
|
|
3517
|
+
return server.location?.name ?? server.datacenter?.location.name ?? "unknown";
|
|
3518
|
+
}
|
|
3519
|
+
function positive(value) {
|
|
3520
|
+
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
|
3521
|
+
}
|
|
3522
|
+
function assertCompatibleUpgrade(current, target) {
|
|
3523
|
+
if (current.architecture && target.architecture && current.architecture !== target.architecture) {
|
|
3524
|
+
throw new Error(`Cannot resize ${current.name} (${current.architecture}) to ${target.name} (${target.architecture}) in place.`);
|
|
3525
|
+
}
|
|
3526
|
+
const regressions = [];
|
|
3527
|
+
if (positive(target.cores) < positive(current.cores))
|
|
3528
|
+
regressions.push("CPU");
|
|
3529
|
+
if (positive(target.memory) < positive(current.memory))
|
|
3530
|
+
regressions.push("memory");
|
|
3531
|
+
if (positive(target.disk) < positive(current.disk))
|
|
3532
|
+
regressions.push("disk");
|
|
3533
|
+
if (regressions.length > 0) {
|
|
3534
|
+
throw new Error(`Refusing an in-place downgrade from ${current.name} to ${target.name}: ${regressions.join(", ")} would shrink.`);
|
|
3535
|
+
}
|
|
3536
|
+
}
|
|
3537
|
+
async function planHetznerServerResize(client, serverId, targetTypeName, upgradeDisk = true) {
|
|
3538
|
+
const [server, targetType] = await Promise.all([
|
|
3539
|
+
client.getServer(serverId),
|
|
3540
|
+
client.getServerType(targetTypeName)
|
|
3541
|
+
]);
|
|
3542
|
+
if (!targetType)
|
|
3543
|
+
throw new Error(`Hetzner server type ${targetTypeName} was not found.`);
|
|
3544
|
+
const currentType = server.server_type;
|
|
3545
|
+
const alreadyComplete = currentType.name === targetType.name;
|
|
3546
|
+
if (!alreadyComplete)
|
|
3547
|
+
assertCompatibleUpgrade(currentType, targetType);
|
|
3548
|
+
const location = serverLocation(server);
|
|
3549
|
+
const locationState = targetType.locations?.find((item) => item.name === location);
|
|
3550
|
+
return {
|
|
3551
|
+
serverId,
|
|
3552
|
+
serverName: server.name,
|
|
3553
|
+
location,
|
|
3554
|
+
currentType,
|
|
3555
|
+
targetType,
|
|
3556
|
+
capacityAvailable: locationState?.available !== false,
|
|
3557
|
+
alreadyComplete,
|
|
3558
|
+
upgradeDisk
|
|
3559
|
+
};
|
|
3445
3560
|
}
|
|
3446
|
-
|
|
3447
|
-
|
|
3448
|
-
|
|
3449
|
-
|
|
3450
|
-
|
|
3561
|
+
function isHetznerCapacityError(error) {
|
|
3562
|
+
if (error instanceof HetznerApiError && error.code === "resource_unavailable")
|
|
3563
|
+
return true;
|
|
3564
|
+
if (error instanceof HetznerActionError) {
|
|
3565
|
+
if (error.action.error?.code === "resource_unavailable")
|
|
3566
|
+
return true;
|
|
3567
|
+
if (/no fitting vhost|placement|free resources/i.test(error.action.error?.message ?? ""))
|
|
3568
|
+
return true;
|
|
3451
3569
|
}
|
|
3452
|
-
|
|
3453
|
-
return { id: firewall.id, name: firewall.name, created: true };
|
|
3570
|
+
return /resource_unavailable|no fitting vhost|error during placement|free resources/i.test(error instanceof Error ? error.message : String(error));
|
|
3454
3571
|
}
|
|
3455
|
-
async function
|
|
3456
|
-
|
|
3457
|
-
let server = (await client.listServers()).find((s) => s.name === options.name);
|
|
3458
|
-
const created = !server;
|
|
3459
|
-
if (!server)
|
|
3460
|
-
server = (await client.createServer(createOptions)).server;
|
|
3461
|
-
if (waitForRunning)
|
|
3462
|
-
server = await client.waitForServerRunning(server.id);
|
|
3463
|
-
return { server, created };
|
|
3572
|
+
async function phase(hooks, value, detail) {
|
|
3573
|
+
await hooks?.onPhase?.(value, detail);
|
|
3464
3574
|
}
|
|
3465
|
-
function
|
|
3466
|
-
|
|
3467
|
-
|
|
3468
|
-
|
|
3469
|
-
|
|
3575
|
+
async function waitForAction(client, action, options) {
|
|
3576
|
+
await client.waitForAction(action.id, {
|
|
3577
|
+
pollIntervalMs: options.actionPollIntervalMs,
|
|
3578
|
+
maxWaitMs: options.actionTimeoutMs
|
|
3579
|
+
});
|
|
3580
|
+
}
|
|
3581
|
+
var ensurePoweredOn = async ({
|
|
3582
|
+
client,
|
|
3583
|
+
hooks,
|
|
3584
|
+
actionPollIntervalMs,
|
|
3585
|
+
actionTimeoutMs,
|
|
3586
|
+
serverPollIntervalMs,
|
|
3587
|
+
serverTimeoutMs
|
|
3588
|
+
}, plan, manifest, recovered) => {
|
|
3589
|
+
let server = await client.getServer(plan.serverId);
|
|
3590
|
+
if (server.status !== "running") {
|
|
3591
|
+
await phase(hooks, recovered ? "recovering" : "powering-on", {
|
|
3592
|
+
serverId: plan.serverId
|
|
3593
|
+
});
|
|
3594
|
+
const action = await client.powerOnServer(plan.serverId);
|
|
3595
|
+
await waitForAction(client, action, { actionPollIntervalMs, actionTimeoutMs });
|
|
3596
|
+
server = await client.waitForServerRunning(plan.serverId, {
|
|
3597
|
+
pollIntervalMs: serverPollIntervalMs,
|
|
3598
|
+
maxWaitMs: serverTimeoutMs
|
|
3599
|
+
});
|
|
3600
|
+
}
|
|
3601
|
+
await phase(hooks, "verifying", {
|
|
3602
|
+
serverId: plan.serverId,
|
|
3603
|
+
recovered
|
|
3604
|
+
});
|
|
3605
|
+
const verification = await hooks?.afterBoot?.(server, plan.targetType, manifest, { recovered });
|
|
3606
|
+
return { server, verification };
|
|
3607
|
+
};
|
|
3608
|
+
async function executeHetznerServerResize(options) {
|
|
3609
|
+
const { client, hooks } = options;
|
|
3610
|
+
const upgradeDisk = options.upgradeDisk ?? true;
|
|
3611
|
+
await phase(hooks, "planning", {
|
|
3612
|
+
serverId: options.serverId,
|
|
3613
|
+
targetType: options.targetType
|
|
3614
|
+
});
|
|
3615
|
+
const plan = await planHetznerServerResize(client, options.serverId, options.targetType, upgradeDisk);
|
|
3616
|
+
let server = await client.getServer(plan.serverId);
|
|
3617
|
+
if (plan.alreadyComplete) {
|
|
3618
|
+
const boot2 = await ensurePoweredOn(options, plan, options.manifest, false);
|
|
3619
|
+
await phase(hooks, "complete", { alreadyComplete: true });
|
|
3620
|
+
return { status: "already-complete", plan, verification: boot2.verification };
|
|
3621
|
+
}
|
|
3622
|
+
if (!plan.capacityAvailable && !options.attemptUnavailable) {
|
|
3623
|
+
await phase(hooks, "waiting-capacity", {
|
|
3624
|
+
location: plan.location,
|
|
3625
|
+
targetType: plan.targetType.name
|
|
3626
|
+
});
|
|
3627
|
+
return { status: "waiting-capacity", plan };
|
|
3628
|
+
}
|
|
3629
|
+
let manifest = options.manifest;
|
|
3630
|
+
if (server.status === "off" && !options.resumeFromOff) {
|
|
3631
|
+
const recovered = await ensurePoweredOn(options, plan, undefined, true);
|
|
3632
|
+
return {
|
|
3633
|
+
status: "recovered",
|
|
3634
|
+
plan,
|
|
3635
|
+
server: recovered.server,
|
|
3636
|
+
verification: recovered.verification,
|
|
3637
|
+
retryable: true,
|
|
3638
|
+
error: "Server was already off without a resumable resize checkpoint; powered it back on."
|
|
3639
|
+
};
|
|
3640
|
+
}
|
|
3641
|
+
if (server.status === "running") {
|
|
3642
|
+
await phase(hooks, "preflight", { serverId: plan.serverId });
|
|
3643
|
+
manifest = await hooks?.preflight?.(server, plan.targetType);
|
|
3644
|
+
await phase(hooks, "shutting-down", { serverId: plan.serverId });
|
|
3645
|
+
const shutdown = await client.shutdownServer(plan.serverId);
|
|
3646
|
+
await waitForAction(client, shutdown, options);
|
|
3647
|
+
server = await client.waitForServerStatus(plan.serverId, "off", {
|
|
3648
|
+
pollIntervalMs: options.serverPollIntervalMs,
|
|
3649
|
+
maxWaitMs: options.serverTimeoutMs
|
|
3650
|
+
});
|
|
3651
|
+
}
|
|
3652
|
+
try {
|
|
3653
|
+
await phase(hooks, "resizing", {
|
|
3654
|
+
serverId: plan.serverId,
|
|
3655
|
+
from: plan.currentType.name,
|
|
3656
|
+
to: plan.targetType.name,
|
|
3657
|
+
upgradeDisk
|
|
3658
|
+
});
|
|
3659
|
+
const change = await client.changeServerType(plan.serverId, plan.targetType.name, upgradeDisk);
|
|
3660
|
+
await waitForAction(client, change, options);
|
|
3661
|
+
} catch (error) {
|
|
3662
|
+
const recovered = await ensurePoweredOn(options, plan, manifest, true);
|
|
3663
|
+
return {
|
|
3664
|
+
status: "recovered",
|
|
3665
|
+
plan,
|
|
3666
|
+
server: recovered.server,
|
|
3667
|
+
manifest,
|
|
3668
|
+
verification: recovered.verification,
|
|
3669
|
+
retryable: isHetznerCapacityError(error),
|
|
3670
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3671
|
+
};
|
|
3672
|
+
}
|
|
3673
|
+
const boot = await ensurePoweredOn(options, plan, manifest, false);
|
|
3674
|
+
if (boot.verification && !boot.verification.ok) {
|
|
3675
|
+
throw new Error(`Resize completed but verification failed: ${(boot.verification.failures ?? []).join("; ")}`);
|
|
3676
|
+
}
|
|
3677
|
+
await phase(hooks, "complete", {
|
|
3678
|
+
serverId: plan.serverId,
|
|
3679
|
+
targetType: plan.targetType.name
|
|
3680
|
+
});
|
|
3681
|
+
return {
|
|
3682
|
+
status: "completed",
|
|
3683
|
+
plan,
|
|
3684
|
+
server: boot.server,
|
|
3685
|
+
manifest,
|
|
3686
|
+
verification: boot.verification
|
|
3687
|
+
};
|
|
3470
3688
|
}
|
|
3471
3689
|
// src/drivers/shared/remote-exec.ts
|
|
3472
3690
|
function buildSshArgs(options = {}) {
|
|
@@ -3528,6 +3746,519 @@ async function waitForCloudInit(host, options = {}) {
|
|
|
3528
3746
|
}
|
|
3529
3747
|
throw new Error(`cloud-init did not finish on ${host} after ${timeoutMs}ms`);
|
|
3530
3748
|
}
|
|
3749
|
+
|
|
3750
|
+
// src/drivers/hetzner/resize-remote.ts
|
|
3751
|
+
var INVENTORY_SCRIPT = String.raw`
|
|
3752
|
+
const { readdir, readFile, readlink, stat } = await import('node:fs/promises')
|
|
3753
|
+
|
|
3754
|
+
const text = (command) => {
|
|
3755
|
+
const result = Bun.spawnSync(['bash', '-lc', command], { stdout: 'pipe', stderr: 'pipe' })
|
|
3756
|
+
return result.exitCode === 0 ? result.stdout.toString().trim() : ''
|
|
3757
|
+
}
|
|
3758
|
+
const lines = (command) => text(command).split('\n').map(value => value.trim()).filter(Boolean).sort()
|
|
3759
|
+
const number = (value) => Number.parseInt(value || '0', 10) || 0
|
|
3760
|
+
|
|
3761
|
+
const root = text('findmnt -n -o SOURCE,FSTYPE /').split(/\s+/)
|
|
3762
|
+
const rootSource = root[0] || ''
|
|
3763
|
+
const rootFsType = root[1] || ''
|
|
3764
|
+
const rootParent = text('lsblk -n -o PKNAME ' + rootSource + ' | head -1')
|
|
3765
|
+
const rootDiskBytes = number(text('lsblk -b -n -o SIZE /dev/' + rootParent + ' | head -1'))
|
|
3766
|
+
const rootFilesystemBytes = number(text("df -B1 --output=size / | tail -1"))
|
|
3767
|
+
const memoryBytes = number(text("awk '/MemTotal/{print $2 * 1024}' /proc/meminfo"))
|
|
3768
|
+
const allRunningServices = lines("systemctl list-units --type=service --state=running --no-legend --plain | awk '{print $1}'")
|
|
3769
|
+
const runningServices = allRunningServices.filter((unit) => {
|
|
3770
|
+
if (/^(containerd|docker|localtunnel|mail|mysql|mariadb|meilisearch|postgresql|redis|rpx-gateway)\b/.test(unit)) return true
|
|
3771
|
+
const source = text('systemctl cat ' + unit + ' 2>/dev/null')
|
|
3772
|
+
return /\/var\/www|\/usr\/local\/bin\/rpx|\/opt\/ts-cloud/.test(source)
|
|
3773
|
+
})
|
|
3774
|
+
const failedUnits = lines("systemctl --failed --type=service --no-legend --plain | awk '{print $1}'")
|
|
3775
|
+
const releaseLinks = []
|
|
3776
|
+
for (const path of lines("find /var/www -mindepth 2 -maxdepth 2 -type l -name current -print 2>/dev/null")) {
|
|
3777
|
+
try {
|
|
3778
|
+
releaseLinks.push(path + '=' + await readlink(path))
|
|
3779
|
+
} catch {}
|
|
3780
|
+
}
|
|
3781
|
+
releaseLinks.sort()
|
|
3782
|
+
|
|
3783
|
+
const routeFragments = []
|
|
3784
|
+
const routeIds = []
|
|
3785
|
+
const routeDomains = new Set()
|
|
3786
|
+
for (const path of lines("find /etc/rpx/sites.d -maxdepth 1 -type f -name '*.json' -print 2>/dev/null")) {
|
|
3787
|
+
try {
|
|
3788
|
+
const source = await readFile(path, 'utf8')
|
|
3789
|
+
const digest = new Bun.CryptoHasher('sha256').update(source).digest('hex')
|
|
3790
|
+
routeFragments.push(path + '=' + digest)
|
|
3791
|
+
const value = JSON.parse(source)
|
|
3792
|
+
for (const route of value.proxies || []) {
|
|
3793
|
+
if (typeof route.to === 'string' && route.to) {
|
|
3794
|
+
routeDomains.add(route.to)
|
|
3795
|
+
routeIds.push(path + ':' + (route.id || route.to) + ':' + (route.path || '/'))
|
|
3796
|
+
}
|
|
3797
|
+
}
|
|
3798
|
+
} catch {}
|
|
3799
|
+
}
|
|
3800
|
+
routeFragments.sort()
|
|
3801
|
+
routeIds.sort()
|
|
3802
|
+
|
|
3803
|
+
const persistentData = []
|
|
3804
|
+
for (const path of ['/var/lib/postgresql', '/var/lib/mysql', '/var/lib/redis', '/var/lib/docker', '/var/mail', '/var/vmail']) {
|
|
3805
|
+
try {
|
|
3806
|
+
const value = await stat(path)
|
|
3807
|
+
if (value.isDirectory()) {
|
|
3808
|
+
const bytes = number(text('du -sx --block-size=1 ' + path + " 2>/dev/null | awk '{print $1}'"))
|
|
3809
|
+
persistentData.push(path + '=' + bytes)
|
|
3810
|
+
}
|
|
3811
|
+
} catch {}
|
|
3812
|
+
}
|
|
3813
|
+
|
|
3814
|
+
const dataCatalog = []
|
|
3815
|
+
for (const database of lines("command -v psql >/dev/null && sudo -u postgres psql -Atqc 'select datname from pg_database where datistemplate = false order by datname' 2>/dev/null || true")) {
|
|
3816
|
+
dataCatalog.push('postgres:' + database)
|
|
3817
|
+
}
|
|
3818
|
+
for (const database of lines("command -v mysql >/dev/null && mysql -NBe 'show databases' 2>/dev/null || true")) {
|
|
3819
|
+
dataCatalog.push('mysql:' + database)
|
|
3820
|
+
}
|
|
3821
|
+
for (const volume of lines("command -v docker >/dev/null && docker volume ls --format '{{.Name}}' 2>/dev/null || true")) {
|
|
3822
|
+
dataCatalog.push('docker-volume:' + volume)
|
|
3823
|
+
}
|
|
3824
|
+
|
|
3825
|
+
console.log(JSON.stringify({
|
|
3826
|
+
capturedAt: new Date().toISOString(),
|
|
3827
|
+
hostname: text('hostname'),
|
|
3828
|
+
cpuCores: number(text('nproc')),
|
|
3829
|
+
memoryBytes,
|
|
3830
|
+
rootSource,
|
|
3831
|
+
rootFsType,
|
|
3832
|
+
rootDiskBytes,
|
|
3833
|
+
rootFilesystemBytes,
|
|
3834
|
+
failedUnits,
|
|
3835
|
+
runningServices,
|
|
3836
|
+
releaseLinks,
|
|
3837
|
+
routeFragments,
|
|
3838
|
+
routeIds,
|
|
3839
|
+
routeDomains: [...routeDomains].sort(),
|
|
3840
|
+
persistentData: persistentData.sort(),
|
|
3841
|
+
dataCatalog: dataCatalog.sort(),
|
|
3842
|
+
}))
|
|
3843
|
+
`;
|
|
3844
|
+
function remoteBunCommand(script) {
|
|
3845
|
+
return `printf %s ${Buffer.from(script).toString("base64")} | base64 -d | bun -`;
|
|
3846
|
+
}
|
|
3847
|
+
async function collectHostManifest(options) {
|
|
3848
|
+
const raw = await sshExecOrThrow(options.host, remoteBunCommand(INVENTORY_SCRIPT), options);
|
|
3849
|
+
return JSON.parse(raw);
|
|
3850
|
+
}
|
|
3851
|
+
async function probeRoute(domain, timeoutMs) {
|
|
3852
|
+
let lastError = "";
|
|
3853
|
+
for (let attempt = 1;attempt <= 3; attempt++) {
|
|
3854
|
+
const process2 = Bun.spawn([
|
|
3855
|
+
"curl",
|
|
3856
|
+
"--location",
|
|
3857
|
+
"--silent",
|
|
3858
|
+
"--show-error",
|
|
3859
|
+
"--output",
|
|
3860
|
+
"/dev/null",
|
|
3861
|
+
"--max-time",
|
|
3862
|
+
String(Math.max(1, Math.ceil(timeoutMs / 1000))),
|
|
3863
|
+
"--write-out",
|
|
3864
|
+
"%{http_code}",
|
|
3865
|
+
`https://${domain}/`
|
|
3866
|
+
], { stdout: "pipe", stderr: "pipe" });
|
|
3867
|
+
const [stdout, stderr, exitCode] = await Promise.all([
|
|
3868
|
+
new Response(process2.stdout).text(),
|
|
3869
|
+
new Response(process2.stderr).text(),
|
|
3870
|
+
process2.exited
|
|
3871
|
+
]);
|
|
3872
|
+
const status = Number.parseInt(stdout.trim(), 10);
|
|
3873
|
+
if (exitCode === 0 && status > 0)
|
|
3874
|
+
return { domain, ok: status < 500, status };
|
|
3875
|
+
lastError = stderr.trim() || `curl exited ${exitCode}`;
|
|
3876
|
+
if (attempt < 3)
|
|
3877
|
+
await Bun.sleep(500 * attempt);
|
|
3878
|
+
}
|
|
3879
|
+
return { domain, ok: false, error: lastError };
|
|
3880
|
+
}
|
|
3881
|
+
async function probeRoutes(domains, timeoutMs) {
|
|
3882
|
+
const probes = [];
|
|
3883
|
+
let cursor = 0;
|
|
3884
|
+
const workers = Array.from({ length: Math.min(4, domains.length) }, async () => {
|
|
3885
|
+
for (;; ) {
|
|
3886
|
+
const index = cursor++;
|
|
3887
|
+
const domain = domains[index];
|
|
3888
|
+
if (!domain)
|
|
3889
|
+
return;
|
|
3890
|
+
probes[index] = await probeRoute(domain, timeoutMs);
|
|
3891
|
+
}
|
|
3892
|
+
});
|
|
3893
|
+
await Promise.all(workers);
|
|
3894
|
+
return probes;
|
|
3895
|
+
}
|
|
3896
|
+
async function collectHetznerResizeManifest(options) {
|
|
3897
|
+
const host = await collectHostManifest(options);
|
|
3898
|
+
const routeProbes = await probeRoutes(host.routeDomains, options.routeTimeoutMs ?? 15000);
|
|
3899
|
+
return { ...host, routeProbes };
|
|
3900
|
+
}
|
|
3901
|
+
async function prepareHetznerResize(options) {
|
|
3902
|
+
const before = await collectHetznerResizeManifest(options);
|
|
3903
|
+
const unhealthy = before.routeProbes.filter((probe) => !probe.ok);
|
|
3904
|
+
if (before.failedUnits.length > 0) {
|
|
3905
|
+
throw new Error(`Preflight found failed services: ${before.failedUnits.join(", ")}`);
|
|
3906
|
+
}
|
|
3907
|
+
if (unhealthy.length > 0) {
|
|
3908
|
+
throw new Error(`Preflight found unhealthy routes: ${unhealthy.map((probe) => probe.domain).join(", ")}`);
|
|
3909
|
+
}
|
|
3910
|
+
await sshExecOrThrow(options.host, String.raw`systemctl list-units --type=service --state=running --no-legend --plain | awk '{print $1}' | while read -r unit; do if systemctl cat "$unit" 2>/dev/null | grep -Eq '/var/www|/usr/local/bin/rpx|/opt/ts-cloud'; then systemctl enable "$unit" >/dev/null 2>&1 || true; fi; done; systemctl daemon-reload; sync`, options);
|
|
3911
|
+
return before;
|
|
3912
|
+
}
|
|
3913
|
+
function missing(expected, actual) {
|
|
3914
|
+
const values = new Set(actual);
|
|
3915
|
+
return expected.filter((value) => !values.has(value));
|
|
3916
|
+
}
|
|
3917
|
+
function targetMemoryBytes(target) {
|
|
3918
|
+
return Math.floor((target.memory ?? 0) * 1024 * 1024 * 1024);
|
|
3919
|
+
}
|
|
3920
|
+
async function expandRootFilesystem(options) {
|
|
3921
|
+
await sshExecOrThrow(options.host, String.raw`set -e
|
|
3922
|
+
root_source="$(findmnt -n -o SOURCE /)"
|
|
3923
|
+
root_fs="$(findmnt -n -o FSTYPE /)"
|
|
3924
|
+
parent="$(lsblk -n -o PKNAME "$root_source" | head -1)"
|
|
3925
|
+
partition="$(lsblk -n -o PARTN "$root_source" | head -1)"
|
|
3926
|
+
if [ -n "$parent" ] && [ -n "$partition" ]; then
|
|
3927
|
+
growpart "/dev/$parent" "$partition" || true
|
|
3928
|
+
fi
|
|
3929
|
+
case "$root_fs" in
|
|
3930
|
+
ext2|ext3|ext4) resize2fs "$root_source" ;;
|
|
3931
|
+
xfs) xfs_growfs / ;;
|
|
3932
|
+
*) echo "Unsupported root filesystem: $root_fs" >&2; exit 1 ;;
|
|
3933
|
+
esac`, options);
|
|
3934
|
+
}
|
|
3935
|
+
async function restoreExpectedServices(options, expected) {
|
|
3936
|
+
const encoded = Buffer.from(expected.join(`
|
|
3937
|
+
`)).toString("base64");
|
|
3938
|
+
await sshExecOrThrow(options.host, `printf %s ${encoded} | base64 -d | while read -r unit; do [ -z "$unit" ] || systemctl is-active --quiet "$unit" || systemctl start "$unit"; done`, options);
|
|
3939
|
+
}
|
|
3940
|
+
async function verifyHetznerResize(options, target, before, context = { recovered: false }) {
|
|
3941
|
+
await waitForSsh(options.host, {
|
|
3942
|
+
...options,
|
|
3943
|
+
timeoutMs: options.sshTimeoutMs ?? 10 * 60000
|
|
3944
|
+
});
|
|
3945
|
+
if (!context.recovered)
|
|
3946
|
+
await expandRootFilesystem(options);
|
|
3947
|
+
if (before)
|
|
3948
|
+
await restoreExpectedServices(options, before.runningServices);
|
|
3949
|
+
const after = await collectHetznerResizeManifest(options);
|
|
3950
|
+
const failures = [];
|
|
3951
|
+
const missingServices = before ? missing(before.runningServices, after.runningServices) : [];
|
|
3952
|
+
const missingReleases = before ? missing(before.releaseLinks, after.releaseLinks) : [];
|
|
3953
|
+
const changedRoutes = before ? missing(before.routeFragments, after.routeFragments) : [];
|
|
3954
|
+
const missingRouteIds = before ? missing(before.routeIds, after.routeIds) : [];
|
|
3955
|
+
const beforeDataPaths = before?.persistentData.map((value) => value.split("=")[0]) ?? [];
|
|
3956
|
+
const afterDataPaths = after.persistentData.map((value) => value.split("=")[0]);
|
|
3957
|
+
const missingDataPaths = missing(beforeDataPaths, afterDataPaths);
|
|
3958
|
+
const missingDataCatalog = before ? missing(before.dataCatalog, after.dataCatalog) : [];
|
|
3959
|
+
const unhealthyRoutes = after.routeProbes.filter((probe) => !probe.ok);
|
|
3960
|
+
const expectedCores = context.recovered && before ? before.cpuCores : target.cores ?? 0;
|
|
3961
|
+
const expectedMemory = context.recovered && before ? before.memoryBytes : targetMemoryBytes(target);
|
|
3962
|
+
const cpuOk = after.cpuCores >= expectedCores;
|
|
3963
|
+
const memoryOk = after.memoryBytes >= expectedMemory * 0.9;
|
|
3964
|
+
const diskOk = context.recovered || after.rootDiskBytes >= (target.disk ?? 0) * 1e9 * 0.95;
|
|
3965
|
+
if (after.failedUnits.length > 0)
|
|
3966
|
+
failures.push(`Failed services: ${after.failedUnits.join(", ")}`);
|
|
3967
|
+
if (missingServices.length > 0)
|
|
3968
|
+
failures.push(`Services did not return: ${missingServices.join(", ")}`);
|
|
3969
|
+
if (missingReleases.length > 0)
|
|
3970
|
+
failures.push(`Release links changed: ${missingReleases.join(", ")}`);
|
|
3971
|
+
if (changedRoutes.length > 0)
|
|
3972
|
+
failures.push(`Route manifests changed: ${changedRoutes.join(", ")}`);
|
|
3973
|
+
if (missingRouteIds.length > 0)
|
|
3974
|
+
failures.push(`Routes disappeared: ${missingRouteIds.join(", ")}`);
|
|
3975
|
+
if (missingDataPaths.length > 0)
|
|
3976
|
+
failures.push(`Persistent data paths disappeared: ${missingDataPaths.join(", ")}`);
|
|
3977
|
+
if (missingDataCatalog.length > 0)
|
|
3978
|
+
failures.push(`Databases or volumes disappeared: ${missingDataCatalog.join(", ")}`);
|
|
3979
|
+
if (unhealthyRoutes.length > 0)
|
|
3980
|
+
failures.push(`Unhealthy routes: ${unhealthyRoutes.map((probe) => probe.domain).join(", ")}`);
|
|
3981
|
+
if (!cpuOk)
|
|
3982
|
+
failures.push(`Expected at least ${expectedCores} CPU cores, found ${after.cpuCores}`);
|
|
3983
|
+
if (!memoryOk)
|
|
3984
|
+
failures.push(`Expected at least ${expectedMemory} bytes of RAM`);
|
|
3985
|
+
if (!diskOk)
|
|
3986
|
+
failures.push(`Expected approximately ${target.disk ?? 0} GB root disk`);
|
|
3987
|
+
return {
|
|
3988
|
+
ok: failures.length === 0,
|
|
3989
|
+
checks: {
|
|
3990
|
+
cpuCores: after.cpuCores,
|
|
3991
|
+
memoryBytes: after.memoryBytes,
|
|
3992
|
+
rootDiskBytes: after.rootDiskBytes,
|
|
3993
|
+
rootFilesystemBytes: after.rootFilesystemBytes,
|
|
3994
|
+
runningServices: after.runningServices.length,
|
|
3995
|
+
releaseLinks: after.releaseLinks.length,
|
|
3996
|
+
routeFragments: after.routeFragments.length,
|
|
3997
|
+
routes: after.routeIds.length,
|
|
3998
|
+
healthyRoutes: after.routeProbes.filter((probe) => probe.ok).length,
|
|
3999
|
+
persistentDataPaths: after.persistentData.length,
|
|
4000
|
+
databasesAndVolumes: after.dataCatalog.length,
|
|
4001
|
+
failedUnits: after.failedUnits.length
|
|
4002
|
+
},
|
|
4003
|
+
failures
|
|
4004
|
+
};
|
|
4005
|
+
}
|
|
4006
|
+
// src/drivers/hetzner/host-optimization.ts
|
|
4007
|
+
function computeConfig(config) {
|
|
4008
|
+
return config.infrastructure?.compute ?? {};
|
|
4009
|
+
}
|
|
4010
|
+
function resolveHetznerHostOptimizationPlan(config) {
|
|
4011
|
+
const compute = computeConfig(config);
|
|
4012
|
+
const monitoring = compute.monitoring;
|
|
4013
|
+
const firewallPorts = [...new Set([22, 80, 443, ...compute.firewall?.allowedPorts ?? []])].filter((port) => Number.isInteger(port) && port >= 1 && port <= 65535).sort((a, b) => a - b);
|
|
4014
|
+
return {
|
|
4015
|
+
firewallPorts,
|
|
4016
|
+
monitoring: monitoring !== false && (typeof monitoring !== "object" || monitoring.enabled !== false),
|
|
4017
|
+
autoUpdates: compute.autoUpdates !== false,
|
|
4018
|
+
swapGb: Math.max(0, Math.floor(compute.swapGb ?? 2)),
|
|
4019
|
+
sshPasswordAuthentication: false,
|
|
4020
|
+
journalMaxUse: "256M",
|
|
4021
|
+
journalRetention: "14day"
|
|
4022
|
+
};
|
|
4023
|
+
}
|
|
4024
|
+
function hardeningScript(plan) {
|
|
4025
|
+
return [
|
|
4026
|
+
"install -d -m 0755 /etc/ssh/sshd_config.d /etc/systemd/journald.conf.d /etc/sysctl.d /etc/fail2ban/jail.d",
|
|
4027
|
+
`cat > /etc/ssh/sshd_config.d/99-ts-cloud-hardening.conf <<'TS_CLOUD_SSH_EOF'`,
|
|
4028
|
+
"# Managed by ts-cloud host optimization.",
|
|
4029
|
+
"PasswordAuthentication no",
|
|
4030
|
+
"KbdInteractiveAuthentication no",
|
|
4031
|
+
"PermitRootLogin prohibit-password",
|
|
4032
|
+
"MaxAuthTries 4",
|
|
4033
|
+
"LoginGraceTime 30",
|
|
4034
|
+
"TS_CLOUD_SSH_EOF",
|
|
4035
|
+
"sshd -t",
|
|
4036
|
+
"systemctl reload ssh",
|
|
4037
|
+
`cat > /etc/systemd/journald.conf.d/99-ts-cloud-retention.conf <<'TS_CLOUD_JOURNAL_EOF'`,
|
|
4038
|
+
"[Journal]",
|
|
4039
|
+
`SystemMaxUse=${plan.journalMaxUse}`,
|
|
4040
|
+
`MaxRetentionSec=${plan.journalRetention}`,
|
|
4041
|
+
"Compress=yes",
|
|
4042
|
+
"TS_CLOUD_JOURNAL_EOF",
|
|
4043
|
+
"systemctl restart systemd-journald",
|
|
4044
|
+
"journalctl --vacuum-size=256M --vacuum-time=14d >/dev/null",
|
|
4045
|
+
`cat > /etc/sysctl.d/99-ts-cloud-production.conf <<'TS_CLOUD_SYSCTL_EOF'`,
|
|
4046
|
+
"# Managed by ts-cloud host optimization.",
|
|
4047
|
+
"vm.swappiness=10",
|
|
4048
|
+
"vm.dirty_background_ratio=5",
|
|
4049
|
+
"vm.dirty_ratio=15",
|
|
4050
|
+
"net.core.somaxconn=4096",
|
|
4051
|
+
"net.ipv4.tcp_max_syn_backlog=4096",
|
|
4052
|
+
"net.ipv4.tcp_fin_timeout=30",
|
|
4053
|
+
"net.ipv4.tcp_keepalive_time=300",
|
|
4054
|
+
"net.ipv4.tcp_keepalive_intvl=30",
|
|
4055
|
+
"net.ipv4.tcp_keepalive_probes=5",
|
|
4056
|
+
"fs.inotify.max_user_watches=524288",
|
|
4057
|
+
"TS_CLOUD_SYSCTL_EOF",
|
|
4058
|
+
"sysctl --system >/dev/null",
|
|
4059
|
+
"export DEBIAN_FRONTEND=noninteractive",
|
|
4060
|
+
"command -v fail2ban-client >/dev/null 2>&1 || apt-get install -y fail2ban",
|
|
4061
|
+
`cat > /etc/fail2ban/jail.d/99-ts-cloud-sshd.conf <<'TS_CLOUD_FAIL2BAN_EOF'`,
|
|
4062
|
+
"[sshd]",
|
|
4063
|
+
"enabled = true",
|
|
4064
|
+
"backend = systemd",
|
|
4065
|
+
"bantime = 1h",
|
|
4066
|
+
"findtime = 10m",
|
|
4067
|
+
"maxretry = 5",
|
|
4068
|
+
"TS_CLOUD_FAIL2BAN_EOF",
|
|
4069
|
+
"systemctl enable --now fail2ban",
|
|
4070
|
+
"systemctl restart fail2ban"
|
|
4071
|
+
];
|
|
4072
|
+
}
|
|
4073
|
+
function swapScript(sizeGb) {
|
|
4074
|
+
if (sizeGb === 0)
|
|
4075
|
+
return [];
|
|
4076
|
+
return [
|
|
4077
|
+
`if ! swapon --show=NAME --noheadings 2>/dev/null | grep -qx '/swapfile'; then`,
|
|
4078
|
+
` if [ ! -f /swapfile ]; then fallocate -l ${sizeGb}G /swapfile || dd if=/dev/zero of=/swapfile bs=1M count=${sizeGb * 1024} status=none; fi`,
|
|
4079
|
+
" chmod 600 /swapfile",
|
|
4080
|
+
" mkswap /swapfile >/dev/null",
|
|
4081
|
+
" swapon /swapfile",
|
|
4082
|
+
"fi",
|
|
4083
|
+
`grep -qE '^/swapfile\\s' /etc/fstab || echo '/swapfile none swap sw 0 0' >> /etc/fstab`
|
|
4084
|
+
];
|
|
4085
|
+
}
|
|
4086
|
+
function buildHetznerHostOptimizationScript(config) {
|
|
4087
|
+
const compute = computeConfig(config);
|
|
4088
|
+
const plan = resolveHetznerHostOptimizationPlan(config);
|
|
4089
|
+
const monitoring = compute.monitoring ?? true;
|
|
4090
|
+
const firewall = compute.firewall ?? { enabled: true };
|
|
4091
|
+
return [
|
|
4092
|
+
"set -euo pipefail",
|
|
4093
|
+
...usesRpxProxy(compute) ? [
|
|
4094
|
+
"systemctl disable --now nginx.service bun-gateway.service ts-cloud-nginx.service 2>/dev/null || true",
|
|
4095
|
+
"systemctl reset-failed nginx.service bun-gateway.service ts-cloud-nginx.service 2>/dev/null || true"
|
|
4096
|
+
] : [],
|
|
4097
|
+
...swapScript(plan.swapGb),
|
|
4098
|
+
...buildAutoUpdatesScript(plan.autoUpdates),
|
|
4099
|
+
...buildMonitoringScript(monitoring),
|
|
4100
|
+
...hardeningScript(plan),
|
|
4101
|
+
...buildUfwScript(firewall),
|
|
4102
|
+
"systemctl daemon-reload",
|
|
4103
|
+
"systemctl reset-failed"
|
|
4104
|
+
];
|
|
4105
|
+
}
|
|
4106
|
+
var REPORT_SCRIPT = String.raw`
|
|
4107
|
+
const text = (command) => {
|
|
4108
|
+
const result = Bun.spawnSync(['bash', '-lc', command], { stdout: 'pipe', stderr: 'pipe' })
|
|
4109
|
+
return result.stdout.toString().trim()
|
|
4110
|
+
}
|
|
4111
|
+
const lines = command => text(command).split('\n').map(value => value.trim()).filter(Boolean)
|
|
4112
|
+
const number = value => Number.parseInt(value || '0', 10) || 0
|
|
4113
|
+
const ssh = Object.fromEntries(lines("sshd -T 2>/dev/null | grep -E '^(passwordauthentication|permitrootlogin) '").map(line => line.split(/\s+/, 2)))
|
|
4114
|
+
const firewallPorts = lines("ufw status 2>/dev/null | awk '/ALLOW/{print $1}'")
|
|
4115
|
+
.flatMap(value => value.startsWith('OpenSSH') ? [22] : (value.match(/^\d+/) ? [number(value)] : []))
|
|
4116
|
+
const publicTcpPorts = lines("ss -lntH | awk '{print $4}'")
|
|
4117
|
+
.filter(address => !address.startsWith('127.') && !address.startsWith('[::1]'))
|
|
4118
|
+
.flatMap(address => {
|
|
4119
|
+
const match = address.match(/:(\d+)$/)
|
|
4120
|
+
return match ? [number(match[1])] : []
|
|
4121
|
+
})
|
|
4122
|
+
const metricsMtime = number(text("stat -c %Y /var/lib/ts-cloud/metrics.json 2>/dev/null"))
|
|
4123
|
+
console.log(JSON.stringify({
|
|
4124
|
+
firewallActive: text("ufw status 2>/dev/null | head -1").includes('active'),
|
|
4125
|
+
firewallPorts: [...new Set(firewallPorts)].sort((a, b) => a - b),
|
|
4126
|
+
passwordAuthentication: ssh.passwordauthentication !== 'no',
|
|
4127
|
+
rootPasswordLogin: ssh.permitrootlogin === 'yes',
|
|
4128
|
+
fail2banActive: text('systemctl is-active fail2ban 2>/dev/null') === 'active',
|
|
4129
|
+
unattendedUpgradesActive: text('systemctl is-active unattended-upgrades 2>/dev/null') === 'active',
|
|
4130
|
+
metricsTimerActive: text('systemctl is-active ts-cloud-metrics.timer 2>/dev/null') === 'active',
|
|
4131
|
+
metricsSnapshotFresh: metricsMtime > 0 && Math.floor(Date.now() / 1000) - metricsMtime < 180,
|
|
4132
|
+
swapBytes: number(text("free -b | awk '/^Swap:/{print $2}'")),
|
|
4133
|
+
journalBytes: number(text("journalctl --disk-usage 2>/dev/null | grep -Eo '[0-9.]+[KMG]' | tail -1 | numfmt --from=iec 2>/dev/null")),
|
|
4134
|
+
failedUnits: lines("systemctl --failed --type=service --no-legend --plain | awk '{print $1}'"),
|
|
4135
|
+
publicTcpPorts: [...new Set(publicTcpPorts)].sort((a, b) => a - b),
|
|
4136
|
+
}))
|
|
4137
|
+
`;
|
|
4138
|
+
function remoteBunCommand2(script) {
|
|
4139
|
+
return `printf %s ${Buffer.from(script).toString("base64")} | base64 -d | bun -`;
|
|
4140
|
+
}
|
|
4141
|
+
async function applyHetznerHostOptimization(config, options) {
|
|
4142
|
+
const script = buildHetznerHostOptimizationScript(config).join(`
|
|
4143
|
+
`);
|
|
4144
|
+
const command = `printf %s ${Buffer.from(script).toString("base64")} | base64 -d | bash`;
|
|
4145
|
+
await sshExecOrThrow(options.host, command, options);
|
|
4146
|
+
}
|
|
4147
|
+
async function collectHetznerHostOptimizationReport(options) {
|
|
4148
|
+
const raw = await sshExecOrThrow(options.host, remoteBunCommand2(REPORT_SCRIPT), options);
|
|
4149
|
+
return JSON.parse(raw);
|
|
4150
|
+
}
|
|
4151
|
+
function verifyHetznerHostOptimization(plan, report) {
|
|
4152
|
+
const failures = [];
|
|
4153
|
+
const actualPorts = new Set(report.firewallPorts);
|
|
4154
|
+
const missingPorts = plan.firewallPorts.filter((port) => !actualPorts.has(port));
|
|
4155
|
+
const unexpectedPorts = report.firewallPorts.filter((port) => !plan.firewallPorts.includes(port));
|
|
4156
|
+
if (!report.firewallActive)
|
|
4157
|
+
failures.push("UFW is not active");
|
|
4158
|
+
if (missingPorts.length > 0)
|
|
4159
|
+
failures.push(`UFW is missing ports: ${missingPorts.join(", ")}`);
|
|
4160
|
+
if (unexpectedPorts.length > 0)
|
|
4161
|
+
failures.push(`UFW has unexpected ports: ${unexpectedPorts.join(", ")}`);
|
|
4162
|
+
if (report.passwordAuthentication)
|
|
4163
|
+
failures.push("SSH password authentication remains enabled");
|
|
4164
|
+
if (report.rootPasswordLogin)
|
|
4165
|
+
failures.push("SSH root password login remains enabled");
|
|
4166
|
+
if (!report.fail2banActive)
|
|
4167
|
+
failures.push("fail2ban is not active");
|
|
4168
|
+
if (plan.autoUpdates && !report.unattendedUpgradesActive)
|
|
4169
|
+
failures.push("unattended upgrades are not active");
|
|
4170
|
+
if (plan.monitoring && !report.metricsTimerActive)
|
|
4171
|
+
failures.push("metrics timer is not active");
|
|
4172
|
+
if (plan.monitoring && !report.metricsSnapshotFresh)
|
|
4173
|
+
failures.push("metrics snapshot is stale");
|
|
4174
|
+
if (plan.swapGb > 0 && report.swapBytes < plan.swapGb * 1024 ** 3 * 0.9)
|
|
4175
|
+
failures.push("configured swap is missing");
|
|
4176
|
+
if (report.failedUnits.length > 0)
|
|
4177
|
+
failures.push(`failed services: ${report.failedUnits.join(", ")}`);
|
|
4178
|
+
return failures;
|
|
4179
|
+
}
|
|
4180
|
+
// src/drivers/hetzner/resize-state.ts
|
|
4181
|
+
import { mkdir as mkdir2, readFile as readFile2, rename as rename2, rmdir, stat, unlink, writeFile as writeFile2 } from "node:fs/promises";
|
|
4182
|
+
import { join as join4 } from "node:path";
|
|
4183
|
+
function resizeCheckpointPath(stackName) {
|
|
4184
|
+
return join4(process.cwd(), STATE_DIR, `${stackName}-resize.json`);
|
|
4185
|
+
}
|
|
4186
|
+
function resizeLockPath(stackName) {
|
|
4187
|
+
return join4(process.cwd(), STATE_DIR, `${stackName}-resize.lock`);
|
|
4188
|
+
}
|
|
4189
|
+
async function readResizeCheckpoint(stackName) {
|
|
4190
|
+
try {
|
|
4191
|
+
return JSON.parse(await readFile2(resizeCheckpointPath(stackName), "utf8"));
|
|
4192
|
+
} catch {
|
|
4193
|
+
return null;
|
|
4194
|
+
}
|
|
4195
|
+
}
|
|
4196
|
+
async function writeResizeCheckpoint(checkpoint) {
|
|
4197
|
+
const path = resizeCheckpointPath(checkpoint.stackName);
|
|
4198
|
+
await mkdir2(join4(process.cwd(), STATE_DIR), { recursive: true });
|
|
4199
|
+
const tempPath = `${path}.${process.pid}.tmp`;
|
|
4200
|
+
await writeFile2(tempPath, `${JSON.stringify(checkpoint, null, 2)}
|
|
4201
|
+
`, "utf8");
|
|
4202
|
+
await rename2(tempPath, path);
|
|
4203
|
+
}
|
|
4204
|
+
async function acquireResizeLock(stackName, staleAfterMs = 30 * 60000) {
|
|
4205
|
+
const path = resizeLockPath(stackName);
|
|
4206
|
+
await mkdir2(join4(process.cwd(), STATE_DIR), { recursive: true });
|
|
4207
|
+
try {
|
|
4208
|
+
await mkdir2(path);
|
|
4209
|
+
} catch (error) {
|
|
4210
|
+
const age = Date.now() - (await stat(path)).mtimeMs;
|
|
4211
|
+
if (age <= staleAfterMs) {
|
|
4212
|
+
throw new Error(`Another resize process holds ${path}.`);
|
|
4213
|
+
}
|
|
4214
|
+
await rename2(path, `${path}.stale-${Date.now()}`);
|
|
4215
|
+
await mkdir2(path);
|
|
4216
|
+
}
|
|
4217
|
+
await writeFile2(join4(path, "owner.json"), `${JSON.stringify({ pid: process.pid, acquiredAt: new Date().toISOString() }, null, 2)}
|
|
4218
|
+
`, "utf8");
|
|
4219
|
+
return async () => {
|
|
4220
|
+
await unlink(join4(path, "owner.json")).catch(() => {});
|
|
4221
|
+
await rmdir(path).catch(() => {});
|
|
4222
|
+
};
|
|
4223
|
+
}
|
|
4224
|
+
// src/drivers/hetzner/provision.ts
|
|
4225
|
+
async function ensureSshKey(client, options) {
|
|
4226
|
+
const body = normalizeSshPublicKey(options.publicKey);
|
|
4227
|
+
const existing = (await client.listSshKeys()).find((k) => normalizeSshPublicKey(k.public_key) === body);
|
|
4228
|
+
if (existing)
|
|
4229
|
+
return { id: existing.id, name: existing.name, created: false };
|
|
4230
|
+
const created = await client.createSshKey({
|
|
4231
|
+
name: options.name,
|
|
4232
|
+
publicKey: options.publicKey.trim(),
|
|
4233
|
+
labels: options.labels
|
|
4234
|
+
});
|
|
4235
|
+
return { id: created.id, name: created.name, created: true };
|
|
4236
|
+
}
|
|
4237
|
+
async function ensureFirewall(client, options) {
|
|
4238
|
+
const existing = (await client.listFirewalls()).find((f) => f.name === options.name);
|
|
4239
|
+
if (existing) {
|
|
4240
|
+
await client.setFirewallRules(existing.id, options.rules);
|
|
4241
|
+
return { id: existing.id, name: existing.name, created: false };
|
|
4242
|
+
}
|
|
4243
|
+
const { firewall } = await client.createFirewall({ name: options.name, rules: options.rules, labels: options.labels });
|
|
4244
|
+
return { id: firewall.id, name: firewall.name, created: true };
|
|
4245
|
+
}
|
|
4246
|
+
async function ensureServer(client, options) {
|
|
4247
|
+
const { waitForRunning = true, ...createOptions } = options;
|
|
4248
|
+
let server = (await client.listServers()).find((s) => s.name === options.name);
|
|
4249
|
+
const created = !server;
|
|
4250
|
+
if (!server)
|
|
4251
|
+
server = (await client.createServer(createOptions)).server;
|
|
4252
|
+
if (waitForRunning)
|
|
4253
|
+
server = await client.waitForServerRunning(server.id);
|
|
4254
|
+
return { server, created };
|
|
4255
|
+
}
|
|
4256
|
+
function serverPublicIpv4(server) {
|
|
4257
|
+
const ip = server.public_net.ipv4?.ip;
|
|
4258
|
+
if (!ip)
|
|
4259
|
+
throw new Error(`Hetzner server "${server.name}" (#${server.id}) has no public IPv4`);
|
|
4260
|
+
return ip;
|
|
4261
|
+
}
|
|
3531
4262
|
// src/drivers/shared/box-provision.ts
|
|
3532
4263
|
var UBUNTU_2404_AMI_PARAM = "/aws/service/canonical/ubuntu/server/24.04/stable/current/amd64/hvm/ebs-gp3/ami-id";
|
|
3533
4264
|
function buildBoxUserData(spec) {
|
|
@@ -4098,16 +4829,16 @@ function buildHostCleanupScript() {
|
|
|
4098
4829
|
];
|
|
4099
4830
|
}
|
|
4100
4831
|
// src/drivers/shared/compute-deploy.ts
|
|
4101
|
-
import { copyFileSync } from "node:fs";
|
|
4832
|
+
import { copyFileSync as copyFileSync2 } from "node:fs";
|
|
4102
4833
|
import { tmpdir as tmpdir2 } from "node:os";
|
|
4103
|
-
import { join as
|
|
4834
|
+
import { join as join6 } from "node:path";
|
|
4104
4835
|
|
|
4105
4836
|
// src/deploy/management-dashboard.ts
|
|
4106
4837
|
import { execSync as execSync2 } from "node:child_process";
|
|
4107
4838
|
import { randomBytes } from "node:crypto";
|
|
4108
|
-
import { chmodSync, existsSync as existsSync2, mkdirSync, readFileSync as readFileSync3, writeFileSync } from "node:fs";
|
|
4839
|
+
import { chmodSync, copyFileSync, existsSync as existsSync2, mkdirSync, readFileSync as readFileSync3, statSync, writeFileSync } from "node:fs";
|
|
4109
4840
|
import { tmpdir } from "node:os";
|
|
4110
|
-
import { dirname, isAbsolute, join as
|
|
4841
|
+
import { basename, dirname, isAbsolute, join as join5, resolve } from "node:path";
|
|
4111
4842
|
import { fileURLToPath } from "node:url";
|
|
4112
4843
|
|
|
4113
4844
|
// src/deploy/dashboard-config-module.ts
|
|
@@ -4199,7 +4930,7 @@ function resolveDashboardVersion() {
|
|
|
4199
4930
|
if (explicit)
|
|
4200
4931
|
return explicit;
|
|
4201
4932
|
const here = dirname(fileURLToPath(import.meta.url));
|
|
4202
|
-
for (const candidate of [
|
|
4933
|
+
for (const candidate of [join5(here, "..", "..", "package.json"), join5(here, "..", "package.json")]) {
|
|
4203
4934
|
try {
|
|
4204
4935
|
const pkg = JSON.parse(readFileSync3(candidate, "utf8"));
|
|
4205
4936
|
if (pkg.name === "@stacksjs/ts-cloud" && pkg.version)
|
|
@@ -4216,14 +4947,27 @@ function stageLiveDashboardRoot(config, cwd, logger) {
|
|
|
4216
4947
|
const stage = resolveStatePath(cwd, "dashboard-release");
|
|
4217
4948
|
try {
|
|
4218
4949
|
mkdirSync(stage, { recursive: true });
|
|
4219
|
-
writeFileSync(
|
|
4950
|
+
writeFileSync(join5(stage, "cloud.config.ts"), serializeDashboardConfig(config));
|
|
4951
|
+
let dependency = resolveDashboardVersion();
|
|
4952
|
+
if (dependency.startsWith("file:")) {
|
|
4953
|
+
const requested = dependency.slice("file:".length);
|
|
4954
|
+
const source = isAbsolute(requested) ? requested : resolve(cwd, requested);
|
|
4955
|
+
if (!existsSync2(source) || !statSync(source).isFile()) {
|
|
4956
|
+
throw new Error(`TS_CLOUD_UI_VERSION points to a missing package file: ${source}`);
|
|
4957
|
+
}
|
|
4958
|
+
const name = basename(source);
|
|
4959
|
+
const destination = join5(stage, name);
|
|
4960
|
+
if (source !== destination)
|
|
4961
|
+
copyFileSync(source, destination);
|
|
4962
|
+
dependency = `file:./${name}`;
|
|
4963
|
+
}
|
|
4220
4964
|
const pkg = {
|
|
4221
4965
|
name: "ts-cloud-dashboard",
|
|
4222
4966
|
private: true,
|
|
4223
4967
|
type: "module",
|
|
4224
|
-
dependencies: { "@stacksjs/ts-cloud":
|
|
4968
|
+
dependencies: { "@stacksjs/ts-cloud": dependency }
|
|
4225
4969
|
};
|
|
4226
|
-
writeFileSync(
|
|
4970
|
+
writeFileSync(join5(stage, "package.json"), `${JSON.stringify(pkg, null, 2)}
|
|
4227
4971
|
`);
|
|
4228
4972
|
return liveStageDir();
|
|
4229
4973
|
} catch (error) {
|
|
@@ -4232,18 +4976,18 @@ function stageLiveDashboardRoot(config, cwd, logger) {
|
|
|
4232
4976
|
}
|
|
4233
4977
|
}
|
|
4234
4978
|
function resolveUiSource(cwd) {
|
|
4235
|
-
if (existsSync2(
|
|
4979
|
+
if (existsSync2(join5(cwd, "packages", "ui", "pages")) || existsSync2(join5(cwd, "packages", "ui", "package.json"))) {
|
|
4236
4980
|
return { uiRoot: "packages/ui/dist", build: "cd packages/ui && bun install && bun run build" };
|
|
4237
4981
|
}
|
|
4238
4982
|
const here = dirname(fileURLToPath(import.meta.url));
|
|
4239
4983
|
const candidates = [
|
|
4240
|
-
|
|
4241
|
-
|
|
4242
|
-
|
|
4243
|
-
|
|
4984
|
+
join5(here, "ui"),
|
|
4985
|
+
join5(here, "..", "ui"),
|
|
4986
|
+
join5(here, "..", "..", "ui"),
|
|
4987
|
+
join5(here, "..", "dist", "ui")
|
|
4244
4988
|
];
|
|
4245
4989
|
for (const dir of candidates) {
|
|
4246
|
-
if (existsSync2(
|
|
4990
|
+
if (existsSync2(join5(dir, "index.html")) || existsSync2(join5(dir, "serverless.html")))
|
|
4247
4991
|
return { uiRoot: dir, build: false };
|
|
4248
4992
|
}
|
|
4249
4993
|
return null;
|
|
@@ -4326,12 +5070,12 @@ function buildManagementDashboardArtifact(site, options) {
|
|
|
4326
5070
|
logger.info(`Management dashboard: building UI (${site.build})`);
|
|
4327
5071
|
execSync2(site.build, { cwd, stdio: "inherit" });
|
|
4328
5072
|
}
|
|
4329
|
-
const root = isAbsolute(site.root) ? site.root :
|
|
5073
|
+
const root = isAbsolute(site.root) ? site.root : join5(cwd, site.root);
|
|
4330
5074
|
if (!existsSync2(root)) {
|
|
4331
5075
|
logger.warn(`Management dashboard: build output not found at ${root} — skipping dashboard artifact.`);
|
|
4332
5076
|
return null;
|
|
4333
5077
|
}
|
|
4334
|
-
const tarball =
|
|
5078
|
+
const tarball = join5(tmpdir(), `${options.slug}-${options.siteName ?? MANAGEMENT_DASHBOARD_SITE}-${options.sha}.tar.gz`);
|
|
4335
5079
|
execSync2(`tar czf "${tarball}" -C "${root}" .`, { stdio: "inherit" });
|
|
4336
5080
|
return tarball;
|
|
4337
5081
|
} catch (error) {
|
|
@@ -4839,10 +5583,11 @@ async function deploySiteRelease(driver, options, logger = noopLogger2) {
|
|
|
4839
5583
|
return { success: false, error: hint };
|
|
4840
5584
|
}
|
|
4841
5585
|
const dbEnv = outputs.servicesPrivateIp ? buildFleetServicesEnv(outputs.servicesPrivateIp, resolveAppDatabase(config)) : buildManagedDbEnv(resolveAppDatabase(config));
|
|
4842
|
-
const
|
|
5586
|
+
const compute = config.infrastructure?.compute;
|
|
5587
|
+
const proxyEnv = usesRpxProxy(compute) ? { HOST: "127.0.0.1" } : {};
|
|
5588
|
+
const envWithServices = { ...dbEnv, ...proxyEnv, ...site.env || {} };
|
|
4843
5589
|
if (isPhpSite(site)) {
|
|
4844
|
-
const
|
|
4845
|
-
const phpVersion = site.phpVersion ?? compute2?.php?.default ?? compute2?.php?.versions?.[0];
|
|
5590
|
+
const phpVersion = site.phpVersion ?? compute?.php?.default ?? compute?.php?.versions?.[0];
|
|
4846
5591
|
const appBase2 = siteInstallBase(slug, siteName);
|
|
4847
5592
|
const siteWithEnv = Object.keys(dbEnv).length > 0 ? { ...site, env: envWithServices } : site;
|
|
4848
5593
|
const deployScript = buildLaravelDeployScript({
|
|
@@ -4852,7 +5597,7 @@ async function deploySiteRelease(driver, options, logger = noopLogger2) {
|
|
|
4852
5597
|
appBase: appBase2,
|
|
4853
5598
|
defaultPhpVersion: phpVersion
|
|
4854
5599
|
});
|
|
4855
|
-
const useNginx = !usesRpxProxy(
|
|
5600
|
+
const useNginx = !usesRpxProxy(compute);
|
|
4856
5601
|
const sslProvider = resolveSslProvider(site);
|
|
4857
5602
|
const customCert = sslProvider === "custom" && site.ssl?.certPath && site.ssl?.keyPath ? { certPath: site.ssl.certPath, keyPath: site.ssl.keyPath } : undefined;
|
|
4858
5603
|
const poolScript = site.isolation ? buildPhpFpmPoolScript({ siteName, appBase: appBase2 }) : [];
|
|
@@ -4868,7 +5613,7 @@ async function deploySiteRelease(driver, options, logger = noopLogger2) {
|
|
|
4868
5613
|
redirects: site.redirects,
|
|
4869
5614
|
ssl: customCert,
|
|
4870
5615
|
auth: site.auth && site.auth.enabled !== false && site.auth.password ? { username: site.auth.username || "admin", password: site.auth.password, realm: site.auth.realm } : undefined,
|
|
4871
|
-
serverSnippet: resolveNginxSnippet(site.nginx,
|
|
5616
|
+
serverSnippet: resolveNginxSnippet(site.nginx, compute?.nginxTemplates),
|
|
4872
5617
|
clientMaxBodySize: site.nginx?.clientMaxBodySize,
|
|
4873
5618
|
hsts: site.ssl?.hsts,
|
|
4874
5619
|
tlsProtocols: site.ssl?.tlsProtocols,
|
|
@@ -4944,7 +5689,6 @@ async function deploySiteRelease(driver, options, logger = noopLogger2) {
|
|
|
4944
5689
|
zeroDowntime: site.zeroDowntime !== false,
|
|
4945
5690
|
healthCheckPath: site.healthCheck?.path
|
|
4946
5691
|
});
|
|
4947
|
-
const compute = config.infrastructure?.compute;
|
|
4948
5692
|
const wantsNginxStatic = kind === "server-static" && !usesRpxProxy(compute) && !!site.domain;
|
|
4949
5693
|
const staticVhost = wantsNginxStatic ? buildNginxVhostScript({
|
|
4950
5694
|
siteName,
|
|
@@ -5001,6 +5745,58 @@ async function deploySiteRelease(driver, options, logger = noopLogger2) {
|
|
|
5001
5745
|
perInstance: result.perInstance
|
|
5002
5746
|
};
|
|
5003
5747
|
}
|
|
5748
|
+
function shellSingleQuote(value) {
|
|
5749
|
+
return `'${value.replaceAll("'", "'\\''")}'`;
|
|
5750
|
+
}
|
|
5751
|
+
function buildManagementDashboardServiceReconciliationScript(slug, desiredSiteNames) {
|
|
5752
|
+
const prefix = `${slug}-dashboard-`;
|
|
5753
|
+
const desiredUnits = desiredSiteNames.map((siteName) => `${slug}-${siteName}.service`);
|
|
5754
|
+
return [
|
|
5755
|
+
"set -euo pipefail",
|
|
5756
|
+
`TS_CLOUD_DASHBOARD_PREFIX=${shellSingleQuote(prefix)}`,
|
|
5757
|
+
`TS_CLOUD_DASHBOARD_DESIRED=${shellSingleQuote(` ${desiredUnits.join(" ")} `)}`,
|
|
5758
|
+
"for TS_CLOUD_UNIT_FILE in /etc/systemd/system/*.service; do",
|
|
5759
|
+
' [ -e "$TS_CLOUD_UNIT_FILE" ] || continue',
|
|
5760
|
+
' TS_CLOUD_UNIT=$(basename "$TS_CLOUD_UNIT_FILE")',
|
|
5761
|
+
' case "$TS_CLOUD_UNIT" in',
|
|
5762
|
+
' "$TS_CLOUD_DASHBOARD_PREFIX"*.service)',
|
|
5763
|
+
' case "$TS_CLOUD_DASHBOARD_DESIRED" in',
|
|
5764
|
+
' *" $TS_CLOUD_UNIT "*) ;;',
|
|
5765
|
+
" *)",
|
|
5766
|
+
' systemctl disable --now "$TS_CLOUD_UNIT" 2>/dev/null || true',
|
|
5767
|
+
' rm -f "$TS_CLOUD_UNIT_FILE"',
|
|
5768
|
+
" ;;",
|
|
5769
|
+
" esac",
|
|
5770
|
+
" ;;",
|
|
5771
|
+
" esac",
|
|
5772
|
+
"done",
|
|
5773
|
+
"systemctl daemon-reload",
|
|
5774
|
+
"systemctl reset-failed"
|
|
5775
|
+
];
|
|
5776
|
+
}
|
|
5777
|
+
async function reconcileManagementDashboardServices(driver, options, desiredSiteNames, logger) {
|
|
5778
|
+
const { config, environment } = options;
|
|
5779
|
+
const slug = config.project.slug;
|
|
5780
|
+
const stackName = resolveProjectStackName(config, environment);
|
|
5781
|
+
const targets = await driver.findComputeTargets({ slug, environment, role: "app", stackName });
|
|
5782
|
+
if (targets.length === 0)
|
|
5783
|
+
return true;
|
|
5784
|
+
const result = await driver.runRemoteDeploy({
|
|
5785
|
+
targets,
|
|
5786
|
+
commands: buildManagementDashboardServiceReconciliationScript(slug, desiredSiteNames),
|
|
5787
|
+
comment: `ts-cloud reconcile management dashboard ${slug}`,
|
|
5788
|
+
tags: {
|
|
5789
|
+
Project: slug,
|
|
5790
|
+
Environment: environment,
|
|
5791
|
+
Role: "app"
|
|
5792
|
+
}
|
|
5793
|
+
});
|
|
5794
|
+
if (!result.success) {
|
|
5795
|
+
logger.error(`Management dashboard reconciliation failed: ${result.error || "unknown error"}`);
|
|
5796
|
+
return false;
|
|
5797
|
+
}
|
|
5798
|
+
return true;
|
|
5799
|
+
}
|
|
5004
5800
|
async function ensureAttachModeDatabase(driver, options, logger) {
|
|
5005
5801
|
const { config, environment } = options;
|
|
5006
5802
|
if (!config.cloud?.attachTo)
|
|
@@ -5043,11 +5839,15 @@ async function deployAllComputeSites(options) {
|
|
|
5043
5839
|
const { config, environment, driver, sha, runtime, tarballForSite, logger = noopLogger2 } = options;
|
|
5044
5840
|
const slug = config.project.slug;
|
|
5045
5841
|
const cwd = options.cwd ?? process.cwd();
|
|
5842
|
+
const autoDashboard = options.managementDashboard !== false;
|
|
5046
5843
|
const hadDashboard = hasManagementDashboardSite(config);
|
|
5047
|
-
|
|
5048
|
-
|
|
5049
|
-
|
|
5844
|
+
if (autoDashboard)
|
|
5845
|
+
ensureManagementDashboard(config, { cwd, logger: { info: logger.info, warn: logger.warn } });
|
|
5846
|
+
const injectedDashboardSites = !autoDashboard || hadDashboard ? [] : managementDashboardSiteNames(config);
|
|
5847
|
+
if (autoDashboard && options.rpxConfig && options.rpxConfig !== config && !hasManagementDashboardSite(options.rpxConfig))
|
|
5050
5848
|
ensureManagementDashboard(options.rpxConfig, { cwd, logger: { info: () => {}, warn: () => {} } });
|
|
5849
|
+
if (autoDashboard && !await reconcileManagementDashboardServices(driver, options, managementDashboardSiteNames(config), logger))
|
|
5850
|
+
return false;
|
|
5051
5851
|
let dashboardTarball = null;
|
|
5052
5852
|
const dashboardTarballBySite = new Map;
|
|
5053
5853
|
if (injectedDashboardSites.length > 0) {
|
|
@@ -5071,8 +5871,8 @@ async function deployAllComputeSites(options) {
|
|
|
5071
5871
|
continue;
|
|
5072
5872
|
}
|
|
5073
5873
|
try {
|
|
5074
|
-
const copy =
|
|
5075
|
-
|
|
5874
|
+
const copy = join6(tmpdir2(), `${slug}-${name}-${sha}.tar.gz`);
|
|
5875
|
+
copyFileSync2(dashboardTarball, copy);
|
|
5076
5876
|
dashboardTarballBySite.set(name, copy);
|
|
5077
5877
|
} catch {
|
|
5078
5878
|
dashboardTarballBySite.set(name, dashboardTarball);
|
|
@@ -5339,4 +6139,4 @@ function buildCloudFrontOriginConfig(options) {
|
|
|
5339
6139
|
CustomErrorResponses: { Quantity: 0 }
|
|
5340
6140
|
};
|
|
5341
6141
|
}
|
|
5342
|
-
export { PANTRY_PROJECT_DIR, pgAdminCommand, BACKUP_RUNNER_PATH, buildBackupRestoreScript, siteInstallBase, isPhpSite, resolveSiteDeployTarget, resolveSiteKind, validateDeploymentConfig, shipsARelease, DEFAULT_RPX_CERTS_DIR, normalizeRoutePath, deriveRouteId, buildRpxConfig, buildRpxLbConfig, renderRpxLauncher, RPX_DIR, RPX_LAUNCHER_PATH, RPX_SERVICE_NAME, buildRpxProvisionScript, buildRpxFragmentRefreshScript, buildUbuntuBootstrapScript, AwsDriver, resolveHetznerLocation, HetznerClient, resolveHetznerApiToken2 as resolveHetznerApiToken, normalizeSshPublicKey, wrapCloudInitUserData, HetznerDriver, LocalBoxDriver, isBoxMode, createCloudDriver, CloudDriverFactory, cloudDrivers,
|
|
6142
|
+
export { PANTRY_PROJECT_DIR, pgAdminCommand, BACKUP_RUNNER_PATH, buildBackupRestoreScript, siteInstallBase, isPhpSite, resolveSiteDeployTarget, resolveSiteKind, validateDeploymentConfig, shipsARelease, DEFAULT_RPX_CERTS_DIR, normalizeRoutePath, deriveRouteId, buildRpxConfig, buildRpxLbConfig, renderRpxLauncher, RPX_DIR, RPX_LAUNCHER_PATH, RPX_SERVICE_NAME, buildRpxProvisionScript, buildRpxFragmentRefreshScript, buildUbuntuBootstrapScript, AwsDriver, resolveHetznerLocation, HetznerClient, resolveHetznerApiToken2 as resolveHetznerApiToken, normalizeSshPublicKey, wrapCloudInitUserData, HetznerDriver, LocalBoxDriver, isBoxMode, createCloudDriver, CloudDriverFactory, cloudDrivers, planHetznerServerResize, isHetznerCapacityError, executeHetznerServerResize, buildSshArgs, sshExec, sshExecOrThrow, scpUpload, waitForSsh, waitForCloudInit, collectHetznerResizeManifest, prepareHetznerResize, verifyHetznerResize, resolveHetznerHostOptimizationPlan, buildHetznerHostOptimizationScript, applyHetznerHostOptimization, collectHetznerHostOptimizationReport, verifyHetznerHostOptimization, resizeCheckpointPath, resizeLockPath, readResizeCheckpoint, writeResizeCheckpoint, acquireResizeLock, ensureSshKey, ensureFirewall, ensureServer, serverPublicIpv4, UBUNTU_2404_AMI_PARAM, buildBoxUserData, HetznerBoxProvisioner, AwsBoxProvisioner, createBoxProvisioner, releasePaths, buildRollbackScript, resolveExecStart, buildSiteDeployScript, buildStaticSiteDeployScript, releaseTarballTmpPath, buildAwsArtifactFetch, buildLocalArtifactFetch, buildHostCleanupScript, MANAGEMENT_DASHBOARD_SITE, dashboardCredentialsFile, resolveDashboardAuth, resolveUiSource, ensureManagementDashboard, buildManagementDashboardArtifact, resolveSiteFramework, deploySiteRelease, deployAllComputeSites, reloadRpxGateway, renewRpxCertificates, MANAGED_CACHE_POLICY_OPTIMIZED, MANAGED_CACHE_POLICY_DISABLED, MANAGED_ORIGIN_REQUEST_POLICY_ALL_VIEWER, buildCloudFrontOriginConfig };
|