@stacksjs/ts-cloud 0.7.62 → 0.7.64
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/cli.js +1556 -1284
- package/dist/{chunk-vx2yva1a.js → chunk-0z7ry8fm.js} +897 -93
- package/dist/{chunk-zvk0f89t.js → chunk-28vh1h91.js} +1 -1
- package/dist/{chunk-ff77qy0c.js → chunk-3c6pp063.js} +519 -143
- package/dist/{chunk-tq46tnxv.js → chunk-ecsyc3p0.js} +4 -1
- package/dist/{chunk-rxhqny6z.js → chunk-zcvdzvmz.js} +2 -2
- package/dist/deploy/dashboard-data-server.d.ts +27 -1
- package/dist/deploy/index.js +4 -4
- package/dist/deploy/telemetry-collection.d.ts +2 -0
- package/dist/drivers/hetzner/client.d.ts +84 -2
- package/dist/drivers/hetzner/host-optimization.d.ts +33 -0
- package/dist/drivers/hetzner/monitoring.d.ts +46 -0
- package/dist/drivers/hetzner/resize-remote.d.ts +38 -0
- package/dist/drivers/hetzner/resize-state.d.ts +24 -0
- package/dist/drivers/hetzner/resize.d.ts +86 -0
- package/dist/drivers/index.d.ts +8 -0
- package/dist/drivers/index.js +35 -3
- package/dist/drivers/shared/compute-deploy.d.ts +15 -0
- package/dist/drivers/shared/releases.d.ts +11 -0
- package/dist/drivers/shared/ufw.d.ts +4 -2
- package/dist/index.js +4 -4
- package/dist/ui/access-denied.html +2 -2
- package/dist/ui/account/automation.html +3 -3
- package/dist/ui/account/security.html +2 -2
- package/dist/ui/applications/compose.html +4 -4
- package/dist/ui/applications/new.html +2 -2
- package/dist/ui/data/backups.html +4 -4
- package/dist/ui/data/services.html +4 -4
- package/dist/ui/data/volumes.html +4 -4
- package/dist/ui/index.html +12 -7
- package/dist/ui/integrations.html +2 -2
- package/dist/ui/operations/alerts.html +4 -4
- package/dist/ui/operations/configuration.html +4 -4
- package/dist/ui/operations/jobs.html +4 -4
- package/dist/ui/operations/maintenance.html +3 -3
- package/dist/ui/operations/observability.html +13 -7
- package/dist/ui/operations/previews.html +4 -4
- package/dist/ui/operations/queue.html +4 -4
- package/dist/ui/operations/regions.html +4 -4
- package/dist/ui/operations/releases.html +4 -4
- package/dist/ui/operations/workloads.html +3 -3
- package/dist/ui/security.html +2 -2
- package/dist/ui/server/actions.html +4 -4
- package/dist/ui/server/activity.html +2 -2
- package/dist/ui/server/capacity.html +4 -4
- package/dist/ui/server/database.html +3 -3
- package/dist/ui/server/deployments.html +4 -4
- package/dist/ui/server/diagnostics.html +2 -2
- package/dist/ui/server/firewall.html +4 -4
- package/dist/ui/server/fleet.html +4 -4
- package/dist/ui/server/logs.html +4 -4
- package/dist/ui/server/metrics.html +203 -3
- package/dist/ui/server/security.html +2 -2
- package/dist/ui/server/services.html +2 -2
- package/dist/ui/server/sites.html +102 -6
- package/dist/ui/server/ssh-keys.html +4 -4
- package/dist/ui/server/team.html +4 -4
- package/dist/ui/server/terminal.html +2 -2
- package/dist/ui/serverless/alarms.html +3 -3
- package/dist/ui/serverless/assets.html +2 -2
- package/dist/ui/serverless/cost.html +2 -2
- package/dist/ui/serverless/data.html +4 -4
- package/dist/ui/serverless/deployments.html +2 -2
- package/dist/ui/serverless/firewall.html +2 -2
- package/dist/ui/serverless/functions.html +4 -4
- package/dist/ui/serverless/logs.html +3 -3
- package/dist/ui/serverless/metrics.html +2 -2
- package/dist/ui/serverless/queues.html +4 -4
- package/dist/ui/serverless/secrets.html +4 -4
- package/dist/ui/serverless/traces.html +4 -4
- package/dist/ui/serverless.html +4 -4
- package/dist/ui-src/pages/index.stx +12 -10
- package/dist/ui-src/pages/operations/observability.stx +6 -4
- package/dist/ui-src/pages/server/metrics.stx +262 -106
- package/dist/ui-src/pages/server/sites.stx +98 -6
- package/package.json +3 -3
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
resolveProjectStackName,
|
|
9
9
|
resolveStatePath,
|
|
10
10
|
statePath
|
|
11
|
-
} from "./chunk-
|
|
11
|
+
} from "./chunk-ecsyc3p0.js";
|
|
12
12
|
import {
|
|
13
13
|
EC2Client,
|
|
14
14
|
SSMClient
|
|
@@ -346,15 +346,14 @@ function buildAutoUpdatesScript(enabled2 = true) {
|
|
|
346
346
|
return [];
|
|
347
347
|
return [
|
|
348
348
|
"export DEBIAN_FRONTEND=noninteractive",
|
|
349
|
-
|
|
350
|
-
|
|
349
|
+
`dpkg-query -W -f='\${Status}' unattended-upgrades 2>/dev/null | grep -q 'ok installed' || apt-get install -y unattended-upgrades`,
|
|
350
|
+
`cat > /etc/apt/apt.conf.d/20auto-upgrades <<'TS_CLOUD_AUTOUPD_EOF'`,
|
|
351
351
|
'APT::Periodic::Update-Package-Lists "1";',
|
|
352
352
|
'APT::Periodic::Unattended-Upgrade "1";',
|
|
353
353
|
'APT::Periodic::Download-Upgradeable-Packages "1";',
|
|
354
354
|
'APT::Periodic::AutocleanInterval "7";',
|
|
355
355
|
"TS_CLOUD_AUTOUPD_EOF",
|
|
356
|
-
"systemctl enable unattended-upgrades 2>/dev/null || true"
|
|
357
|
-
"systemctl restart unattended-upgrades 2>/dev/null || true"
|
|
356
|
+
"systemctl enable --now unattended-upgrades 2>/dev/null || true"
|
|
358
357
|
];
|
|
359
358
|
}
|
|
360
359
|
|
|
@@ -366,11 +365,14 @@ var DEFAULT_MEM_PERCENT = 90;
|
|
|
366
365
|
var DEFAULT_DISK_PERCENT = 90;
|
|
367
366
|
var SERVICE_PROBES = [
|
|
368
367
|
["nginx", 80],
|
|
368
|
+
["https", 443],
|
|
369
369
|
["phpFpm", 9074],
|
|
370
370
|
["mysql", 3306],
|
|
371
371
|
["postgres", 5432],
|
|
372
372
|
["redis", 6379],
|
|
373
|
-
["meilisearch", 7700]
|
|
373
|
+
["meilisearch", 7700],
|
|
374
|
+
["typesense", 8108],
|
|
375
|
+
["smtp", 25]
|
|
374
376
|
];
|
|
375
377
|
function resolveMonitoring(monitoring = true) {
|
|
376
378
|
const obj = typeof monitoring === "object" ? monitoring : {};
|
|
@@ -389,18 +391,20 @@ function buildMonitoringScript(monitoring = true) {
|
|
|
389
391
|
const probeLines = SERVICE_PROBES.map(([name, port]) => `SVC_${name.toUpperCase()}=$(probe ${port})`);
|
|
390
392
|
const servicesJson = SERVICE_PROBES.map(([name]) => `"${name}":"$SVC_${name.toUpperCase()}"`).join(",");
|
|
391
393
|
return [
|
|
394
|
+
"if ! command -v sadf >/dev/null 2>&1; then export DEBIAN_FRONTEND=noninteractive; apt-get update -qq; apt-get install -y -qq sysstat; fi",
|
|
395
|
+
"systemctl enable --now sysstat.service 2>/dev/null || true",
|
|
392
396
|
"mkdir -p /var/lib/ts-cloud",
|
|
393
|
-
|
|
397
|
+
`cat > /usr/local/bin/ts-cloud-metrics.sh <<'TS_CLOUD_METRICS_EOF'`,
|
|
394
398
|
"#!/bin/bash",
|
|
395
399
|
"set -uo pipefail",
|
|
396
|
-
|
|
400
|
+
`LOAD=$(cut -d' ' -f1 /proc/loadavg)`,
|
|
397
401
|
"CPUS=$(nproc)",
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
+
`MEM_TOTAL=$(free -m | awk '/^Mem:/{print $2}')`,
|
|
403
|
+
`MEM_USED=$(free -m | awk '/^Mem:/{print $3}')`,
|
|
404
|
+
`SWAP_TOTAL=$(free -m | awk '/^Swap:/{print $2}')`,
|
|
405
|
+
`SWAP_USED=$(free -m | awk '/^Swap:/{print $3}')`,
|
|
402
406
|
`DISK_PCT=$(df -P / | awk 'NR==2{gsub("%","",$5); print $5}')`,
|
|
403
|
-
|
|
407
|
+
`UPTIME_SEC=$(cut -d' ' -f1 /proc/uptime | cut -d. -f1)`,
|
|
404
408
|
`RX_BYTES=$(awk -F'[: ]+' 'NR>2 && $2!="lo"{rx+=$3} END{print rx+0}' /proc/net/dev)`,
|
|
405
409
|
`TX_BYTES=$(awk -F'[: ]+' 'NR>2 && $2!="lo"{tx+=$11} END{print tx+0}' /proc/net/dev)`,
|
|
406
410
|
"probe(){ (exec 3<>/dev/tcp/127.0.0.1/$1) 2>/dev/null && echo up || echo down; }",
|
|
@@ -427,7 +431,7 @@ function buildMonitoringScript(monitoring = true) {
|
|
|
427
431
|
"fi",
|
|
428
432
|
"TS_CLOUD_METRICS_EOF",
|
|
429
433
|
"chmod +x /usr/local/bin/ts-cloud-metrics.sh",
|
|
430
|
-
|
|
434
|
+
`cat > /etc/systemd/system/ts-cloud-metrics.service <<'TS_CLOUD_METRICS_SVC_EOF'`,
|
|
431
435
|
"[Unit]",
|
|
432
436
|
"Description=ts-cloud metrics collector",
|
|
433
437
|
"",
|
|
@@ -435,20 +439,22 @@ function buildMonitoringScript(monitoring = true) {
|
|
|
435
439
|
"Type=oneshot",
|
|
436
440
|
"ExecStart=/usr/local/bin/ts-cloud-metrics.sh",
|
|
437
441
|
"TS_CLOUD_METRICS_SVC_EOF",
|
|
438
|
-
|
|
442
|
+
`cat > /etc/systemd/system/ts-cloud-metrics.timer <<'TS_CLOUD_METRICS_TMR_EOF'`,
|
|
439
443
|
"[Unit]",
|
|
440
444
|
"Description=Run ts-cloud metrics collector every minute",
|
|
441
445
|
"",
|
|
442
446
|
"[Timer]",
|
|
443
|
-
"
|
|
444
|
-
"
|
|
447
|
+
"OnCalendar=*-*-* *:*:00",
|
|
448
|
+
"AccuracySec=1s",
|
|
449
|
+
"RandomizedDelaySec=5s",
|
|
450
|
+
"Persistent=true",
|
|
445
451
|
"",
|
|
446
452
|
"[Install]",
|
|
447
453
|
"WantedBy=timers.target",
|
|
448
454
|
"TS_CLOUD_METRICS_TMR_EOF",
|
|
449
455
|
"systemctl daemon-reload",
|
|
450
|
-
"systemctl enable ts-cloud-metrics.timer",
|
|
451
|
-
"systemctl start ts-cloud-metrics.
|
|
456
|
+
"systemctl enable --now ts-cloud-metrics.timer",
|
|
457
|
+
"systemctl start ts-cloud-metrics.service"
|
|
452
458
|
];
|
|
453
459
|
}
|
|
454
460
|
|
|
@@ -1170,6 +1176,7 @@ import { readdirSync, readFileSync } from 'node:fs'
|
|
|
1170
1176
|
const dir = ${JSON.stringify(sitesDir)}
|
|
1171
1177
|
const proxies = []
|
|
1172
1178
|
const seen = new Set()
|
|
1179
|
+
const owners = new Map()
|
|
1173
1180
|
const suffixes = new Set()
|
|
1174
1181
|
const guardHosts = new Set()
|
|
1175
1182
|
let email
|
|
@@ -1190,8 +1197,12 @@ for (const f of files) {
|
|
|
1190
1197
|
catch (err) { console.error('[rpx-assembler] SKIPPING malformed fragment ' + f + ' — its host(s) will 404 until fixed: ' + err); continue }
|
|
1191
1198
|
for (const p of frag.proxies ?? []) {
|
|
1192
1199
|
const key = p.id || (p.to + (p.path ?? ''))
|
|
1193
|
-
if (seen.has(key))
|
|
1200
|
+
if (seen.has(key)) {
|
|
1201
|
+
console.warn('[rpx-assembler] duplicate route ' + key + ' in ' + f + ' ignored; first declared by ' + owners.get(key))
|
|
1202
|
+
continue
|
|
1203
|
+
}
|
|
1194
1204
|
seen.add(key)
|
|
1205
|
+
owners.set(key, f)
|
|
1195
1206
|
proxies.push(p)
|
|
1196
1207
|
}
|
|
1197
1208
|
for (const s of frag.onDemandTls?.allowedSuffixes ?? []) suffixes.add(s)
|
|
@@ -1427,7 +1438,8 @@ function buildUfwScript(firewall = {}) {
|
|
|
1427
1438
|
const ports = [...new Set([...UFW_BASE_PORTS, ...firewall.allowedPorts || []])].filter((p) => Number.isInteger(p) && p >= 1 && p <= 65535).sort((a, b) => a - b);
|
|
1428
1439
|
const lines = [
|
|
1429
1440
|
"export DEBIAN_FRONTEND=noninteractive",
|
|
1430
|
-
"apt-get install -y ufw",
|
|
1441
|
+
"command -v ufw >/dev/null 2>&1 || apt-get install -y ufw",
|
|
1442
|
+
"ufw --force reset",
|
|
1431
1443
|
"ufw default deny incoming",
|
|
1432
1444
|
"ufw default allow outgoing",
|
|
1433
1445
|
"ufw allow OpenSSH"
|
|
@@ -2132,6 +2144,28 @@ function resolveHetznerSettings(config, overrides = {}) {
|
|
|
2132
2144
|
// src/drivers/hetzner/client.ts
|
|
2133
2145
|
var DEFAULT_API_URL = "https://api.hetzner.cloud/v1";
|
|
2134
2146
|
|
|
2147
|
+
class HetznerApiError extends Error {
|
|
2148
|
+
status;
|
|
2149
|
+
code;
|
|
2150
|
+
details;
|
|
2151
|
+
constructor(message, status, code, details) {
|
|
2152
|
+
super(message);
|
|
2153
|
+
this.status = status;
|
|
2154
|
+
this.code = code;
|
|
2155
|
+
this.details = details;
|
|
2156
|
+
this.name = "HetznerApiError";
|
|
2157
|
+
}
|
|
2158
|
+
}
|
|
2159
|
+
|
|
2160
|
+
class HetznerActionError extends Error {
|
|
2161
|
+
action;
|
|
2162
|
+
constructor(message, action) {
|
|
2163
|
+
super(message);
|
|
2164
|
+
this.action = action;
|
|
2165
|
+
this.name = "HetznerActionError";
|
|
2166
|
+
}
|
|
2167
|
+
}
|
|
2168
|
+
|
|
2135
2169
|
class HetznerClient {
|
|
2136
2170
|
name = "hetzner";
|
|
2137
2171
|
apiToken;
|
|
@@ -2165,7 +2199,7 @@ class HetznerClient {
|
|
|
2165
2199
|
if (!response.ok) {
|
|
2166
2200
|
const message = data.error?.message || response.statusText || "Hetzner API error";
|
|
2167
2201
|
const code = data.error?.code ? ` [${data.error.code}]` : "";
|
|
2168
|
-
throw new
|
|
2202
|
+
throw new HetznerApiError(`Hetzner API ${method} ${path} (${response.status})${code}: ${message}`, response.status, data.error?.code, data.error?.details);
|
|
2169
2203
|
}
|
|
2170
2204
|
return data;
|
|
2171
2205
|
}
|
|
@@ -2189,6 +2223,23 @@ class HetznerClient {
|
|
|
2189
2223
|
const data = await this.request("GET", `/servers/${id}`);
|
|
2190
2224
|
return data.server;
|
|
2191
2225
|
}
|
|
2226
|
+
async getServerMetrics(id, options) {
|
|
2227
|
+
const query = new URLSearchParams({
|
|
2228
|
+
type: options.types.join(","),
|
|
2229
|
+
start: options.start.toISOString(),
|
|
2230
|
+
end: options.end.toISOString(),
|
|
2231
|
+
step: String(Math.max(60, Math.floor(options.step ?? 60)))
|
|
2232
|
+
});
|
|
2233
|
+
const data = await this.request("GET", `/servers/${id}/metrics?${query}`);
|
|
2234
|
+
return data.metrics;
|
|
2235
|
+
}
|
|
2236
|
+
async listServerTypes(name) {
|
|
2237
|
+
const query = name ? `?name=${encodeURIComponent(name)}` : "";
|
|
2238
|
+
return this.requestAll(`/server_types${query}`, "server_types");
|
|
2239
|
+
}
|
|
2240
|
+
async getServerType(name) {
|
|
2241
|
+
return (await this.listServerTypes(name)).find((item) => item.name === name) ?? null;
|
|
2242
|
+
}
|
|
2192
2243
|
async createServer(options) {
|
|
2193
2244
|
const data = await this.request("POST", "/servers", {
|
|
2194
2245
|
name: options.name,
|
|
@@ -2300,16 +2351,35 @@ class HetznerClient {
|
|
|
2300
2351
|
});
|
|
2301
2352
|
return data.ssh_key;
|
|
2302
2353
|
}
|
|
2354
|
+
async getAction(actionId) {
|
|
2355
|
+
const data = await this.request("GET", `/actions/${actionId}`);
|
|
2356
|
+
return data.action;
|
|
2357
|
+
}
|
|
2358
|
+
async shutdownServer(serverId) {
|
|
2359
|
+
const data = await this.request("POST", `/servers/${serverId}/actions/shutdown`, {});
|
|
2360
|
+
return data.action;
|
|
2361
|
+
}
|
|
2362
|
+
async powerOnServer(serverId) {
|
|
2363
|
+
const data = await this.request("POST", `/servers/${serverId}/actions/poweron`, {});
|
|
2364
|
+
return data.action;
|
|
2365
|
+
}
|
|
2366
|
+
async changeServerType(serverId, serverType, upgradeDisk) {
|
|
2367
|
+
const data = await this.request("POST", `/servers/${serverId}/actions/change_type`, {
|
|
2368
|
+
server_type: serverType,
|
|
2369
|
+
upgrade_disk: upgradeDisk
|
|
2370
|
+
});
|
|
2371
|
+
return data.action;
|
|
2372
|
+
}
|
|
2303
2373
|
async waitForAction(actionId, options) {
|
|
2304
2374
|
const pollInterval = options?.pollIntervalMs ?? 2000;
|
|
2305
2375
|
const maxWait = options?.maxWaitMs ?? 300000;
|
|
2306
2376
|
const start = Date.now();
|
|
2307
2377
|
while (Date.now() - start < maxWait) {
|
|
2308
|
-
const
|
|
2309
|
-
if (
|
|
2310
|
-
return
|
|
2311
|
-
if (
|
|
2312
|
-
throw new
|
|
2378
|
+
const action = await this.getAction(actionId);
|
|
2379
|
+
if (action.status === "success")
|
|
2380
|
+
return action;
|
|
2381
|
+
if (action.status === "error") {
|
|
2382
|
+
throw new HetznerActionError(action.error?.message || "Hetzner action failed", action);
|
|
2313
2383
|
}
|
|
2314
2384
|
await new Promise((resolve) => setTimeout(resolve, pollInterval));
|
|
2315
2385
|
}
|
|
@@ -2327,6 +2397,18 @@ class HetznerClient {
|
|
|
2327
2397
|
}
|
|
2328
2398
|
throw new Error(`Timed out waiting for server ${serverId} to reach running state`);
|
|
2329
2399
|
}
|
|
2400
|
+
async waitForServerStatus(serverId, status, options) {
|
|
2401
|
+
const pollInterval = options?.pollIntervalMs ?? 3000;
|
|
2402
|
+
const maxWait = options?.maxWaitMs ?? 600000;
|
|
2403
|
+
const start = Date.now();
|
|
2404
|
+
while (Date.now() - start < maxWait) {
|
|
2405
|
+
const server = await this.getServer(serverId);
|
|
2406
|
+
if (server.status === status)
|
|
2407
|
+
return server;
|
|
2408
|
+
await new Promise((resolve) => setTimeout(resolve, pollInterval));
|
|
2409
|
+
}
|
|
2410
|
+
throw new Error(`Timed out waiting for server ${serverId} to reach ${status} state`);
|
|
2411
|
+
}
|
|
2330
2412
|
}
|
|
2331
2413
|
function resolveHetznerApiToken2(configToken, config) {
|
|
2332
2414
|
const token = resolveHetznerApiToken(configToken, config);
|
|
@@ -2499,7 +2581,7 @@ class HetznerDriver {
|
|
|
2499
2581
|
}
|
|
2500
2582
|
const labels = tsCloudLabels(slug, environment, "app");
|
|
2501
2583
|
const sites = config.sites || {};
|
|
2502
|
-
const sitePorts = this.collectUpstreamPorts(sites);
|
|
2584
|
+
const sitePorts = usesRpxProxy(compute) ? [] : this.collectUpstreamPorts(sites);
|
|
2503
2585
|
const firewallName = `${slug}-${environment}-app-fw`;
|
|
2504
2586
|
const { firewall } = await this.ensureFirewall(firewallName, labels, buildHetznerFirewallRules({
|
|
2505
2587
|
allowSsh: compute.allowSsh !== false,
|
|
@@ -3430,43 +3512,179 @@ class CloudDriverFactory {
|
|
|
3430
3512
|
}
|
|
3431
3513
|
}
|
|
3432
3514
|
var cloudDrivers = new CloudDriverFactory;
|
|
3433
|
-
// src/drivers/hetzner/
|
|
3434
|
-
|
|
3435
|
-
|
|
3436
|
-
|
|
3437
|
-
|
|
3438
|
-
|
|
3439
|
-
|
|
3440
|
-
|
|
3441
|
-
|
|
3442
|
-
|
|
3443
|
-
}
|
|
3444
|
-
|
|
3515
|
+
// src/drivers/hetzner/resize.ts
|
|
3516
|
+
function serverLocation(server) {
|
|
3517
|
+
return server.location?.name ?? server.datacenter?.location.name ?? "unknown";
|
|
3518
|
+
}
|
|
3519
|
+
function positive(value) {
|
|
3520
|
+
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
|
3521
|
+
}
|
|
3522
|
+
function assertCompatibleUpgrade(current, target) {
|
|
3523
|
+
if (current.architecture && target.architecture && current.architecture !== target.architecture) {
|
|
3524
|
+
throw new Error(`Cannot resize ${current.name} (${current.architecture}) to ${target.name} (${target.architecture}) in place.`);
|
|
3525
|
+
}
|
|
3526
|
+
const regressions = [];
|
|
3527
|
+
if (positive(target.cores) < positive(current.cores))
|
|
3528
|
+
regressions.push("CPU");
|
|
3529
|
+
if (positive(target.memory) < positive(current.memory))
|
|
3530
|
+
regressions.push("memory");
|
|
3531
|
+
if (positive(target.disk) < positive(current.disk))
|
|
3532
|
+
regressions.push("disk");
|
|
3533
|
+
if (regressions.length > 0) {
|
|
3534
|
+
throw new Error(`Refusing an in-place downgrade from ${current.name} to ${target.name}: ${regressions.join(", ")} would shrink.`);
|
|
3535
|
+
}
|
|
3536
|
+
}
|
|
3537
|
+
async function planHetznerServerResize(client, serverId, targetTypeName, upgradeDisk = true) {
|
|
3538
|
+
const [server, targetType] = await Promise.all([
|
|
3539
|
+
client.getServer(serverId),
|
|
3540
|
+
client.getServerType(targetTypeName)
|
|
3541
|
+
]);
|
|
3542
|
+
if (!targetType)
|
|
3543
|
+
throw new Error(`Hetzner server type ${targetTypeName} was not found.`);
|
|
3544
|
+
const currentType = server.server_type;
|
|
3545
|
+
const alreadyComplete = currentType.name === targetType.name;
|
|
3546
|
+
if (!alreadyComplete)
|
|
3547
|
+
assertCompatibleUpgrade(currentType, targetType);
|
|
3548
|
+
const location = serverLocation(server);
|
|
3549
|
+
const locationState = targetType.locations?.find((item) => item.name === location);
|
|
3550
|
+
return {
|
|
3551
|
+
serverId,
|
|
3552
|
+
serverName: server.name,
|
|
3553
|
+
location,
|
|
3554
|
+
currentType,
|
|
3555
|
+
targetType,
|
|
3556
|
+
capacityAvailable: locationState?.available !== false,
|
|
3557
|
+
alreadyComplete,
|
|
3558
|
+
upgradeDisk
|
|
3559
|
+
};
|
|
3445
3560
|
}
|
|
3446
|
-
|
|
3447
|
-
|
|
3448
|
-
|
|
3449
|
-
|
|
3450
|
-
|
|
3561
|
+
function isHetznerCapacityError(error) {
|
|
3562
|
+
if (error instanceof HetznerApiError && error.code === "resource_unavailable")
|
|
3563
|
+
return true;
|
|
3564
|
+
if (error instanceof HetznerActionError) {
|
|
3565
|
+
if (error.action.error?.code === "resource_unavailable")
|
|
3566
|
+
return true;
|
|
3567
|
+
if (/no fitting vhost|placement|free resources/i.test(error.action.error?.message ?? ""))
|
|
3568
|
+
return true;
|
|
3451
3569
|
}
|
|
3452
|
-
|
|
3453
|
-
return { id: firewall.id, name: firewall.name, created: true };
|
|
3570
|
+
return /resource_unavailable|no fitting vhost|error during placement|free resources/i.test(error instanceof Error ? error.message : String(error));
|
|
3454
3571
|
}
|
|
3455
|
-
async function
|
|
3456
|
-
|
|
3457
|
-
let server = (await client.listServers()).find((s) => s.name === options.name);
|
|
3458
|
-
const created = !server;
|
|
3459
|
-
if (!server)
|
|
3460
|
-
server = (await client.createServer(createOptions)).server;
|
|
3461
|
-
if (waitForRunning)
|
|
3462
|
-
server = await client.waitForServerRunning(server.id);
|
|
3463
|
-
return { server, created };
|
|
3572
|
+
async function phase(hooks, value, detail) {
|
|
3573
|
+
await hooks?.onPhase?.(value, detail);
|
|
3464
3574
|
}
|
|
3465
|
-
function
|
|
3466
|
-
|
|
3467
|
-
|
|
3468
|
-
|
|
3469
|
-
|
|
3575
|
+
async function waitForAction(client, action, options) {
|
|
3576
|
+
await client.waitForAction(action.id, {
|
|
3577
|
+
pollIntervalMs: options.actionPollIntervalMs,
|
|
3578
|
+
maxWaitMs: options.actionTimeoutMs
|
|
3579
|
+
});
|
|
3580
|
+
}
|
|
3581
|
+
var ensurePoweredOn = async ({
|
|
3582
|
+
client,
|
|
3583
|
+
hooks,
|
|
3584
|
+
actionPollIntervalMs,
|
|
3585
|
+
actionTimeoutMs,
|
|
3586
|
+
serverPollIntervalMs,
|
|
3587
|
+
serverTimeoutMs
|
|
3588
|
+
}, plan, manifest, recovered) => {
|
|
3589
|
+
let server = await client.getServer(plan.serverId);
|
|
3590
|
+
if (server.status !== "running") {
|
|
3591
|
+
await phase(hooks, recovered ? "recovering" : "powering-on", {
|
|
3592
|
+
serverId: plan.serverId
|
|
3593
|
+
});
|
|
3594
|
+
const action = await client.powerOnServer(plan.serverId);
|
|
3595
|
+
await waitForAction(client, action, { actionPollIntervalMs, actionTimeoutMs });
|
|
3596
|
+
server = await client.waitForServerRunning(plan.serverId, {
|
|
3597
|
+
pollIntervalMs: serverPollIntervalMs,
|
|
3598
|
+
maxWaitMs: serverTimeoutMs
|
|
3599
|
+
});
|
|
3600
|
+
}
|
|
3601
|
+
await phase(hooks, "verifying", {
|
|
3602
|
+
serverId: plan.serverId,
|
|
3603
|
+
recovered
|
|
3604
|
+
});
|
|
3605
|
+
const verification = await hooks?.afterBoot?.(server, plan.targetType, manifest, { recovered });
|
|
3606
|
+
return { server, verification };
|
|
3607
|
+
};
|
|
3608
|
+
async function executeHetznerServerResize(options) {
|
|
3609
|
+
const { client, hooks } = options;
|
|
3610
|
+
const upgradeDisk = options.upgradeDisk ?? true;
|
|
3611
|
+
await phase(hooks, "planning", {
|
|
3612
|
+
serverId: options.serverId,
|
|
3613
|
+
targetType: options.targetType
|
|
3614
|
+
});
|
|
3615
|
+
const plan = await planHetznerServerResize(client, options.serverId, options.targetType, upgradeDisk);
|
|
3616
|
+
let server = await client.getServer(plan.serverId);
|
|
3617
|
+
if (plan.alreadyComplete) {
|
|
3618
|
+
const boot2 = await ensurePoweredOn(options, plan, options.manifest, false);
|
|
3619
|
+
await phase(hooks, "complete", { alreadyComplete: true });
|
|
3620
|
+
return { status: "already-complete", plan, verification: boot2.verification };
|
|
3621
|
+
}
|
|
3622
|
+
if (!plan.capacityAvailable && !options.attemptUnavailable) {
|
|
3623
|
+
await phase(hooks, "waiting-capacity", {
|
|
3624
|
+
location: plan.location,
|
|
3625
|
+
targetType: plan.targetType.name
|
|
3626
|
+
});
|
|
3627
|
+
return { status: "waiting-capacity", plan };
|
|
3628
|
+
}
|
|
3629
|
+
let manifest = options.manifest;
|
|
3630
|
+
if (server.status === "off" && !options.resumeFromOff) {
|
|
3631
|
+
const recovered = await ensurePoweredOn(options, plan, undefined, true);
|
|
3632
|
+
return {
|
|
3633
|
+
status: "recovered",
|
|
3634
|
+
plan,
|
|
3635
|
+
server: recovered.server,
|
|
3636
|
+
verification: recovered.verification,
|
|
3637
|
+
retryable: true,
|
|
3638
|
+
error: "Server was already off without a resumable resize checkpoint; powered it back on."
|
|
3639
|
+
};
|
|
3640
|
+
}
|
|
3641
|
+
if (server.status === "running") {
|
|
3642
|
+
await phase(hooks, "preflight", { serverId: plan.serverId });
|
|
3643
|
+
manifest = await hooks?.preflight?.(server, plan.targetType);
|
|
3644
|
+
await phase(hooks, "shutting-down", { serverId: plan.serverId });
|
|
3645
|
+
const shutdown = await client.shutdownServer(plan.serverId);
|
|
3646
|
+
await waitForAction(client, shutdown, options);
|
|
3647
|
+
server = await client.waitForServerStatus(plan.serverId, "off", {
|
|
3648
|
+
pollIntervalMs: options.serverPollIntervalMs,
|
|
3649
|
+
maxWaitMs: options.serverTimeoutMs
|
|
3650
|
+
});
|
|
3651
|
+
}
|
|
3652
|
+
try {
|
|
3653
|
+
await phase(hooks, "resizing", {
|
|
3654
|
+
serverId: plan.serverId,
|
|
3655
|
+
from: plan.currentType.name,
|
|
3656
|
+
to: plan.targetType.name,
|
|
3657
|
+
upgradeDisk
|
|
3658
|
+
});
|
|
3659
|
+
const change = await client.changeServerType(plan.serverId, plan.targetType.name, upgradeDisk);
|
|
3660
|
+
await waitForAction(client, change, options);
|
|
3661
|
+
} catch (error) {
|
|
3662
|
+
const recovered = await ensurePoweredOn(options, plan, manifest, true);
|
|
3663
|
+
return {
|
|
3664
|
+
status: "recovered",
|
|
3665
|
+
plan,
|
|
3666
|
+
server: recovered.server,
|
|
3667
|
+
manifest,
|
|
3668
|
+
verification: recovered.verification,
|
|
3669
|
+
retryable: isHetznerCapacityError(error),
|
|
3670
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3671
|
+
};
|
|
3672
|
+
}
|
|
3673
|
+
const boot = await ensurePoweredOn(options, plan, manifest, false);
|
|
3674
|
+
if (boot.verification && !boot.verification.ok) {
|
|
3675
|
+
throw new Error(`Resize completed but verification failed: ${(boot.verification.failures ?? []).join("; ")}`);
|
|
3676
|
+
}
|
|
3677
|
+
await phase(hooks, "complete", {
|
|
3678
|
+
serverId: plan.serverId,
|
|
3679
|
+
targetType: plan.targetType.name
|
|
3680
|
+
});
|
|
3681
|
+
return {
|
|
3682
|
+
status: "completed",
|
|
3683
|
+
plan,
|
|
3684
|
+
server: boot.server,
|
|
3685
|
+
manifest,
|
|
3686
|
+
verification: boot.verification
|
|
3687
|
+
};
|
|
3470
3688
|
}
|
|
3471
3689
|
// src/drivers/shared/remote-exec.ts
|
|
3472
3690
|
function buildSshArgs(options = {}) {
|
|
@@ -3528,6 +3746,519 @@ async function waitForCloudInit(host, options = {}) {
|
|
|
3528
3746
|
}
|
|
3529
3747
|
throw new Error(`cloud-init did not finish on ${host} after ${timeoutMs}ms`);
|
|
3530
3748
|
}
|
|
3749
|
+
|
|
3750
|
+
// src/drivers/hetzner/resize-remote.ts
|
|
3751
|
+
var INVENTORY_SCRIPT = String.raw`
|
|
3752
|
+
const { readdir, readFile, readlink, stat } = await import('node:fs/promises')
|
|
3753
|
+
|
|
3754
|
+
const text = (command) => {
|
|
3755
|
+
const result = Bun.spawnSync(['bash', '-lc', command], { stdout: 'pipe', stderr: 'pipe' })
|
|
3756
|
+
return result.exitCode === 0 ? result.stdout.toString().trim() : ''
|
|
3757
|
+
}
|
|
3758
|
+
const lines = (command) => text(command).split('\n').map(value => value.trim()).filter(Boolean).sort()
|
|
3759
|
+
const number = (value) => Number.parseInt(value || '0', 10) || 0
|
|
3760
|
+
|
|
3761
|
+
const root = text('findmnt -n -o SOURCE,FSTYPE /').split(/\s+/)
|
|
3762
|
+
const rootSource = root[0] || ''
|
|
3763
|
+
const rootFsType = root[1] || ''
|
|
3764
|
+
const rootParent = text('lsblk -n -o PKNAME ' + rootSource + ' | head -1')
|
|
3765
|
+
const rootDiskBytes = number(text('lsblk -b -n -o SIZE /dev/' + rootParent + ' | head -1'))
|
|
3766
|
+
const rootFilesystemBytes = number(text("df -B1 --output=size / | tail -1"))
|
|
3767
|
+
const memoryBytes = number(text("awk '/MemTotal/{print $2 * 1024}' /proc/meminfo"))
|
|
3768
|
+
const allRunningServices = lines("systemctl list-units --type=service --state=running --no-legend --plain | awk '{print $1}'")
|
|
3769
|
+
const runningServices = allRunningServices.filter((unit) => {
|
|
3770
|
+
if (/^(containerd|docker|localtunnel|mail|mysql|mariadb|meilisearch|postgresql|redis|rpx-gateway)\b/.test(unit)) return true
|
|
3771
|
+
const source = text('systemctl cat ' + unit + ' 2>/dev/null')
|
|
3772
|
+
return /\/var\/www|\/usr\/local\/bin\/rpx|\/opt\/ts-cloud/.test(source)
|
|
3773
|
+
})
|
|
3774
|
+
const failedUnits = lines("systemctl --failed --type=service --no-legend --plain | awk '{print $1}'")
|
|
3775
|
+
const releaseLinks = []
|
|
3776
|
+
for (const path of lines("find /var/www -mindepth 2 -maxdepth 2 -type l -name current -print 2>/dev/null")) {
|
|
3777
|
+
try {
|
|
3778
|
+
releaseLinks.push(path + '=' + await readlink(path))
|
|
3779
|
+
} catch {}
|
|
3780
|
+
}
|
|
3781
|
+
releaseLinks.sort()
|
|
3782
|
+
|
|
3783
|
+
const routeFragments = []
|
|
3784
|
+
const routeIds = []
|
|
3785
|
+
const routeDomains = new Set()
|
|
3786
|
+
for (const path of lines("find /etc/rpx/sites.d -maxdepth 1 -type f -name '*.json' -print 2>/dev/null")) {
|
|
3787
|
+
try {
|
|
3788
|
+
const source = await readFile(path, 'utf8')
|
|
3789
|
+
const digest = new Bun.CryptoHasher('sha256').update(source).digest('hex')
|
|
3790
|
+
routeFragments.push(path + '=' + digest)
|
|
3791
|
+
const value = JSON.parse(source)
|
|
3792
|
+
for (const route of value.proxies || []) {
|
|
3793
|
+
if (typeof route.to === 'string' && route.to) {
|
|
3794
|
+
routeDomains.add(route.to)
|
|
3795
|
+
routeIds.push(path + ':' + (route.id || route.to) + ':' + (route.path || '/'))
|
|
3796
|
+
}
|
|
3797
|
+
}
|
|
3798
|
+
} catch {}
|
|
3799
|
+
}
|
|
3800
|
+
routeFragments.sort()
|
|
3801
|
+
routeIds.sort()
|
|
3802
|
+
|
|
3803
|
+
const persistentData = []
|
|
3804
|
+
for (const path of ['/var/lib/postgresql', '/var/lib/mysql', '/var/lib/redis', '/var/lib/docker', '/var/mail', '/var/vmail']) {
|
|
3805
|
+
try {
|
|
3806
|
+
const value = await stat(path)
|
|
3807
|
+
if (value.isDirectory()) {
|
|
3808
|
+
const bytes = number(text('du -sx --block-size=1 ' + path + " 2>/dev/null | awk '{print $1}'"))
|
|
3809
|
+
persistentData.push(path + '=' + bytes)
|
|
3810
|
+
}
|
|
3811
|
+
} catch {}
|
|
3812
|
+
}
|
|
3813
|
+
|
|
3814
|
+
const dataCatalog = []
|
|
3815
|
+
for (const database of lines("command -v psql >/dev/null && sudo -u postgres psql -Atqc 'select datname from pg_database where datistemplate = false order by datname' 2>/dev/null || true")) {
|
|
3816
|
+
dataCatalog.push('postgres:' + database)
|
|
3817
|
+
}
|
|
3818
|
+
for (const database of lines("command -v mysql >/dev/null && mysql -NBe 'show databases' 2>/dev/null || true")) {
|
|
3819
|
+
dataCatalog.push('mysql:' + database)
|
|
3820
|
+
}
|
|
3821
|
+
for (const volume of lines("command -v docker >/dev/null && docker volume ls --format '{{.Name}}' 2>/dev/null || true")) {
|
|
3822
|
+
dataCatalog.push('docker-volume:' + volume)
|
|
3823
|
+
}
|
|
3824
|
+
|
|
3825
|
+
console.log(JSON.stringify({
|
|
3826
|
+
capturedAt: new Date().toISOString(),
|
|
3827
|
+
hostname: text('hostname'),
|
|
3828
|
+
cpuCores: number(text('nproc')),
|
|
3829
|
+
memoryBytes,
|
|
3830
|
+
rootSource,
|
|
3831
|
+
rootFsType,
|
|
3832
|
+
rootDiskBytes,
|
|
3833
|
+
rootFilesystemBytes,
|
|
3834
|
+
failedUnits,
|
|
3835
|
+
runningServices,
|
|
3836
|
+
releaseLinks,
|
|
3837
|
+
routeFragments,
|
|
3838
|
+
routeIds,
|
|
3839
|
+
routeDomains: [...routeDomains].sort(),
|
|
3840
|
+
persistentData: persistentData.sort(),
|
|
3841
|
+
dataCatalog: dataCatalog.sort(),
|
|
3842
|
+
}))
|
|
3843
|
+
`;
|
|
3844
|
+
function remoteBunCommand(script) {
|
|
3845
|
+
return `printf %s ${Buffer.from(script).toString("base64")} | base64 -d | bun -`;
|
|
3846
|
+
}
|
|
3847
|
+
async function collectHostManifest(options) {
|
|
3848
|
+
const raw = await sshExecOrThrow(options.host, remoteBunCommand(INVENTORY_SCRIPT), options);
|
|
3849
|
+
return JSON.parse(raw);
|
|
3850
|
+
}
|
|
3851
|
+
async function probeRoute(domain, timeoutMs) {
|
|
3852
|
+
let lastError = "";
|
|
3853
|
+
for (let attempt = 1;attempt <= 3; attempt++) {
|
|
3854
|
+
const process2 = Bun.spawn([
|
|
3855
|
+
"curl",
|
|
3856
|
+
"--location",
|
|
3857
|
+
"--silent",
|
|
3858
|
+
"--show-error",
|
|
3859
|
+
"--output",
|
|
3860
|
+
"/dev/null",
|
|
3861
|
+
"--max-time",
|
|
3862
|
+
String(Math.max(1, Math.ceil(timeoutMs / 1000))),
|
|
3863
|
+
"--write-out",
|
|
3864
|
+
"%{http_code}",
|
|
3865
|
+
`https://${domain}/`
|
|
3866
|
+
], { stdout: "pipe", stderr: "pipe" });
|
|
3867
|
+
const [stdout, stderr, exitCode] = await Promise.all([
|
|
3868
|
+
new Response(process2.stdout).text(),
|
|
3869
|
+
new Response(process2.stderr).text(),
|
|
3870
|
+
process2.exited
|
|
3871
|
+
]);
|
|
3872
|
+
const status = Number.parseInt(stdout.trim(), 10);
|
|
3873
|
+
if (exitCode === 0 && status > 0)
|
|
3874
|
+
return { domain, ok: status < 500, status };
|
|
3875
|
+
lastError = stderr.trim() || `curl exited ${exitCode}`;
|
|
3876
|
+
if (attempt < 3)
|
|
3877
|
+
await Bun.sleep(500 * attempt);
|
|
3878
|
+
}
|
|
3879
|
+
return { domain, ok: false, error: lastError };
|
|
3880
|
+
}
|
|
3881
|
+
async function probeRoutes(domains, timeoutMs) {
|
|
3882
|
+
const probes = [];
|
|
3883
|
+
let cursor = 0;
|
|
3884
|
+
const workers = Array.from({ length: Math.min(4, domains.length) }, async () => {
|
|
3885
|
+
for (;; ) {
|
|
3886
|
+
const index = cursor++;
|
|
3887
|
+
const domain = domains[index];
|
|
3888
|
+
if (!domain)
|
|
3889
|
+
return;
|
|
3890
|
+
probes[index] = await probeRoute(domain, timeoutMs);
|
|
3891
|
+
}
|
|
3892
|
+
});
|
|
3893
|
+
await Promise.all(workers);
|
|
3894
|
+
return probes;
|
|
3895
|
+
}
|
|
3896
|
+
async function collectHetznerResizeManifest(options) {
|
|
3897
|
+
const host = await collectHostManifest(options);
|
|
3898
|
+
const routeProbes = await probeRoutes(host.routeDomains, options.routeTimeoutMs ?? 15000);
|
|
3899
|
+
return { ...host, routeProbes };
|
|
3900
|
+
}
|
|
3901
|
+
async function prepareHetznerResize(options) {
|
|
3902
|
+
const before = await collectHetznerResizeManifest(options);
|
|
3903
|
+
const unhealthy = before.routeProbes.filter((probe) => !probe.ok);
|
|
3904
|
+
if (before.failedUnits.length > 0) {
|
|
3905
|
+
throw new Error(`Preflight found failed services: ${before.failedUnits.join(", ")}`);
|
|
3906
|
+
}
|
|
3907
|
+
if (unhealthy.length > 0) {
|
|
3908
|
+
throw new Error(`Preflight found unhealthy routes: ${unhealthy.map((probe) => probe.domain).join(", ")}`);
|
|
3909
|
+
}
|
|
3910
|
+
await sshExecOrThrow(options.host, String.raw`systemctl list-units --type=service --state=running --no-legend --plain | awk '{print $1}' | while read -r unit; do if systemctl cat "$unit" 2>/dev/null | grep -Eq '/var/www|/usr/local/bin/rpx|/opt/ts-cloud'; then systemctl enable "$unit" >/dev/null 2>&1 || true; fi; done; systemctl daemon-reload; sync`, options);
|
|
3911
|
+
return before;
|
|
3912
|
+
}
|
|
3913
|
+
function missing(expected, actual) {
|
|
3914
|
+
const values = new Set(actual);
|
|
3915
|
+
return expected.filter((value) => !values.has(value));
|
|
3916
|
+
}
|
|
3917
|
+
function targetMemoryBytes(target) {
|
|
3918
|
+
return Math.floor((target.memory ?? 0) * 1024 * 1024 * 1024);
|
|
3919
|
+
}
|
|
3920
|
+
async function expandRootFilesystem(options) {
|
|
3921
|
+
await sshExecOrThrow(options.host, String.raw`set -e
|
|
3922
|
+
root_source="$(findmnt -n -o SOURCE /)"
|
|
3923
|
+
root_fs="$(findmnt -n -o FSTYPE /)"
|
|
3924
|
+
parent="$(lsblk -n -o PKNAME "$root_source" | head -1)"
|
|
3925
|
+
partition="$(lsblk -n -o PARTN "$root_source" | head -1)"
|
|
3926
|
+
if [ -n "$parent" ] && [ -n "$partition" ]; then
|
|
3927
|
+
growpart "/dev/$parent" "$partition" || true
|
|
3928
|
+
fi
|
|
3929
|
+
case "$root_fs" in
|
|
3930
|
+
ext2|ext3|ext4) resize2fs "$root_source" ;;
|
|
3931
|
+
xfs) xfs_growfs / ;;
|
|
3932
|
+
*) echo "Unsupported root filesystem: $root_fs" >&2; exit 1 ;;
|
|
3933
|
+
esac`, options);
|
|
3934
|
+
}
|
|
3935
|
+
async function restoreExpectedServices(options, expected) {
|
|
3936
|
+
const encoded = Buffer.from(expected.join(`
|
|
3937
|
+
`)).toString("base64");
|
|
3938
|
+
await sshExecOrThrow(options.host, `printf %s ${encoded} | base64 -d | while read -r unit; do [ -z "$unit" ] || systemctl is-active --quiet "$unit" || systemctl start "$unit"; done`, options);
|
|
3939
|
+
}
|
|
3940
|
+
async function verifyHetznerResize(options, target, before, context = { recovered: false }) {
|
|
3941
|
+
await waitForSsh(options.host, {
|
|
3942
|
+
...options,
|
|
3943
|
+
timeoutMs: options.sshTimeoutMs ?? 10 * 60000
|
|
3944
|
+
});
|
|
3945
|
+
if (!context.recovered)
|
|
3946
|
+
await expandRootFilesystem(options);
|
|
3947
|
+
if (before)
|
|
3948
|
+
await restoreExpectedServices(options, before.runningServices);
|
|
3949
|
+
const after = await collectHetznerResizeManifest(options);
|
|
3950
|
+
const failures = [];
|
|
3951
|
+
const missingServices = before ? missing(before.runningServices, after.runningServices) : [];
|
|
3952
|
+
const missingReleases = before ? missing(before.releaseLinks, after.releaseLinks) : [];
|
|
3953
|
+
const changedRoutes = before ? missing(before.routeFragments, after.routeFragments) : [];
|
|
3954
|
+
const missingRouteIds = before ? missing(before.routeIds, after.routeIds) : [];
|
|
3955
|
+
const beforeDataPaths = before?.persistentData.map((value) => value.split("=")[0]) ?? [];
|
|
3956
|
+
const afterDataPaths = after.persistentData.map((value) => value.split("=")[0]);
|
|
3957
|
+
const missingDataPaths = missing(beforeDataPaths, afterDataPaths);
|
|
3958
|
+
const missingDataCatalog = before ? missing(before.dataCatalog, after.dataCatalog) : [];
|
|
3959
|
+
const unhealthyRoutes = after.routeProbes.filter((probe) => !probe.ok);
|
|
3960
|
+
const expectedCores = context.recovered && before ? before.cpuCores : target.cores ?? 0;
|
|
3961
|
+
const expectedMemory = context.recovered && before ? before.memoryBytes : targetMemoryBytes(target);
|
|
3962
|
+
const cpuOk = after.cpuCores >= expectedCores;
|
|
3963
|
+
const memoryOk = after.memoryBytes >= expectedMemory * 0.9;
|
|
3964
|
+
const diskOk = context.recovered || after.rootDiskBytes >= (target.disk ?? 0) * 1e9 * 0.95;
|
|
3965
|
+
if (after.failedUnits.length > 0)
|
|
3966
|
+
failures.push(`Failed services: ${after.failedUnits.join(", ")}`);
|
|
3967
|
+
if (missingServices.length > 0)
|
|
3968
|
+
failures.push(`Services did not return: ${missingServices.join(", ")}`);
|
|
3969
|
+
if (missingReleases.length > 0)
|
|
3970
|
+
failures.push(`Release links changed: ${missingReleases.join(", ")}`);
|
|
3971
|
+
if (changedRoutes.length > 0)
|
|
3972
|
+
failures.push(`Route manifests changed: ${changedRoutes.join(", ")}`);
|
|
3973
|
+
if (missingRouteIds.length > 0)
|
|
3974
|
+
failures.push(`Routes disappeared: ${missingRouteIds.join(", ")}`);
|
|
3975
|
+
if (missingDataPaths.length > 0)
|
|
3976
|
+
failures.push(`Persistent data paths disappeared: ${missingDataPaths.join(", ")}`);
|
|
3977
|
+
if (missingDataCatalog.length > 0)
|
|
3978
|
+
failures.push(`Databases or volumes disappeared: ${missingDataCatalog.join(", ")}`);
|
|
3979
|
+
if (unhealthyRoutes.length > 0)
|
|
3980
|
+
failures.push(`Unhealthy routes: ${unhealthyRoutes.map((probe) => probe.domain).join(", ")}`);
|
|
3981
|
+
if (!cpuOk)
|
|
3982
|
+
failures.push(`Expected at least ${expectedCores} CPU cores, found ${after.cpuCores}`);
|
|
3983
|
+
if (!memoryOk)
|
|
3984
|
+
failures.push(`Expected at least ${expectedMemory} bytes of RAM`);
|
|
3985
|
+
if (!diskOk)
|
|
3986
|
+
failures.push(`Expected approximately ${target.disk ?? 0} GB root disk`);
|
|
3987
|
+
return {
|
|
3988
|
+
ok: failures.length === 0,
|
|
3989
|
+
checks: {
|
|
3990
|
+
cpuCores: after.cpuCores,
|
|
3991
|
+
memoryBytes: after.memoryBytes,
|
|
3992
|
+
rootDiskBytes: after.rootDiskBytes,
|
|
3993
|
+
rootFilesystemBytes: after.rootFilesystemBytes,
|
|
3994
|
+
runningServices: after.runningServices.length,
|
|
3995
|
+
releaseLinks: after.releaseLinks.length,
|
|
3996
|
+
routeFragments: after.routeFragments.length,
|
|
3997
|
+
routes: after.routeIds.length,
|
|
3998
|
+
healthyRoutes: after.routeProbes.filter((probe) => probe.ok).length,
|
|
3999
|
+
persistentDataPaths: after.persistentData.length,
|
|
4000
|
+
databasesAndVolumes: after.dataCatalog.length,
|
|
4001
|
+
failedUnits: after.failedUnits.length
|
|
4002
|
+
},
|
|
4003
|
+
failures
|
|
4004
|
+
};
|
|
4005
|
+
}
|
|
4006
|
+
// src/drivers/hetzner/host-optimization.ts
|
|
4007
|
+
function computeConfig(config) {
|
|
4008
|
+
return config.infrastructure?.compute ?? {};
|
|
4009
|
+
}
|
|
4010
|
+
function resolveHetznerHostOptimizationPlan(config) {
|
|
4011
|
+
const compute = computeConfig(config);
|
|
4012
|
+
const monitoring = compute.monitoring;
|
|
4013
|
+
const firewallPorts = [...new Set([22, 80, 443, ...compute.firewall?.allowedPorts ?? []])].filter((port) => Number.isInteger(port) && port >= 1 && port <= 65535).sort((a, b) => a - b);
|
|
4014
|
+
return {
|
|
4015
|
+
firewallPorts,
|
|
4016
|
+
monitoring: monitoring !== false && (typeof monitoring !== "object" || monitoring.enabled !== false),
|
|
4017
|
+
autoUpdates: compute.autoUpdates !== false,
|
|
4018
|
+
swapGb: Math.max(0, Math.floor(compute.swapGb ?? 2)),
|
|
4019
|
+
sshPasswordAuthentication: false,
|
|
4020
|
+
journalMaxUse: "256M",
|
|
4021
|
+
journalRetention: "14day"
|
|
4022
|
+
};
|
|
4023
|
+
}
|
|
4024
|
+
function hardeningScript(plan) {
|
|
4025
|
+
return [
|
|
4026
|
+
"install -d -m 0755 /etc/ssh/sshd_config.d /etc/systemd/journald.conf.d /etc/sysctl.d /etc/fail2ban/jail.d",
|
|
4027
|
+
`cat > /etc/ssh/sshd_config.d/99-ts-cloud-hardening.conf <<'TS_CLOUD_SSH_EOF'`,
|
|
4028
|
+
"# Managed by ts-cloud host optimization.",
|
|
4029
|
+
"PasswordAuthentication no",
|
|
4030
|
+
"KbdInteractiveAuthentication no",
|
|
4031
|
+
"PermitRootLogin prohibit-password",
|
|
4032
|
+
"MaxAuthTries 4",
|
|
4033
|
+
"LoginGraceTime 30",
|
|
4034
|
+
"TS_CLOUD_SSH_EOF",
|
|
4035
|
+
"sshd -t",
|
|
4036
|
+
"systemctl reload ssh",
|
|
4037
|
+
`cat > /etc/systemd/journald.conf.d/99-ts-cloud-retention.conf <<'TS_CLOUD_JOURNAL_EOF'`,
|
|
4038
|
+
"[Journal]",
|
|
4039
|
+
`SystemMaxUse=${plan.journalMaxUse}`,
|
|
4040
|
+
`MaxRetentionSec=${plan.journalRetention}`,
|
|
4041
|
+
"Compress=yes",
|
|
4042
|
+
"TS_CLOUD_JOURNAL_EOF",
|
|
4043
|
+
"systemctl restart systemd-journald",
|
|
4044
|
+
"journalctl --vacuum-size=256M --vacuum-time=14d >/dev/null",
|
|
4045
|
+
`cat > /etc/sysctl.d/99-ts-cloud-production.conf <<'TS_CLOUD_SYSCTL_EOF'`,
|
|
4046
|
+
"# Managed by ts-cloud host optimization.",
|
|
4047
|
+
"vm.swappiness=10",
|
|
4048
|
+
"vm.dirty_background_ratio=5",
|
|
4049
|
+
"vm.dirty_ratio=15",
|
|
4050
|
+
"net.core.somaxconn=4096",
|
|
4051
|
+
"net.ipv4.tcp_max_syn_backlog=4096",
|
|
4052
|
+
"net.ipv4.tcp_fin_timeout=30",
|
|
4053
|
+
"net.ipv4.tcp_keepalive_time=300",
|
|
4054
|
+
"net.ipv4.tcp_keepalive_intvl=30",
|
|
4055
|
+
"net.ipv4.tcp_keepalive_probes=5",
|
|
4056
|
+
"fs.inotify.max_user_watches=524288",
|
|
4057
|
+
"TS_CLOUD_SYSCTL_EOF",
|
|
4058
|
+
"sysctl --system >/dev/null",
|
|
4059
|
+
"export DEBIAN_FRONTEND=noninteractive",
|
|
4060
|
+
"command -v fail2ban-client >/dev/null 2>&1 || apt-get install -y fail2ban",
|
|
4061
|
+
`cat > /etc/fail2ban/jail.d/99-ts-cloud-sshd.conf <<'TS_CLOUD_FAIL2BAN_EOF'`,
|
|
4062
|
+
"[sshd]",
|
|
4063
|
+
"enabled = true",
|
|
4064
|
+
"backend = systemd",
|
|
4065
|
+
"bantime = 1h",
|
|
4066
|
+
"findtime = 10m",
|
|
4067
|
+
"maxretry = 5",
|
|
4068
|
+
"TS_CLOUD_FAIL2BAN_EOF",
|
|
4069
|
+
"systemctl enable --now fail2ban",
|
|
4070
|
+
"systemctl restart fail2ban"
|
|
4071
|
+
];
|
|
4072
|
+
}
|
|
4073
|
+
function swapScript(sizeGb) {
|
|
4074
|
+
if (sizeGb === 0)
|
|
4075
|
+
return [];
|
|
4076
|
+
return [
|
|
4077
|
+
`if ! swapon --show=NAME --noheadings 2>/dev/null | grep -qx '/swapfile'; then`,
|
|
4078
|
+
` if [ ! -f /swapfile ]; then fallocate -l ${sizeGb}G /swapfile || dd if=/dev/zero of=/swapfile bs=1M count=${sizeGb * 1024} status=none; fi`,
|
|
4079
|
+
" chmod 600 /swapfile",
|
|
4080
|
+
" mkswap /swapfile >/dev/null",
|
|
4081
|
+
" swapon /swapfile",
|
|
4082
|
+
"fi",
|
|
4083
|
+
`grep -qE '^/swapfile\\s' /etc/fstab || echo '/swapfile none swap sw 0 0' >> /etc/fstab`
|
|
4084
|
+
];
|
|
4085
|
+
}
|
|
4086
|
+
function buildHetznerHostOptimizationScript(config) {
|
|
4087
|
+
const compute = computeConfig(config);
|
|
4088
|
+
const plan = resolveHetznerHostOptimizationPlan(config);
|
|
4089
|
+
const monitoring = compute.monitoring ?? true;
|
|
4090
|
+
const firewall = compute.firewall ?? { enabled: true };
|
|
4091
|
+
return [
|
|
4092
|
+
"set -euo pipefail",
|
|
4093
|
+
...usesRpxProxy(compute) ? [
|
|
4094
|
+
"systemctl disable --now nginx.service bun-gateway.service ts-cloud-nginx.service 2>/dev/null || true",
|
|
4095
|
+
"systemctl reset-failed nginx.service bun-gateway.service ts-cloud-nginx.service 2>/dev/null || true"
|
|
4096
|
+
] : [],
|
|
4097
|
+
...swapScript(plan.swapGb),
|
|
4098
|
+
...buildAutoUpdatesScript(plan.autoUpdates),
|
|
4099
|
+
...buildMonitoringScript(monitoring),
|
|
4100
|
+
...hardeningScript(plan),
|
|
4101
|
+
...buildUfwScript(firewall),
|
|
4102
|
+
"systemctl daemon-reload",
|
|
4103
|
+
"systemctl reset-failed"
|
|
4104
|
+
];
|
|
4105
|
+
}
|
|
4106
|
+
var REPORT_SCRIPT = String.raw`
|
|
4107
|
+
const text = (command) => {
|
|
4108
|
+
const result = Bun.spawnSync(['bash', '-lc', command], { stdout: 'pipe', stderr: 'pipe' })
|
|
4109
|
+
return result.stdout.toString().trim()
|
|
4110
|
+
}
|
|
4111
|
+
const lines = command => text(command).split('\n').map(value => value.trim()).filter(Boolean)
|
|
4112
|
+
const number = value => Number.parseInt(value || '0', 10) || 0
|
|
4113
|
+
const ssh = Object.fromEntries(lines("sshd -T 2>/dev/null | grep -E '^(passwordauthentication|permitrootlogin) '").map(line => line.split(/\s+/, 2)))
|
|
4114
|
+
const firewallPorts = lines("ufw status 2>/dev/null | awk '/ALLOW/{print $1}'")
|
|
4115
|
+
.flatMap(value => value.startsWith('OpenSSH') ? [22] : (value.match(/^\d+/) ? [number(value)] : []))
|
|
4116
|
+
const publicTcpPorts = lines("ss -lntH | awk '{print $4}'")
|
|
4117
|
+
.filter(address => !address.startsWith('127.') && !address.startsWith('[::1]'))
|
|
4118
|
+
.flatMap(address => {
|
|
4119
|
+
const match = address.match(/:(\d+)$/)
|
|
4120
|
+
return match ? [number(match[1])] : []
|
|
4121
|
+
})
|
|
4122
|
+
const metricsMtime = number(text("stat -c %Y /var/lib/ts-cloud/metrics.json 2>/dev/null"))
|
|
4123
|
+
console.log(JSON.stringify({
|
|
4124
|
+
firewallActive: text("ufw status 2>/dev/null | head -1").includes('active'),
|
|
4125
|
+
firewallPorts: [...new Set(firewallPorts)].sort((a, b) => a - b),
|
|
4126
|
+
passwordAuthentication: ssh.passwordauthentication !== 'no',
|
|
4127
|
+
rootPasswordLogin: ssh.permitrootlogin === 'yes',
|
|
4128
|
+
fail2banActive: text('systemctl is-active fail2ban 2>/dev/null') === 'active',
|
|
4129
|
+
unattendedUpgradesActive: text('systemctl is-active unattended-upgrades 2>/dev/null') === 'active',
|
|
4130
|
+
metricsTimerActive: text('systemctl is-active ts-cloud-metrics.timer 2>/dev/null') === 'active',
|
|
4131
|
+
metricsSnapshotFresh: metricsMtime > 0 && Math.floor(Date.now() / 1000) - metricsMtime < 180,
|
|
4132
|
+
swapBytes: number(text("free -b | awk '/^Swap:/{print $2}'")),
|
|
4133
|
+
journalBytes: number(text("journalctl --disk-usage 2>/dev/null | grep -Eo '[0-9.]+[KMG]' | tail -1 | numfmt --from=iec 2>/dev/null")),
|
|
4134
|
+
failedUnits: lines("systemctl --failed --type=service --no-legend --plain | awk '{print $1}'"),
|
|
4135
|
+
publicTcpPorts: [...new Set(publicTcpPorts)].sort((a, b) => a - b),
|
|
4136
|
+
}))
|
|
4137
|
+
`;
|
|
4138
|
+
function remoteBunCommand2(script) {
|
|
4139
|
+
return `printf %s ${Buffer.from(script).toString("base64")} | base64 -d | bun -`;
|
|
4140
|
+
}
|
|
4141
|
+
async function applyHetznerHostOptimization(config, options) {
|
|
4142
|
+
const script = buildHetznerHostOptimizationScript(config).join(`
|
|
4143
|
+
`);
|
|
4144
|
+
const command = `printf %s ${Buffer.from(script).toString("base64")} | base64 -d | bash`;
|
|
4145
|
+
await sshExecOrThrow(options.host, command, options);
|
|
4146
|
+
}
|
|
4147
|
+
async function collectHetznerHostOptimizationReport(options) {
|
|
4148
|
+
const raw = await sshExecOrThrow(options.host, remoteBunCommand2(REPORT_SCRIPT), options);
|
|
4149
|
+
return JSON.parse(raw);
|
|
4150
|
+
}
|
|
4151
|
+
function verifyHetznerHostOptimization(plan, report) {
|
|
4152
|
+
const failures = [];
|
|
4153
|
+
const actualPorts = new Set(report.firewallPorts);
|
|
4154
|
+
const missingPorts = plan.firewallPorts.filter((port) => !actualPorts.has(port));
|
|
4155
|
+
const unexpectedPorts = report.firewallPorts.filter((port) => !plan.firewallPorts.includes(port));
|
|
4156
|
+
if (!report.firewallActive)
|
|
4157
|
+
failures.push("UFW is not active");
|
|
4158
|
+
if (missingPorts.length > 0)
|
|
4159
|
+
failures.push(`UFW is missing ports: ${missingPorts.join(", ")}`);
|
|
4160
|
+
if (unexpectedPorts.length > 0)
|
|
4161
|
+
failures.push(`UFW has unexpected ports: ${unexpectedPorts.join(", ")}`);
|
|
4162
|
+
if (report.passwordAuthentication)
|
|
4163
|
+
failures.push("SSH password authentication remains enabled");
|
|
4164
|
+
if (report.rootPasswordLogin)
|
|
4165
|
+
failures.push("SSH root password login remains enabled");
|
|
4166
|
+
if (!report.fail2banActive)
|
|
4167
|
+
failures.push("fail2ban is not active");
|
|
4168
|
+
if (plan.autoUpdates && !report.unattendedUpgradesActive)
|
|
4169
|
+
failures.push("unattended upgrades are not active");
|
|
4170
|
+
if (plan.monitoring && !report.metricsTimerActive)
|
|
4171
|
+
failures.push("metrics timer is not active");
|
|
4172
|
+
if (plan.monitoring && !report.metricsSnapshotFresh)
|
|
4173
|
+
failures.push("metrics snapshot is stale");
|
|
4174
|
+
if (plan.swapGb > 0 && report.swapBytes < plan.swapGb * 1024 ** 3 * 0.9)
|
|
4175
|
+
failures.push("configured swap is missing");
|
|
4176
|
+
if (report.failedUnits.length > 0)
|
|
4177
|
+
failures.push(`failed services: ${report.failedUnits.join(", ")}`);
|
|
4178
|
+
return failures;
|
|
4179
|
+
}
|
|
4180
|
+
// src/drivers/hetzner/resize-state.ts
|
|
4181
|
+
import { mkdir as mkdir2, readFile as readFile2, rename as rename2, rmdir, stat, unlink, writeFile as writeFile2 } from "node:fs/promises";
|
|
4182
|
+
import { join as join4 } from "node:path";
|
|
4183
|
+
function resizeCheckpointPath(stackName) {
|
|
4184
|
+
return join4(process.cwd(), STATE_DIR, `${stackName}-resize.json`);
|
|
4185
|
+
}
|
|
4186
|
+
function resizeLockPath(stackName) {
|
|
4187
|
+
return join4(process.cwd(), STATE_DIR, `${stackName}-resize.lock`);
|
|
4188
|
+
}
|
|
4189
|
+
async function readResizeCheckpoint(stackName) {
|
|
4190
|
+
try {
|
|
4191
|
+
return JSON.parse(await readFile2(resizeCheckpointPath(stackName), "utf8"));
|
|
4192
|
+
} catch {
|
|
4193
|
+
return null;
|
|
4194
|
+
}
|
|
4195
|
+
}
|
|
4196
|
+
async function writeResizeCheckpoint(checkpoint) {
|
|
4197
|
+
const path = resizeCheckpointPath(checkpoint.stackName);
|
|
4198
|
+
await mkdir2(join4(process.cwd(), STATE_DIR), { recursive: true });
|
|
4199
|
+
const tempPath = `${path}.${process.pid}.tmp`;
|
|
4200
|
+
await writeFile2(tempPath, `${JSON.stringify(checkpoint, null, 2)}
|
|
4201
|
+
`, "utf8");
|
|
4202
|
+
await rename2(tempPath, path);
|
|
4203
|
+
}
|
|
4204
|
+
async function acquireResizeLock(stackName, staleAfterMs = 30 * 60000) {
|
|
4205
|
+
const path = resizeLockPath(stackName);
|
|
4206
|
+
await mkdir2(join4(process.cwd(), STATE_DIR), { recursive: true });
|
|
4207
|
+
try {
|
|
4208
|
+
await mkdir2(path);
|
|
4209
|
+
} catch (error) {
|
|
4210
|
+
const age = Date.now() - (await stat(path)).mtimeMs;
|
|
4211
|
+
if (age <= staleAfterMs) {
|
|
4212
|
+
throw new Error(`Another resize process holds ${path}.`);
|
|
4213
|
+
}
|
|
4214
|
+
await rename2(path, `${path}.stale-${Date.now()}`);
|
|
4215
|
+
await mkdir2(path);
|
|
4216
|
+
}
|
|
4217
|
+
await writeFile2(join4(path, "owner.json"), `${JSON.stringify({ pid: process.pid, acquiredAt: new Date().toISOString() }, null, 2)}
|
|
4218
|
+
`, "utf8");
|
|
4219
|
+
return async () => {
|
|
4220
|
+
await unlink(join4(path, "owner.json")).catch(() => {});
|
|
4221
|
+
await rmdir(path).catch(() => {});
|
|
4222
|
+
};
|
|
4223
|
+
}
|
|
4224
|
+
// src/drivers/hetzner/provision.ts
|
|
4225
|
+
async function ensureSshKey(client, options) {
|
|
4226
|
+
const body = normalizeSshPublicKey(options.publicKey);
|
|
4227
|
+
const existing = (await client.listSshKeys()).find((k) => normalizeSshPublicKey(k.public_key) === body);
|
|
4228
|
+
if (existing)
|
|
4229
|
+
return { id: existing.id, name: existing.name, created: false };
|
|
4230
|
+
const created = await client.createSshKey({
|
|
4231
|
+
name: options.name,
|
|
4232
|
+
publicKey: options.publicKey.trim(),
|
|
4233
|
+
labels: options.labels
|
|
4234
|
+
});
|
|
4235
|
+
return { id: created.id, name: created.name, created: true };
|
|
4236
|
+
}
|
|
4237
|
+
async function ensureFirewall(client, options) {
|
|
4238
|
+
const existing = (await client.listFirewalls()).find((f) => f.name === options.name);
|
|
4239
|
+
if (existing) {
|
|
4240
|
+
await client.setFirewallRules(existing.id, options.rules);
|
|
4241
|
+
return { id: existing.id, name: existing.name, created: false };
|
|
4242
|
+
}
|
|
4243
|
+
const { firewall } = await client.createFirewall({ name: options.name, rules: options.rules, labels: options.labels });
|
|
4244
|
+
return { id: firewall.id, name: firewall.name, created: true };
|
|
4245
|
+
}
|
|
4246
|
+
async function ensureServer(client, options) {
|
|
4247
|
+
const { waitForRunning = true, ...createOptions } = options;
|
|
4248
|
+
let server = (await client.listServers()).find((s) => s.name === options.name);
|
|
4249
|
+
const created = !server;
|
|
4250
|
+
if (!server)
|
|
4251
|
+
server = (await client.createServer(createOptions)).server;
|
|
4252
|
+
if (waitForRunning)
|
|
4253
|
+
server = await client.waitForServerRunning(server.id);
|
|
4254
|
+
return { server, created };
|
|
4255
|
+
}
|
|
4256
|
+
function serverPublicIpv4(server) {
|
|
4257
|
+
const ip = server.public_net.ipv4?.ip;
|
|
4258
|
+
if (!ip)
|
|
4259
|
+
throw new Error(`Hetzner server "${server.name}" (#${server.id}) has no public IPv4`);
|
|
4260
|
+
return ip;
|
|
4261
|
+
}
|
|
3531
4262
|
// src/drivers/shared/box-provision.ts
|
|
3532
4263
|
var UBUNTU_2404_AMI_PARAM = "/aws/service/canonical/ubuntu/server/24.04/stable/current/amd64/hvm/ebs-gp3/ami-id";
|
|
3533
4264
|
function buildBoxUserData(spec) {
|
|
@@ -3832,6 +4563,12 @@ function buildResetReleaseDir(paths) {
|
|
|
3832
4563
|
"fi"
|
|
3833
4564
|
];
|
|
3834
4565
|
}
|
|
4566
|
+
function buildStrandedReleaseTrap(paths) {
|
|
4567
|
+
const live = `TS_CLOUD_TRAP_LIVE="$(readlink -f ${paths.current} 2>/dev/null || true)"`;
|
|
4568
|
+
const target = `TS_CLOUD_TRAP_TARGET="$(readlink -f ${paths.release} 2>/dev/null || echo ${paths.release})"`;
|
|
4569
|
+
const body = `if [ $? -ne 0 ]; then ${live}; ${target}; [ "$TS_CLOUD_TRAP_LIVE" = "$TS_CLOUD_TRAP_TARGET" ] || rm -rf ${paths.release}; fi`;
|
|
4570
|
+
return `trap '${body}' EXIT`;
|
|
4571
|
+
}
|
|
3835
4572
|
function buildIsReleaseLive(paths) {
|
|
3836
4573
|
return [
|
|
3837
4574
|
`TS_CLOUD_LIVE_PATH="$(readlink -f ${paths.current} 2>/dev/null || true)"`,
|
|
@@ -3964,8 +4701,7 @@ function buildSiteDeployScript(options) {
|
|
|
3964
4701
|
const stageRelease = [
|
|
3965
4702
|
"set -euo pipefail",
|
|
3966
4703
|
...buildDeployLock(paths),
|
|
3967
|
-
|
|
3968
|
-
`trap 'if [ $? -ne 0 ] && [ "$TS_CLOUD_IS_LIVE" != "yes" ]; then rm -rf ${paths.release}; fi' EXIT`,
|
|
4704
|
+
buildStrandedReleaseTrap(paths),
|
|
3969
4705
|
...artifactFetch,
|
|
3970
4706
|
...buildEnsureReleaseLayout(paths, sharedPaths),
|
|
3971
4707
|
...buildResetReleaseDir(paths),
|
|
@@ -4054,8 +4790,7 @@ function buildStaticSiteDeployScript(options) {
|
|
|
4054
4790
|
return [
|
|
4055
4791
|
"set -euo pipefail",
|
|
4056
4792
|
...buildDeployLock(paths),
|
|
4057
|
-
|
|
4058
|
-
`trap 'if [ $? -ne 0 ] && [ "$TS_CLOUD_IS_LIVE" != "yes" ]; then rm -rf ${paths.release}; fi' EXIT`,
|
|
4793
|
+
buildStrandedReleaseTrap(paths),
|
|
4059
4794
|
...artifactFetch,
|
|
4060
4795
|
...buildEnsureReleaseLayout(paths, []),
|
|
4061
4796
|
...buildResetReleaseDir(paths),
|
|
@@ -4094,16 +4829,16 @@ function buildHostCleanupScript() {
|
|
|
4094
4829
|
];
|
|
4095
4830
|
}
|
|
4096
4831
|
// src/drivers/shared/compute-deploy.ts
|
|
4097
|
-
import { copyFileSync } from "node:fs";
|
|
4832
|
+
import { copyFileSync as copyFileSync2 } from "node:fs";
|
|
4098
4833
|
import { tmpdir as tmpdir2 } from "node:os";
|
|
4099
|
-
import { join as
|
|
4834
|
+
import { join as join6 } from "node:path";
|
|
4100
4835
|
|
|
4101
4836
|
// src/deploy/management-dashboard.ts
|
|
4102
4837
|
import { execSync as execSync2 } from "node:child_process";
|
|
4103
4838
|
import { randomBytes } from "node:crypto";
|
|
4104
|
-
import { chmodSync, existsSync as existsSync2, mkdirSync, readFileSync as readFileSync3, writeFileSync } from "node:fs";
|
|
4839
|
+
import { chmodSync, copyFileSync, existsSync as existsSync2, mkdirSync, readFileSync as readFileSync3, statSync, writeFileSync } from "node:fs";
|
|
4105
4840
|
import { tmpdir } from "node:os";
|
|
4106
|
-
import { dirname, isAbsolute, join as
|
|
4841
|
+
import { basename, dirname, isAbsolute, join as join5, resolve } from "node:path";
|
|
4107
4842
|
import { fileURLToPath } from "node:url";
|
|
4108
4843
|
|
|
4109
4844
|
// src/deploy/dashboard-config-module.ts
|
|
@@ -4195,7 +4930,7 @@ function resolveDashboardVersion() {
|
|
|
4195
4930
|
if (explicit)
|
|
4196
4931
|
return explicit;
|
|
4197
4932
|
const here = dirname(fileURLToPath(import.meta.url));
|
|
4198
|
-
for (const candidate of [
|
|
4933
|
+
for (const candidate of [join5(here, "..", "..", "package.json"), join5(here, "..", "package.json")]) {
|
|
4199
4934
|
try {
|
|
4200
4935
|
const pkg = JSON.parse(readFileSync3(candidate, "utf8"));
|
|
4201
4936
|
if (pkg.name === "@stacksjs/ts-cloud" && pkg.version)
|
|
@@ -4212,14 +4947,27 @@ function stageLiveDashboardRoot(config, cwd, logger) {
|
|
|
4212
4947
|
const stage = resolveStatePath(cwd, "dashboard-release");
|
|
4213
4948
|
try {
|
|
4214
4949
|
mkdirSync(stage, { recursive: true });
|
|
4215
|
-
writeFileSync(
|
|
4950
|
+
writeFileSync(join5(stage, "cloud.config.ts"), serializeDashboardConfig(config));
|
|
4951
|
+
let dependency = resolveDashboardVersion();
|
|
4952
|
+
if (dependency.startsWith("file:")) {
|
|
4953
|
+
const requested = dependency.slice("file:".length);
|
|
4954
|
+
const source = isAbsolute(requested) ? requested : resolve(cwd, requested);
|
|
4955
|
+
if (!existsSync2(source) || !statSync(source).isFile()) {
|
|
4956
|
+
throw new Error(`TS_CLOUD_UI_VERSION points to a missing package file: ${source}`);
|
|
4957
|
+
}
|
|
4958
|
+
const name = basename(source);
|
|
4959
|
+
const destination = join5(stage, name);
|
|
4960
|
+
if (source !== destination)
|
|
4961
|
+
copyFileSync(source, destination);
|
|
4962
|
+
dependency = `file:./${name}`;
|
|
4963
|
+
}
|
|
4216
4964
|
const pkg = {
|
|
4217
4965
|
name: "ts-cloud-dashboard",
|
|
4218
4966
|
private: true,
|
|
4219
4967
|
type: "module",
|
|
4220
|
-
dependencies: { "@stacksjs/ts-cloud":
|
|
4968
|
+
dependencies: { "@stacksjs/ts-cloud": dependency }
|
|
4221
4969
|
};
|
|
4222
|
-
writeFileSync(
|
|
4970
|
+
writeFileSync(join5(stage, "package.json"), `${JSON.stringify(pkg, null, 2)}
|
|
4223
4971
|
`);
|
|
4224
4972
|
return liveStageDir();
|
|
4225
4973
|
} catch (error) {
|
|
@@ -4228,18 +4976,18 @@ function stageLiveDashboardRoot(config, cwd, logger) {
|
|
|
4228
4976
|
}
|
|
4229
4977
|
}
|
|
4230
4978
|
function resolveUiSource(cwd) {
|
|
4231
|
-
if (existsSync2(
|
|
4979
|
+
if (existsSync2(join5(cwd, "packages", "ui", "pages")) || existsSync2(join5(cwd, "packages", "ui", "package.json"))) {
|
|
4232
4980
|
return { uiRoot: "packages/ui/dist", build: "cd packages/ui && bun install && bun run build" };
|
|
4233
4981
|
}
|
|
4234
4982
|
const here = dirname(fileURLToPath(import.meta.url));
|
|
4235
4983
|
const candidates = [
|
|
4236
|
-
|
|
4237
|
-
|
|
4238
|
-
|
|
4239
|
-
|
|
4984
|
+
join5(here, "ui"),
|
|
4985
|
+
join5(here, "..", "ui"),
|
|
4986
|
+
join5(here, "..", "..", "ui"),
|
|
4987
|
+
join5(here, "..", "dist", "ui")
|
|
4240
4988
|
];
|
|
4241
4989
|
for (const dir of candidates) {
|
|
4242
|
-
if (existsSync2(
|
|
4990
|
+
if (existsSync2(join5(dir, "index.html")) || existsSync2(join5(dir, "serverless.html")))
|
|
4243
4991
|
return { uiRoot: dir, build: false };
|
|
4244
4992
|
}
|
|
4245
4993
|
return null;
|
|
@@ -4322,12 +5070,12 @@ function buildManagementDashboardArtifact(site, options) {
|
|
|
4322
5070
|
logger.info(`Management dashboard: building UI (${site.build})`);
|
|
4323
5071
|
execSync2(site.build, { cwd, stdio: "inherit" });
|
|
4324
5072
|
}
|
|
4325
|
-
const root = isAbsolute(site.root) ? site.root :
|
|
5073
|
+
const root = isAbsolute(site.root) ? site.root : join5(cwd, site.root);
|
|
4326
5074
|
if (!existsSync2(root)) {
|
|
4327
5075
|
logger.warn(`Management dashboard: build output not found at ${root} — skipping dashboard artifact.`);
|
|
4328
5076
|
return null;
|
|
4329
5077
|
}
|
|
4330
|
-
const tarball =
|
|
5078
|
+
const tarball = join5(tmpdir(), `${options.slug}-${options.siteName ?? MANAGEMENT_DASHBOARD_SITE}-${options.sha}.tar.gz`);
|
|
4331
5079
|
execSync2(`tar czf "${tarball}" -C "${root}" .`, { stdio: "inherit" });
|
|
4332
5080
|
return tarball;
|
|
4333
5081
|
} catch (error) {
|
|
@@ -4835,10 +5583,11 @@ async function deploySiteRelease(driver, options, logger = noopLogger2) {
|
|
|
4835
5583
|
return { success: false, error: hint };
|
|
4836
5584
|
}
|
|
4837
5585
|
const dbEnv = outputs.servicesPrivateIp ? buildFleetServicesEnv(outputs.servicesPrivateIp, resolveAppDatabase(config)) : buildManagedDbEnv(resolveAppDatabase(config));
|
|
4838
|
-
const
|
|
5586
|
+
const compute = config.infrastructure?.compute;
|
|
5587
|
+
const proxyEnv = usesRpxProxy(compute) ? { HOST: "127.0.0.1" } : {};
|
|
5588
|
+
const envWithServices = { ...dbEnv, ...proxyEnv, ...site.env || {} };
|
|
4839
5589
|
if (isPhpSite(site)) {
|
|
4840
|
-
const
|
|
4841
|
-
const phpVersion = site.phpVersion ?? compute2?.php?.default ?? compute2?.php?.versions?.[0];
|
|
5590
|
+
const phpVersion = site.phpVersion ?? compute?.php?.default ?? compute?.php?.versions?.[0];
|
|
4842
5591
|
const appBase2 = siteInstallBase(slug, siteName);
|
|
4843
5592
|
const siteWithEnv = Object.keys(dbEnv).length > 0 ? { ...site, env: envWithServices } : site;
|
|
4844
5593
|
const deployScript = buildLaravelDeployScript({
|
|
@@ -4848,7 +5597,7 @@ async function deploySiteRelease(driver, options, logger = noopLogger2) {
|
|
|
4848
5597
|
appBase: appBase2,
|
|
4849
5598
|
defaultPhpVersion: phpVersion
|
|
4850
5599
|
});
|
|
4851
|
-
const useNginx = !usesRpxProxy(
|
|
5600
|
+
const useNginx = !usesRpxProxy(compute);
|
|
4852
5601
|
const sslProvider = resolveSslProvider(site);
|
|
4853
5602
|
const customCert = sslProvider === "custom" && site.ssl?.certPath && site.ssl?.keyPath ? { certPath: site.ssl.certPath, keyPath: site.ssl.keyPath } : undefined;
|
|
4854
5603
|
const poolScript = site.isolation ? buildPhpFpmPoolScript({ siteName, appBase: appBase2 }) : [];
|
|
@@ -4864,7 +5613,7 @@ async function deploySiteRelease(driver, options, logger = noopLogger2) {
|
|
|
4864
5613
|
redirects: site.redirects,
|
|
4865
5614
|
ssl: customCert,
|
|
4866
5615
|
auth: site.auth && site.auth.enabled !== false && site.auth.password ? { username: site.auth.username || "admin", password: site.auth.password, realm: site.auth.realm } : undefined,
|
|
4867
|
-
serverSnippet: resolveNginxSnippet(site.nginx,
|
|
5616
|
+
serverSnippet: resolveNginxSnippet(site.nginx, compute?.nginxTemplates),
|
|
4868
5617
|
clientMaxBodySize: site.nginx?.clientMaxBodySize,
|
|
4869
5618
|
hsts: site.ssl?.hsts,
|
|
4870
5619
|
tlsProtocols: site.ssl?.tlsProtocols,
|
|
@@ -4940,7 +5689,6 @@ async function deploySiteRelease(driver, options, logger = noopLogger2) {
|
|
|
4940
5689
|
zeroDowntime: site.zeroDowntime !== false,
|
|
4941
5690
|
healthCheckPath: site.healthCheck?.path
|
|
4942
5691
|
});
|
|
4943
|
-
const compute = config.infrastructure?.compute;
|
|
4944
5692
|
const wantsNginxStatic = kind === "server-static" && !usesRpxProxy(compute) && !!site.domain;
|
|
4945
5693
|
const staticVhost = wantsNginxStatic ? buildNginxVhostScript({
|
|
4946
5694
|
siteName,
|
|
@@ -4997,6 +5745,58 @@ async function deploySiteRelease(driver, options, logger = noopLogger2) {
|
|
|
4997
5745
|
perInstance: result.perInstance
|
|
4998
5746
|
};
|
|
4999
5747
|
}
|
|
5748
|
+
function shellSingleQuote(value) {
|
|
5749
|
+
return `'${value.replaceAll("'", "'\\''")}'`;
|
|
5750
|
+
}
|
|
5751
|
+
function buildManagementDashboardServiceReconciliationScript(slug, desiredSiteNames) {
|
|
5752
|
+
const prefix = `${slug}-dashboard-`;
|
|
5753
|
+
const desiredUnits = desiredSiteNames.map((siteName) => `${slug}-${siteName}.service`);
|
|
5754
|
+
return [
|
|
5755
|
+
"set -euo pipefail",
|
|
5756
|
+
`TS_CLOUD_DASHBOARD_PREFIX=${shellSingleQuote(prefix)}`,
|
|
5757
|
+
`TS_CLOUD_DASHBOARD_DESIRED=${shellSingleQuote(` ${desiredUnits.join(" ")} `)}`,
|
|
5758
|
+
"for TS_CLOUD_UNIT_FILE in /etc/systemd/system/*.service; do",
|
|
5759
|
+
' [ -e "$TS_CLOUD_UNIT_FILE" ] || continue',
|
|
5760
|
+
' TS_CLOUD_UNIT=$(basename "$TS_CLOUD_UNIT_FILE")',
|
|
5761
|
+
' case "$TS_CLOUD_UNIT" in',
|
|
5762
|
+
' "$TS_CLOUD_DASHBOARD_PREFIX"*.service)',
|
|
5763
|
+
' case "$TS_CLOUD_DASHBOARD_DESIRED" in',
|
|
5764
|
+
' *" $TS_CLOUD_UNIT "*) ;;',
|
|
5765
|
+
" *)",
|
|
5766
|
+
' systemctl disable --now "$TS_CLOUD_UNIT" 2>/dev/null || true',
|
|
5767
|
+
' rm -f "$TS_CLOUD_UNIT_FILE"',
|
|
5768
|
+
" ;;",
|
|
5769
|
+
" esac",
|
|
5770
|
+
" ;;",
|
|
5771
|
+
" esac",
|
|
5772
|
+
"done",
|
|
5773
|
+
"systemctl daemon-reload",
|
|
5774
|
+
"systemctl reset-failed"
|
|
5775
|
+
];
|
|
5776
|
+
}
|
|
5777
|
+
async function reconcileManagementDashboardServices(driver, options, desiredSiteNames, logger) {
|
|
5778
|
+
const { config, environment } = options;
|
|
5779
|
+
const slug = config.project.slug;
|
|
5780
|
+
const stackName = resolveProjectStackName(config, environment);
|
|
5781
|
+
const targets = await driver.findComputeTargets({ slug, environment, role: "app", stackName });
|
|
5782
|
+
if (targets.length === 0)
|
|
5783
|
+
return true;
|
|
5784
|
+
const result = await driver.runRemoteDeploy({
|
|
5785
|
+
targets,
|
|
5786
|
+
commands: buildManagementDashboardServiceReconciliationScript(slug, desiredSiteNames),
|
|
5787
|
+
comment: `ts-cloud reconcile management dashboard ${slug}`,
|
|
5788
|
+
tags: {
|
|
5789
|
+
Project: slug,
|
|
5790
|
+
Environment: environment,
|
|
5791
|
+
Role: "app"
|
|
5792
|
+
}
|
|
5793
|
+
});
|
|
5794
|
+
if (!result.success) {
|
|
5795
|
+
logger.error(`Management dashboard reconciliation failed: ${result.error || "unknown error"}`);
|
|
5796
|
+
return false;
|
|
5797
|
+
}
|
|
5798
|
+
return true;
|
|
5799
|
+
}
|
|
5000
5800
|
async function ensureAttachModeDatabase(driver, options, logger) {
|
|
5001
5801
|
const { config, environment } = options;
|
|
5002
5802
|
if (!config.cloud?.attachTo)
|
|
@@ -5039,11 +5839,15 @@ async function deployAllComputeSites(options) {
|
|
|
5039
5839
|
const { config, environment, driver, sha, runtime, tarballForSite, logger = noopLogger2 } = options;
|
|
5040
5840
|
const slug = config.project.slug;
|
|
5041
5841
|
const cwd = options.cwd ?? process.cwd();
|
|
5842
|
+
const autoDashboard = options.managementDashboard !== false;
|
|
5042
5843
|
const hadDashboard = hasManagementDashboardSite(config);
|
|
5043
|
-
|
|
5044
|
-
|
|
5045
|
-
|
|
5844
|
+
if (autoDashboard)
|
|
5845
|
+
ensureManagementDashboard(config, { cwd, logger: { info: logger.info, warn: logger.warn } });
|
|
5846
|
+
const injectedDashboardSites = !autoDashboard || hadDashboard ? [] : managementDashboardSiteNames(config);
|
|
5847
|
+
if (autoDashboard && options.rpxConfig && options.rpxConfig !== config && !hasManagementDashboardSite(options.rpxConfig))
|
|
5046
5848
|
ensureManagementDashboard(options.rpxConfig, { cwd, logger: { info: () => {}, warn: () => {} } });
|
|
5849
|
+
if (autoDashboard && !await reconcileManagementDashboardServices(driver, options, managementDashboardSiteNames(config), logger))
|
|
5850
|
+
return false;
|
|
5047
5851
|
let dashboardTarball = null;
|
|
5048
5852
|
const dashboardTarballBySite = new Map;
|
|
5049
5853
|
if (injectedDashboardSites.length > 0) {
|
|
@@ -5067,8 +5871,8 @@ async function deployAllComputeSites(options) {
|
|
|
5067
5871
|
continue;
|
|
5068
5872
|
}
|
|
5069
5873
|
try {
|
|
5070
|
-
const copy =
|
|
5071
|
-
|
|
5874
|
+
const copy = join6(tmpdir2(), `${slug}-${name}-${sha}.tar.gz`);
|
|
5875
|
+
copyFileSync2(dashboardTarball, copy);
|
|
5072
5876
|
dashboardTarballBySite.set(name, copy);
|
|
5073
5877
|
} catch {
|
|
5074
5878
|
dashboardTarballBySite.set(name, dashboardTarball);
|
|
@@ -5335,4 +6139,4 @@ function buildCloudFrontOriginConfig(options) {
|
|
|
5335
6139
|
CustomErrorResponses: { Quantity: 0 }
|
|
5336
6140
|
};
|
|
5337
6141
|
}
|
|
5338
|
-
export { PANTRY_PROJECT_DIR, pgAdminCommand, BACKUP_RUNNER_PATH, buildBackupRestoreScript, siteInstallBase, isPhpSite, resolveSiteDeployTarget, resolveSiteKind, validateDeploymentConfig, shipsARelease, DEFAULT_RPX_CERTS_DIR, normalizeRoutePath, deriveRouteId, buildRpxConfig, buildRpxLbConfig, renderRpxLauncher, RPX_DIR, RPX_LAUNCHER_PATH, RPX_SERVICE_NAME, buildRpxProvisionScript, buildRpxFragmentRefreshScript, buildUbuntuBootstrapScript, AwsDriver, resolveHetznerLocation, HetznerClient, resolveHetznerApiToken2 as resolveHetznerApiToken, normalizeSshPublicKey, wrapCloudInitUserData, HetznerDriver, LocalBoxDriver, isBoxMode, createCloudDriver, CloudDriverFactory, cloudDrivers,
|
|
6142
|
+
export { PANTRY_PROJECT_DIR, pgAdminCommand, BACKUP_RUNNER_PATH, buildBackupRestoreScript, siteInstallBase, isPhpSite, resolveSiteDeployTarget, resolveSiteKind, validateDeploymentConfig, shipsARelease, DEFAULT_RPX_CERTS_DIR, normalizeRoutePath, deriveRouteId, buildRpxConfig, buildRpxLbConfig, renderRpxLauncher, RPX_DIR, RPX_LAUNCHER_PATH, RPX_SERVICE_NAME, buildRpxProvisionScript, buildRpxFragmentRefreshScript, buildUbuntuBootstrapScript, AwsDriver, resolveHetznerLocation, HetznerClient, resolveHetznerApiToken2 as resolveHetznerApiToken, normalizeSshPublicKey, wrapCloudInitUserData, HetznerDriver, LocalBoxDriver, isBoxMode, createCloudDriver, CloudDriverFactory, cloudDrivers, planHetznerServerResize, isHetznerCapacityError, executeHetznerServerResize, buildSshArgs, sshExec, sshExecOrThrow, scpUpload, waitForSsh, waitForCloudInit, collectHetznerResizeManifest, prepareHetznerResize, verifyHetznerResize, resolveHetznerHostOptimizationPlan, buildHetznerHostOptimizationScript, applyHetznerHostOptimization, collectHetznerHostOptimizationReport, verifyHetznerHostOptimization, resizeCheckpointPath, resizeLockPath, readResizeCheckpoint, writeResizeCheckpoint, acquireResizeLock, ensureSshKey, ensureFirewall, ensureServer, serverPublicIpv4, UBUNTU_2404_AMI_PARAM, buildBoxUserData, HetznerBoxProvisioner, AwsBoxProvisioner, createBoxProvisioner, releasePaths, buildRollbackScript, resolveExecStart, buildSiteDeployScript, buildStaticSiteDeployScript, releaseTarballTmpPath, buildAwsArtifactFetch, buildLocalArtifactFetch, buildHostCleanupScript, MANAGEMENT_DASHBOARD_SITE, dashboardCredentialsFile, resolveDashboardAuth, resolveUiSource, ensureManagementDashboard, buildManagementDashboardArtifact, resolveSiteFramework, deploySiteRelease, deployAllComputeSites, reloadRpxGateway, renewRpxCertificates, MANAGED_CACHE_POLICY_OPTIMIZED, MANAGED_CACHE_POLICY_DISABLED, MANAGED_ORIGIN_REQUEST_POLICY_ALL_VIEWER, buildCloudFrontOriginConfig };
|