@studiometa/trafic-agent 0.1.46 → 0.1.48

package/dist/setup/migrations/0019__recover_orphan_containers.d.ts

@@ -0,0 +1,36 @@
+import { type SetupIo } from "../io.js";
+import type { Migration } from "../types.js";
+/** How long a container gets to stop after SIGTERM, in seconds. */
+export declare const STOP_TIMEOUT = 600;
+/**
+ * Migration 0019: stop the containers Docker lost track of in migration 0018.
+ *
+ * 0018 restarted Docker to apply the address pools. On a server set up before
+ * 0.1.47, Docker ran the containerd image store (`overlayfs`) although
+ * daemon.json asked for `overlay2`, and the restart switched the driver.
+ * dockerd skips containers created with another driver, so it forgot every
+ * running container — but `live-restore` had left their processes running.
+ * Those orphans keep the locks on the project volumes: each new database
+ * container exits on "Can't lock aria control file", each Elasticsearch on
+ * "failed to obtain node locks", and the project never starts again. Seen
+ * on a live server: 66 orphans, 27 of them databases.
+ *
+ * containerd still knows the orphans: they are the tasks in the `moby`
+ * namespace with no matching Docker container. Each one gets SIGTERM, which
+ * MariaDB, MySQL and Elasticsearch handle as a clean shutdown, and as long as
+ * it needs. No SIGKILL: a database killed mid-write is the one outcome worse
+ * than the outage, so a process that does not stop is reported and left
+ * alone. The stopped tasks are then deleted, which ends their shims.
+ *
+ * Nothing is restarted here: the volumes are free again, and the agent starts
+ * each project on its next request, on the new driver.
+ *
+ * Idempotent: does nothing when every containerd task belongs to a container
+ * Docker knows.
+ */
+export declare const migration0019RecoverOrphanContainers: Migration;
+/**
+ * The migration body, with its effects injected so tests can drive it.
+ */
+export declare function recoverOrphanContainers(io?: SetupIo): void;
+//# sourceMappingURL=0019__recover_orphan_containers.d.ts.map

package/dist/setup/migrations/0019__recover_orphan_containers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"0019__recover_orphan_containers.d.ts","sourceRoot":"","sources":["../../../src/setup/migrations/0019__recover_orphan_containers.ts"],"names":[],"mappings":"AACA,OAAO,EAAU,KAAK,OAAO,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAE7C,mEAAmE;AACnE,eAAO,MAAM,YAAY,MAAM,CAAC;AAEhC;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,eAAO,MAAM,oCAAoC,EAAE,SAOlD,CAAC;AASF;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,EAAE,GAAE,OAAgB,GAAG,IAAI,CA8BlE"}

package/dist/setup/migrations/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/setup/migrations/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAkBjF,iDAAiD;AACjD,eAAO,MAAM,qBAAqB,iCAAiC,CAAC;AAEpE,+CAA+C;AAC/C,eAAO,MAAM,cAAc,EAAE,SAAS,EAiBrC,CAAC;AAMF;;;GAGG;AACH,wBAAgB,kBAAkB,IAAI,cAAc,CAqBnD;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,cAAc,GAAG,IAAI,CAG9D;AAMD;;GAEG;AACH,wBAAgB,cAAc,IAAI,kBAAkB,EAAE,CAMrD;AAED;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,UAAQ,GAAG,IAAI,CAyBzD;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,IAAI,IAAI,CAU/C"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/setup/migrations/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAqBjF,iDAAiD;AACjD,eAAO,MAAM,qBAAqB,iCAAiC,CAAC;AAEpE,+CAA+C;AAC/C,eAAO,MAAM,cAAc,EAAE,SAAS,EAoBrC,CAAC;AAMF;;;GAGG;AACH,wBAAgB,kBAAkB,IAAI,cAAc,CAqBnD;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,cAAc,GAAG,IAAI,CAG9D;AAMD;;GAEG;AACH,wBAAgB,cAAc,IAAI,kBAAkB,EAAE,CAMrD;AAED;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,UAAQ,GAAG,IAAI,CAyBzD;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,IAAI,IAAI,CAU/C"}

package/dist/{setup-B02lDDZU.js → setup-DWgb6dAD.js}

@@ -1,4 +1,4 @@
-import { B as resetSteps, C as shouldNeverStop, F as error, H as step, I as exec, L as execSilent, M as setProjectStatus, O as getIdleProjects, P as nodeIo, R as info, S as loadProjectConfig, U as success, V as setDryRun, W as warn, Y as loadProjectList, Z as stopProject, _ as installDdev, at as parseDuration, b as readToolPorts, d as TLS_STORE_CONFIG, f as addDdevAptRepo, g as getDockerGatewayIp, h as findRunningProject, it as loadConfig, l as ROUTER_COMPOSE_OVERRIDE, m as configureTraefik, p as configureDdev, q as getProjectInfo, u as STATIC_CONFIG, v as installSystemDeps, x as getIdleTimeoutMs, y as purgeTraefikAcmeStorage, z as isRoot } from "./server-EPU5dtPx.js";
+import { $ as stopProject, A as getIdleProjects, B as info, C as getIdleTimeoutMs, G as success, H as resetSteps, I as nodeIo, K as warn, L as error, P as setProjectStatus, R as exec, S as readToolPorts, T as shouldNeverStop, U as setDryRun, V as isRoot, W as step, Y as getProjectInfo, Z as loadProjectList, _ as findRunningProjects, b as installSystemDeps, d as STATIC_CONFIG, f as TLS_STORE_CONFIG, g as findRunningProject, h as configureTraefik, m as configureDdev, ot as loadConfig, p as addDdevAptRepo, st as parseDuration, u as ROUTER_COMPOSE_OVERRIDE, v as getDockerGatewayIp, w as loadProjectConfig, x as purgeTraefikAcmeStorage, y as installDdev, z as execSilent } from "./server-DZAo8Clg.js";
 import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 //#region src/setup/docker-firewall.ts
 /** Where the rule script lives, so systemd and a human can both run it. */
@@ -226,6 +226,44 @@ function installDocker(io = nodeIo) {
 	io.exec("usermod -aG docker ddev", { silent: true });
 	success("Added ddev user to docker group");
 }
+/** Docker daemon config file */
+var DAEMON_CONFIG = "/etc/docker/daemon.json";
+/**
+* Address pools for the networks Docker creates.
+*
+* Every DDEV project gets its own network. Docker's built-in pools hold only
+* about 31 of them (15 × /16 in 172.17–31.x, 16 × /20 in 192.168.x), so a
+* server with ~30 projects fails with "all predefined address pools have
+* been fully subnetted". /24 subnets in 172.16.0.0/12 give 4096 networks,
+* and keep every network inside the range the firewall rules already allow
+* (hardening.ts, docker-firewall.ts) — the built-in 192.168.x pools are not.
+*/
+var DOCKER_ADDRESS_POOLS = [{
+	base: "172.16.0.0/12",
+	size: 24
+}];
+/**
+* Add the address pools to an existing daemon config.
+*
+* Returns true when the config was changed. Leaves a config that already
+* sets pools alone (an operator choice), and one it cannot parse.
+*/
+function addDockerAddressPools(io = nodeIo) {
+	let config = {};
+	if (io.fileExists("/etc/docker/daemon.json")) try {
+		config = JSON.parse(io.readFile(DAEMON_CONFIG));
+	} catch {
+		warn("Could not parse existing Docker config, skipping");
+		return false;
+	}
+	if ("default-address-pools" in config) return false;
+	io.exec("mkdir -p /etc/docker");
+	io.writeFile(DAEMON_CONFIG, JSON.stringify({
+		...config,
+		"default-address-pools": DOCKER_ADDRESS_POOLS
+	}, null, 2));
+	return true;
+}
 /**
 * Configure Docker for production use
 */
@@ -238,10 +276,11 @@ function configureDocker(io = nodeIo) {
 			"max-file": "3"
 		},
 		"storage-driver": "overlay2",
-		"live-restore": true
+		"live-restore": true,
+		"default-address-pools": DOCKER_ADDRESS_POOLS
 	};
 	io.exec("mkdir -p /etc/docker");
-	const configPath = "/etc/docker/daemon.json";
+	const configPath = DAEMON_CONFIG;
 	if (io.fileExists(configPath)) try {
 		const merged = {
 			...JSON.parse(io.readFile(configPath)),
@@ -254,10 +293,59 @@ function configureDocker(io = nodeIo) {
 		return;
 	}
 	else io.writeFile(configPath, JSON.stringify(daemonConfig, null, 2));
-	io.exec("systemctl reload docker || systemctl restart docker");
+	restartDocker(io);
 	success("Docker configured with production settings");
 }
 /**
+* Restart Docker to apply daemon.json, without orphaning running containers.
+*
+* `live-restore` keeps containers running across a restart, but only when
+* the storage driver stays the same. Before 0.1.47, setup applied daemon.json
+* with a reload, and a reload does not apply `storage-driver`. Docker 29
+* defaults a fresh install to the containerd image store (`overlayfs`), so
+* such a server runs `overlayfs` while daemon.json says `overlay2`. The first
+* full restart switches the driver, and dockerd silently skips every
+* container created with the other one. Their processes keep running and
+* keep the locks on the database volumes. DDEV then creates new containers
+* that exit at once on "Can't lock aria control file". Seen on a live server
+* after migration 0018: 66 containers lost, 27 of them databases.
+*
+* So when the driver is about to change, the DDEV projects are stopped first,
+* which shuts their databases down cleanly, and the projects that were running
+* are started again on the new driver. Volumes do not depend on the storage
+* driver, so no data moves. Images are pulled again for the new driver.
+*/
+function restartDocker(io = nodeIo) {
+	if (!switchesStorageDriver(io)) {
+		io.exec("systemctl restart docker");
+		return;
+	}
+	const running = findRunningProjects(io);
+	warn("Docker changes its storage driver on restart, stopping DDEV projects first");
+	if (io.commandExists("ddev")) io.exec("su - ddev -c 'DDEV_NONINTERACTIVE=true ddev poweroff'", { silent: true });
+	io.exec("systemctl restart docker");
+	if (running.length > 0) {
+		info(`Starting ${running.length} project(s) again: ${running.join(", ")}`);
+		io.exec(`su - ddev -c 'DDEV_NONINTERACTIVE=true ddev start ${running.join(" ")}'`, { silent: true });
+	}
+}
+/**
+* Whether a restart makes Docker use another storage driver.
+*
+* Compares the driver the daemon runs now with the one daemon.json asks for.
+* A config without `storage-driver` keeps the current one.
+*/
+function switchesStorageDriver(io) {
+	const active = io.execSilent("docker info --format '{{.Driver}}'");
+	if (!active || !io.fileExists("/etc/docker/daemon.json")) return false;
+	try {
+		const configured = JSON.parse(io.readFile(DAEMON_CONFIG))["storage-driver"];
+		return typeof configured === "string" && configured !== active;
+	} catch {
+		return false;
+	}
+}
+/**
 * Setup Docker system prune cron job
 */
 function setupDockerPrune(io = nodeIo) {
@@ -1391,9 +1479,9 @@ var migration0013SshKexDefaults = {
 * and its dist is the Ubuntu codename, so
 * `"origin=Docker,codename=${distro_codename}"` is the entry that matches it.
 *
-* DDEV, installed from pkg.ddev.com, is deliberately not added: a DDEV major
-* landing unattended can break running previews. DDEV is updated by
-* `trafic-agent upgrade` instead, when an operator is there to see it.
+* DDEV, installed from its own apt repository, is deliberately not added: a
+* DDEV major landing unattended can break running previews. DDEV is updated
+* by `trafic-agent upgrade` instead, when an operator is there to see it.
 *
 * `configureUnattendedUpgrades` now writes the Docker entry, which covers
 * fresh installs. Servers set up by an earlier release still carry the old
@@ -1548,6 +1636,152 @@ function findRouterContainer(io = nodeIo) {
 	return io.execSilent("docker ps -aq --filter label=com.docker.compose.service=ddev-router").trim().split("\n")[0] || void 0;
 }
 //#endregion
+//#region src/setup/migrations/0017__ddev_apt_cloudsmith.ts
+/** The Gemfury source file setup wrote before this release. */
+var OLD_SOURCES = "/etc/apt/sources.list.d/ddev.list";
+/** The dearmoured keyring that source was signed by. */
+var OLD_KEYRING = "/etc/apt/keyrings/ddev.gpg";
+/**
+* Migration 0017: move the DDEV apt repository from Gemfury to Cloudsmith.
+*
+* DDEV 1.25.4 moved its Debian and RPM packages to Cloudsmith
+* (ddev/ddev#8698) and its install docs now name
+* `https://packages.ddev.com/public/deb/ubuntu` with the key at
+* `https://packages.ddev.com/public/gpg.key`. The former Gemfury host,
+* `pkg.ddev.com`, still receives packages and DDEV has announced no
+* retirement date, so nothing is broken today — but a server left on the old
+* repository would stop seeing new DDEV releases the day it is switched off,
+* and `trafic-agent upgrade` runs `apt-get install --only-upgrade ddev`
+* against whatever repository is configured.
+*
+* The old files are removed rather than left alongside the new ones, as
+* DDEV's own instructions do: two sources offering the same package make apt
+* pick by version and would leave the Gemfury host in the picture for as
+* long as it answers. `setup` writes only the Cloudsmith source, so a fresh
+* server and an upgraded one end up identical.
+*
+* Idempotent: skipped once the Cloudsmith source file is in place, and each
+* removal is guarded by the file's existence.
+*/
+var migration0017DdevAptCloudsmith = {
+	id: "0017__ddev_apt_cloudsmith",
+	description: "Move the DDEV apt repository from Gemfury to Cloudsmith",
+	run() {
+		runDdevAptCloudsmithMigration();
+	}
+};
+/** The migration body, with its effects injected so tests can drive it. */
+function runDdevAptCloudsmithMigration(io = nodeIo) {
+	if (io.fileExists("/etc/apt/sources.list.d/ddev.sources")) return;
+	if (io.fileExists(OLD_SOURCES)) io.exec(`rm -f ${OLD_SOURCES}`, { silent: true });
+	if (io.fileExists(OLD_KEYRING)) io.exec(`rm -f ${OLD_KEYRING}`, { silent: true });
+	addDdevAptRepo(io);
+	io.exec("apt-get update -qq", { silent: true });
+}
+//#endregion
+//#region src/setup/migrations/0018__docker_address_pools.ts
+/**
+* Migration 0018: give Docker enough address space for one network per
+* project.
+*
+* `setup` wrote a daemon.json without `default-address-pools`, so Docker
+* used its built-in pools: about 31 networks. Each DDEV project creates its
+* own `ddev-<name>_default` network, and a server with ~30 projects failed
+* to start the next one with "all predefined address pools have been fully
+* subnetted". The last 16 of those built-in networks are in 192.168.x,
+* outside the 172.16.0.0/12 range the firewall allows to reach the agent.
+*
+* `configureDocker` now writes /24 pools in 172.16.0.0/12, which covers
+* fresh installs. This migration adds them to existing servers.
+*
+* Docker reads the pools only at start, so the daemon is restarted.
+* `live-restore`, set by setup, keeps the running containers up. Existing
+* networks keep their subnets; only new networks get a /24, and Docker skips
+* subnets an existing network already uses.
+*
+* Changed after release: the first version ran `systemctl restart docker`
+* directly. On servers set up before 0.1.47 that restart also switched the
+* storage driver, which `live-restore` cannot survive, and orphaned every
+* running container (see restartDocker). The result on a server is the same
+* either way; only servers that have not run it yet get the safe restart.
+* Servers that already ran it are repaired by migration 0019.
+*
+* Idempotent: does nothing when the config already sets pools.
+*/
+var migration0018DockerAddressPools = {
+	id: "0018__docker_address_pools",
+	description: "Give Docker address pools for one network per project",
+	run() {
+		runDockerAddressPoolsMigration();
+	}
+};
+/**
+* The migration body, with its effects injected so tests can drive it.
+*/
+function runDockerAddressPoolsMigration(io = nodeIo) {
+	if (addDockerAddressPools(io)) restartDocker(io);
+}
+/**
+* Migration 0019: stop the containers Docker lost track of in migration 0018.
+*
+* 0018 restarted Docker to apply the address pools. On a server set up before
+* 0.1.47, Docker ran the containerd image store (`overlayfs`) although
+* daemon.json asked for `overlay2`, and the restart switched the driver.
+* dockerd skips containers created with another driver, so it forgot every
+* running container — but `live-restore` had left their processes running.
+* Those orphans keep the locks on the project volumes: each new database
+* container exits on "Can't lock aria control file", each Elasticsearch on
+* "failed to obtain node locks", and the project never starts again. Seen
+* on a live server: 66 orphans, 27 of them databases.
+*
+* containerd still knows the orphans: they are the tasks in the `moby`
+* namespace with no matching Docker container. Each one gets SIGTERM, which
+* MariaDB, MySQL and Elasticsearch handle as a clean shutdown, and as long as
+* it needs. No SIGKILL: a database killed mid-write is the one outcome worse
+* than the outage, so a process that does not stop is reported and left
+* alone. The stopped tasks are then deleted, which ends their shims.
+*
+* Nothing is restarted here: the volumes are free again, and the agent starts
+* each project on its next request, on the new driver.
+*
+* Idempotent: does nothing when every containerd task belongs to a container
+* Docker knows.
+*/
+var migration0019RecoverOrphanContainers = {
+	id: "0019__recover_orphan_containers",
+	description: "Stop the containers Docker lost track of when it changed storage driver",
+	run() {
+		recoverOrphanContainers();
+	}
+};
+/**
+* The migration body, with its effects injected so tests can drive it.
+*/
+function recoverOrphanContainers(io = nodeIo) {
+	const known = new Set(io.execSilent("docker ps -aq --no-trunc").split("\n").filter(Boolean));
+	const orphans = parseTasks(io.execSilent("ctr -n moby tasks ls")).filter((task) => !known.has(task.id));
+	if (orphans.length === 0) return;
+	info(`Stopping ${orphans.length} container(s) Docker no longer tracks`);
+	const running = orphans.filter((task) => task.status === "RUNNING");
+	for (const task of running) io.execSilent(`kill -TERM ${task.pid}`);
+	const stuck = running.filter((task) => {
+		io.execSilent(`timeout 600 tail --pid=${task.pid} -f /dev/null`);
+		return io.fileExists(`/proc/${task.pid}`);
+	});
+	for (const task of orphans.filter((task) => !stuck.includes(task))) io.execSilent(`ctr -n moby tasks delete ${task.id}`);
+	for (const task of stuck) warn(`Container ${task.id.slice(0, 12)} (PID ${task.pid}) did not stop, left running`);
+}
+/**
+* Parse `ctr tasks ls`: a header, then one `TASK PID STATUS` row per task.
+*/
+function parseTasks(output) {
+	return output.split("\n").slice(1).map((line) => line.trim().split(/\s+/)).filter((columns) => columns.length >= 3).map(([id, pid, status]) => ({
+		id,
+		pid,
+		status
+	}));
+}
+//#endregion
 //#region src/setup/migrations/index.ts
 /** Path to the persisted migration state file */
 var MIGRATIONS_STATE_FILE = "/etc/trafic/.migrations.json";
@@ -1568,7 +1802,10 @@ var ALL_MIGRATIONS = [
 	migration0013SshKexDefaults,
 	migration0014UnattendedUpgradesDockerOrigin,
 	migration0015CatchallPriority,
-	migration0016WildcardDnsChallenge
+	migration0016WildcardDnsChallenge,
+	migration0018DockerAddressPools,
+	migration0017DdevAptCloudsmith,
+	migration0019RecoverOrphanContainers
 ];
 /**
 * Load the current migration state from disk.
@@ -1726,4 +1963,4 @@ async function audit() {
 //#endregion
 export { printAuditResults as a, stopIdleProjects as c, buildDockerFirewallScript as d, buildDockerFirewallUnit as f, runPendingMigrations as i, DOCKER_FIREWALL_SCRIPT as l, setup as n, runAudit as o, configureDockerFirewall as p, listMigrations as r, startIdleScheduler as s, audit as t, DOCKER_FIREWALL_UNIT as u };
 
-//# sourceMappingURL=setup-B02lDDZU.js.map
+//# sourceMappingURL=setup-DWgb6dAD.js.map