@studiometa/trafic-agent 0.1.49 → 0.1.51

package/dist/{server-DZAo8Clg.js → server-Cfk-O4DJ.js}

@@ -1,10 +1,17 @@
 import { existsSync, readFileSync, watch, writeFileSync } from "node:fs";
+import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { parse } from "smol-toml";
 import { execFile, execSync } from "node:child_process";
 import { DatabaseSync } from "node:sqlite";
-import { dirname, join, resolve } from "node:path";
 import { createServer } from "node:http";
 import { fileURLToPath } from "node:url";
+//#region src/utils/paths.ts
+/** Whether `child` is strictly inside `parent`. Both are resolved paths. */
+function isInside(parent, child) {
+	const path = relative(parent, child);
+	return Boolean(path) && !isAbsolute(path) && path.split(sep)[0] !== "..";
+}
+//#endregion
 //#region src/utils/config.ts
 var DEFAULT_CONFIG_PATHS = [
 	"/etc/trafic/config.toml",
@@ -19,6 +26,7 @@ var DEFAULT_CONFIG = {
 	projectsDir: "/home/ddev/www",
 	idleTimeout: "30m",
 	idleCheckInterval: "5m",
+	maxRunningProjects: 0,
 	auth: {
 		defaultPolicy: "basic",
 		allowedIps: [],
@@ -27,7 +35,11 @@ var DEFAULT_CONFIG = {
 		rules: [],
 		trustedProxyHops: 1
 	},
-	tls: { dnsEnv: {} }
+	tls: { dnsEnv: {} },
+	backup: {
+		dir: "/var/backups/trafic",
+		retainDays: 14
+	}
 };
 /**
 * Parse a duration string (e.g., "30m", "1h", "2h30m") to milliseconds
@@ -40,6 +52,22 @@ function parseDuration(duration) {
 	const seconds = parseInt(match[3] ?? "0", 10);
 	return (hours * 3600 + minutes * 60 + seconds) * 1e3;
 }
+/** Duration suffixes accepted in a timeout, in milliseconds. */
+var TIMEOUT_UNITS = {
+	s: 1e3,
+	m: 6e4,
+	h: 36e5
+};
+/**
+* Parse a command-line timeout such as "10m", "90s" or "1h"; a bare number
+* is minutes. The grammar is the one of `trafic --timeout`, which the CLI
+* passes on as is. Returns undefined for anything else, or for zero.
+*/
+function parseTimeout(value) {
+	const match = /^(\d+)([smh])?$/.exec(value.trim());
+	if (!match || Number(match[1]) <= 0) return;
+	return Number(match[1]) * TIMEOUT_UNITS[match[2] ?? "m"];
+}
 /**
 * Load configuration from TOML file
 */
@@ -54,6 +82,7 @@ function loadConfig(configPath) {
 	const parsed = parse(content);
 	const auth = mergeAuthConfig(parsed.auth);
 	const tls = mergeTlsConfig(parsed.tls);
+	const backup = mergeBackupConfig(parsed.backup);
 	return {
 		tld: parsed.tld ?? DEFAULT_CONFIG.tld,
 		port: parsed.port ?? DEFAULT_CONFIG.port,
@@ -62,8 +91,20 @@ function loadConfig(configPath) {
 		projectsDir: parsed.projects_dir ?? DEFAULT_CONFIG.projectsDir,
 		idleTimeout: parsed.idle_timeout ?? DEFAULT_CONFIG.idleTimeout,
 		idleCheckInterval: parsed.idle_check_interval ?? DEFAULT_CONFIG.idleCheckInterval,
+		maxRunningProjects: parsed.max_running_projects ?? DEFAULT_CONFIG.maxRunningProjects,
 		auth,
-		tls
+		tls,
+		backup
+	};
+}
+/**
+* Read the [backup] section. validateConfig reports a bad value.
+*/
+function mergeBackupConfig(raw) {
+	if (!raw) return DEFAULT_CONFIG.backup;
+	return {
+		dir: raw.dir ?? DEFAULT_CONFIG.backup.dir,
+		retainDays: raw.retain_days ?? DEFAULT_CONFIG.backup.retainDays
 	};
 }
 /**
@@ -134,11 +175,19 @@ function validateConfig(config) {
 	const errors = [];
 	if (!config.tld) errors.push("tld is required");
 	if (config.port < 1 || config.port > 65535) errors.push("port must be between 1 and 65535");
+	if (!Number.isInteger(config.maxRunningProjects) || config.maxRunningProjects < 0) errors.push("max_running_projects must be a whole number, 0 or more");
 	if (config.auth.defaultPolicy !== "allow" && config.auth.defaultPolicy !== "deny" && config.auth.defaultPolicy !== "basic" && config.auth.defaultPolicy !== "token") errors.push("auth.default_policy must be \"allow\", \"deny\", \"basic\", or \"token\"");
 	for (const [key, value] of Object.entries(config.tls.dnsEnv)) {
 		if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key)) errors.push(`tls.dns_env key "${key}" is not a valid environment variable name`);
 		if (typeof value !== "string") errors.push(`tls.dns_env.${key} must be a string`);
 	}
+	if (typeof config.backup.dir !== "string" || !isAbsolute(config.backup.dir)) errors.push("backup.dir must be an absolute path");
+	else {
+		const backupDir = resolve(config.backup.dir);
+		const projectsDir = resolve(config.projectsDir);
+		if (backupDir === projectsDir || isInside(projectsDir, backupDir) || isInside(backupDir, projectsDir)) errors.push("backup.dir must not be projects_dir, be inside it or contain it");
+	}
+	if (!Number.isInteger(config.backup.retainDays) || config.backup.retainDays < 1) errors.push("backup.retain_days must be a whole number of 1 or more");
 	return errors;
 }
 //#endregion
@@ -377,8 +426,8 @@ var APPROOT_LINE = /^\s+approot:\s*(.+?)\s*$/;
 *
 * The file nests the path under the project name:
 *
-*   scalian:
-*       approot: /home/ddev/www/scalian
+*   my-app:
+*       approot: /home/ddev/www/my-app
 *
 * Indentation therefore has to be read. A flat key/value pass produces one
 * entry per line instead: the project mapped to an empty string, plus a
@@ -433,10 +482,11 @@ function watchProjectList(projectListPath, onChange) {
 	});
 }
 /** The real runner, used unless a caller passes its own. */
-var nodeDdevRunner = (args, timeoutMs) => new Promise((resolve) => {
+var nodeDdevRunner = (args, timeoutMs, cwd) => new Promise((resolve) => {
 	execFile("ddev", args, {
 		encoding: "utf-8",
-		timeout: timeoutMs
+		timeout: timeoutMs,
+		cwd
 	}, (error, stdout) => {
 		if (error) {
 			console.error(`ddev ${args.join(" ")} failed:`, error.message);
@@ -474,7 +524,7 @@ async function getProjectInfo(name, run = nodeDdevRunner) {
 			httpsURLs: raw.httpsURLs ?? [],
 			type: raw.type,
 			phpVersion: raw.php_version,
-			dbType: raw.dbinfo?.dbType
+			dbType: raw.dbinfo?.database_type
 		};
 	} catch {
 		return;
@@ -498,17 +548,19 @@ async function stopProject(name, run = nodeDdevRunner) {
 	return ok;
 }
 /**
-* List all DDEV projects with their status
+* List all DDEV projects with their live status.
+*
+* `ddev list -j` asks Docker, so a project started or stopped outside the
+* agent (by `trafic deploy`, say) shows its real state here. It wraps the
+* entries in a `raw` array.
 */
-function listProjects() {
+async function listProjects(run = nodeDdevRunner) {
+	const { ok, stdout } = await run(["list", "-j"], 3e4);
+	if (!ok) return [];
 	try {
-		const output = execSync("ddev list -j", {
-			encoding: "utf-8",
-			timeout: 3e4
-		});
-		const data = JSON.parse(output);
-		if (!Array.isArray(data)) return [];
-		return data.map((p) => ({
+		const data = JSON.parse(stdout);
+		if (!Array.isArray(data.raw)) return [];
+		return data.raw.map((p) => ({
 			name: p.name,
 			status: p.status,
 			appRoot: p.approot,
@@ -651,16 +703,25 @@ var nodeIo = {
 var systemClock = () => Date.now();
 var db = null;
 /**
+* How long a write waits for another connection to release the lock.
+*
+* The service and a `trafic-agent destroy` run over SSH open the same file.
+* WAL lets readers and one writer work together, but a second writer fails
+* at once with "database is locked" unless it is told to wait.
+*/
+var BUSY_TIMEOUT_MS = 5e3;
+/**
 * Initialize the database connection and schema
 */
 function initDb(dbPath) {
-	db = new DatabaseSync(dbPath);
+	db = new DatabaseSync(dbPath, { timeout: BUSY_TIMEOUT_MS });
 	db.exec("PRAGMA journal_mode=WAL");
 	db.exec(`
     CREATE TABLE IF NOT EXISTS projects (
       name TEXT PRIMARY KEY,
       last_access INTEGER NOT NULL,
-      status TEXT NOT NULL DEFAULT 'stopped'
+      status TEXT NOT NULL DEFAULT 'stopped',
+      status_pid INTEGER
     );
 
     CREATE TABLE IF NOT EXISTS access_logs (
@@ -674,10 +735,31 @@ function initDb(dbPath) {
 
     CREATE INDEX IF NOT EXISTS idx_access_logs_project ON access_logs(project);
     CREATE INDEX IF NOT EXISTS idx_access_logs_timestamp ON access_logs(timestamp);
+
+    CREATE TABLE IF NOT EXISTS scheduled_destroys (
+      name TEXT PRIMARY KEY,
+      destroy_at INTEGER NOT NULL,
+      backup INTEGER NOT NULL DEFAULT 0,
+      timeout_ms INTEGER,
+      removing INTEGER NOT NULL DEFAULT 0
+    );
   `);
+	addColumn(db, "projects", "status_pid INTEGER");
 	return db;
 }
 /**
+* Add a column to a table that does not have it yet. The service and a
+* `trafic-agent destroy` run can both get here first, so a column that
+* exists already is not an error.
+*/
+function addColumn(database, table, column) {
+	try {
+		database.exec(`ALTER TABLE ${table} ADD COLUMN ${column}`);
+	} catch (error) {
+		if (!/duplicate column name/i.test(error.message)) throw error;
+	}
+}
+/**
 * Get the database instance
 */
 function getDb() {
@@ -710,14 +792,63 @@ function getProject(name) {
 	return getDb().prepare("SELECT name, last_access as lastAccess, status FROM projects WHERE name = ?").get(name);
 }
 /**
+* The statuses a process sets while it works on a project: "starting" (the
+* waiting page, in the service) and "destroying" (`trafic-agent destroy`).
+*/
+var BUSY_STATUSES = ["starting", "destroying"];
+/**
 * Set project status
+*
+* A busy status ("starting", "destroying") records the id of this process,
+* so releaseStaleStatuses can tell when the process that set it is gone.
 */
 function setProjectStatus(name, status, now = systemClock) {
 	getDb().prepare(`
-    INSERT INTO projects (name, last_access, status)
-    VALUES (?, ?, ?)
-    ON CONFLICT(name) DO UPDATE SET status = excluded.status
-  `).run(name, now(), status);
+    INSERT INTO projects (name, last_access, status, status_pid)
+    VALUES (?, ?, ?, ?)
+    ON CONFLICT(name) DO UPDATE SET
+      status = excluded.status,
+      status_pid = excluded.status_pid
+  `).run(name, now(), status, BUSY_STATUSES.includes(status) ? process.pid : null);
+}
+/**
+* Whether a process is alive. EPERM means it is, but belongs to another user.
+*/
+function isProcessAlive(pid) {
+	try {
+		process.kill(pid, 0);
+		return true;
+	} catch (error) {
+		return error.code === "EPERM";
+	}
+}
+/**
+* Set back to "running" the busy statuses ("starting", "destroying") whose
+* process is gone, and return the names of those projects.
+*
+* The process that sets a busy status sets it back when it ends. A process
+* that is killed (a cancelled CI job, a dropped SSH connection, a restart of
+* the service) cannot, and the status would stay: the sweeps never stop a
+* busy project, max_running_projects counts it as running, and the waiting
+* page never starts it. "running" lets the idle sweep check the project
+* against DDEV and stop it, also when it is stopped already.
+*
+* `isAlive` says whether a process still works on its project: the service
+* passes its own check at start, where its own id is not alive yet.
+*/
+function releaseStaleStatuses(isAlive = isProcessAlive) {
+	const database = getDb();
+	const rows = database.prepare("SELECT name, status, status_pid AS pid FROM projects WHERE status IN ('starting', 'destroying')").all();
+	const release = database.prepare(`
+    UPDATE projects SET status = 'running', status_pid = NULL
+    WHERE name = ? AND status = ? AND status_pid IS ?
+  `);
+	const released = [];
+	for (const { name, status, pid } of rows) {
+		if (pid !== null && isAlive(pid)) continue;
+		if (Number(release.run(name, status, pid).changes) > 0) released.push(name);
+	}
+	return released;
 }
 /**
 * Get idle projects (last access older than threshold)
@@ -732,6 +863,118 @@ function getIdleProjects(thresholdMs, now = systemClock) {
   `).all(cutoff);
 }
 /**
+* Delete every record of a project: its status row, its access logs and its
+* scheduled removal. Returns whether there was any.
+*/
+function deleteProject(name) {
+	const database = getDb();
+	return [
+		database.prepare("DELETE FROM projects WHERE name = ?").run(name),
+		database.prepare("DELETE FROM access_logs WHERE project = ?").run(name),
+		database.prepare("DELETE FROM scheduled_destroys WHERE name = ?").run(name)
+	].reduce((total, result) => total + Number(result.changes), 0) > 0;
+}
+/**
+* Delete the records of the projects that DDEV no longer lists, and return
+* their names.
+*
+* A project removed outside the agent (`ddev delete` by hand, or a
+* `trafic destroy` older than the agent's own removal) leaves its row behind.
+* releaseStaleStatuses would set that row back to "running", and every sweep
+* would then ask DDEV about a project it does not know.
+*
+* A project with a busy status whose process lives is kept: `ddev delete`
+* takes it out of the list before its removal deletes the records. A project
+* with a scheduled removal is kept too: `ddev delete` by hand leaves its
+* directory, and the scheduled removal still has to remove it. An empty
+* list is kept as is too: a missing or unreadable project list is more likely
+* than a server with no project, and keeping the rows is harmless.
+*/
+function deleteUnlistedProjects(listed, isAlive = isProcessAlive) {
+	if (listed.size === 0) return [];
+	const rows = getDb().prepare("SELECT name, status, status_pid AS pid FROM projects WHERE name NOT IN (SELECT name FROM scheduled_destroys)").all();
+	const deleted = [];
+	for (const { name, status, pid } of rows) {
+		if (listed.has(name)) continue;
+		if (BUSY_STATUSES.includes(status) && pid !== null && isAlive(pid)) continue;
+		deleteProject(name);
+		deleted.push(name);
+	}
+	return deleted;
+}
+/**
+* Schedule the removal of a project, or replace a removal already
+* scheduled. `timeoutMs` is the time limit of the backup.
+*/
+function scheduleDestroy(name, destroyAt, backup, timeoutMs) {
+	getDb().prepare(`
+    INSERT INTO scheduled_destroys (name, destroy_at, backup, timeout_ms)
+    VALUES (?, ?, ?, ?)
+    ON CONFLICT(name) DO UPDATE SET
+      destroy_at = excluded.destroy_at,
+      backup = excluded.backup,
+      timeout_ms = excluded.timeout_ms
+  `).run(name, destroyAt, backup ? 1 : 0, timeoutMs ?? null);
+}
+function toScheduledDestroy({ name, destroyAt, backup, timeoutMs, removing }) {
+	return {
+		name,
+		destroyAt,
+		backup: backup === 1,
+		...timeoutMs === null ? {} : { timeoutMs },
+		removing: removing === 1
+	};
+}
+/** The columns of a ScheduledDestroy. */
+var SCHEDULED_DESTROY_COLUMNS = "name, destroy_at as destroyAt, backup, timeout_ms as timeoutMs, removing";
+/**
+* Cancel the scheduled removal of a project:
+* - "cancelled" when it was scheduled and is now cancelled
+* - "none" when there was none
+* - "removing" when the sweep already removes the project (see
+*   startScheduledDestroy): it is too late, and the removal stays
+*/
+function cancelScheduledDestroy(name) {
+	const result = getDb().prepare("DELETE FROM scheduled_destroys WHERE name = ? AND removing = 0").run(name);
+	if (Number(result.changes) > 0) return "cancelled";
+	return getScheduledDestroy(name) ? "removing" : "none";
+}
+/**
+* Mark a scheduled removal as started, when it is still due at `dueBy`.
+* From then on, a cancel is refused, and a retry after a failure does not
+* back the project up again. Returns false when the removal was cancelled
+* or moved to a later date meanwhile. One statement, so a cancel from
+* another process comes either before (false) or after (refused).
+*/
+function startScheduledDestroy(name, dueBy) {
+	const result = getDb().prepare("UPDATE scheduled_destroys SET removing = 1 WHERE name = ? AND destroy_at <= ?").run(name, dueBy);
+	return Number(result.changes) > 0;
+}
+/**
+* Get the scheduled removals, soonest first. With `dueBy`, only the removals
+* due at or before that time.
+*/
+function getScheduledDestroys(dueBy = Number.MAX_SAFE_INTEGER) {
+	return getDb().prepare(`
+    SELECT ${SCHEDULED_DESTROY_COLUMNS}
+    FROM scheduled_destroys
+    WHERE destroy_at <= ?
+    ORDER BY destroy_at, name
+  `).all(dueBy).map(toScheduledDestroy);
+}
+/**
+* Get the scheduled removal of a project when it is due at or before
+* `dueBy`, or undefined when there is none.
+*/
+function getScheduledDestroy(name, dueBy = Number.MAX_SAFE_INTEGER) {
+	const row = getDb().prepare(`
+    SELECT ${SCHEDULED_DESTROY_COLUMNS}
+    FROM scheduled_destroys
+    WHERE name = ? AND destroy_at <= ?
+  `).get(name, dueBy);
+	return row && toScheduledDestroy(row);
+}
+/**
 * Log an access
 */
 function logAccess(log) {
@@ -828,6 +1071,86 @@ function getIdleTimeoutMs(config) {
 	return ms > 0 ? ms : void 0;
 }
 //#endregion
+//#region src/tasks/running-limit.ts
+/**
+* The real dependencies: the agent database, and DDEV through `run`. The
+* default runner is the real one.
+*/
+function ddevRunningLimitDeps(run = nodeDdevRunner) {
+	return {
+		listProjects: () => listProjects(run),
+		stopProject: (name) => stopProject(name, run),
+		getProject,
+		setProjectStatus: (name, status) => setProjectStatus(name, status),
+		updateProjectAccess: (name) => updateProjectAccess(name)
+	};
+}
+/** Whether the agent works on a project, so it must not be stopped. */
+function isBusy(record) {
+	return record.status === "starting" || record.status === "destroying";
+}
+/** DDEV states where the project's containers are up and use memory. */
+function isUp(project) {
+	return project.status === "running" || project.status === "starting" || project.status === "unhealthy";
+}
+/**
+* Stop the least recently used projects until `max` or fewer run.
+*
+* A project runs when `ddev list` shows it up, or when the agent is starting
+* it (status "starting" in the database). `ddev list` is the source because
+* `trafic deploy` runs `ddev start` itself, so the database does not know
+* about every running project. The projects the agent is starting count too,
+* because DDEV shows them stopped until their containers exist: the waiting
+* page marks its project "starting" before it calls this, so that project
+* counts once whether DDEV shows it up or not, and calls made for starts at
+* the same time each see all of them.
+*
+* Because of deploys, the database is first brought in line with DDEV. A
+* project that runs while the database says it is stopped (or has no record
+* of it) was started outside the agent, most likely by a deploy. Its last
+* access is then old or missing, which would make it the first one stopped,
+* right after the deploy. It is recorded as accessed now instead. A project
+* the database says runs while DDEV says it does not was stopped outside the
+* agent, and is recorded as stopped, so its next start is seen too.
+*
+* Projects with `idle_timeout: never`, projects the agent is starting and
+* projects `trafic-agent destroy` is removing (status "destroying": its
+* backup starts a stopped project) are never stopped. When no other project
+* can be stopped, a warning is logged and the server stays over the limit.
+*
+* Returns the names of the projects it stopped.
+*/
+async function enforceRunningLimit(options, deps = ddevRunningLimitDeps()) {
+	const { max, neverStops } = options;
+	if (max <= 0) return [];
+	const projects = await deps.listProjects();
+	for (const project of projects) {
+		const status = deps.getProject(project.name)?.status;
+		if (isUp(project) && (status === void 0 || status === "stopped")) {
+			deps.updateProjectAccess(project.name);
+			deps.setProjectStatus(project.name, "running");
+		} else if (!isUp(project) && status === "running") deps.setProjectStatus(project.name, "stopped");
+	}
+	const running = [];
+	for (const project of projects) {
+		const record = deps.getProject(project.name);
+		if (record && (isUp(project) || record.status === "starting")) running.push(record);
+	}
+	const excess = running.length - max;
+	if (excess <= 0) return [];
+	const candidates = running.filter((record) => !isBusy(record) && !neverStops(record.name)).sort((a, b) => a.lastAccess - b.lastAccess).slice(0, excess);
+	const stopped = [];
+	for (const record of candidates) {
+		console.log(`Stopping ${record.name}: ${running.length} projects running, max_running_projects is ${max}`);
+		if (await deps.stopProject(record.name)) {
+			deps.setProjectStatus(record.name, "stopped");
+			stopped.push(record.name);
+		} else console.error(`Failed to stop: ${record.name}`);
+	}
+	if (stopped.length < excess) console.warn(`${running.length - stopped.length} projects running, max_running_projects is ${max}: no other project can be stopped`);
+	return stopped;
+}
+//#endregion
 //#region src/setup/ddev.ts
 /**
 * Install system dependencies required by Trafic
@@ -1435,7 +1758,15 @@ async function handleErrors(req, res, deps) {
 		res.end(template.replace("{{message}}", "Project not found"));
 		return;
 	}
-	if (deps.getProject(projectName)?.status === "starting") {
+	deps.releaseStaleStatuses();
+	const record = deps.getProject(projectName);
+	if (record?.status === "destroying") {
+		const template = deps.loadTemplate("error");
+		res.writeHead(503, { "Content-Type": "text/html" });
+		res.end(template.replace("{{message}}", "This project is being removed"));
+		return;
+	}
+	if (record?.status === "starting") {
 		const template = deps.loadTemplate("wait");
 		res.writeHead(503, {
 			"Content-Type": "text/html",
@@ -1451,12 +1782,37 @@ async function handleErrors(req, res, deps) {
 		"Retry-After": "5"
 	});
 	res.end(template.replace(/\{\{project\}\}/g, projectName).replace(/\{\{hostname\}\}/g, hostname));
-	deps.startProject(projectName).then((success) => {
+	startWithinLimit(projectName, deps);
+}
+/**
+* Stop projects until the running ones fit under `max`, before the waiting
+* page starts a project. A project's own config says whether it never stops.
+*/
+function makeRoomUnderLimit(max, projectConfigs, run = nodeDdevRunner) {
+	return enforceRunningLimit({
+		max,
+		neverStops: (name) => shouldNeverStop(projectConfigs.get(name) ?? {})
+	}, ddevRunningLimitDeps(run));
+}
+/**
+* Make room under max_running_projects, then start the project.
+*
+* A failure to make room does not block the start: a page that never comes
+* up is worse than a server one project over its limit.
+*/
+async function startWithinLimit(projectName, deps) {
+	try {
+		await deps.makeRoom();
+	} catch (error) {
+		console.error(`Could not make room for ${projectName}:`, error);
+	}
+	try {
+		const success = await deps.startProject(projectName);
 		deps.setProjectStatus(projectName, success ? "running" : "stopped");
-	}).catch((error) => {
+	} catch (error) {
 		deps.setProjectStatus(projectName, "stopped");
 		console.error(`Could not start ${projectName}:`, error);
-	});
+	}
 }
 /**
 * Handle status polling requests
@@ -1506,7 +1862,7 @@ async function handleRequest(req, res, deps) {
 			res.writeHead(200, { "Content-Type": "application/json" });
 			res.end(JSON.stringify({
 				status: "ok",
-				version: "__VERSION__"
+				version: "0.1.51"
 			}));
 		} else await handleErrors(req, res, deps);
 	} catch (error) {
@@ -1532,11 +1888,29 @@ function reloadProjects() {
 	console.log(`Loaded ${projectList.size} projects (${projectConfigs.size} with custom config)`);
 }
 /**
+* Set back the busy statuses that a stopped service or a killed removal left
+* behind, when the service starts. No start or removal of this new process
+* is in flight yet, so its own id counts as gone too: after a reboot, the
+* new service can have the id of the old one.
+*/
+function releaseStatusesAtStart(isAlive = isProcessAlive) {
+	return releaseStaleStatuses((pid) => pid !== process.pid && isAlive(pid));
+}
+/**
+* Delete the records of the projects that DDEV no longer lists, when the
+* service starts. Its own id counts as gone, as in releaseStatusesAtStart.
+*/
+function deleteUnlistedAtStart(projectListPath, isAlive = isProcessAlive) {
+	return deleteUnlistedProjects(new Set(loadProjectList(projectListPath).keys()), (pid) => pid !== process.pid && isAlive(pid));
+}
+/**
 * Start the agent server
 */
 function startServer(agentConfig) {
 	config = agentConfig;
 	initDb(config.dbPath);
+	for (const name of deleteUnlistedAtStart(config.projectListPath)) console.log(`Removed ${name} from the agent database: DDEV no longer lists it`);
+	for (const name of releaseStatusesAtStart()) console.log(`Set ${name} back to running: the process that started or removed it is gone`);
 	syncForwardAuthAddress(config.projectListPath);
 	warnIfWildcardNotApplied(config);
 	reloadProjects();
@@ -1553,7 +1927,9 @@ function startServer(agentConfig) {
 		logAccess,
 		getProject,
 		setProjectStatus,
+		releaseStaleStatuses,
 		startProject,
+		makeRoom: () => makeRoomUnderLimit(config.maxRunningProjects, projectConfigs),
 		getProjectInfo
 	});
 	createServer((req, res) => {
@@ -1571,6 +1947,6 @@ function startServer(agentConfig) {
 	});
 }
 //#endregion
-export { stopProject as $, getIdleProjects as A, info as B, getIdleTimeoutMs as C, closeDb as D, cleanOldLogs as E, updateProjectAccess as F, success as G, resetSteps as H, nodeIo as I, getProjectFromHostname as J, warn as K, error as L, initDb as M, logAccess as N, getAccessLogs as O, setProjectStatus as P, startProject as Q, exec as R, readToolPorts as S, shouldNeverStop as T, setDryRun as U, isRoot as V, step as W, listProjects as X, getProjectInfo as Y, loadProjectList as Z, findRunningProjects as _, handleStatus as a, parseBearerToken as at, installSystemDeps as b, DDEV_SOURCES as c, validateConfig as ct, STATIC_CONFIG as d, watchProjectList as et, TLS_STORE_CONFIG as f, findRunningProject as g, configureTraefik as h, handleRequest as i, parseBasicAuth as it, getProject as j, getDb as k, DNS_RESOLVER as l, configureDdev as m, handleAuth as n, matchHostname as nt, routePath as o, loadConfig as ot, addDdevAptRepo as p, buildHostnameIndex as q, handleErrors as r, matchIp as rt, startServer as s, parseDuration as st, getEffectiveAuthConfig as t, checkAuth as tt, ROUTER_COMPOSE_OVERRIDE as u, getDockerGatewayIp as v, loadProjectConfig as w, purgeTraefikAcmeStorage as x, installDdev as y, execSilent as z };
+export { exec as $, shouldNeverStop as A, getScheduledDestroy as B, installSystemDeps as C, parseDuration as Ct, enforceRunningLimit as D, ddevRunningLimitDeps as E, isInside as Et, deleteUnlistedProjects as F, releaseStaleStatuses as G, initDb as H, getAccessLogs as I, startScheduledDestroy as J, scheduleDestroy as K, getDb as L, cleanOldLogs as M, closeDb as N, getIdleTimeoutMs as O, deleteProject as P, error as Q, getIdleProjects as R, installDdev as S, loadConfig as St, readToolPorts as T, validateConfig as Tt, isProcessAlive as U, getScheduledDestroys as V, logAccess as W, updateProjectAccess as X, systemClock as Y, nodeIo as Z, configureDdev as _, checkAuth as _t, handleRequest as a, step as at, findRunningProjects as b, parseBasicAuth as bt, releaseStatusesAtStart as c, buildHostnameIndex as ct, DDEV_SOURCES as d, listProjects as dt, execSilent as et, DNS_RESOLVER as f, loadProjectList as ft, addDdevAptRepo as g, watchProjectList as gt, TLS_STORE_CONFIG as h, stopProject as ht, handleErrors as i, setDryRun as it, cancelScheduledDestroy as j, loadProjectConfig as k, routePath as l, getProjectFromHostname as lt, STATIC_CONFIG as m, startProject as mt, getEffectiveAuthConfig as n, isRoot as nt, handleStatus as o, success as ot, ROUTER_COMPOSE_OVERRIDE as p, nodeDdevRunner as pt, setProjectStatus as q, handleAuth as r, resetSteps as rt, makeRoomUnderLimit as s, warn as st, deleteUnlistedAtStart as t, info as tt, startServer as u, getProjectInfo as ut, configureTraefik as v, matchHostname as vt, purgeTraefikAcmeStorage as w, parseTimeout as wt, getDockerGatewayIp as x, parseBearerToken as xt, findRunningProject as y, matchIp as yt, getProject as z };
 
-//# sourceMappingURL=server-DZAo8Clg.js.map
+//# sourceMappingURL=server-Cfk-O4DJ.js.map