kempo 4.8.0 → 4.9.0

package/server/utils/updates/prepareUpdate.js

@@ -0,0 +1,109 @@
+import { rmSync } from 'fs';
+import { join } from 'path';
+import { compareVersions } from './helpers.js';
+import { PACKAGE_ROOT, RUNNING_VERSION } from './runtime.js';
+import extensionTargets, { extensionPlan } from './extensionTargets.js';
+import fetchManifest from './fetchManifest.js';
+import getInstallInfo from './getInstallInfo.js';
+import { installedVersion, npmFailure, runNpm, targetsFor } from './npm.js';
+import { paths, stage, writeState } from './siteLayout.js';
+import { forcedFiles, planStarterFiles } from './starterFiles.js';
+import { pendingUpgrades } from '../setup/runUpgrades.js';
+
+let preparing = false;
+
+const NO_FILES = { add: [], update: [], current: [], edited: [], deleted: [], forced: [] };
+
+/*
+  What switching to a staged version would change besides the code, worked out from the staged copy: for kempo, the starter
+  files it would replace, add, keep (edited) or need confirmation for, and the data migrations it carries; for each updated
+  extension, what it adds and whether it is breaking. `breaking` is true when anything in it can only be undone from a backup.
+*/
+export const buildPlan = async ({ cwd, oldRoot, newRoot, extensions = [], executor }) => {
+  const files = newRoot ? planStarterFiles({ cwd, oldRoot, newRoot, forced: forcedFiles(newRoot) }) : NO_FILES;
+  const upgrades = newRoot ? await pendingUpgrades({ dir: join(newRoot, 'server', 'db', 'upgrades'), ...(executor ? { executor } : {}) }) : [];
+  return { files, upgrades, extensions, breaking: upgrades.some(upgrade => upgrade.breaking) || extensions.some(extension => extension.breaking) };
+};
+
+/*
+  Installs newer versions of kempo and/or of installed extensions into a staging folder next to the running site, which keeps
+  running from its own files, and works out what switching to them would change. Nothing the site uses is touched, so it
+  can fail, or be abandoned, without a trace. Applying it is applyStagedUpdate. Only for a site that runs from a Docker
+  volume (KEMPO_SITE_ROOT); any other site is updated in place, by applyUpdate.
+
+  `version` is the kempo version, "latest" by default, or null to leave kempo as it is; `extensions` are names. All are
+  installed in one npm run so they are consistent with each other. Resolves to [error, { version, packages, plan }]. `run`,
+  `install`, `oldRoot`, `fetcher` and `executor` are for tests.
+*/
+export default async ({ version = 'latest', extensions = [], siteRoot = process.env.KEMPO_SITE_ROOT, cwd = process.cwd(), oldRoot = PACKAGE_ROOT, install = getInstallInfo({ cwd }), run = runNpm, fetcher, executor } = {}) => {
+  if(!siteRoot){
+    return [{ code: 409, msg: 'This site is not set up to prepare an update ahead of switching to it' }, null];
+  }
+  if(!install.canUpdate){
+    return [{ code: 409, msg: install.reason }, null];
+  }
+  if(version === null && !extensions.length){
+    return [{ code: 400, msg: 'Choose something to update' }, null];
+  }
+  if(preparing){
+    return [{ code: 409, msg: 'An update is already being prepared' }, null];
+  }
+
+  preparing = true;
+  try {
+    let manifest = null;
+    if(version !== null){
+      const [manifestError, found] = await fetchManifest({ version, fetcher });
+      if(manifestError) return [manifestError, null];
+      if(compareVersions(found.version, RUNNING_VERSION) <= 0){
+        return [{ code: 409, msg: `Version ${RUNNING_VERSION} is already running` }, null];
+      }
+      manifest = found;
+    }
+
+    const kempoVersion = manifest?.version ?? RUNNING_VERSION;
+    const [targetsError, wanted] = await extensionTargets({ names: extensions, kempoVersion, root: cwd, fetcher, ...(executor ? { executor } : {}) });
+    if(targetsError) return [targetsError, null];
+
+    let staged;
+    try {
+      staged = stage({ siteRoot, version: kempoVersion });
+    } catch(error) {
+      return [{ code: 500, msg: `Space for the new version could not be made: ${error.message}` }, null];
+    }
+
+    const discard = () => rmSync(paths(siteRoot).staging, { recursive: true, force: true });
+    const targets = [...(manifest ? targetsFor({ manifest, dir: staged.dir }) : []), ...wanted.targets];
+    try {
+      await run(targets, staged.dir);
+    } catch(error) {
+      discard();
+      return [{ code: 500, msg: `The update could not be downloaded: ${npmFailure(error)}` }, null];
+    }
+
+    const missing = [...(manifest ? [{ name: 'kempo', to: manifest.version }] : []), ...wanted.items].find(item => installedVersion(staged.dir, item.name) !== item.to);
+    if(missing){
+      discard();
+      return [{ code: 500, msg: `npm finished but ${missing.name} ${missing.to} is not in the staged copy` }, null];
+    }
+
+    const [planError, extensionEntries] = await extensionPlan({ dir: staged.dir, items: wanted.items, executor });
+    if(planError){
+      discard();
+      return [planError, null];
+    }
+
+    let plan;
+    try {
+      plan = await buildPlan({ cwd, oldRoot, newRoot: manifest ? join(staged.dir, 'node_modules', 'kempo') : null, extensions: extensionEntries, executor });
+    } catch(error) {
+      discard();
+      return [{ code: 500, msg: `What this update would change could not be worked out: ${error.message}` }, null];
+    }
+
+    writeState(siteRoot, 'staged', { id: staged.id, dir: staged.dir, version: kempoVersion, kempoUpdate: Boolean(manifest), from: RUNNING_VERSION, packages: targets, plan, preparedAt: new Date().toISOString() });
+    return [null, { version: kempoVersion, kempoUpdate: Boolean(manifest), packages: targets, plan }];
+  } finally {
+    preparing = false;
+  }
+};

package/server/utils/updates/releaseStatus.js

@@ -0,0 +1,23 @@
+import { existsSync } from 'fs';
+import { join } from 'path';
+import { getCurrent, paths, readState } from './siteLayout.js';
+
+/*
+  The part of the update status that only a site running from a Docker volume has: whether a version is staged and what
+  switching to it would change, whether there is an earlier version to go back to, and how the last switch ended.
+*/
+export default (siteRoot = process.env.KEMPO_SITE_ROOT) => {
+  if(!siteRoot) return { mode: 'in-place' };
+
+  const staged = readState(siteRoot, 'staged');
+  const { previous } = getCurrent(siteRoot);
+  const last = readState(siteRoot, 'last');
+  return {
+    mode: 'release',
+    staged: staged && existsSync(join(staged.dir, 'node_modules')) ? { version: staged.version, kempoUpdate: Boolean(staged.kempoUpdate), packages: staged.packages ?? [], plan: staged.plan, preparedAt: staged.preparedAt } : null,
+    canGoBack: Boolean(previous?.id && existsSync(join(paths(siteRoot).releases, previous.id, 'node_modules'))),
+    previousVersion: previous?.version ?? null,
+    switching: readState(siteRoot, 'switch')?.status ?? null,
+    lastUpdate: last ? { ok: last.ok, from: last.from, to: last.to, packages: last.packages ?? [], reason: last.reason ?? null, breaking: Boolean(last.breaking), at: last.at } : null
+  };
+};

package/server/utils/updates/restart.js

@@ -0,0 +1,64 @@
+import { spawn } from 'child_process';
+import { existsSync } from 'fs';
+
+/*
+  How the server comes back after an update.
+
+    respawn  this process starts its own replacement and exits. For `npm start` or `npm run dev` run by hand.
+    exit     this process only exits and whatever started it starts it again (pm2, systemd, a container's
+             restart policy). Respawning there would fight the supervisor for the port, and a container
+             stops when its main process does.
+
+  Chosen by the KEMPO_RESTART environment variable ("respawn" or "exit"), otherwise "exit" when pm2, systemd,
+  Kubernetes or Docker is detected and "respawn" when none is.
+*/
+export const restartMode = (env = process.env, inContainer = () => existsSync('/.dockerenv')) => {
+  if(env.KEMPO_RESTART === 'exit' || env.KEMPO_RESTART === 'respawn') return env.KEMPO_RESTART;
+  return env.pm_id !== undefined || env.INVOCATION_ID || env.KUBERNETES_SERVICE_HOST || inContainer() ? 'exit' : 'respawn';
+};
+
+/*
+  The replacement cannot be started by the process it replaces: the port is still taken. A small detached
+  watcher waits for that process to be gone and then starts the same command in the same directory with
+  the same environment. It gives up waiting after a minute and starts anyway.
+*/
+const WATCHER = `
+const { spawn } = require('child_process');
+const pid = Number(process.argv[1]);
+const command = JSON.parse(process.argv[2]);
+const started = Date.now();
+const alive = () => {
+  try { process.kill(pid, 0); return true; } catch(error) { return error.code === 'EPERM'; }
+};
+const wait = () => {
+  if(alive() && Date.now() - started < 60000) return setTimeout(wait, 200);
+  setTimeout(() => {
+    spawn(command[0], command.slice(1), { detached: true, stdio: 'ignore', cwd: process.cwd(), env: process.env, windowsHide: true }).unref();
+    process.exit(0);
+  }, 500);
+};
+wait();
+`;
+
+/*
+  Under a supervisor the exit code is nonzero on purpose (75, "try again later"): a Docker `restart: on-failure` or a
+  systemd `Restart=on-failure` policy restarts only after a failure, while `always` and `unless-stopped` restart after any exit.
+*/
+export const RESTART_EXIT_CODE = 75;
+
+export const restartServer = ({ mode = restartMode(), exit = code => process.exit(code), spawner = spawn } = {}) => {
+  if(mode === 'respawn'){
+    const command = [process.execPath, ...process.execArgv, ...process.argv.slice(1)];
+    spawner(process.execPath, ['-e', WATCHER, String(process.pid), JSON.stringify(command)], {
+      detached: true,
+      stdio: 'ignore',
+      cwd: process.cwd(),
+      env: process.env,
+      windowsHide: true
+    }).unref();
+  }
+  exit(mode === 'respawn' ? 0 : RESTART_EXIT_CODE);
+};
+
+/* Restarts after `delayMs`, so the response that announced it has been sent first. */
+export const scheduleRestart = ({ delayMs = 1000, ...options } = {}) => setTimeout(() => restartServer(options), delayMs);

package/server/utils/updates/rollbackUpdate.js

@@ -0,0 +1,56 @@
+import { existsSync } from 'fs';
+import { join } from 'path';
+import { RUNNING_VERSION } from './runtime.js';
+import { restartMode, scheduleRestart } from './restart.js';
+import { getCurrent, paths, readState, requestSwitch } from './siteLayout.js';
+
+let going = false;
+
+/*
+  Puts a site that runs from a Docker volume back on the version it had before the last update: the code, and the
+  starter files the update replaced. If that update changed data, going back also restores the database from the backup
+  taken before it, which loses whatever was entered since, so that needs `confirm.data`. Otherwise (the usual case: an
+  update only added to the database) the earlier version runs on the database as it is and nothing is lost.
+
+  Resolves to [error, { from, to, restart, restoresData }].
+*/
+export default async ({ confirm = {}, siteRoot = process.env.KEMPO_SITE_ROOT, restart = scheduleRestart } = {}) => {
+  if(!siteRoot){
+    return [{ code: 409, msg: 'This site is not set up to go back to an earlier version' }, null];
+  }
+  if(going){
+    return [{ code: 409, msg: 'A switch is already under way' }, null];
+  }
+  if(readState(siteRoot, 'switch')){
+    return [{ code: 409, msg: 'The last switch has not finished proving itself yet' }, null];
+  }
+
+  const { previous, version } = getCurrent(siteRoot);
+  const source = previous?.id ? join(paths(siteRoot).releases, previous.id) : null;
+  if(!source || !existsSync(join(source, 'node_modules'))){
+    return [{ code: 409, msg: 'There is no earlier version to go back to' }, null];
+  }
+
+  const last = readState(siteRoot, 'last');
+  const restoresData = Boolean(last?.ok && last.breaking && last.to === version && last.dbBackup && existsSync(last.dbBackup));
+  if(restoresData && !confirm.data){
+    return [{ code: 409, msg: 'Going back restores the database from before the update and loses what was entered since', details: { confirm: 'data' } }, null];
+  }
+
+  going = true;
+  try {
+    requestSwitch({
+      siteRoot,
+      id: previous.id,
+      version: previous.version,
+      source,
+      kind: 'rollback',
+      filesBackup: last?.ok && last.to === version ? last.filesBackup : null,
+      dbBackup: restoresData ? last.dbBackup : null
+    });
+    restart();
+    return [null, { from: RUNNING_VERSION, to: previous.version, restart: restartMode(), restoresData }];
+  } finally {
+    going = false;
+  }
+};

package/server/utils/updates/runUpdateCheck.js

@@ -0,0 +1,52 @@
+import { createHash } from 'crypto';
+import { and, eq } from 'drizzle-orm';
+import db from '../../db/index.js';
+import { notification } from '../../db/schema.js';
+import getSetting from '../settings/getSetting.js';
+import createNotification from '../notifications/createNotification.js';
+import getUpdateStatus from './getUpdateStatus.js';
+import { PACKAGE } from './helpers.js';
+
+/*
+  Raising a notification again would mark it unread for everyone, so each one has a key that names what it is about, and
+  one that already exists is left alone: one for each new version of kempo, and one for each distinct set of extension
+  updates that are waiting.
+*/
+const raiseOnce = async ({ dedupeKey, ...notice }) => {
+  const [existing] = await db.select({ id: notification.id }).from(notification).where(and(eq(notification.owner, PACKAGE), eq(notification.dedupeKey, dedupeKey))).limit(1);
+  if(existing) return;
+  const [error] = await createNotification({ owner: PACKAGE, permission: 'system:updates:manage', dedupeKey, actions: [{ label: 'View updates', href: '/admin/updates' }], ...notice });
+  if(error) console.error(`[kempo] Update notification: ${error.msg}`);
+};
+
+/*
+  The scheduled check (kempo's `update-check` job). Asks npm for the newest version of kempo and of every installed extension
+  and tells everyone who can install them what is waiting, once for each new version of kempo and once for each distinct set
+  of extension updates. Nothing is said about a site that cannot install updates (a linked checkout). The
+  `system:update_check_enabled` setting turns this off.
+*/
+export default async ({ fetcher, install } = {}) => {
+  const [, enabled] = await getSetting('system', 'update_check_enabled', true);
+  if(enabled === false) return;
+
+  const [, status] = await getUpdateStatus({ force: true, fetcher, install });
+  if(!status?.canUpdate) return;
+
+  if(status.updateAvailable){
+    await raiseOnce({
+      dedupeKey: `update:${status.latestVersion}`,
+      title: `Kempo ${status.latestVersion} is available`,
+      message: `This site runs ${status.runningVersion}.${status.major ? ' This is a major release and may include breaking changes.' : ''}`
+    });
+  }
+
+  const { updates } = status.extensionUpdates;
+  if(updates.length){
+    const list = updates.map(update => `${update.name}@${update.latestVersion}`);
+    await raiseOnce({
+      dedupeKey: `extension-updates:${createHash('sha256').update(list.join('\n')).digest('hex').slice(0, 16)}`,
+      title: updates.length === 1 ? `${updates[0].name} ${updates[0].latestVersion} is available` : `${updates.length} extension updates are available`,
+      message: `${updates.slice(0, 5).map(update => `${update.name} ${update.installedVersion} -> ${update.latestVersion}${update.major ? ' (major)' : ''}`).join(', ')}${updates.length > 5 ? ', and more' : ''}.`
+    });
+  }
+};

package/server/utils/updates/runtime.js

@@ -0,0 +1,15 @@
+import { readFileSync } from 'fs';
+import { randomUUID } from 'crypto';
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'url';
+
+/*
+  What this process is running, read once when it starts. After an update the files on disk are the new
+  version while this process is still the old one, so these are what tell the two apart: the running
+  version, and a boot id that is different in every process.
+*/
+export const PACKAGE_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..', '..');
+
+export const RUNNING_VERSION = JSON.parse(readFileSync(join(PACKAGE_ROOT, 'package.json'), 'utf8')).version;
+
+export const BOOT_ID = randomUUID();

package/server/utils/updates/siteLayout.js

@@ -0,0 +1,200 @@
+import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from 'fs';
+import { dirname, join } from 'path';
+
+/*
+  The folders of a site that runs from a Docker volume (KEMPO_SITE_ROOT, normally /site), and the steps that move between
+  versions of kempo. It uses nothing but Node's own modules, because the image's startup script runs it before any of the
+  site's code can be trusted, and so a copy of it that is older than the site's still understands the state it left.
+
+    app/        the site. Its files are the owner's (public, templates, uploads, .env) and are never moved. Its
+                node_modules and package.json are "the code": the only part a version change replaces
+    staging/    a new version being installed, with its own node_modules and copy of the manifests, while the site
+                keeps running from app/
+    releases/   the code the site had before the last switch, kept so it can go back
+    state/      what the startup script and the site tell each other (see below)
+
+  A switch is two renames inside one volume, done by the startup script while no site process is running, so the site never
+  loads files from two versions at once. Each step can be repeated, so a switch that was cut short finishes at the next start.
+
+  state/switch.json says where a switch is: "requested" (the site asked, nothing moved yet), "switching" (moving),
+  "switched" (the new code is in place and has not yet proved itself) and is removed once the site has stayed up for a
+  minute. A site that stops while it is "switched" more than MAX_ATTEMPTS times, or says it failed, is put back.
+*/
+export const MAX_ATTEMPTS = 3;
+
+const MANIFESTS = ['package.json', 'package-lock.json'];
+
+export const paths = siteRoot => ({
+  siteRoot,
+  app: join(siteRoot, 'app'),
+  staging: join(siteRoot, 'staging'),
+  releases: join(siteRoot, 'releases'),
+  state: join(siteRoot, 'state'),
+});
+
+/*
+  State files
+*/
+const readJson = file => {
+  try {
+    return JSON.parse(readFileSync(file, 'utf8'));
+  } catch {
+    return null;
+  }
+};
+
+const writeJson = (file, data) => {
+  mkdirSync(dirname(file), { recursive: true });
+  writeFileSync(`${file}.tmp`, JSON.stringify(data, null, 2));
+  renameSync(`${file}.tmp`, file);
+};
+
+export const readState = (siteRoot, name) => readJson(join(paths(siteRoot).state, `${name}.json`));
+
+export const writeState = (siteRoot, name, data) => writeJson(join(paths(siteRoot).state, `${name}.json`), data);
+
+export const clearState = (siteRoot, name) => rmSync(join(paths(siteRoot).state, `${name}.json`), { force: true });
+
+const versionIn = dir => readJson(join(dir, 'node_modules', 'kempo', 'package.json'))?.version ?? null;
+
+const releaseId = version => `${Date.now()}-${version}`;
+
+const copyManifests = (from, to) => {
+  for(const name of MANIFESTS){
+    if(existsSync(join(from, name))) cpSync(join(from, name), join(to, name));
+  }
+};
+
+/*
+  The first start: the image holds the site as it was built (its files and its node_modules) and the volume is empty,
+  so the volume is filled from it. After that the volume is the site and the image is only the runtime.
+*/
+export const seedSite = ({ siteRoot, seed }) => {
+  const { app } = paths(siteRoot);
+  if(existsSync(join(app, 'package.json'))) return false;
+  mkdirSync(app, { recursive: true });
+  cpSync(seed, app, { recursive: true, verbatimSymlinks: true });
+  const version = versionIn(app);
+  writeState(siteRoot, 'current', { id: releaseId(version ?? 'seed'), version, previous: null });
+  return true;
+};
+
+export const getCurrent = siteRoot => readState(siteRoot, 'current') ?? { id: null, version: versionIn(paths(siteRoot).app), previous: null };
+
+/*
+  Staging a new version: a copy of the code the site has now, which the caller then installs into. The running site is
+  not touched. Only one is kept: staging again replaces the one before.
+*/
+export const stage = ({ siteRoot, version }) => {
+  const { app, staging } = paths(siteRoot);
+  rmSync(staging, { recursive: true, force: true });
+  const id = releaseId(version);
+  const dir = join(staging, id);
+  mkdirSync(dir, { recursive: true });
+  copyManifests(app, dir);
+  cpSync(join(app, 'node_modules'), join(dir, 'node_modules'), { recursive: true, verbatimSymlinks: true });
+  return { id, dir };
+};
+
+/*
+  Asking for a switch. `source` is a folder holding node_modules and the manifests: a staged version, or an earlier
+  release to go back to. The startup script does it at the next start.
+*/
+export const requestSwitch = ({ siteRoot, id, version, source, kind = 'update', ...details }) => {
+  const current = getCurrent(siteRoot);
+  writeState(siteRoot, 'switch', { status: 'requested', kind, id, version, source, from: { id: current.id, version: current.version }, attempts: 0, requestedAt: new Date().toISOString(), ...details });
+};
+
+const swapIn = ({ siteRoot, sw }) => {
+  const { app, releases } = paths(siteRoot);
+  const old = join(releases, sw.from.id ?? `${Date.now()}-unknown`);
+  const oldModules = join(app, 'node_modules');
+
+  if(existsSync(oldModules) && !existsSync(join(old, 'node_modules'))){
+    mkdirSync(old, { recursive: true });
+    renameSync(oldModules, join(old, 'node_modules'));
+    copyManifests(app, old);
+  }
+  if(existsSync(join(sw.source, 'node_modules'))){
+    renameSync(join(sw.source, 'node_modules'), oldModules);
+    copyManifests(sw.source, app);
+  }
+  rmSync(sw.source, { recursive: true, force: true });
+  rmSync(paths(siteRoot).staging, { recursive: true, force: true });
+  writeState(siteRoot, 'current', { id: sw.id, version: sw.version, previous: { id: sw.from.id, version: sw.from.version } });
+};
+
+/* Carries out a requested (or half-done) switch. Returns the switch, or null when there was nothing to do. */
+export const performSwitch = siteRoot => {
+  const sw = readState(siteRoot, 'switch');
+  if(!sw || !['requested', 'switching'].includes(sw.status)) return null;
+  writeState(siteRoot, 'switch', { ...sw, status: 'switching' });
+  swapIn({ siteRoot, sw });
+  const done = { ...sw, status: 'switched', attempts: 0 };
+  writeState(siteRoot, 'switch', done);
+  return done;
+};
+
+/*
+  Files the update changed outside the code (starter files), saved before they were overwritten, put back: those that
+  existed are restored and those the update created are removed.
+*/
+export const restoreFiles = ({ siteRoot, backup }) => {
+  if(!backup?.dir) return;
+  const { app } = paths(siteRoot);
+  for(const { path, existed } of backup.files ?? []){
+    if(existed) cpSync(join(backup.dir, path), join(app, path), { force: true });
+    else rmSync(join(app, path), { force: true });
+  }
+};
+
+/*
+  Back to the release before this one. Used by the startup script when a new version keeps failing, and for going back by
+  hand. Records why in state/last.json.
+*/
+export const rollBack = ({ siteRoot, reason }) => {
+  const sw = readState(siteRoot, 'switch');
+  const current = getCurrent(siteRoot);
+  const previous = current.previous;
+  const source = previous?.id ? join(paths(siteRoot).releases, previous.id) : null;
+  if(!source || !existsSync(join(source, 'node_modules'))) return { ok: false, reason: 'There is no earlier version to go back to' };
+
+  swapIn({ siteRoot, sw: { id: previous.id, version: previous.version, source, from: { id: current.id, version: current.version } } });
+  restoreFiles({ siteRoot, backup: sw?.filesBackup });
+  writeState(siteRoot, 'last', { ok: false, from: current.version, to: previous.version, reason, at: new Date().toISOString(), dbBackup: sw?.dbBackup ?? null });
+  clearState(siteRoot, 'switch');
+  return { ok: true, from: current.version, to: previous.version, dbBackup: sw?.dbBackup ?? null };
+};
+
+/* True when a switch has not proved itself and the site has failed to stay up. Going back by hand is never undone by itself. */
+export const shouldRollBack = sw => sw?.status === 'switched' && sw.kind !== 'rollback' && (Boolean(sw.failed) || sw.attempts > MAX_ATTEMPTS);
+
+export const countAttempt = siteRoot => {
+  const sw = readState(siteRoot, 'switch');
+  if(sw?.status !== 'switched') return sw;
+  const next = { ...sw, attempts: sw.attempts + 1 };
+  writeState(siteRoot, 'switch', next);
+  return next;
+};
+
+/* The site says its new code cannot work (a data migration failed). It is put back at the next start. */
+export const flagFailed = ({ siteRoot, reason }) => {
+  const sw = readState(siteRoot, 'switch');
+  if(sw?.status === 'switched') writeState(siteRoot, 'switch', { ...sw, failed: true, failure: reason });
+};
+
+/* The new code has stayed up: the switch is final, and every release but the one before this is deleted. */
+export const markHealthy = siteRoot => {
+  const sw = readState(siteRoot, 'switch');
+  if(sw?.status !== 'switched') return false;
+  const { releases } = paths(siteRoot);
+  const current = getCurrent(siteRoot);
+  writeState(siteRoot, 'last', { ok: true, from: sw.from.version, to: sw.version, kind: sw.kind, breaking: Boolean(sw.breaking), packages: sw.packages ?? [], dbBackup: sw.dbBackup ?? null, filesBackup: sw.filesBackup ?? null, at: new Date().toISOString() });
+  clearState(siteRoot, 'switch');
+  if(existsSync(releases)){
+    for(const name of readdirSync(releases)){
+      if(name !== current.previous?.id) rmSync(join(releases, name), { recursive: true, force: true });
+    }
+  }
+  return true;
+};

package/server/utils/updates/starterFiles.js

@@ -0,0 +1,118 @@
+import { createHash } from 'crypto';
+import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'fs';
+import { dirname, join, relative, sep } from 'path';
+
+/*
+  The starter files: what `npx kempo init` copies from the kempo package into a site (app-public/ into public/, and
+  app-templates/ into templates/). They start as kempo's and become the owner's the moment anyone edits one.
+
+  Kempo remembers the fingerprint (SHA-256) of each starter file it put in the site, in .kempo/shipped.json. An update then
+  treats a file in the site like this:
+
+    identical to the new version's       nothing to do
+    still what kempo put there           never edited: replaced by the new version
+    anything else                        edited, by anyone (the admin, FTP, a shell): kept, and offered as "newer" only
+    missing, but kempo shipped it before the owner deleted it: it stays deleted
+    missing and new in this version      added
+
+  A site made before this record existed has none, so a file is judged against what the version that is running ships.
+  A version can name files it must replace even when they were edited (a security fix) in its package.json:
+  { "kempo": { "forceUpdate": ["public/login/", "public/init.js"] } }. Those wait for the owner's confirmation, and the
+  file as it was is saved first.
+*/
+export const SOURCES = [['app-public', 'public'], ['app-templates', 'templates']];
+
+const RECORD = ['.kempo', 'shipped.json'];
+
+const hashOf = file => createHash('sha256').update(readFileSync(file)).digest('hex');
+
+const walk = dir => readdirSync(dir, { withFileTypes: true }).flatMap(entry => (entry.isDirectory() ? walk(join(dir, entry.name)) : [join(dir, entry.name)]));
+
+/* [{ path, source, hash }] for every starter file a kempo package ships, `path` being where it goes in the site. */
+export const listShipped = packageRoot => SOURCES.flatMap(([from, to]) => {
+  const base = join(packageRoot, from);
+  if(!existsSync(base)) return [];
+  return walk(base).map(file => ({ path: `${to}/${relative(base, file).split(sep).join('/')}`, source: file, hash: hashOf(file) }));
+});
+
+export const readRecord = cwd => {
+  try {
+    return JSON.parse(readFileSync(join(cwd, ...RECORD), 'utf8'));
+  } catch {
+    return { version: null, files: {} };
+  }
+};
+
+const writeRecord = (cwd, record) => {
+  mkdirSync(join(cwd, '.kempo'), { recursive: true });
+  writeFileSync(join(cwd, ...RECORD), JSON.stringify(record, null, 2));
+};
+
+/* The files a version must replace, from its package.json. */
+export const forcedFiles = packageRoot => {
+  try {
+    return JSON.parse(readFileSync(join(packageRoot, 'package.json'), 'utf8')).kempo?.forceUpdate ?? [];
+  } catch {
+    return [];
+  }
+};
+
+/* For `npx kempo init`: remembers what was copied. */
+export const recordShipped = ({ cwd, packageRoot, version }) => {
+  const files = {};
+  for(const file of listShipped(packageRoot)){
+    if(existsSync(join(cwd, file.path)) && hashOf(join(cwd, file.path)) === file.hash) files[file.path] = file.hash;
+  }
+  writeRecord(cwd, { version, files });
+};
+
+/* What an update would do to the site's starter files. Changes nothing. */
+export const planStarterFiles = ({ cwd, oldRoot, newRoot, forced = forcedFiles(newRoot) }) => {
+  const recorded = readRecord(cwd).files;
+  const before = new Map(listShipped(oldRoot).map(file => [file.path, file.hash]));
+  const plan = { add: [], update: [], current: [], edited: [], deleted: [], forced: [] };
+
+  for(const file of listShipped(newRoot)){
+    const target = join(cwd, file.path);
+    if(!existsSync(target)){
+      (recorded[file.path] || before.has(file.path) ? plan.deleted : plan.add).push(file.path);
+    } else if(before.get(file.path) === file.hash || hashOf(target) === file.hash){
+      plan.current.push(file.path);
+    } else {
+      const shipped = recorded[file.path] ?? before.get(file.path);
+      if(shipped && hashOf(target) === shipped) plan.update.push(file.path);
+      else if(forced.some(entry => entry === file.path || (entry.endsWith('/') && file.path.startsWith(entry)))) plan.forced.push(file.path);
+      else plan.edited.push(file.path);
+    }
+  }
+  return plan;
+};
+
+/*
+  Does it. Every file it replaces is copied to `backupDir` first. Resolves to { dir, files: [{ path, existed }] }, which is
+  what putting them back needs (restoreFiles in siteLayout.js).
+*/
+export const applyStarterFiles = ({ cwd, newRoot, version, plan, includeForced = false, backupDir }) => {
+  const shipped = new Map(listShipped(newRoot).map(file => [file.path, file]));
+  const changed = [...plan.add, ...plan.update, ...(includeForced ? plan.forced : [])];
+  const files = [];
+
+  for(const path of changed){
+    const target = join(cwd, path);
+    const existed = existsSync(target);
+    if(existed){
+      mkdirSync(dirname(join(backupDir, path)), { recursive: true });
+      cpSync(target, join(backupDir, path));
+    }
+    mkdirSync(dirname(target), { recursive: true });
+    cpSync(shipped.get(path).source, target);
+    files.push({ path, existed });
+  }
+
+  const record = readRecord(cwd);
+  for(const path of [...changed, ...plan.current]){
+    if(existsSync(join(cwd, path)) && hashOf(join(cwd, path)) === shipped.get(path).hash) record.files[path] = shipped.get(path).hash;
+  }
+  writeRecord(cwd, { version, files: record.files });
+  return { dir: backupDir, files };
+};