kempo 4.8.0 → 4.9.0

package/server/utils/extensions/updateExtension.js

@@ -1,29 +1,29 @@
 import db from '../../db/index.js';
-import { extension, hook, permission, group, groupPermission } from '../../db/schema.js';
-import { eq, and } from 'drizzle-orm';
+import { extension } from '../../db/schema.js';
+import { eq } from 'drizzle-orm';
 import { createRequire } from 'module';
 import { dirname, join } from 'path';
-import { randomUUID } from 'crypto';
 import { execSync } from 'child_process';
-import triggerHook, { clearHandlerCache } from '../hooks/triggerHook.js';
-import createHook from '../hooks/createHook.js';
-import registerDashboardWidget from '../dashboard/registerDashboardWidget.js';
-import unregisterDashboardWidget from '../dashboard/unregisterDashboardWidget.js';
-import registerJobs from '../scheduler/registerJobs.js';
-import setSetting from '../settings/setSetting.js';
-import declaredValue from '../settings/declaredValue.js';
-import createTablesFromSchema from './createTablesFromSchema.js';
 import readKempoConfig from './readKempoConfig.js';
+import upgradeExtension from './upgradeExtension.js';
 import { normalizeAliases } from './aliases.js';
 import { normalizeScopes, scopeNames } from './scopes.js';
 import checkClaims from './checkClaims.js';
 
 const requireFromProject = createRequire(join(process.cwd(), 'package.json'));
 
+/*
+  Installs a newer version of one extension in place, and brings the database in step with it straight away, without a
+  restart (see upgradeExtension). For a site that runs from a Docker volume updates go through the Updates screen instead,
+  which stages them, shows what they change and can go back; this refuses there so it cannot bypass that.
+*/
 export default async ({ name, version = 'latest' }) => {
   if(!name){
     return [{ code: 400, msg: 'Extension name is required' }, null];
   }
+  if(process.env.KEMPO_SITE_ROOT){
+    return [{ code: 409, msg: 'This site updates extensions from the Updates screen' }, null];
+  }
 
   let oldRecord;
   try {
@@ -35,27 +35,18 @@ export default async ({ name, version = 'latest' }) => {
     return [{ code: 500, msg: 'Failed to find extension' }, null];
   }
 
-  const oldKempo = oldRecord.kempo || {};
   const oldVersion = oldRecord.version;
 
-  /*
-    Install updated package
-  */
   try {
     execSync(`npm install ${name}@${version}`, { cwd: process.cwd(), stdio: 'pipe' });
   } catch(error){
     return [{ code: 500, msg: `Failed to update package: ${error.message}` }, null];
   }
 
-  /*
-    What the new version declares comes from its kempo-config.json (package.json's own `kempo` field is
-    only the fallback for an older extension). Reading package.json alone meant that no extension's new
-    permissions, settings, groups, hooks or widgets ever reached the database through this update.
-  */
-  let extRoot, pkg, newKempo;
+  let extRoot, newKempo;
   try {
     extRoot = dirname(requireFromProject.resolve(`${name}/package.json`));
-    ({ pkg, kempo: newKempo } = await readKempoConfig(extRoot));
+    ({ kempo: newKempo } = await readKempoConfig(extRoot));
   } catch {
     return [{ code: 404, msg: `Updated package "${name}" not found` }, null];
   }
@@ -81,135 +72,5 @@ export default async ({ name, version = 'latest' }) => {
     return [{ ...refusal, msg }, null];
   }
 
-  /*
-    Create any new tables from schema
-  */
-  if(newKempo.schema){
-    try {
-      const schemaModule = await import(`${join(extRoot, newKempo.schema)}?t=${Date.now()}`);
-      const [err] = await createTablesFromSchema(schemaModule);
-      if(err) return [err, null];
-    } catch(error){
-      return [{ code: 500, msg: `Failed to load extension schema: ${error.message}` }, null];
-    }
-  }
-
-  /*
-    Run update script with diff context
-  */
-  try {
-    const updateModule = await import(`${join(extRoot, 'update.js')}?t=${Date.now()}`).catch(() => null);
-    if(updateModule?.default){
-      await updateModule.default({ oldVersion, newVersion: pkg.version, oldKempo, newKempo });
-    }
-  } catch(error){
-    return [{ code: 500, msg: `Extension update script failed: ${error.message}` }, null];
-  }
-
-  /*
-    Add new permissions (never delete — user data may depend on them)
-  */
-  const oldPermNames = new Set((oldKempo.permissions || []).map(p => p.name));
-  for(const { name: permName, description } of newKempo.permissions || []){
-    if(!oldPermNames.has(permName)){
-      await db.insert(permission).values({
-        name: permName,
-        description: description || null,
-        owner: name,
-        createdAt: new Date(),
-      }).onConflictDoNothing();
-    }
-  }
-
-  /*
-    Add new settings only (never overwrite — user may have changed the value)
-  */
-  const oldSettingNames = new Set((oldKempo.settings || []).map(s => s.name));
-  for(const { name: settingName, value, type, description } of newKempo.settings || []){
-    if(!oldSettingNames.has(settingName)){
-      await setSetting(name, settingName, declaredValue(value, type), type || null, false, description || null);
-    }
-  }
-
-  /*
-    Add new groups
-  */
-  const oldGroupNames = new Set((oldKempo.groups || []).map(g => g.name));
-  for(const { name: groupName, description, permissions: groupPerms = [] } of newKempo.groups || []){
-    if(!oldGroupNames.has(groupName)){
-      await db.insert(group).values({
-        name: groupName,
-        description: description || null,
-        owner: name,
-        createdAt: new Date(),
-      }).onConflictDoNothing();
-      for(const permName of groupPerms){
-        await db.insert(groupPermission).values({
-          id: randomUUID(),
-          groupName,
-          permissionName: permName,
-          createdAt: new Date(),
-        }).onConflictDoNothing();
-      }
-    }
-  }
-
-  /*
-    Sync hooks: add new, remove deleted
-  */
-  const oldHooks = oldKempo.hooks || {};
-  const newHooks = newKempo.hooks || {};
-  for(const [event, callback] of Object.entries(newHooks)){
-    if(!oldHooks[event]){
-      await createHook({ owner: name, event, callback });
-    }
-  }
-  for(const event of Object.keys(oldHooks)){
-    if(!newHooks[event]){
-      await db.delete(hook).where(and(eq(hook.owner, name), eq(hook.event, event))).catch(() => {});
-    }
-  }
-
-  /*
-    Sync dashboard widgets: register or refresh every declared one, remove those no longer declared
-  */
-  const newWidgetKeys = new Set((newKempo.widgets || []).map(w => w.key));
-  for(const widget of newKempo.widgets || []){
-    const [widgetError] = await registerDashboardWidget({ ...widget, owner: name });
-    if(widgetError){
-      console.warn(`[${name}] dashboard widget "${widget.key}" was not registered: ${widgetError.msg}`);
-    }
-  }
-  for(const { key } of oldKempo.widgets || []){
-    if(!newWidgetKeys.has(key)){
-      await unregisterDashboardWidget({ owner: name, key });
-    }
-  }
-
-  /*
-    Sync scheduled jobs: the schedule becomes exactly what is declared now
-  */
-  if((newKempo.jobs || []).length || (oldKempo.jobs || []).length){
-    const [jobsError] = await registerJobs({ owner: name, jobs: newKempo.jobs || [] });
-    if(jobsError){
-      console.warn(`[${name}] scheduled jobs were not registered: ${jobsError.msg}`);
-    }
-  }
-
-  const now = new Date();
-  try {
-    await db.update(extension).set({
-      version: pkg.version || null,
-      kempo: newKempo,
-      updatedAt: now,
-    }).where(eq(extension.name, name));
-  } catch(error){
-    return [{ code: 500, msg: 'Failed to update extension record' }, null];
-  }
-
-  clearHandlerCache();
-  await triggerHook('extension:updated', { name, oldVersion, newVersion: pkg.version });
-
-  return [null, { name, oldVersion, newVersion: pkg.version }];
+  return upgradeExtension({ name, record: oldRecord, extRoot });
 };
-

package/server/utils/extensions/upgradeExtension.js

@@ -0,0 +1,117 @@
+import db from '../../db/index.js';
+import { extension, hook, permission, group, groupPermission } from '../../db/schema.js';
+import { eq, and } from 'drizzle-orm';
+import { join } from 'path';
+import { pathToFileURL } from 'url';
+import { randomUUID } from 'crypto';
+import triggerHook, { clearHandlerCache } from '../hooks/triggerHook.js';
+import createHook from '../hooks/createHook.js';
+import registerDashboardWidget from '../dashboard/registerDashboardWidget.js';
+import unregisterDashboardWidget from '../dashboard/unregisterDashboardWidget.js';
+import registerJobs from '../scheduler/registerJobs.js';
+import setSetting from '../settings/setSetting.js';
+import declaredValue from '../settings/declaredValue.js';
+import syncCoreSchema from '../setup/syncCoreSchema.js';
+import readKempoConfig from './readKempoConfig.js';
+import { invalidateScopeCache } from './scopeCache.js';
+
+/*
+  Brings the database in step with a newer version of an extension that is already on disk: its tables (additions only: new
+  tables, columns and indexes), its own `update.js` (given the old and new version and what each declared), and whatever the
+  new version declares that the old one did not (permissions, settings, groups, hooks, dashboard widgets and scheduled jobs).
+  It only adds: a permission, setting or group an extension no longer declares stays, because data may depend on it, and a
+  setting the owner changed keeps their value. Hooks, widgets and jobs follow what is declared now.
+
+  `record` is the extension's row as it was before. Resolves to [error, { name, oldVersion, newVersion }]. Run by the startup
+  step (reconcileExtensions) and by updateExtension, never on a version that is older than the row's.
+*/
+export default async ({ name, record, extRoot = join(process.cwd(), 'node_modules', name) }) => {
+  const oldKempo = record.kempo || {};
+  const oldVersion = record.version;
+
+  let pkg, newKempo;
+  try {
+    ({ pkg, kempo: newKempo } = await readKempoConfig(extRoot));
+  } catch {
+    return [{ code: 404, msg: `Package "${name}" is not installed` }, null];
+  }
+
+  if(newKempo.schema){
+    try {
+      const schemaModule = await import(`${pathToFileURL(join(extRoot, newKempo.schema)).href}?t=${Date.now()}`);
+      const [schemaError, synced] = await syncCoreSchema({ schema: schemaModule });
+      if(schemaError) return [schemaError, null];
+      for(const { change, reason } of synced.failed) console.warn(`[${name}] database change not applied: ${change}: ${reason}`);
+    } catch(error) {
+      return [{ code: 500, msg: `Failed to load extension schema: ${error.message}` }, null];
+    }
+  }
+
+  try {
+    const updateModule = await import(`${pathToFileURL(join(extRoot, 'update.js')).href}?t=${Date.now()}`).catch(() => null);
+    if(updateModule?.default){
+      await updateModule.default({ oldVersion, newVersion: pkg.version, oldKempo, newKempo });
+    }
+  } catch(error) {
+    return [{ code: 500, msg: `Extension update script failed: ${error.message}` }, null];
+  }
+
+  const oldPermNames = new Set((oldKempo.permissions || []).map(p => p.name));
+  for(const { name: permName, description } of newKempo.permissions || []){
+    if(!oldPermNames.has(permName)){
+      await db.insert(permission).values({ name: permName, description: description || null, owner: name, createdAt: new Date() }).onConflictDoNothing();
+    }
+  }
+
+  const oldSettingNames = new Set((oldKempo.settings || []).map(s => s.name));
+  for(const { name: settingName, value, type, description } of newKempo.settings || []){
+    if(!oldSettingNames.has(settingName)){
+      await setSetting(name, settingName, declaredValue(value, type), type || null, false, description || null);
+    }
+  }
+
+  const oldGroupNames = new Set((oldKempo.groups || []).map(g => g.name));
+  for(const { name: groupName, description, permissions: groupPerms = [] } of newKempo.groups || []){
+    if(!oldGroupNames.has(groupName)){
+      await db.insert(group).values({ name: groupName, description: description || null, owner: name, createdAt: new Date() }).onConflictDoNothing();
+      for(const permName of groupPerms){
+        await db.insert(groupPermission).values({ id: randomUUID(), groupName, permissionName: permName, createdAt: new Date() }).onConflictDoNothing();
+      }
+    }
+  }
+
+  const oldHooks = oldKempo.hooks || {};
+  const newHooks = newKempo.hooks || {};
+  for(const [event, callback] of Object.entries(newHooks)){
+    if(!oldHooks[event]) await createHook({ owner: name, event, callback });
+  }
+  for(const event of Object.keys(oldHooks)){
+    if(!newHooks[event]) await db.delete(hook).where(and(eq(hook.owner, name), eq(hook.event, event))).catch(() => {});
+  }
+
+  const newWidgetKeys = new Set((newKempo.widgets || []).map(w => w.key));
+  for(const widget of newKempo.widgets || []){
+    const [widgetError] = await registerDashboardWidget({ ...widget, owner: name });
+    if(widgetError) console.warn(`[${name}] dashboard widget "${widget.key}" was not registered: ${widgetError.msg}`);
+  }
+  for(const { key } of oldKempo.widgets || []){
+    if(!newWidgetKeys.has(key)) await unregisterDashboardWidget({ owner: name, key });
+  }
+
+  if((newKempo.jobs || []).length || (oldKempo.jobs || []).length){
+    const [jobsError] = await registerJobs({ owner: name, jobs: newKempo.jobs || [] });
+    if(jobsError) console.warn(`[${name}] scheduled jobs were not registered: ${jobsError.msg}`);
+  }
+
+  try {
+    await db.update(extension).set({ version: pkg.version || null, kempo: newKempo, updatedAt: new Date() }).where(eq(extension.name, name));
+  } catch {
+    return [{ code: 500, msg: 'Failed to update extension record' }, null];
+  }
+
+  clearHandlerCache();
+  invalidateScopeCache();
+  await triggerHook('extension:updated', { name, oldVersion, newVersion: pkg.version });
+
+  return [null, { name, oldVersion, newVersion: pkg.version }];
+};

package/server/utils/extensions/writeSiteFile.js

@@ -0,0 +1,66 @@
+import { createHash } from 'crypto';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
+import { dirname, join, normalize, sep } from 'path';
+
+/*
+  For an extension that has to put a file into the site (a generated template, a starter page): writes it the way kempo's own
+  starter files are handled, so an update of the extension never overwrites what the owner changed.
+
+    not there, never written by this extension     written
+    not there, but this extension wrote it before  the owner deleted it, so it stays deleted
+    identical to what is being written             nothing to do
+    still exactly what this extension last wrote   never edited, so replaced
+    anything else                                  edited by the owner (or anyone), so kept
+
+  The extension's fingerprints are in .kempo/shipped/<owner>.json. `path` is relative to the site and must be inside public/
+  or templates/. Most extensions need none of this, because their own public/ is served from the package; this is for the
+  few that generate something into the site. Resolves to [error, { status }] where status is "written", "current", "kept"
+  or "deleted".
+*/
+const hashOf = content => createHash('sha256').update(content).digest('hex');
+
+export default ({ owner, path, content, cwd = process.cwd() } = {}) => {
+  if(typeof owner !== 'string' || !/^(@[a-z0-9-~][a-z0-9-._~]*\/)?[a-z0-9-~][a-z0-9-._~]*$/.test(owner)){
+    return [{ code: 400, msg: 'owner must be the extension name' }, null];
+  }
+  const relative = typeof path === 'string' ? normalize(path).split(sep).join('/') : '';
+  if(!relative || relative.startsWith('..') || relative.startsWith('/') || !/^(public|templates)\//.test(relative)){
+    return [{ code: 400, msg: 'path must be a file inside public/ or templates/' }, null];
+  }
+  if(typeof content !== 'string' && !Buffer.isBuffer(content)){
+    return [{ code: 400, msg: 'content must be text or a buffer' }, null];
+  }
+
+  const recordFile = join(cwd, '.kempo', 'shipped', `${owner.replace('/', '__')}.json`);
+  let files = {};
+  try {
+    files = JSON.parse(readFileSync(recordFile, 'utf8')).files ?? {};
+  } catch {
+    files = {};
+  }
+  const save = () => {
+    mkdirSync(dirname(recordFile), { recursive: true });
+    writeFileSync(recordFile, JSON.stringify({ files }, null, 2));
+  };
+
+  const target = join(cwd, relative);
+  const wanted = hashOf(content);
+
+  if(!existsSync(target)){
+    if(files[relative]) return [null, { status: 'deleted' }];
+  } else {
+    const here = hashOf(readFileSync(target));
+    if(here === wanted){
+      files[relative] = wanted;
+      save();
+      return [null, { status: 'current' }];
+    }
+    if(files[relative] !== here) return [null, { status: 'kept' }];
+  }
+
+  mkdirSync(dirname(target), { recursive: true });
+  writeFileSync(target, content);
+  files[relative] = wanted;
+  save();
+  return [null, { status: 'written' }];
+};

package/server/utils/settings/systemSettings.js

@@ -20,6 +20,7 @@ export const SYSTEM_SETTINGS = [
   { owner: 'system', name: 'email_retention_days', value: 30, type: 'number', isPublic: false, description: 'Days to keep the log of sent emails before it is deleted (0 keeps it forever)' },
   { owner: 'system', name: 'email_retention_by_type', value: {}, type: 'json', isPublic: false, description: 'Days to keep a particular kind of email, instead of the number above, as { "verify-email": 7, "kempo-commerce/order-processing": 365 } (0 keeps that kind forever)' },
   { owner: 'system', name: 'email_from', value: '', type: 'string', isPublic: false, description: 'The address emails are sent from, as name@example.com or Shop Name <name@example.com>. It must be on a domain verified with the email provider. Empty uses EMAIL_FROM from the server environment' },
+  { owner: 'system', name: 'update_check_enabled', value: true, type: 'boolean', isPublic: false, description: 'Check npm every 12 hours for a newer kempo and tell the people who can install it' },
 ];
 
 /* Resolves to [error, { added: [names] }]. */

package/server/utils/setup/runUpgrades.js

@@ -0,0 +1,70 @@
+import { existsSync, readdirSync } from 'fs';
+import { dirname, join } from 'path';
+import { fileURLToPath, pathToFileURL } from 'url';
+import { sql } from 'drizzle-orm';
+import db from '../../db/index.js';
+import { upgrade } from '../../db/schema.js';
+
+/*
+  Kempo's own data migrations: the files in server/db/upgrades, which restructure or move data that the additive sync of
+  the tables (syncCoreSchema) cannot. See that folder's README for how to write one.
+
+  They run in file-name order, each once, in a transaction, when the server starts and after the tables are in step. A
+  database that has never had the `upgrade` table is taken to already hold what the shipped migrations make (a new site,
+  or a site on its first version with this): they are recorded as run without being run, under a row called 'baseline'.
+*/
+export const DEFAULT_DIR = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'db', 'upgrades');
+
+const filesIn = dir => (existsSync(dir) ? readdirSync(dir).filter(name => name.endsWith('.js')).sort() : []);
+
+const load = (dir, file) => import(`${pathToFileURL(join(dir, file)).href}?t=${Date.now()}`);
+
+const appliedIds = async executor => new Set((await executor.select({ id: upgrade.id }).from(upgrade)).map(row => row.id));
+
+/*
+  What would run, in order, as [{ id, description, breaking }]. `dir` is another version's folder, to look ahead at an
+  update before it is switched to. Nothing is run or recorded.
+*/
+export const pendingUpgrades = async ({ dir = DEFAULT_DIR, executor = db } = {}) => {
+  const applied = await appliedIds(executor);
+  if(!applied.has('baseline')) return [];
+  const pending = [];
+  for(const file of filesIn(dir).filter(name => !applied.has(name))){
+    const { meta = {} } = await load(dir, file);
+    pending.push({ id: file, description: meta.description ?? '', breaking: Boolean(meta.breaking) });
+  }
+  return pending;
+};
+
+/*
+  Resolves to [error, { ran }]. An error is the first migration that failed, with `breaking` saying whether it was one the
+  previous version cannot run without; nothing after it runs, and what it did is rolled back.
+*/
+export default async ({ dir = DEFAULT_DIR, executor = db } = {}) => {
+  try {
+    const applied = await appliedIds(executor);
+    if(!applied.has('baseline')){
+      await executor.transaction(async tx => {
+        for(const id of [...filesIn(dir), 'baseline']) await tx.insert(upgrade).values({ id, description: id === 'baseline' ? 'Existing migrations were already part of this database' : 'Part of the baseline' }).onConflictDoNothing();
+      });
+      return [null, { ran: [] }];
+    }
+
+    const ran = [];
+    for(const file of filesIn(dir).filter(name => !applied.has(name))){
+      const { meta = {}, up } = await load(dir, file);
+      try {
+        await executor.transaction(async tx => {
+          await up({ db: tx, sql });
+          await tx.insert(upgrade).values({ id: file, description: meta.description ?? null, breaking: Boolean(meta.breaking) });
+        });
+        ran.push(file);
+      } catch(error) {
+        return [{ code: 500, msg: `The data migration ${file} failed: ${String(error.cause?.message || error.message).slice(0, 300)}`, breaking: Boolean(meta.breaking), id: file }, { ran }];
+      }
+    }
+    return [null, { ran }];
+  } catch(error) {
+    return [{ code: 500, msg: `The data migrations could not be checked: ${String(error.cause?.message || error.message).slice(0, 300)}`, breaking: false }, null];
+  }
+};

package/server/utils/setup/seedDatabase.js

@@ -22,7 +22,7 @@ const DEFAULT_GROUPS = {
       'system:fragments:read', 'system:fragments:create', 'system:fragments:update', 'system:fragments:delete',
       'system:globals:read', 'system:globals:create', 'system:globals:update', 'system:globals:delete',
       'system:extensions:read', 'system:extensions:install', 'system:extensions:uninstall', 'system:extensions:manage',
-      'system:realtime:read', 'system:emails:read', 'system:emails:manage'
+      'system:realtime:read', 'system:emails:read', 'system:emails:manage', 'system:updates:manage'
     ]
   }
 };

package/server/utils/setup/seedIfNew.js

@@ -0,0 +1,21 @@
+import { eq } from 'drizzle-orm';
+import db from '../../db/index.js';
+import { group } from '../../db/schema.js';
+import seedDatabase from './seedDatabase.js';
+
+/*
+  A database with no Administrators group is a new one (the group cannot be deleted), so kempo's groups, permissions and
+  settings are put in it, which is what `scripts/init-db.js` does by hand. It is what lets a new site start from an empty
+  database with no command typed: the tables come from syncCoreSchema, this fills them, and the /setup page makes the
+  first administrator. A database that has the group is never touched.
+*/
+export default async () => {
+  try {
+    const [existing] = await db.select({ name: group.name }).from(group).where(eq(group.name, 'system:Administrators')).limit(1);
+    if(existing) return [null, { seeded: false }];
+    await seedDatabase({ log: () => {} });
+    return [null, { seeded: true }];
+  } catch(error) {
+    return [{ code: 500, msg: `A new database could not be set up: ${String(error.cause?.message || error.message).slice(0, 300)}` }, null];
+  }
+};

package/server/utils/setup/syncCoreSchema.js

@@ -0,0 +1,130 @@
+import { sql } from 'drizzle-orm';
+import db from '../../db/index.js';
+import * as coreSchema from '../../db/schema.js';
+
+/*
+  Brings kempo's own tables in step with the installed version of kempo, so an update never needs a command typed.
+  It runs when the server starts, before the site serves anything.
+
+  It only ever adds. It looks at which of the schema's tables, columns, indexes and constraints the database already
+  has, and asks drizzle-kit's migration generator (the one `drizzle-kit push` is built on) for the difference between
+  "what the database has" and "what the schema says", so the new indexes and foreign keys come out exactly as a push
+  would make them. What the database has is the schema with every missing piece cut out, so the difference can only
+  be additions: it can never contain a DROP or a TRUNCATE, and a table that an extension or the site made is never
+  looked at. A column that kempo no longer has stays where it is. A change to something that already exists (a new
+  type, a NOT NULL) is not detected here: that is what a release's own migration step is for.
+
+  Each statement is applied on its own, so one the database refuses (a new unique index over rows that already break
+  it, a NOT NULL column with no default on a table that has rows) is reported without holding back the rest. Several
+  processes starting together queue on an advisory lock, and the one after the first finds nothing left to do.
+
+  (drizzle-kit's `pushSchema` is not used: it drops the parameters of its own introspection queries, which fails on any
+  table with a composite key and then ends the process, and for a type change it proposes TRUNCATE ... CASCADE.)
+*/
+const LOCK_KEY = 'kempo:sync-core-schema';
+
+const ADDITIVE = [
+  /^CREATE TABLE /i,
+  /^CREATE (UNIQUE )?INDEX /i,
+  /^CREATE TYPE /i,
+  /^ALTER TABLE .+ ADD COLUMN /i,
+  /^ALTER TABLE .+ ADD CONSTRAINT /i,
+];
+
+/* A last check on what the generator produced: anything that is not an addition is never run. */
+export const isAdditive = statement => ADDITIVE.some(pattern => pattern.test(statement.trim()));
+
+/* One plain sentence for a statement, for the log and the Updates screen. */
+export const describe = statement => {
+  const text = statement.trim();
+  const named = (pattern, make) => {
+    const match = text.match(pattern);
+    return match ? make(...match.slice(1)) : null;
+  };
+  return named(/^CREATE TABLE "([^"]+)"/i, table => `Created table ${table}`)
+    ?? named(/^ALTER TABLE "([^"]+)" ADD COLUMN "([^"]+)"/i, (table, column) => `Added column ${table}.${column}`)
+    ?? named(/^CREATE (?:UNIQUE )?INDEX "([^"]+)"/i, index => `Created index ${index}`)
+    ?? named(/^ALTER TABLE "([^"]+)" ADD CONSTRAINT "([^"]+)"/i, (table, constraint) => `Added ${constraint} to ${table}`)
+    ?? text.replace(/\s+/g, ' ').slice(0, 120);
+};
+
+const rowsOf = result => (Array.isArray(result) ? result : result.rows);
+
+/* The names of what the database already has in the public schema. */
+const readDatabase = async executor => {
+  const names = async (query, field) => new Set(rowsOf(await executor.execute(query)).map(row => row[field]));
+  const columns = rowsOf(await executor.execute(sql`select table_name, column_name from information_schema.columns where table_schema = 'public'`));
+  return {
+    tables: await names(sql`select table_name from information_schema.tables where table_schema = 'public'`, 'table_name'),
+    columns: new Set(columns.map(row => `${row.table_name}.${row.column_name}`)),
+    indexes: await names(sql`select indexname from pg_indexes where schemaname = 'public'`, 'indexname'),
+    constraints: await names(sql`select conname from pg_constraint join pg_namespace on pg_namespace.oid = pg_constraint.connamespace where nspname = 'public'`, 'conname'),
+    enums: await names(sql`select typname from pg_type join pg_namespace on pg_namespace.oid = pg_type.typnamespace where typtype = 'e' and nspname = 'public'`, 'typname'),
+  };
+};
+
+/* The schema snapshot with everything the database lacks cut out: what drizzle-kit would see if it had introspected the database. */
+const withoutMissing = (snapshot, existing) => {
+  const prev = structuredClone(snapshot);
+  for(const key of Object.keys(prev.enums)){
+    if(!existing.enums.has(prev.enums[key].name)) delete prev.enums[key];
+  }
+  for(const [key, table] of Object.entries(prev.tables)){
+    if(!existing.tables.has(table.name)){
+      delete prev.tables[key];
+      continue;
+    }
+    for(const name of Object.keys(table.columns)){
+      if(!existing.columns.has(`${table.name}.${name}`)) delete table.columns[name];
+    }
+    for(const name of Object.keys(table.indexes)){
+      if(!existing.indexes.has(name)) delete table.indexes[name];
+    }
+    for(const group of [table.foreignKeys, table.uniqueConstraints, table.compositePrimaryKeys ?? {}]){
+      for(const id of Object.keys(group)){
+        if(!existing.constraints.has(group[id].name)) delete group[id];
+      }
+    }
+  }
+  return prev;
+};
+
+/* What the last run did, for the Updates screen. */
+let lastRun = { ran: false, applied: [], failed: [] };
+
+export const getSchemaSyncResult = () => lastRun;
+
+/*
+  Resolves to [error, { applied, failed }]: `applied` is a list of sentences, `failed` is { change, reason } for what
+  the database refused. `schema` and `executor` are for tests and extensions: other tables, in another database.
+*/
+export default async ({ schema = coreSchema, executor = db } = {}) => {
+  try {
+    const { generateDrizzleJson, generateMigration } = await import('drizzle-kit/api');
+    const wanted = generateDrizzleJson(schema);
+    const result = { applied: [], failed: [] };
+
+    await executor.transaction(async tx => {
+      await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${LOCK_KEY}::text))`);
+      const statements = await generateMigration(withoutMissing(wanted, await readDatabase(tx)), wanted);
+
+      for(const statement of statements){
+        if(!isAdditive(statement)){
+          result.failed.push({ change: describe(statement), reason: 'Kempo only adds to its tables, so this was not run' });
+          continue;
+        }
+        try {
+          await tx.transaction(inner => inner.execute(sql.raw(statement)));
+          result.applied.push(describe(statement));
+        } catch(error) {
+          result.failed.push({ change: describe(statement), reason: String(error.cause?.message || error.message).slice(0, 300) });
+        }
+      }
+    });
+
+    if(schema === coreSchema) lastRun = { ran: true, ...result };
+    return [null, result];
+  } catch(error) {
+    return [{ code: 500, msg: `The database could not be checked against this version of kempo: ${String(error.cause?.message || error.message).slice(0, 300)}` }, null];
+  }
+};

package/server/utils/setup/systemPermissions.js

@@ -55,6 +55,8 @@ export const SYSTEM_PERMISSIONS = [
   { name: 'system:emails:read', description: 'View the log of emails, including what they said' },
   { name: 'system:emails:manage', description: 'Send a test email and try failed emails again' },
 
+  { name: 'system:updates:manage', description: 'See, check for and install kempo updates, which restarts the server' },
+
   { name: 'system:admin:access', description: 'Access admin panel' },
 ];