kempo 4.6.1 → 4.7.0

package/server/utils/menus/store.js

@@ -0,0 +1,90 @@
+import { readFile, writeFile } from 'fs/promises';
+import { join } from 'path';
+import { existsSync } from 'fs';
+import { GLOBAL_FILE, GLOBAL_DISABLED_FILE, parseEntries, serializeEntries } from '../global-content/helpers.js';
+import findNavItems from './findNavItems.js';
+
+const read = async path => existsSync(path) ? parseEntries(await readFile(path, 'utf-8')) : [];
+
+const write = async (path, entries) => {
+  if(!entries.length && !existsSync(path)) return;
+  await writeFile(path, entries.length ? serializeEntries(entries) + '\n' : '', 'utf-8');
+};
+
+/*
+  Where a menu's own entries are filed. It is not a place any template has, so nothing is rendered
+  from them directly: what shows up on the site is the copies below.
+*/
+export const menuHome = menuId => `menu:${menuId}`;
+
+export const isMenuDefinition = entry => entry.menuLocations !== null;
+
+export const isMenuItem = entry => Boolean(entry.menu) && entry.menuLocations === null && !entry.menuCopyOf;
+
+/*
+  The page shows a menu in as many places as it is assigned, but a piece of global content fills one
+  location. So each enabled item gets a copy in every location its menu is assigned to, and the copies
+  are what a template renders. They are rebuilt from the items on every write, never edited, which is
+  what keeps a menu's places in step with its items.
+
+  An item an extension supplies keeps the extension's markup as it was when last written, so it renders
+  even though kempo-server cannot look it up at render time. `navItems` (name to item) is how it is kept
+  current: the markup is refreshed from the extension, and the item gets no copies, so it disappears from
+  the site, while the extension is not available. The item itself stays in its menu, so it returns with
+  the extension.
+*/
+const rebuildCopies = (active, navItems) => {
+  const entries = active.filter(e => !e.menuCopyOf);
+  const copies = [];
+
+  for(const entry of entries.filter(e => e.navItem && navItems?.has(e.navItem))){
+    const current = navItems.get(entry.navItem);
+    entry.name = current.label;
+    entry.markup = current.markup;
+  }
+
+  for(const definition of entries.filter(isMenuDefinition)){
+    const items = entries.filter(e => isMenuItem(e) && e.menu === definition.id && !(e.navItem && navItems && !navItems.has(e.navItem)));
+    for(const location of definition.menuLocations){
+      for(const item of items){
+        copies.push({
+          ...item,
+          id: `${item.id}@${location}`,
+          locked: false,
+          location,
+          menuCopyOf: item.id,
+          navItem: '',
+          extra: {}
+        });
+      }
+    }
+  }
+  return [...entries, ...copies];
+};
+
+/*
+  Both files a site's global content lives in: what is showing, and what has been switched off.
+  An item can be in either, so menu operations always look at both. The copies are left out of
+  `active`, as they only exist to be rendered.
+*/
+export const readStores = async rootDir => ({
+  active: (await read(join(rootDir, GLOBAL_FILE))).filter(e => !e.menuCopyOf),
+  disabled: await read(join(rootDir, GLOBAL_DISABLED_FILE))
+});
+
+export const writeStores = async (rootDir, { active, disabled }, extensionDirs) => {
+  const hasNavItems = [...active, ...disabled].some(e => e.navItem);
+  const navItems = hasNavItems ? new Map((await findNavItems({ extensionDirs }))[1].map(item => [item.key, item])) : null;
+  await write(join(rootDir, GLOBAL_FILE), rebuildCopies(active, navItems));
+  await write(join(rootDir, GLOBAL_DISABLED_FILE), disabled);
+};
+
+/*
+  Brings every menu up to date with the extensions that are installed now. Run when extensions are
+  installed, enabled, disabled or removed, so what an extension supplied leaves the site with it.
+*/
+export const resyncMenus = async (rootDir, extensionDirs) => {
+  const stores = await readStores(rootDir);
+  if(![...stores.active, ...stores.disabled].some(e => e.navItem)) return;
+  await writeStores(rootDir, stores, extensionDirs);
+};

package/server/utils/scheduler/helpers.js

@@ -0,0 +1,36 @@
+import parseDuration from '../realtime/duration.js';
+
+export const MIN_INTERVAL_MS = 1000;
+
+/* How long a process may hold a job before another may take it over, in case the first one died mid-run. */
+export const LEASE_MS = 5 * 60 * 1000;
+
+export const jobId = (owner, name) => `${owner}:${name}`;
+
+/*
+  Checks the `jobs` an extension declares: [{ name, every, callback }]. `every` is a number of
+  milliseconds or a string such as "30s", "1m" or "6h". Returns [error, jobs] where each job has
+  `everyMs` filled in.
+*/
+export const normalizeJobs = jobs => {
+  if(jobs === undefined || jobs === null) return [null, []];
+  if(!Array.isArray(jobs)) return [{ code: 400, msg: 'jobs must be a list' }, null];
+
+  const seen = new Set();
+  const result = [];
+  for(const job of jobs){
+    const name = String(job?.name ?? '');
+    if(!/^[a-z0-9][a-z0-9-]*$/.test(name)) return [{ code: 400, msg: `Job name "${name}" must be lowercase letters, numbers and dashes` }, null];
+    if(seen.has(name)) return [{ code: 400, msg: `Job "${name}" is declared twice` }, null];
+    seen.add(name);
+
+    const everyMs = parseDuration(job.every);
+    if(everyMs === null || everyMs < MIN_INTERVAL_MS) return [{ code: 400, msg: `Job "${name}" needs an "every" of at least one second, such as "1m"` }, null];
+
+    const callback = String(job.callback ?? '');
+    if(!callback.startsWith('./')) return [{ code: 400, msg: `Job "${name}" needs a "callback" path starting with ./` }, null];
+
+    result.push({ name, every: job.every, everyMs, callback });
+  }
+  return [null, result];
+};

package/server/utils/scheduler/listJobs.js

@@ -0,0 +1,11 @@
+import db from '../../db/index.js';
+import { scheduledJob } from '../../db/schema.js';
+import { asc } from 'drizzle-orm';
+
+export default async () => {
+  try {
+    return [null, await db.select().from(scheduledJob).orderBy(asc(scheduledJob.owner), asc(scheduledJob.name))];
+  } catch(error) {
+    return [{ code: 500, msg: 'Failed to list jobs' }, null];
+  }
+};

package/server/utils/scheduler/registerJobs.js

@@ -0,0 +1,37 @@
+import db from '../../db/index.js';
+import { scheduledJob } from '../../db/schema.js';
+import { and, eq, notInArray } from 'drizzle-orm';
+import { normalizeJobs, jobId } from './helpers.js';
+
+/*
+  Makes an extension's schedule match what it declares: adds new jobs, refreshes the interval and
+  callback of existing ones (keeping when they are next due), and removes the ones no longer declared.
+  Safe to call from install and again from update.
+*/
+export default async ({ owner, jobs }) => {
+  if(!owner){
+    return [{ code: 400, msg: 'Owner is required' }, null];
+  }
+
+  const [invalid, declared] = normalizeJobs(jobs);
+  if(invalid){
+    return [invalid, null];
+  }
+
+  try {
+    const now = new Date();
+    for(const job of declared){
+      const row = { owner, name: job.name, everyMs: job.everyMs, callback: job.callback };
+      await db.insert(scheduledJob)
+        .values({ id: jobId(owner, job.name), ...row, nextRunAt: now, createdAt: now })
+        .onConflictDoUpdate({ target: scheduledJob.id, set: row });
+    }
+    const names = declared.map(job => job.name);
+    await db.delete(scheduledJob).where(names.length
+      ? and(eq(scheduledJob.owner, owner), notInArray(scheduledJob.name, names))
+      : eq(scheduledJob.owner, owner));
+    return [null, { registered: declared.length }];
+  } catch(error) {
+    return [{ code: 500, msg: 'Failed to register jobs' }, null];
+  }
+};

package/server/utils/scheduler/runJobs.js

@@ -0,0 +1,90 @@
+import db from '../../db/index.js';
+import { scheduledJob } from '../../db/schema.js';
+import { and, eq, lte, or, isNull, asc } from 'drizzle-orm';
+import { join } from 'path';
+import { pathToFileURL } from 'url';
+import { getDisabledExtensionNames } from '../extensions/scopeCache.js';
+import { LEASE_MS, jobId } from './helpers.js';
+
+const resolveHandler = (owner, callback) => join(process.cwd(), 'node_modules', owner, callback);
+
+/*
+  Takes a job for this process: one conditional UPDATE, so of any number of processes asking at once
+  exactly one gets the row. It then moves `nextRunAt` on by an interval. `force` skips the "is it due"
+  test (for running a job by hand) but never the lease.
+*/
+const claim = async (id, { force = false } = {}) => {
+  const now = new Date();
+  const conditions = [eq(scheduledJob.id, id), or(isNull(scheduledJob.lockedUntil), lte(scheduledJob.lockedUntil, now))];
+  if(!force) conditions.push(lte(scheduledJob.nextRunAt, now));
+
+  const [row] = await db.update(scheduledJob)
+    .set({ lockedUntil: new Date(now.getTime() + LEASE_MS) })
+    .where(and(...conditions))
+    .returning();
+  if(!row) return null;
+
+  if(!force){
+    await db.update(scheduledJob).set({ nextRunAt: new Date(now.getTime() + row.everyMs) }).where(eq(scheduledJob.id, id));
+  }
+  return row;
+};
+
+/* Runs a claimed job's handler, then releases the lease and records how it went. A throwing job never stops anything else. */
+const execute = async row => {
+  let status = 'ok';
+  let message = null;
+  try {
+    const mod = await import(pathToFileURL(resolveHandler(row.owner, row.callback)).href);
+    await (mod.default || mod)({ owner: row.owner, name: row.name });
+  } catch(error) {
+    status = 'error';
+    message = String(error?.message ?? error).slice(0, 500);
+    console.error(`[kempo] Job "${row.name}" (${row.owner}) failed: ${message}`);
+  }
+  await db.update(scheduledJob)
+    .set({ lockedUntil: null, lastRunAt: new Date(), lastStatus: status, lastError: message })
+    .where(eq(scheduledJob.id, row.id))
+    .catch(() => {});
+  return { status, error: message };
+};
+
+/*
+  Runs every job that is due and not held by another process. A disabled extension's jobs stay
+  registered but never run, the same as its hooks. Returns the ones this process ran.
+*/
+export const runDueJobs = async () => {
+  try {
+    const disabled = await getDisabledExtensionNames();
+    const candidates = await db.select().from(scheduledJob).where(lte(scheduledJob.nextRunAt, new Date())).orderBy(asc(scheduledJob.nextRunAt));
+    const ran = [];
+    for(const candidate of candidates){
+      if(disabled.has(candidate.owner)) continue;
+      const row = await claim(candidate.id);
+      if(!row) continue;
+      ran.push({ owner: row.owner, name: row.name, ...(await execute(row)) });
+    }
+    return [null, ran];
+  } catch(error) {
+    return [{ code: 500, msg: 'Failed to run jobs' }, null];
+  }
+};
+
+/* Runs one job now, whatever the schedule says. Refused with 409 while another process is running it. */
+export const runJob = async ({ owner, name } = {}) => {
+  if(!owner || !name){
+    return [{ code: 400, msg: 'Owner and name are required' }, null];
+  }
+
+  try {
+    const id = jobId(owner, name);
+    const [exists] = await db.select({ id: scheduledJob.id }).from(scheduledJob).where(eq(scheduledJob.id, id));
+    if(!exists) return [{ code: 404, msg: 'Job not found' }, null];
+    if((await getDisabledExtensionNames()).has(owner)) return [{ code: 409, msg: 'The extension is disabled' }, null];
+    const row = await claim(id, { force: true });
+    if(!row) return [{ code: 409, msg: 'The job is already running' }, null];
+    return [null, await execute(row)];
+  } catch(error) {
+    return [{ code: 500, msg: 'Failed to run the job' }, null];
+  }
+};

package/server/utils/scheduler/scheduler.js

@@ -0,0 +1,37 @@
+import { runDueJobs } from './runJobs.js';
+
+/*
+  One ticker per process, held on a Symbol.for global so that the middleware and an extension reach the
+  same one even if kempo was resolved twice. Each tick only asks the database what is due; the lease on
+  each job row is what stops several processes running the same job. Set KEMPO_SCHEDULER=off to keep a
+  process from running jobs.
+*/
+const SCHEDULER = Symbol.for('kempo.scheduler');
+
+export const TICK_MS = 15000;
+
+export const startScheduler = ({ tickMs = TICK_MS } = {}) => {
+  if(globalThis[SCHEDULER] || process.env.KEMPO_SCHEDULER === 'off') return false;
+
+  let running = false;
+  const timer = setInterval(async () => {
+    if(running) return;
+    running = true;
+    try {
+      const [error] = await runDueJobs();
+      if(error) console.error(`[kempo] Scheduler: ${error.msg}`);
+    } finally {
+      running = false;
+    }
+  }, tickMs);
+  timer.unref?.();
+  globalThis[SCHEDULER] = timer;
+  return true;
+};
+
+export const stopScheduler = () => {
+  if(!globalThis[SCHEDULER]) return false;
+  clearInterval(globalThis[SCHEDULER]);
+  globalThis[SCHEDULER] = null;
+  return true;
+};

package/server/utils/scheduler/unregisterJobs.js

@@ -0,0 +1,16 @@
+import db from '../../db/index.js';
+import { scheduledJob } from '../../db/schema.js';
+import { eq } from 'drizzle-orm';
+
+export default async ({ owner }) => {
+  if(!owner){
+    return [{ code: 400, msg: 'Owner is required' }, null];
+  }
+
+  try {
+    await db.delete(scheduledJob).where(eq(scheduledJob.owner, owner));
+    return [null, { success: true }];
+  } catch(error) {
+    return [{ code: 500, msg: 'Failed to remove jobs' }, null];
+  }
+};

package/server/utils/settings/ensureEncryptionKey.js

@@ -0,0 +1,132 @@
+import { readFile, writeFile, chmod } from 'fs/promises';
+import { join } from 'path';
+import crypto from 'crypto';
+import { and, eq, isNotNull, ne, sql } from 'drizzle-orm';
+import db from '../../db/index.js';
+import { setting } from '../../db/schema.js';
+
+/*
+  Makes sure SETTINGS_ENCRYPTION_KEY exists before anything needs it, so a site never has to set one up by
+  hand. In order:
+
+    1. the environment, when it has one (a secret manager, docker compose, the host). It always wins
+    2. the site's `.env` file, for a key written after the process started
+    3. a new random key, but only when nothing is encrypted yet. Written to `.env` (owner-only, and a
+       .gitignore that names it is kept honest), then set for this process, so there is no restart
+
+  A key is never generated while the database already holds encrypted settings: a new one would make every
+  one of them unreadable. That case is an error that says so. A key is also never kept only in memory:
+  when `.env` cannot be written the answer is an error, because a key that vanishes on the next restart
+  is worse than none.
+
+  Anyone with shell access to the server can read the key in `.env`. Back it up with the database: losing
+  it makes the stored credentials unrecoverable (they can be typed in again, nothing else is lost).
+*/
+
+const NAME = 'SETTINGS_ENCRYPTION_KEY';
+const VALID = /^[0-9a-fA-F]{64}$/;
+const LINE = new RegExp(`^[ \\t]*(?:export[ \\t]+)?${NAME}[ \\t]*=[ \\t]*(.*?)[ \\t]*$`, 'm');
+const COMMENT = '# Encrypts the API credentials saved in the admin. Back this up with your database: if it is lost they cannot be recovered.';
+
+const unquote = value => value.replace(/^(['"])(.*)\1$/, '$2');
+
+const readText = async path => {
+  try {
+    return await readFile(path, 'utf8');
+  } catch(error) {
+    if(error.code === 'ENOENT') return null;
+    throw error;
+  }
+};
+
+/* How many settings are stored encrypted. A site that has not created its tables yet has none. */
+const storedSecrets = async () => {
+  try {
+    const [{ total }] = await db.select({ total: sql`count(*)::int` }).from(setting)
+      .where(and(eq(setting.type, 'secret'), isNotNull(setting.value), ne(setting.value, '')));
+    return [null, total];
+  } catch(error) {
+    if((error?.code || error?.cause?.code) === '42P01') return [null, 0];
+    return [{ code: 500, msg: 'Could not check whether this site already has encrypted settings, so no encryption key was created' }, null];
+  }
+};
+
+/* Puts the key in `.env`: replaces an empty `SETTINGS_ENCRYPTION_KEY=` line, or adds one. */
+const saveKey = async (path, key, text) => {
+  const line = `${NAME}=${key}`;
+  let next;
+  if(text === null) next = `${COMMENT}\n${line}\n`;
+  else if(LINE.test(text)) next = text.replace(LINE, line);
+  else next = `${text}${text === '' || text.endsWith('\n') ? '' : '\n'}\n${COMMENT}\n${line}\n`;
+  await writeFile(path, next, { mode: 0o600 });
+  await chmod(path, 0o600).catch(() => {});
+};
+
+/* A .gitignore that exists but does not name `.env` would let the key be committed. */
+const keepOutOfGit = async dir => {
+  const path = join(dir, '.gitignore');
+  try {
+    const text = await readText(path);
+    if(text === null || /^\/?\.env\s*$/m.test(text)) return;
+    await writeFile(path, `${text}${text.endsWith('\n') ? '' : '\n'}.env\n`);
+  } catch {
+    /* Not being able to tidy .gitignore is not worth refusing a key over. */
+  }
+};
+
+const resolve = async ({ cwd }) => {
+  const fromEnvironment = process.env[NAME];
+  if(fromEnvironment){
+    return VALID.test(fromEnvironment)
+      ? [null, { source: 'environment' }]
+      : [{ code: 500, msg: `${NAME} must be a 64-character hex string (32 bytes). Remove it to have one generated, or replace it with the key this site was created with` }, null];
+  }
+
+  const path = join(cwd, '.env');
+  let text;
+  try {
+    text = await readText(path);
+  } catch(error) {
+    return [{ code: 500, msg: `Could not read ${path} (${error.code ?? error.message}), so no encryption key was created` }, null];
+  }
+
+  const inFile = text === null ? '' : unquote(LINE.exec(text)?.[1] ?? '');
+  if(inFile){
+    if(!VALID.test(inFile)) return [{ code: 500, msg: `${NAME} in ${path} must be a 64-character hex string (32 bytes)` }, null];
+    process.env[NAME] = inFile;
+    return [null, { source: 'file', path }];
+  }
+
+  const [countError, count] = await storedSecrets();
+  if(countError) return [countError, null];
+  if(count > 0){
+    return [{ code: 500, msg: `This site has ${count} encrypted setting${count === 1 ? '' : 's'} but ${NAME} is not set. Restore the key it was created with (your .env backup, or the host's environment). A new key would make them unreadable` }, null];
+  }
+
+  const key = crypto.randomBytes(32).toString('hex');
+  try {
+    await saveKey(path, key, text);
+  } catch(error) {
+    return [{ code: 500, msg: `Could not save a new ${NAME} to ${path} (${error.code ?? error.message}). Set ${NAME} in the server's environment instead` }, null];
+  }
+  await keepOutOfGit(cwd);
+  process.env[NAME] = key;
+  console.warn(`[kempo] ${NAME} was not set, so one was generated and saved to ${path}. Back it up with your database: if it is lost, stored credentials (Stripe keys and the like) cannot be recovered and must be entered again.`);
+  return [null, { source: 'generated', path }];
+};
+
+/* One answer per process: concurrent callers share the work, and a failure can be tried again. */
+let pending = null;
+
+export const ensureEncryptionKey = ({ cwd = process.cwd() } = {}) => {
+  pending ??= resolve({ cwd }).then(result => {
+    if(result[0]) pending = null;
+    return result;
+  });
+  return pending;
+};
+
+/* For tests: forget the answer so the next call decides again. */
+export const resetEncryptionKey = () => {
+  pending = null;
+};

package/server/utils/settings/secretCrypto.js

@@ -37,5 +37,9 @@ export const decryptSecret = encoded => {
   const ciphertext = data.subarray(28);
   const decipher = crypto.createDecipheriv('aes-256-gcm', getKey(), iv);
   decipher.setAuthTag(authTag);
-  return Buffer.concat([decipher.update(ciphertext), decipher.final()]).toString('utf8');
+  try {
+    return Buffer.concat([decipher.update(ciphertext), decipher.final()]).toString('utf8');
+  } catch {
+    throw new Error('A stored secret could not be decrypted: SETTINGS_ENCRYPTION_KEY is not the key it was saved with. Restore the original key, or enter the credential again');
+  }
 };

package/dist/kempo/icons/create_new_folder.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 -960 960 960"><path fill="currentColor" d="M560-320h80v-80h80v-80h-80v-80h-80v80h-80v80h80v80ZM160-160q-33 0-56.5-23.5T80-240v-480q0-33 23.5-56.5T160-800h240l80 80h320q33 0 56.5 23.5T880-640v400q0 33-23.5 56.5T800-160H160Zm0-80h640v-400H447l-80-80H160v480Zm0 0v-480 480Z"/></svg>

package/dist/kempo/icons/credit_card.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 -960 960 960"><path fill="currentColor" d="M880-720v480q0 33-23.5 56.5T800-160H160q-33 0-56.5-23.5T80-240v-480q0-33 23.5-56.5T160-800h640q33 0 56.5 23.5T880-720ZM160-640h640v-80H160v80Zm0 160v240h640v-240H160Zm0 240v-480 480Z"/></svg>

package/dist/kempo/icons/login.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 -960 960 960"><path fill="currentColor" d="M480-120v-80h280v-560H480v-80h280q33 0 56.5 23.5T840-760v560q0 33-23.5 56.5T760-120H480Zm-80-160-55-58 102-102H120v-80h327L345-622l55-58 200 200-200 200Z"/></svg>

package/dist/kempo/icons/payments.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 -960 960 960"><path fill="currentColor" d="M560-440q-50 0-85-35t-35-85q0-50 35-85t85-35q50 0 85 35t35 85q0 50-35 85t-85 35ZM280-320q-33 0-56.5-23.5T200-400v-320q0-33 23.5-56.5T280-800h560q33 0 56.5 23.5T920-720v320q0 33-23.5 56.5T840-320H280Zm80-80h400q0-33 23.5-56.5T840-480v-160q-33 0-56.5-23.5T760-720H360q0 33-23.5 56.5T280-640v160q33 0 56.5 23.5T360-400Zm440 240H120q-33 0-56.5-23.5T40-240v-440h80v440h680v80ZM280-400v-320 320Z"/></svg>

package/dist/kempo/icons/toll.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 -960 960 960"><path fill="currentColor" d="M373-253q-93-93-93-227t93-227q93-93 227-93t227 93q93 93 93 227t-93 227q-93 93-227 93t-227-93Zm-93 83q-106-28-173-114T40-480q0-110 67-196t173-114v84q-72 25-116 87t-44 139q0 77 44 139t116 87v84Zm320-310Zm170 170q70-70 70-170t-70-170q-70-70-170-70t-170 70q-70 70-70 170t70 170q70 70 170 70t170-70Z"/></svg>

package/dist/kempo/icons/wallet.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 -960 960 960"><path fill="currentColor" d="M200-120q-33 0-56.5-23.5T120-200v-560q0-33 23.5-56.5T200-840h560q33 0 56.5 23.5T840-760v80h-80v-80H200v560h560v-80h80v80q0 33-23.5 56.5T760-120H200Zm320-160q-33 0-56.5-23.5T440-360v-240q0-33 23.5-56.5T520-680h280q33 0 56.5 23.5T880-600v240q0 33-23.5 56.5T800-280H520Zm280-80v-240H520v240h280Zm-160-60q25 0 42.5-17.5T700-480q0-25-17.5-42.5T640-540q-25 0-42.5 17.5T580-480q0 25 17.5 42.5T640-420Z"/></svg>