kempo 4.5.7 → 4.6.0

package/server/utils/extensions/disableExtensionTree.js

@@ -0,0 +1,46 @@
+import db from '../../db/index.js';
+import { extension } from '../../db/schema.js';
+import { eq, inArray } from 'drizzle-orm';
+import { invalidateScopeCache } from './scopeCache.js';
+import getEnabledDependentTree from './getDependentTree.js';
+
+/*
+  Switches an extension off. Anything enabled that requires it (directly or through another) cannot
+  keep running without it, so without `cascade` this refuses and returns that whole tree as
+  `dependents` for the caller to show. With `cascade` the extension and everything above it are
+  switched off together, in one statement, so there is no half-disabled state to recover from.
+  Enabling is never cascaded: each of them is turned back on deliberately.
+*/
+export default async ({ name, cascade = false }) => {
+  if(!name){
+    return [{ code: 400, msg: 'Extension name is required' }, null];
+  }
+
+  try {
+    const [existing] = await db.select({ name: extension.name }).from(extension).where(eq(extension.name, name)).limit(1);
+    if(!existing){
+      return [{ code: 404, msg: 'Extension not found' }, null];
+    }
+
+    const dependents = await getEnabledDependentTree(name);
+    if(dependents.length && !cascade){
+      return [{
+        code: 409,
+        msg: `${dependents.map(dependent => dependent.name).join(', ')} depends on this extension and is still enabled`,
+        dependents,
+      }, null];
+    }
+
+    const names = [name, ...dependents.map(dependent => dependent.name)];
+    await db.update(extension)
+      .set({ enabled: false, updatedAt: new Date() })
+      .where(inArray(extension.name, names));
+
+    const [updated] = await db.select().from(extension).where(eq(extension.name, name)).limit(1);
+
+    invalidateScopeCache();
+    return [null, { ...updated, disabled: names }];
+  } catch(error) {
+    return [{ code: 500, msg: 'Failed to disable extension' }, null];
+  }
+};

package/server/utils/extensions/enableExtension.js

@@ -1,33 +1,2 @@
-import db from '../../db/index.js';
-import { extension } from '../../db/schema.js';
-import { eq } from 'drizzle-orm';
-import { invalidateScopeCache } from './scopeCache.js';
-import { getUnmetDependencies } from './dependencies.js';
-
-export default async ({ name }) => {
-  if(!name){
-    return [{ code: 400, msg: 'Extension name is required' }, null];
-  }
-
-  try {
-    const [existing] = await db.select().from(extension).where(eq(extension.name, name)).limit(1);
-    if(!existing){
-      return [{ code: 404, msg: 'Extension not found' }, null];
-    }
-
-    const unmet = await getUnmetDependencies(existing.kempo?.dependencies || []);
-    if(unmet.length){
-      return [{ code: 409, msg: `Requires ${unmet.join(', ')} to be enabled first` }, null];
-    }
-
-    const [updated] = await db.update(extension)
-      .set({ enabled: true, updatedAt: new Date() })
-      .where(eq(extension.name, name))
-      .returning();
-
-    invalidateScopeCache();
-    return [null, updated];
-  } catch(error) {
-    return [{ code: 500, msg: 'Failed to enable extension' }, null];
-  }
-};
+/* The name callers import; the implementation, with its cascade over requirements, is in enableExtensionTree.js. */
+export { default } from './enableExtensionTree.js';

package/server/utils/extensions/enableExtensionTree.js

@@ -0,0 +1,54 @@
+import db from '../../db/index.js';
+import { extension } from '../../db/schema.js';
+import { eq, inArray } from 'drizzle-orm';
+import { invalidateScopeCache } from './scopeCache.js';
+import getRequirementTree from './getRequirementTree.js';
+
+/*
+  Switches an extension on. It cannot run without what it requires, so any of that which is installed
+  but off has to be enabled too: without `cascade` this refuses and returns that whole chain as
+  `requirements` for the caller to show, and with `cascade` they are enabled together with it, in one
+  statement. A requirement that is not installed at all cannot be fixed by enabling, and is refused
+  by name either way.
+
+  It only ever turns on what the extension needs. Whatever else was switched off along with it, for
+  its own reasons, stays off.
+*/
+export default async ({ name, cascade = false }) => {
+  if(!name){
+    return [{ code: 400, msg: 'Extension name is required' }, null];
+  }
+
+  try {
+    const [existing] = await db.select({ name: extension.name }).from(extension).where(eq(extension.name, name)).limit(1);
+    if(!existing){
+      return [{ code: 404, msg: 'Extension not found' }, null];
+    }
+
+    const { disabled, missing } = await getRequirementTree(name);
+
+    if(missing.length){
+      return [{ code: 409, msg: `Requires ${missing.join(', ')}, which ${missing.length === 1 ? 'is' : 'are'} not installed`, missing }, null];
+    }
+
+    if(disabled.length && !cascade){
+      return [{
+        code: 409,
+        msg: `Requires ${disabled.map(requirement => requirement.name).join(', ')} to be enabled first`,
+        requirements: disabled,
+      }, null];
+    }
+
+    const names = [name, ...disabled.map(requirement => requirement.name)];
+    await db.update(extension)
+      .set({ enabled: true, updatedAt: new Date() })
+      .where(inArray(extension.name, names));
+
+    const [updated] = await db.select().from(extension).where(eq(extension.name, name)).limit(1);
+
+    invalidateScopeCache();
+    return [null, { ...updated, enabled: true, enabledNames: names }];
+  } catch(error) {
+    return [{ code: 500, msg: 'Failed to enable extension' }, null];
+  }
+};

package/server/utils/extensions/getDependentTree.js

@@ -0,0 +1,47 @@
+import db from '../../db/index.js';
+import { extension } from '../../db/schema.js';
+import { eq } from 'drizzle-orm';
+
+/*
+  Everything that has to go along with `name`: the extensions that require it, the ones that require
+  those, and so on. Each entry says which members of the affected set (the extension itself included)
+  it requires directly, so the chain can be shown, and whether it is currently enabled. Deepest
+  dependents first, which is the order they would have to be dealt with one at a time.
+
+  By default only enabled extensions count, which is what disabling needs: one that is already off
+  does not have to be switched off again. Uninstalling needs `includeDisabled`, because an extension
+  that is installed but switched off is still installed, and must not be left behind without what it
+  requires.
+*/
+export default async (name, { includeDisabled = false } = {}) => {
+  const rows = await (includeDisabled
+    ? db.select({ name: extension.name, kempo: extension.kempo, enabled: extension.enabled }).from(extension)
+    : db.select({ name: extension.name, kempo: extension.kempo, enabled: extension.enabled }).from(extension).where(eq(extension.enabled, true)));
+
+  const requires = new Map(rows.map(row => [row.name, row.kempo?.dependencies || []]));
+  const enabled = new Map(rows.map(row => [row.name, row.enabled]));
+  const depth = new Map([[name, 0]]);
+  let frontier = [name];
+
+  while(frontier.length){
+    const next = [];
+    for(const [candidate, needs] of requires){
+      if(depth.has(candidate)) continue;
+      const via = frontier.find(need => needs.includes(need));
+      if(via !== undefined){
+        depth.set(candidate, depth.get(via) + 1);
+        next.push(candidate);
+      }
+    }
+    frontier = next;
+  }
+
+  return [...depth.entries()]
+    .filter(([candidate]) => candidate !== name)
+    .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+    .map(([candidate]) => ({
+      name: candidate,
+      requires: requires.get(candidate).filter(need => depth.has(need)),
+      enabled: enabled.get(candidate) === true,
+    }));
+};

package/server/utils/extensions/getExtension.js

@@ -2,7 +2,7 @@ import db from '../../db/index.js';
 import { extension } from '../../db/schema.js';
 import { eq } from 'drizzle-orm';
 import { join } from 'path';
-import { readFile } from 'fs/promises';
+import readKempoConfig from './readKempoConfig.js';
 
 const NODE_MODULES = join(process.cwd(), 'node_modules');
 
@@ -18,15 +18,16 @@ export default async ({ name }) => {
     }
     const ext = results[0];
     let pkg = {};
+    let kempo = {};
     try {
-      pkg = JSON.parse(await readFile(join(NODE_MODULES, name, 'package.json'), 'utf-8'));
+      ({ pkg, kempo } = await readKempoConfig(join(NODE_MODULES, name)));
     } catch { /* package may not be installed */ }
     return [null, {
       ...ext,
       description: pkg.description || null,
       author: pkg.author || null,
       license: pkg.license || null,
-      docs: pkg.kempo?.docs || null,
+      docs: kempo.docs || null,
     }];
   } catch(error) {
     return [{ code: 500, msg: 'Failed to get extension' }, null];

package/server/utils/extensions/getMarketplaceClient.js

@@ -0,0 +1,20 @@
+import getSetting from '../settings/getSetting.js';
+import { createCatalogClient, DEFAULT_CATALOG_URL } from './catalogClient.js';
+
+const clients = new Map();
+
+/*
+  This site's marketplace client. The address is the `system:extensions_catalog_url` setting: it
+  defaults to https://extensions.kempo.dev/api/v1, and an empty value turns the marketplace off. It is
+  deliberately not one of the seeded default settings, because seeding rewrites those on every run and
+  would put a site's chosen address back.
+
+  One client per address, so its in-memory cache outlives a request.
+*/
+export default async () => {
+  const [, configured] = await getSetting('system', 'extensions_catalog_url', DEFAULT_CATALOG_URL);
+  const baseUrl = typeof configured === 'string' ? configured.trim() : DEFAULT_CATALOG_URL;
+
+  if(!clients.has(baseUrl)) clients.set(baseUrl, createCatalogClient({ baseUrl }));
+  return clients.get(baseUrl);
+};

package/server/utils/extensions/getRequirementTree.js

@@ -0,0 +1,43 @@
+import db from '../../db/index.js';
+import { extension } from '../../db/schema.js';
+
+/*
+  What an extension needs in order to run, followed all the way down: the extensions it requires, the
+  ones those require, and so on.
+
+    disabled   installed but switched off, so they have to be enabled first. Each says which of its
+               own requirements it needs, and they come dependencies-first, the order they could be
+               enabled one at a time.
+    missing    not installed at all. Enabling cannot fix that.
+
+  The mirror of getDependentTree.js, which walks the other way (what needs this one).
+*/
+export default async name => {
+  const rows = await db.select({ name: extension.name, kempo: extension.kempo, enabled: extension.enabled }).from(extension);
+  const byName = new Map(rows.map(row => [row.name, row]));
+
+  const disabled = [];
+  const missing = [];
+  const seen = new Set([name]);
+
+  const visit = current => {
+    for(const required of byName.get(current)?.kempo?.dependencies || []){
+      if(seen.has(required)) continue;
+      seen.add(required);
+
+      const row = byName.get(required);
+      if(!row){
+        missing.push(required);
+        continue;
+      }
+
+      visit(required);
+      if(!row.enabled){
+        disabled.push({ name: required, requires: (row.kempo?.dependencies || []).filter(need => byName.has(need)) });
+      }
+    }
+  };
+  visit(name);
+
+  return { disabled, missing };
+};

package/server/utils/extensions/getSiteInstallId.js

@@ -0,0 +1,21 @@
+import { randomUUID } from 'crypto';
+import getSetting from '../settings/getSetting.js';
+import setSetting from '../settings/setSetting.js';
+
+/*
+  The random id this site uses to identify itself to the extension marketplace, so an install is
+  counted once however many times an extension is uninstalled and reinstalled, and a site can have one
+  rating per extension. Generated the first time it is needed and kept in `system:install_id`.
+
+  It identifies nothing but this install: no name, address or account is attached to it.
+*/
+export default async () => {
+  const [error, existing] = await getSetting('system', 'install_id', null);
+  if(error) return [error, null];
+  if(typeof existing === 'string' && existing) return [null, existing];
+
+  const installId = randomUUID();
+  const [setError] = await setSetting('system', 'install_id', installId, 'string', false, 'Random id this site uses to identify itself to the extension marketplace');
+  if(setError) return [setError, null];
+  return [null, installId];
+};

package/server/utils/extensions/installExtension.js

@@ -4,7 +4,6 @@ import { eq } from 'drizzle-orm';
 import { join } from 'path';
 import { pathToFileURL } from 'url';
 import { existsSync } from 'fs';
-import { readFile } from 'fs/promises';
 import { randomUUID } from 'crypto';
 import triggerHook, { clearHandlerCache } from '../hooks/triggerHook.js';
 import createHook from '../hooks/createHook.js';
@@ -12,6 +11,7 @@ import registerDashboardWidget from '../dashboard/registerDashboardWidget.js';
 import setSetting from '../settings/setSetting.js';
 import declaredValue from '../settings/declaredValue.js';
 import createTablesFromSchema from './createTablesFromSchema.js';
+import readKempoConfig from './readKempoConfig.js';
 import { invalidateScopeCache } from './scopeCache.js';
 import { getUnmetDependencies } from './dependencies.js';
 
@@ -31,7 +31,6 @@ export default async ({ name }) => {
 
   const extRoot = join(process.cwd(), 'node_modules', name);
   const extPkgPath = join(extRoot, 'package.json');
-  const kempoConfigPath = join(extRoot, 'kempo-config.json');
 
   if(!existsSync(extPkgPath)){
     return [{ code: 404, msg: `Extension package "${name}" not found` }, null];
@@ -39,10 +38,7 @@ export default async ({ name }) => {
 
   let pkg, kempoConfig;
   try {
-    pkg = JSON.parse(await readFile(extPkgPath, 'utf-8'));
-    kempoConfig = existsSync(kempoConfigPath)
-      ? JSON.parse(await readFile(kempoConfigPath, 'utf-8'))
-      : {};
+    ({ pkg, kempo: kempoConfig } = await readKempoConfig(extRoot));
   } catch {
     return [{ code: 500, msg: `Failed to read extension package files` }, null];
   }

package/server/utils/extensions/installFromCatalog.js

@@ -0,0 +1,127 @@
+import { existsSync } from 'fs';
+import { join } from 'path';
+import db from '../../db/index.js';
+import { extension } from '../../db/schema.js';
+import { eq } from 'drizzle-orm';
+import installExtension from './installExtension.js';
+import installPackage from './installPackage.js';
+import enableExtension from './enableExtension.js';
+import { NAME_PATTERN } from './catalogClient.js';
+
+const isInstalled = async name => {
+  const [row] = await db.select({ name: extension.name }).from(extension).where(eq(extension.name, name)).limit(1);
+  return !!row;
+};
+
+const isEnabled = async name => {
+  const [row] = await db.select({ enabled: extension.enabled }).from(extension).where(eq(extension.name, name)).limit(1);
+  return row?.enabled === true;
+};
+
+const hasPackage = name => existsSync(join(process.cwd(), 'node_modules', name, 'package.json'));
+
+/*
+  Installs an extension, and with `withDependencies` every extension it needs first, in dependency
+  order, downloading what is not in node_modules. What is installed is exactly the marketplace's tree
+  for that extension: the caller is expected to have shown it to the administrator for approval.
+
+  Without `withDependencies` only `name` is installed, and a dependency that is not installed yet is
+  an error that names them (`needs`), so the caller can ask.
+
+  A package that is already in node_modules is not downloaded again. A failure part way leaves what
+  was installed before it installed (each one is a normal, complete install) and says which.
+
+  `deps` exists so the order and the refusals can be tested without npm or a database.
+*/
+export default async ({ name, withDependencies = false, client, installId, report = true, deps = {} }) => {
+  const download = deps.download || installPackage;
+  const install = deps.install || installExtension;
+  const installed = deps.isInstalled || isInstalled;
+  const enabled = deps.isEnabled || isEnabled;
+  const enable = deps.enable || enableExtension;
+  const present = deps.hasPackage || hasPackage;
+
+  if(typeof name !== 'string' || !NAME_PATTERN.test(name)){
+    return [{ code: 400, msg: 'Extension name is not a valid package name' }, null];
+  }
+
+  if(await installed(name)){
+    return [{ code: 409, msg: 'Extension is already installed' }, null];
+  }
+
+  const [detailsError, details] = await client.details(name);
+
+  /*
+    Without the marketplace an extension that is already in node_modules can still be installed the
+    way it always could be; one that has to be downloaded cannot.
+  */
+  if(detailsError){
+    if(!present(name)) return [detailsError, null];
+    const [error, data] = await install({ name });
+    if(error) return [error, null];
+    return [null, { name, installed: [name], ...data }];
+  }
+
+  const plan = [...details.dependencies.extensions, { name, version: details.version, installable: details.installable }];
+
+  for(const missing of details.dependencies.unresolved){
+    if(!(await installed(missing)) && !present(missing)){
+      return [{ code: 409, msg: `Requires ${missing}, which is not in the extension catalogue` }, null];
+    }
+  }
+
+  if(!withDependencies){
+    const needs = [];
+    for(const step of plan.slice(0, -1)){
+      if(!(await installed(step.name)) || !(await enabled(step.name))) needs.push(step.name);
+    }
+    if(needs.length){
+      return [{ code: 409, msg: `Requires ${needs.join(', ')} to be installed and enabled first`, needs }, null];
+    }
+  }
+
+  const done = [];
+  const switchedOn = [];
+  let result = {};
+
+  for(const step of withDependencies ? plan : plan.slice(-1)){
+    /*
+      Installed but switched off is not usable as a requirement. The administrator approved this chain,
+      so it is switched on rather than refused.
+    */
+    if(await installed(step.name)){
+      if(!(await enabled(step.name))){
+        const [enableError] = await enable({ name: step.name, cascade: true });
+        if(enableError) return [{ ...enableError, installed: done, enabled: switchedOn }, null];
+        switchedOn.push(step.name);
+      }
+      continue;
+    }
+
+    if(!present(step.name)){
+      if(!step.installable){
+        return [{ code: 409, msg: `"${step.name}" is listed but not published yet, so it cannot be installed`, installed: done }, null];
+      }
+      const [downloadError] = await download({ name: step.name, version: step.version || 'latest' });
+      if(downloadError) return [{ ...downloadError, installed: done }, null];
+    }
+
+    const [installError, data] = await install({ name: step.name });
+    if(installError) return [{ ...installError, installed: done }, null];
+
+    done.push(step.name);
+    if(step.name === name) result = data;
+  }
+
+  /*
+    Tell the marketplace, once per extension, after the fact and without making the install wait for
+    it: a marketplace that is down must not make an install fail.
+  */
+  if(report && installId){
+    for(const installedName of done){
+      client.report({ installId, extension: installedName }).catch(() => {});
+    }
+  }
+
+  return [null, { name, installed: done, enabled: switchedOn, ...result }];
+};

package/server/utils/extensions/installPackage.js

@@ -0,0 +1,36 @@
+import { execFile } from 'child_process';
+import { promisify } from 'util';
+import { NAME_PATTERN } from './catalogClient.js';
+
+const run = promisify(execFile);
+
+/*
+  Downloads an extension's package into this site's node_modules with `npm install`, which also adds it
+  to the site's package.json. Only an exact package name and a plain version or tag are accepted, which
+  is what makes passing them to a shell on Windows (where npm is a .cmd) safe.
+
+  Note for container deployments: this changes the running container's files. A redeploy that rebuilds
+  the image from the repository's package.json does not include it, so add the package there too.
+*/
+export default async ({ name, version = 'latest' }) => {
+  if(typeof name !== 'string' || !NAME_PATTERN.test(name)){
+    return [{ code: 400, msg: 'Extension name is not a valid package name' }, null];
+  }
+  if(typeof version !== 'string' || !/^[A-Za-z0-9._-]+$/.test(version)){
+    return [{ code: 400, msg: 'Extension version is not valid' }, null];
+  }
+
+  try {
+    await run('npm', ['install', `${name}@${version}`, '--no-audit', '--no-fund'], {
+      cwd: process.cwd(),
+      shell: process.platform === 'win32',
+      timeout: 5 * 60 * 1000,
+      maxBuffer: 10 * 1024 * 1024,
+    });
+  } catch(error) {
+    const detail = String(error.stderr || error.message || '').trim().split('\n').filter(Boolean).slice(-3).join(' ');
+    return [{ code: 500, msg: `Failed to download "${name}": ${detail}` }, null];
+  }
+
+  return [null, { name, version }];
+};

package/server/utils/extensions/listExtensions.js

@@ -2,15 +2,16 @@ import db from '../../db/index.js';
 import { extension } from '../../db/schema.js';
 import { sql } from 'drizzle-orm';
 import { join } from 'path';
-import { readFile } from 'fs/promises';
+import readKempoConfig from './readKempoConfig.js';
 
 const NODE_MODULES = join(process.cwd(), 'node_modules');
 
-const readPkg = async (name) => {
+const readConfig = async name => {
   try {
-    const raw = await readFile(join(NODE_MODULES, name, 'package.json'), 'utf-8');
-    return JSON.parse(raw);
-  } catch { return {}; }
+    return await readKempoConfig(join(NODE_MODULES, name));
+  } catch {
+    return { pkg: {}, kempo: {} };
+  }
 };
 
 export default async ({ limit = 50, offset = 0 } = {}) => {
@@ -20,13 +21,13 @@ export default async ({ limit = 50, offset = 0 } = {}) => {
       db.select({ count: sql`count(*)` }).from(extension),
     ]);
     const enriched = await Promise.all(items.map(async (ext) => {
-      const pkg = await readPkg(ext.name);
+      const { pkg, kempo } = await readConfig(ext.name);
       return {
         ...ext,
         description: pkg.description || null,
         author: pkg.author || null,
         license: pkg.license || null,
-        docs: pkg.kempo?.docs || null,
+        docs: kempo.docs || null,
       };
     }));
     return [null, { items: enriched, total: Number(count), limit, offset }];

package/server/utils/extensions/readKempoConfig.js

@@ -0,0 +1,20 @@
+import { join } from 'path';
+import { readFile } from 'fs/promises';
+
+/*
+  What an installed extension declares to kempo: its `public-scope`, `docs`, `git`, `schema`, `hooks`,
+  `permissions`, `settings`, `groups`, `dependencies` and `widgets`.
+
+  It lives in the extension's kempo-config.json, which is the one place it is written. package.json's own
+  `kempo` field is still read, as the fallback for an older extension that put everything there, and
+  kempo-config.json wins field by field. Nothing should add new keys to package.json.
+
+  Reads both files fresh each time, so an extension replaced on disk is seen without a restart.
+  Throws when package.json cannot be read, which is how a caller knows the package is not installed.
+*/
+export default async extRoot => {
+  const pkg = JSON.parse(await readFile(join(extRoot, 'package.json'), 'utf-8'));
+  const config = await readFile(join(extRoot, 'kempo-config.json'), 'utf-8').then(JSON.parse, () => ({}));
+
+  return { pkg, kempo: { ...(pkg.kempo || {}), ...config } };
+};

package/server/utils/extensions/scopeCache.js

@@ -1,18 +1,27 @@
 import db from '../../db/index.js';
 import { extension } from '../../db/schema.js';
-import { eq } from 'drizzle-orm';
 
 let cache = null;
 
 export const invalidateScopeCache = () => { cache = null; };
 
-export const getEnabledExtensions = async () => {
+/* One read of the extension table serves both questions, until an install/enable/disable/uninstall. */
+const load = async () => {
   if(cache) return cache;
   try {
-    cache = await db.select().from(extension).where(eq(extension.enabled, true));
+    const rows = await db.select().from(extension);
+    cache = {
+      enabled: rows.filter(row => row.enabled),
+      disabled: new Set(rows.filter(row => !row.enabled).map(row => row.name)),
+    };
   } catch(e) {
-    console.error('[scopeCache] DB error fetching enabled extensions:', e.message);
-    return [];
+    console.error('[scopeCache] DB error fetching extensions:', e.message);
+    return { enabled: [], disabled: new Set() };
   }
   return cache;
 };
+
+export const getEnabledExtensions = async () => (await load()).enabled;
+
+/* Names of extensions that are installed but switched off. Not-installed owners (core, a site) are never in it. */
+export const getDisabledExtensionNames = async () => (await load()).disabled;