@crombolo/server 0.9.1 → 0.10.0

package/src/bundles.ts

@@ -2,7 +2,7 @@
  * Server-side bundle extensions: a bundle that has to answer a request, not just
  * render a screen.
  *
- * `docs/17-bundles.md` argued that server-side extensions should not exist here,
+ * `docs/bundles.md` argued that server-side extensions should not exist here,
  * and the objection was specific — loading third-party JS into a running node
  * has no isolation story, and a `role: web` node would never see a runtime
  * install. What answers it is wiring rather than sandboxing: **nothing in this
@@ -28,12 +28,20 @@
  * - A handler that throws becomes a 500 and a log line naming the bundle, never a
  *   broken request path.
  *
- * Deliberately not offered: migrations, raw SQL, and anything that writes the
- * site's files. A bundle that needs a table of its own needs a release of this
- * CMS, because a migration that arrives with a dependency is a schema no
- * `migration_history` can account for.
+ * - Every wired bundle belongs to an npm package the site declares, whose
+ *   `crombolo` manifest names the same id and declares a server half. Its name,
+ *   version and peer range come from there, and a peer range that excludes this
+ *   release refuses the boot.
+ *
+ * - A bundle that declares `tables` owns them: its own ledgered chain, written
+ *   with the declarative builder in `@crombolo/data`, and a `tables`
+ *   capability confined to them (`docs/bundles.md`, Part 4).
+ *
+ * Deliberately not offered: raw SQL, a `Db`, and anything that writes the
+ * site's files.
  */
 import { hasCultureAccess, hasSection, type Principal } from '@crombolo/api-management'
+import { BUNDLE_ID, type BundlePackage, peerRangeProblem } from '@crombolo/backoffice-host'
 import {
   type AppAlias,
   type ExtensionSectionAlias,
@@ -41,6 +49,10 @@ import {
   SECTION_ALIASES,
 } from '@crombolo/core'
 import {
+  type BundleTables,
+  type BundleTablesCapability,
+  bundleTablesCompatibility,
+  compileBundleTables,
   type Db,
   type RedirectMatchKind,
   RedirectRepository,
@@ -52,12 +64,6 @@ import { normaliseRedirectPath, type PublishedCache } from '@crombolo/render'
 /** The segment every bundle's routes hang below, inside the plugin API. */
 export const BUNDLE_API_SEGMENT = 'bundle'
 
-/**
- * A bundle's id, which is also its URL segment: lowercase, no dots or slashes, so
- * it cannot climb out of its namespace however it is spelled.
- */
-const BUNDLE_ID = /^[a-z][a-z0-9-]{1,38}$/
-
 /**
  * A route path below the bundle's namespace. Literal segments, optionally one
  * `:name` parameter each, and no leading or trailing slash — so a path cannot
@@ -133,6 +139,8 @@ export interface BundleHost {
   redirects: BundleRedirects
   documents: BundleDocuments
   log: BundleLog
+  /** The bundle's own tables, and only for a bundle that declares some. */
+  tables: BundleTablesCapability
 }
 
 export type BundleCapability = keyof BundleHost
@@ -166,9 +174,26 @@ export type { AppAlias, ExtensionSectionAlias, Principal }
 export { hasCultureAccess, hasSection }
 
 /** Every capability name, for validating what a bundle asked for. */
-export const BUNDLE_CAPABILITIES: readonly BundleCapability[] = ['redirects', 'documents', 'log']
-
-export interface BundleRequest<C extends BundleCapability = BundleCapability> {
+export const BUNDLE_CAPABILITIES: readonly BundleCapability[] = [
+  'redirects',
+  'documents',
+  'log',
+  'tables',
+]
+
+/** What a handler is handed: the declared capabilities, with `tables` typed from the declaration. */
+export type HostFor<C extends BundleCapability, T extends BundleTables> = Pick<
+  BundleHost,
+  Exclude<C, 'tables'>
+> &
+  ('tables' extends C
+    ? { tables: T extends BundleTables<infer S> ? BundleTablesCapability<S> : never }
+    : unknown)
+
+export interface BundleRequest<
+  C extends BundleCapability = BundleCapability,
+  T extends BundleTables = BundleTables,
+> {
   request: Request
   url: URL
   /** Already authenticated, and already holding the bundle's declared section. */
@@ -176,34 +201,45 @@ export interface BundleRequest<C extends BundleCapability = BundleCapability> {
   /** The `:name` segments of the matched route. */
   params: Record<string, string>
   /** Only what the bundle declared, frozen. */
-  host: Pick<BundleHost, C>
+  host: HostFor<C, T>
 }
 
-export interface ServerBundleRoute<C extends BundleCapability = BundleCapability> {
+export interface ServerBundleRoute<
+  C extends BundleCapability = BundleCapability,
+  T extends BundleTables = BundleTables,
+> {
   method: BundleMethod
   /** Below the bundle's namespace; `''` is the namespace root. */
   path: string
-  handler(context: BundleRequest<C>): Response | Promise<Response>
+  handler(context: BundleRequest<C, T>): Response | Promise<Response>
 }
 
-export interface ServerBundle<C extends BundleCapability = BundleCapability> {
-  /** URL segment and log name; unique across the site's bundles. */
+export interface ServerBundle<
+  C extends BundleCapability = BundleCapability,
+  T extends BundleTables = BundleTables,
+> {
+  /**
+   * URL segment and log name: the `crombolo.id` of the package that exports
+   * this bundle, which is where its name and version are read from.
+   */
   id: string
-  /** What the boot log and an error message call it. */
-  name: string
   /**
    * The section a caller must have access to. Every route is gated on it; there
    * is no per-route override and no public route.
    */
   section: BundleSection
   capabilities: readonly C[]
-  routes: readonly ServerBundleRoute<C>[]
+  routes: readonly ServerBundleRoute<C, T>[]
+  /** Tables the bundle owns, from `defineBundleTables`, under its own id. */
+  tables?: T
 }
 
 /** A bundle as mounted, with the prefix its routes answer below. */
 export interface MountedBundle {
   id: string
+  /** From its package's manifest. */
   name: string
+  package: { name: string; version: string; peerRange: string }
   section: BundleSection
   capabilities: readonly BundleCapability[]
   /** `<backoffice>/crombolo/api/bundle/<id>`. */
@@ -216,19 +252,40 @@ export interface MountedBundle {
  * thrown so the boot can report all of them at once; `createServerBundles`
  * turns a non-empty list into a failed boot, because a bundle that is half
  * mounted is worse than one that is absent.
+ *
+ * `packages` are the bundles the site declares as dependencies. A wired bundle
+ * must be one of them, so its version and peer range are always known.
  */
-export function validateServerBundles(bundles: readonly ServerBundle[]): string[] {
+export function validateServerBundles(
+  bundles: readonly ServerBundle[],
+  packages: readonly BundlePackage[],
+  coreVersion: string,
+): string[] {
   const problems: string[] = []
   const seen = new Set<string>()
   for (const bundle of bundles) {
-    const label = bundle.id || bundle.name || '<unnamed>'
+    const label = bundle.id || '<unnamed>'
     if (!BUNDLE_ID.test(bundle.id))
       problems.push(
         `Bundle "${label}" has the id "${bundle.id}", which is not 2–39 characters of lowercase letters, digits and hyphens starting with a letter.`,
       )
     else if (seen.has(bundle.id)) problems.push(`Two bundles claim the id "${bundle.id}".`)
-    else seen.add(bundle.id)
-    if (!bundle.name) problems.push(`Bundle "${label}" has no name.`)
+    else {
+      seen.add(bundle.id)
+      const owner = packages.find((pkg) => pkg.id === bundle.id)
+      if (!owner)
+        problems.push(
+          `Bundle "${label}" is wired in crombolo.config.ts, but no package the site depends on declares the bundle id "${bundle.id}". Add the package to package.json.`,
+        )
+      else if (!owner.carries.server)
+        problems.push(
+          `Bundle "${label}" is wired in crombolo.config.ts, but ${owner.packageName} declares no "server" in its crombolo manifest.`,
+        )
+      else {
+        const peer = peerRangeProblem(owner, coreVersion)
+        if (peer) problems.push(`Bundle "${label}" cannot run here: ${peer}.`)
+      }
+    }
     if (!(bundle.section in SECTION_ALIASES) && !isExtensionSectionAlias(bundle.section))
       problems.push(
         `Bundle "${label}" is gated on the section "${bundle.section}", which is neither a core section nor a manifest alias of the form Vendor.Section.Name.`,
@@ -238,6 +295,28 @@ export function validateServerBundles(bundles: readonly ServerBundle[]): string[
         problems.push(
           `Bundle "${label}" asked for the capability "${capability}", which no host offers.`,
         )
+    if (bundle.capabilities.includes('tables') && !bundle.tables)
+      problems.push(
+        `Bundle "${label}" asked for the capability "tables", but declares no tables of its own.`,
+      )
+    if (bundle.tables) {
+      if (bundle.tables.id !== bundle.id)
+        problems.push(
+          `Bundle "${label}" declares its tables under the id "${bundle.tables.id}"; a bundle's tables are under its own id.`,
+        )
+      const owner = packages.find((pkg) => pkg.id === bundle.id)
+      for (const problem of compileBundleTables(bundle.tables, owner?.version).problems)
+        problems.push(`Bundle "${label}" tables: ${problem}.`)
+      // Read by an operator's tooling before anything is installed, so it must
+      // say what the plan does — derived, not asserted, as core's is.
+      if (owner) {
+        const derived = bundleTablesCompatibility(bundle.tables, owner.version)
+        if (owner.field.migrations !== derived)
+          problems.push(
+            `Bundle "${label}"'s package.json declares "crombolo.migrations": ${JSON.stringify(owner.field.migrations ?? null)}, but its tables make ${owner.version} "${derived}".`,
+          )
+      }
+    }
     const routes = new Set<string>()
     for (const route of bundle.routes) {
       if (route.path !== '' && !ROUTE_PATH.test(route.path))
@@ -404,11 +483,17 @@ export function createBundleRedirects(options: {
 
 export interface ServerBundlesOptions {
   bundles: readonly ServerBundle[]
+  /** The bundle packages the site depends on, from discovery. */
+  packages: readonly BundlePackage[]
+  /** The running release, which every bundle's peer range must admit. */
+  coreVersion: string
   /** `<backoffice>/crombolo/api`, from `createBackOfficePaths`. */
   pluginApiPath: string
-  host: BundleHost
+  host: Omit<BundleHost, 'tables'>
   /** Named per bundle, so a log line says which one wrote it. */
   log(bundleId: string): BundleLog
+  /** A bundle's own tables, resolved against its schema alone; absent where none are offered. */
+  tables?(bundle: ServerBundle): BundleTablesCapability | undefined
 }
 
 export interface ServerBundles {
@@ -430,7 +515,7 @@ export interface ServerBundles {
  * would fail on its first request rather than at boot.
  */
 export function createServerBundles(options: ServerBundlesOptions): ServerBundles {
-  const problems = validateServerBundles(options.bundles)
+  const problems = validateServerBundles(options.bundles, options.packages, options.coreVersion)
   if (problems.length > 0)
     throw new Error(`The configured bundles cannot be mounted:\n- ${problems.join('\n- ')}`)
 
@@ -438,21 +523,37 @@ export function createServerBundles(options: ServerBundlesOptions): ServerBundle
   const byId = new Map(options.bundles.map((bundle) => [bundle.id, bundle]))
   const logs = new Map(options.bundles.map((bundle) => [bundle.id, options.log(bundle.id)]))
 
-  const mounted: MountedBundle[] = options.bundles.map((bundle) => ({
-    id: bundle.id,
-    name: bundle.name,
-    section: bundle.section,
-    capabilities: bundle.capabilities,
-    prefix: `${basePath}${bundle.id}`,
-    routes: bundle.routes.map((route) => ({ method: route.method, path: route.path })),
-  }))
+  const owners = new Map(options.packages.map((pkg) => [pkg.id, pkg]))
+  const mounted: MountedBundle[] = options.bundles.map((bundle) => {
+    const owner = owners.get(bundle.id) as BundlePackage
+    return {
+      id: bundle.id,
+      name: owner.name,
+      package: { name: owner.packageName, version: owner.version, peerRange: owner.peerRange },
+      section: bundle.section,
+      capabilities: bundle.capabilities,
+      prefix: `${basePath}${bundle.id}`,
+      routes: bundle.routes.map((route) => ({ method: route.method, path: route.path })),
+    }
+  })
+  const names = new Map(mounted.map((bundle) => [bundle.id, bundle.name]))
 
   /** Only the declared capabilities, frozen, so a handler cannot reach further. */
+  // Built once per bundle, each over that bundle's schema alone.
+  const tables = new Map(
+    options.bundles
+      .filter((bundle) => bundle.capabilities.includes('tables'))
+      .map((bundle) => [bundle.id, options.tables?.(bundle)]),
+  )
   const contextFor = (bundle: ServerBundle): Pick<BundleHost, BundleCapability> => {
     const host: Record<string, unknown> = {}
     for (const capability of bundle.capabilities)
       host[capability] =
-        capability === 'log' ? (logs.get(bundle.id) as BundleLog) : options.host[capability]
+        capability === 'log'
+          ? (logs.get(bundle.id) as BundleLog)
+          : capability === 'tables'
+            ? tables.get(bundle.id)
+            : options.host[capability]
     return Object.freeze(host) as Pick<BundleHost, BundleCapability>
   }
 
@@ -492,7 +593,7 @@ export function createServerBundles(options: ServerBundlesOptions): ServerBundle
           // A bundle's mistake is its own 500, named, rather than an unhandled
           // rejection in the request path.
           logs.get(bundle.id)?.error('The bundle {bundle} failed to answer {method} {path}', {
-            bundle: bundle.name,
+            bundle: names.get(bundle.id),
             method,
             path: pathname,
             error,

package/src/config.ts

@@ -205,7 +205,7 @@ export interface CromboloConfig {
    *
    * Nothing here throws: a site with no provider reports the features as
    * unavailable, with the reason, rather than failing when someone presses the
-   * button. See `email.ts` and `docs/14-configuration.md`.
+   * button. See `email.ts` and `docs/configuration.md`.
    */
   email: EmailPort | null | undefined
   /**
@@ -245,9 +245,16 @@ export interface CromboloConfig {
    * factory its package exports. Imported and listed here by hand on purpose:
    * installing a bundle never makes its server code run, so the only way code
    * reaches the request path is a committed, deployed edit to this file
-   * (`bundles.ts`, `docs/17-bundles.md`).
+   * (`bundles.ts`, `docs/bundles.md`).
    */
   bundles: ServerBundle[]
+  /**
+   * Whether installing a bundle from the Marketplace also applies the content it
+   * carries — its schema, components and content — straight after `bun add`. On
+   * by default; a load-balanced site that wants content to arrive only with a
+   * deploy turns it off, and an install can decline it for itself.
+   */
+  applyBundleContentOnInstall: boolean
   /**
    * Where the Packages section's Marketplace sends someone browsing for
    * extensions. Never framed — npm refuses to be framed — so the native view
@@ -463,6 +470,7 @@ export function loadConfig(
     ),
     redirects: [],
     trackRedirects: Bun.env.CROMBOLO_TRACK_REDIRECTS !== 'false',
+    applyBundleContentOnInstall: Bun.env.CROMBOLO_APPLY_BUNDLE_CONTENT !== 'false',
     seo: stripUndefined({
       sitemap: envFlag(Bun.env.CROMBOLO_SITEMAP),
       indexable: envFlag(Bun.env.CROMBOLO_INDEXABLE),
@@ -545,4 +553,4 @@ export function describeDatabase(
   }
 }
 
-export const VERSION = '0.9.1'
+export const VERSION = '0.10.0'

package/src/database.ts

@@ -2,7 +2,7 @@
  * Database bootstrap: connect, migrate, seed.
  *
  * Migration and seeding run at startup rather than through the install wizard;
- * the wizard is Phase 6. Both are idempotent, so restarting is safe.
+ * the wizard is story 080. Both are idempotent, so restarting is safe.
  */
 import { hashPassword, passwordConfigJson } from '@crombolo/auth'
 import {
@@ -20,13 +20,27 @@ import {
   seedContent,
   seedIdentity,
 } from '@crombolo/data'
+import { type BundleChain, migrateBundles, pendingBundleSteps } from './bundle-tables.ts'
 import type { CromboloConfig } from './config.ts'
 
 /** Production never auto-upgrades: an existing database with steps pending refuses to boot. */
 export class UpgradePendingError extends Error {
-  constructor(readonly steps: string[]) {
+  constructor(
+    readonly steps: string[],
+    /** Each bundle with steps pending, and those steps. */
+    readonly bundles: { id: string; steps: string[] }[] = [],
+  ) {
+    const parts: string[] = []
+    if (steps.length > 0)
+      parts.push(
+        `${steps.length} framework migration(s) pending on this database: ${steps.join(', ')}.`,
+      )
+    for (const bundle of bundles)
+      parts.push(
+        `The bundle "${bundle.id}" has ${bundle.steps.length} step(s) pending on this database: ${bundle.steps.join(', ')}.`,
+      )
     super(
-      `${steps.length} framework migration(s) pending on this database: ${steps.join(', ')}. ` +
+      `${parts.join(' ')} ` +
         'Production does not upgrade at boot; run `crombolo upgrade check`, then `crombolo upgrade`.',
     )
   }
@@ -65,16 +79,35 @@ export async function attachDatabase(config: CromboloConfig): Promise<Bootstrap>
   return { db, seededAdminPassword: undefined }
 }
 
-export async function bootstrapDatabase(config: CromboloConfig): Promise<Bootstrap> {
+/**
+ * `chains` are the wired bundles' tables, which only a node that mounts bundles
+ * passes: a bundle's steps pending in production refuse the boot just as
+ * core's do, and a database ahead of a bundle's code refuses it by name.
+ */
+export async function bootstrapDatabase(
+  config: CromboloConfig,
+  chains: readonly BundleChain[] = [],
+): Promise<Bootstrap> {
   const db = await connect({ file: config.sqliteFile })
-  // pendingMigrations throws when the database is newer than this code.
-  const pending = await pendingMigrations(db, cromboloPlan)
-  const fresh = (await readState(db)) === INITIAL_STATE
-  if (pending.length > 0 && !fresh && !config.development) {
+  try {
+    // pendingMigrations throws when the database is newer than this code.
+    const pending = await pendingMigrations(db, cromboloPlan)
+    const fresh = (await readState(db)) === INITIAL_STATE
+    const bundles = fresh ? [] : await pendingBundleSteps(db, chains)
+    const bundlesPending = bundles.filter((bundle) => bundle.pending.length > 0)
+    if ((pending.length > 0 || bundlesPending.length > 0) && !fresh && !config.development)
+      throw new UpgradePendingError(
+        pending.map((m) => m.name),
+        bundlesPending.map((bundle) => ({
+          id: bundle.id,
+          steps: bundle.pending.map((m) => m.name),
+        })),
+      )
+  } catch (error) {
     await db.close()
-    throw new UpgradePendingError(pending.map((m) => m.name))
+    throw error
   }
-  return { db, seededAdminPassword: await installDatabase(db, config) }
+  return { db, seededAdminPassword: await installDatabase(db, config, chains) }
 }
 
 /**
@@ -83,8 +116,14 @@ export async function bootstrapDatabase(config: CromboloConfig): Promise<Bootstr
  * steps is exactly its job. Returns the admin password when this run created
  * the account and none was configured.
  */
-export async function installDatabase(db: Db, config: CromboloConfig): Promise<string | undefined> {
+export async function installDatabase(
+  db: Db,
+  config: CromboloConfig,
+  chains: readonly BundleChain[] = [],
+): Promise<string | undefined> {
   await migrate(db, cromboloPlan, { by: config.nodeId })
+  // Core first, so a bundle step always runs against a core it declared itself compatible with.
+  await migrateBundles(db, chains, { by: config.nodeId })
 
   const password = config.adminPassword ?? crypto.randomUUID().replaceAll('-', '').slice(0, 20)
   const seeded = await seedIdentity(db, {

package/src/dictionary-transfer.ts

@@ -1,7 +1,7 @@
 /**
  * Dictionary items as a `.udt` file, for seeding one environment from another.
  *
- * `09-schema-as-code.md` leaves the dictionary in the database rather than in
+ * `schema-as-code.md` leaves the dictionary in the database rather than in
  * `schema/`, because translators work in production where the schema files are
  * read-only — and promises this pair of commands as the way a lower environment
  * gets the translations.

package/src/email.ts

@@ -221,7 +221,7 @@ export const EMAIL_DEPENDENT_FEATURES = [
 ]
 
 export const EMAIL_SETUP_HINT =
-  'Set CROMBOLO_EMAIL_PROVIDER to resend, postmark or custom with its key and CROMBOLO_EMAIL_FROM, or pass `email` in crombolo.config.ts. See docs/14-configuration.md.'
+  'Set CROMBOLO_EMAIL_PROVIDER to resend, postmark or custom with its key and CROMBOLO_EMAIL_FROM, or pass `email` in crombolo.config.ts. See docs/configuration.md.'
 
 export function emailAvailability(port: EmailPort | undefined): EmailAvailability {
   if (!port)

package/src/extensions-notice.ts

@@ -1,7 +1,7 @@
 /**
  * The one trace App_Plugins leaves behind.
  *
- * Extensions are npm dependencies now (`docs/17-bundles.md`), and nothing
+ * Extensions are npm dependencies now (`docs/bundles.md`), and nothing
  * reads this directory any more. A site upgrading with files still in it would
  * otherwise find its extensions had quietly stopped loading, so the boot says
  * what happened. Remove at 1.0.
@@ -29,7 +29,7 @@ export function warnAboutAppPlugins(siteDir: string, log: NoticeLog): void {
       count: entries.length,
       dir,
       remedy:
-        'Backoffice extensions are npm dependencies now — install one with `bun add`, and see docs/17-bundles.md.',
+        'Backoffice extensions are npm dependencies now — install one with `bun add`, and see docs/bundles.md.',
     },
   )
 }

package/src/form-files.ts

@@ -1,5 +1,5 @@
 /**
- * Writing a form definition back to `schema/forms/`. `docs/18-forms.md`.
+ * Writing a form definition back to `schema/forms/`. `docs/forms.md`.
  *
  * The designer saves a form the way the type editors save a content type: the
  * file is the truth, so the write goes to the file and `onChanged` publishes it
@@ -234,7 +234,7 @@ export async function deleteForm(
       ],
     }
   // The entries stay: they are the business record, and deleting a definition
-  // must not take what people sent it (`docs/18-forms.md`).
+  // must not take what people sent it (`docs/forms.md`).
   rmSync(file)
   await options.onChanged?.()
   return { ok: true, file }

package/src/form-submit.ts

@@ -1,5 +1,5 @@
 /**
- * Taking a form submission. `docs/18-forms.md`.
+ * Taking a form submission. `docs/forms.md`.
  *
  * `POST /crombolo/forms/<key>`, outside the Management API because a visitor is
  * not a backoffice user and there is no operation in the vendored contract for

package/src/form-workflows.ts

@@ -1,5 +1,5 @@
 /**
- * Running a form's workflows. `docs/18-forms.md`.
+ * Running a form's workflows. `docs/forms.md`.
  *
  * The submission is already stored by the time anything here runs: these are
  * the side effects, and a side effect that fails must never cost the entry.
@@ -7,7 +7,7 @@
  * a mail server that is down for ten minutes delays an email rather than
  * losing it — and a multi-node site sends it once.
  *
- * Three types. The XSLT ones Umbraco has are gone (`docs/18-forms.md` says
+ * Three types. The XSLT ones Umbraco has are gone (`docs/forms.md` says
  * why), and Slack is `sendToUrl` with a webhook URL rather than a type of its
  * own.
  */

package/src/forms.ts

@@ -1,5 +1,5 @@
 /**
- * What the backoffice reads about forms. `docs/18-forms.md`.
+ * What the backoffice reads about forms. `docs/forms.md`.
  *
  * Non-contract, under the plugin path, because there is no form operation in
  * the vendored OpenAPI document — Umbraco Forms ships its own API and its own

package/src/index.ts

@@ -11,6 +11,9 @@ export * from './adapters/redirects.ts'
 export * from './adapters/references.ts'
 export * from './adapters/template-files.ts'
 export * from './backup.ts'
+export * from './bundle-install.ts'
+export * from './bundle-sections.ts'
+export * from './bundle-tables.ts'
 export * from './bundles.ts'
 export * from './components.ts'
 export * from './config.ts'