@crombolo/server 0.9.1 → 0.11.0

package/src/server.ts

@@ -1,13 +1,18 @@
 /**
  * The Crombolo server.
  *
- * Route order matters and mirrors docs/01-architecture.md: static assets, the
+ * Route order matters and mirrors docs/architecture.md: static assets, the
  * non-contract security and branding endpoints, the contract-first Management
  * API, the SPA shells, then the front-end site.
  */
 
 import { join } from 'node:path'
-import { createManagementApiRouter, hasSection, type Principal } from '@crombolo/api-management'
+import {
+  type ClientAddress,
+  createManagementApiRouter,
+  hasSection,
+  type Principal,
+} from '@crombolo/api-management'
 import {
   AuthService,
   AuthStore,
@@ -41,11 +46,13 @@ import {
 } from '@crombolo/core'
 import {
   appendCacheInstruction,
+  createTablesCapability,
   type Db,
   FormEntryRepository,
   FormWorkflowRepository,
   RedirectRepository,
   readReport,
+  TransferRunRepository,
   type WritesPaused,
 } from '@crombolo/data'
 import {
@@ -63,9 +70,13 @@ import {
 import type { SyncReport } from '@crombolo/schema'
 import type { Server } from 'bun'
 import { createNodeLookup, loadAccess } from './access.ts'
+import { createAuditHost, createAuditSink, pluginAuditPlan, recordAudit } from './adapters/audit.ts'
+import { readCreatedBundlePackage, writeCreatedBundlePackage } from './adapters/bundles.ts'
 import { createPublishedContentSource, loadDraftNode } from './adapters/documents.ts'
 import { createSchemaProposals } from './adapters/schema-proposals.ts'
-import { createBundleRedirects, createServerBundles } from './bundles.ts'
+import { installBundleContent } from './bundle-install.ts'
+import { bundleChains, bundleTablesReport } from './bundle-tables.ts'
+import { createBundleRedirects, createServerBundles, validateServerBundles } from './bundles.ts'
 import { type CromboloConfig, loadConfig, type ServerRole, VERSION } from './config.ts'
 import { attachDatabase, bootstrapDatabase } from './database.ts'
 import { createDiagnostics } from './diagnostics.ts'
@@ -198,7 +209,7 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
     }
   }
   const domainLog = logger('domains')
-  // Extensions are the site's own npm dependencies (`docs/17-bundles.md`).
+  // Extensions are the site's own npm dependencies (`docs/bundles.md`).
   // Uncached in development so an author editing one sees a reload pick it up.
   const extensions = createExtensionRegistry(config.siteDir, { cache: !config.development })
   const marketplace = createMarketplace({
@@ -207,6 +218,26 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
     marketplaceUrl: config.marketplaceUrl,
     extensions,
     siteDir: config.siteDir,
+    wired: () => bundles?.mounted.map((bundle) => bundle.id) ?? [],
+    contentApplied: async (packageName) =>
+      (await new TransferRunRepository(db).appliedForPackage(packageName))?.packageVersion ?? null,
+    applyContentOnInstall: config.applyBundleContentOnInstall,
+    applyContent: async (packageName) => {
+      const result = await installBundleContent(db, config, { bundle: packageName })
+      if (!result.runId) {
+        const reasons = [
+          ...result.problems,
+          ...result.check.outstanding.map((finding) => finding.message),
+        ]
+        return {
+          applied: false,
+          message: `Its content was not applied: ${reasons.join('; ') || 'the check refused it'}. Resolve that, then run \`crombolo bundle install ${packageName}\`.`,
+        }
+      }
+      // The content landed under a cache this process had already constructed.
+      cache.invalidate()
+      return { applied: true, runId: result.runId, message: 'Its content was applied.' }
+    },
   })
   const paths = createBackOfficePaths({ backOfficePath: config.backOfficePath })
   // Two questions, not one: what this process answers, and what it is
@@ -214,8 +245,17 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
   // `all` and `web` serve the public site.
   const servesBackOffice = config.role !== 'web'
   const owns = config.role !== 'web'
+  // Bundles are checked before the database is touched, because a bundle that
+  // owns tables is migrated at boot, and a bad declaration must not get that far.
+  const bundlePackages = servesBackOffice ? extensions.bundles().bundles : []
+  if (servesBackOffice) {
+    const problems = validateServerBundles(config.bundles, bundlePackages, VERSION)
+    if (problems.length > 0)
+      throw new Error(`The configured bundles cannot be mounted:\n- ${problems.join('\n- ')}`)
+  }
+  const chains = servesBackOffice ? bundleChains(config.bundles, bundlePackages) : []
   const { db, seededAdminPassword } = owns
-    ? await bootstrapDatabase(config)
+    ? await bootstrapDatabase(config, chains)
     : await attachDatabase(config)
   // A configured schema store is copied down before anything reads `schema/`, and
   // becomes the schema directory for this boot. Everything after here — loading,
@@ -236,10 +276,30 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
   const auth = new AuthService(new AuthStore(db))
   const cookies = { siteName: config.siteName, securePrefix: config.secureCookies }
   const names = cookieNames(cookies)
+  // The socket's peer for each request in flight, which only `Bun.serve` knows;
+  // a request that came some other way (a test, the assistant) has none.
+  const peers = new WeakMap<Request, string>()
+  const clientAddress = (request: Request): ClientAddress => ({
+    ipAddress: peers.get(request) ?? null,
+    forwardedFor: request.headers.get('x-forwarded-for'),
+  })
   const authRoutes = createAuthRoutes({
     backOfficePath: paths.backOfficePath,
     service: auth,
     cookies,
+    onAudit: (event, request) =>
+      recordAudit(db, {
+        category: 'sign-in',
+        action: event.action,
+        source: 'request',
+        userKey: event.user?.key ?? null,
+        userName: event.user?.userName ?? event.login ?? null,
+        entityType: event.user ? 'User' : null,
+        entityKey: event.user?.key ?? null,
+        entityName: event.user?.userName ?? null,
+        detail: event.detail ?? null,
+        ...clientAddress(request),
+      }),
   })
 
   /**
@@ -351,6 +411,12 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
   const bundles = servesBackOffice
     ? createServerBundles({
         bundles: config.bundles,
+        packages: bundlePackages,
+        coreVersion: VERSION,
+        tables: (bundle) => {
+          const chain = chains.find((candidate) => candidate.id === bundle.id)
+          return chain ? createTablesCapability(db, chain.compiled.schema) : undefined
+        },
         pluginApiPath: `${paths.pluginPath}/api`,
         host: {
           redirects: createBundleRedirects({ db, cache }),
@@ -495,6 +561,7 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
         schema,
         mediaFiles,
         email,
+        onSearchQueued: () => jobs.kickSearchIndexing(),
         // This node looks again at once; the others are told to. The write has
         // already landed wherever `componentsDir` points, so there is nothing to
         // distribute — only the fact that it happened.
@@ -517,11 +584,15 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
     authenticate,
     nodes: createNodeLookup(db),
     allowLocalLogin: config.allowLocalLogin,
+    audit: createAuditSink(db),
+    clientAddress,
     onNotImplemented: config.development
       ? (operation, request) => notImplemented.record(operation, request)
       : undefined,
     onServerEvent: (event) => {
       events.broadcast('notify', [event])
+      // Saves, deletes and moves all announce themselves; whatever they queued gets indexed now.
+      jobs.kickSearchIndexing()
       // An open preview of the document reloads when it changes.
       if (event.eventSource === 'Umbraco:CMS:Document' && event.eventType !== 'Deleted')
         previewHub.broadcast('refreshed', [event.key])
@@ -566,6 +637,7 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
     nodeState: schema.nodeState,
     nodeId: schema.nodeId,
     versionCleanup: config.versionCleanup,
+    auditRetentionDays: config.auditRetentionDays,
     mediaFiles,
     // The workflow queue drains here rather than in the request: a submission is
     // stored first, and an e-mail that cannot be sent yet must not cost it.
@@ -592,7 +664,7 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
     nodeId: schema.nodeId,
     role: config.role,
   })
-  // App_Plugins is gone (`docs/17-bundles.md`). Saying so beats a site whose
+  // App_Plugins is gone (`docs/bundles.md`). Saying so beats a site whose
   // extensions silently stopped loading after an upgrade; remove at 1.0.
   warnAboutAppPlugins(config.siteDir, log)
   noticeAboutEmail(emailStatus, logger('email'), { development: config.development })
@@ -640,6 +712,9 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
 
   // Imported only when configured, like the assistant: an unconfigured site has no
   // git route and no credentials anywhere near it.
+  const auditPath = `${paths.pluginPath}/api/audit`
+  const auditHost = createAuditHost(db, config.auditRetentionDays)
+
   const gitPath = `${paths.pluginPath}/api/git`
   const git =
     config.git && servesBackOffice
@@ -674,7 +749,10 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
     // there — they are not routes at all, and 404 like any other unknown path.
     if (servesBackOffice) {
       const response = await serveBackOffice(request, url)
-      if (response) return response
+      if (response) {
+        if (response.ok) await auditPluginWrite(request, pathname)
+        return response
+      }
     }
 
     if (pathname.startsWith(`${MEMBER_ROUTE_PREFIX}/`)) {
@@ -709,6 +787,27 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
     return renderSite(request, pathname, url.host)
   }
 
+  /**
+   * The audit entry for a successful write to one of Crombolo's own endpoints,
+   * which the contract router does not see. The assistant's are left out: what it
+   * changes goes through the router, and is recorded there. docs/audit-log.md.
+   */
+  async function auditPluginWrite(request: Request, pathname: string): Promise<void> {
+    const prefix = `${paths.pluginPath}/api/`
+    if (request.method === 'GET' || request.method === 'HEAD') return
+    if (!pathname.startsWith(prefix) || pathname.startsWith(`${assistantPath}/`)) return
+    const principal = await authenticate(request)
+    if (!principal) return
+    const plan = pluginAuditPlan(request.method, pathname.slice(prefix.length))
+    await recordAudit(db, {
+      ...plan,
+      source: 'request',
+      userKey: principal.id,
+      userName: principal.name,
+      ...clientAddress(request),
+    })
+  }
+
   /**
    * The backoffice, the Management API and everything that serves an editor.
    *
@@ -847,24 +946,44 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
         return Response.json({
           url: marketplace.url(),
           keyword: config.packageKeyword,
+          appliesContent: config.applyBundleContentOnInstall,
           items: await marketplace.search(url.searchParams.get('q') ?? undefined),
         })
       }
 
+      // A created bundle's npm name and version: not in Umbraco's definition, so
+      // not in the contract, and set from the Created view beside it.
+      const created = /^created\/([0-9a-f-]{36})\/package$/i.exec(action)
+      if (created) {
+        const id = created[1] as string
+        if (request.method === 'GET') {
+          const found = await readCreatedBundlePackage(db, id)
+          return found ? Response.json(found) : new Response('Not Found', { status: 404 })
+        }
+        if (request.method === 'PUT') {
+          const body = (await request.json().catch(() => ({}))) as Record<string, unknown>
+          const outcome = await writeCreatedBundlePackage(db, id, body)
+          return Response.json(outcome, { status: outcome.ok ? 200 : 400 })
+        }
+      }
+
       if (action === 'installed' && request.method === 'GET') {
-        return Response.json({ items: marketplace.installed() })
+        return Response.json({ items: await marketplace.installed() })
       }
 
       if ((action === 'install' || action === 'uninstall') && request.method === 'POST') {
         const body = (await request.json().catch(() => ({}))) as {
           name?: unknown
           version?: unknown
+          applyContent?: unknown
         }
         const name = typeof body.name === 'string' ? body.name : ''
         const version = typeof body.version === 'string' ? body.version : undefined
         const outcome =
           action === 'install'
-            ? await marketplace.install(name, version)
+            ? await marketplace.install(name, version, {
+                applyContent: body.applyContent !== false,
+              })
             : await marketplace.uninstall(name)
         log.info('{user} {action} the package {name}: {outcome}', {
           user: principal.email,
@@ -901,7 +1020,7 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
     /**
      * Non-contract: forms and their entries. There is no form operation in the
      * vendored contract — Umbraco Forms ships its own API and its own client,
-     * and neither is here — so this is ours (`docs/18-forms.md`).
+     * and neither is here — so this is ours (`docs/forms.md`).
      *
      * Definitions are read from `schema/` per request rather than cached: the
      * files are the truth, the backoffice may have just rewritten one, and a
@@ -924,7 +1043,7 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
       const byKey = formsByKey(definitions)
       // Submissions are personal data, so reading them asks for more than a
       // session. Four verbs, because designing a form and reading what people
-      // sent it are different jobs (`docs/18-forms.md`).
+      // sent it are different jobs (`docs/forms.md`).
       const may = (verb: string) => principal.isAdmin || principal.permissions.includes(verb)
       const formFiles = {
         schemaDir: config.schemaDir,
@@ -1106,6 +1225,16 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
     // and no explanation. The findings name documents and the properties that are
     // unfilled: that is the Settings dashboard's material, and the dashboard's own
     // manifest is already `Umb.Condition.SectionAlias` on Settings.
+    // Non-contract, Settings: the audit log dashboard's reads. Settings, as the
+    // dashboard is: it names who signed in from where, and every change made.
+    if (pathname === auditPath || pathname.startsWith(`${auditPath}/`)) {
+      const principal = await authenticate(request)
+      if (!principal) return new Response('Unauthorized', { status: 401 })
+      if (!hasSection(principal, 'settings')) return new Response('Forbidden', { status: 403 })
+      const response = await auditHost.handle(request, pathname.slice(auditPath.length))
+      if (response) return response
+    }
+
     if (pathname === `${paths.pluginPath}/api/change-report`) {
       const principal = await authenticate(request)
       if (!principal) return new Response('Unauthorized', { status: 401 })
@@ -1122,7 +1251,10 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
         source[f.kind] = (source[f.kind] ?? 0) + 1
         bySource[f.source] = source
       }
-      return Response.json({ health: health(), counts, bySource, findings })
+      // A bundle's tables are part of what an upgrade changes, so the dashboard
+      // that shows an upgrade's findings shows their steps too.
+      const bundles = await bundleTablesReport(db, chains)
+      return Response.json({ health: health(), counts, bySource, findings, bundles })
     }
 
     const staticMatch = resolveStaticFile(paths, pathname, {
@@ -1542,6 +1674,8 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
       port: config.port,
       maxRequestBodySize: config.maxRequestBodyBytes,
       fetch: diagnostics.wrap(async (request, bunServer: Server<HubSocketData>) => {
+        const peer = bunServer.requestIP(request)?.address
+        if (peer) peers.set(request, peer)
         const pathname = new URL(request.url).pathname
         if (pathname === hubPath || pathname === previewHubPath) {
           const principal = await authenticate(request)

package/src/status.ts

@@ -27,6 +27,7 @@ import {
   TransferRunRepository,
 } from '@crombolo/data'
 import { loadSchemaDirectory } from '@crombolo/schema'
+import { BundleConfigurationError, bundleChainsFor, pendingBundleSteps } from './bundle-tables.ts'
 import { type CromboloConfig, describeDatabase, VERSION } from './config.ts'
 import { type EmailAvailability, emailAvailability, resolveEmailPort } from './email.ts'
 import { assertViewRuntime } from './view-runtime.ts'
@@ -60,6 +61,12 @@ export interface SiteStatus {
     pending: boolean
   }
   framework: { pending: string[] }
+  /**
+   * Each wired bundle that owns tables, by id: its chain's state and the steps
+   * still to run. Only a node that mounts bundles applies them, so this is what
+   * such a node would refuse to boot over in production.
+   */
+  bundles: Record<string, { name: string; state: string | null; pending: string[] }>
   content: { documents: number; media: number; elements: number; lastImport?: string }
   /**
    * Open findings by kind. `auto` ones are reported, not counted against the
@@ -140,6 +147,7 @@ export async function siteStatus(config: CromboloConfig): Promise<SiteStatus> {
       pending: false,
     },
     framework: { pending: [] },
+    bundles: {},
     content: { documents: 0, media: 0, elements: 0 },
     findings: { blocking: 0, person: 0, auto: 0, resolved: 0 },
     versions: { crombolo: VERSION, backoffice: backofficeVersion(), bun: Bun.version },
@@ -189,6 +197,27 @@ export async function siteStatus(config: CromboloConfig): Promise<SiteStatus> {
         `${status.framework.pending.length} framework migration(s) pending: run \`crombolo upgrade\``,
       )
 
+    try {
+      const chains = bundleChainsFor(config)
+      for (const chain of chains)
+        status.bundles[chain.id] = { name: chain.name, state: null, pending: [] }
+      for (const bundle of await pendingBundleSteps(db, chains)) {
+        status.bundles[bundle.id] = {
+          name: bundle.name,
+          state: bundle.state,
+          pending: bundle.pending.map((step) => step.name),
+        }
+        if (bundle.pending.length > 0 && !config.development)
+          status.problems.push(
+            `the bundle "${bundle.id}" has ${bundle.pending.length} step(s) pending: run \`crombolo upgrade\``,
+          )
+      }
+    } catch (error) {
+      if (error instanceof BundleConfigurationError)
+        for (const problem of error.problems) status.problems.push(problem)
+      else status.problems.push((error as Error).message)
+    }
+
     const state = await currentSchemaState(db)
     status.schema.database = { version: state.version, revision: state.revision }
     if (loaded) {

package/src/tar.ts

@@ -0,0 +1,85 @@
+/**
+ * A minimal npm package tarball writer: a gzipped ustar archive with every entry
+ * under `package/`, which is the layout `npm pack` produces and `bun add
+ * ./file.tgz` and `bun publish` read (`docs/bundles.md`).
+ *
+ * No dependency: the format is 512-byte headers and padded file bodies, and
+ * `Bun.gzipSync` does the compression. Timestamps are fixed, as npm fixes them,
+ * so the same selection packs to the same bytes.
+ */
+
+export interface TarEntry {
+  /** The path inside the package, `/`-separated and never absolute. */
+  path: string
+  bytes: Uint8Array
+}
+
+const BLOCK = 512
+
+/** 1985-10-26T08:15:00Z, the mtime npm writes into every packed file. */
+const NPM_MTIME = 499162500
+
+const encoder = new TextEncoder()
+
+/** An octal field, NUL-terminated and zero-padded to its width. */
+function octal(value: number, width: number): Uint8Array {
+  return encoder.encode(`${value.toString(8).padStart(width - 1, '0')}\0`)
+}
+
+/**
+ * Splits a path across ustar's `prefix` (155 bytes) and `name` (100 bytes) at a
+ * slash, which is how a path longer than 100 bytes is written without an
+ * extension header.
+ */
+function splitPath(path: string): { name: string; prefix: string } {
+  if (encoder.encode(path).length <= 100) return { name: path, prefix: '' }
+  for (let i = path.lastIndexOf('/'); i > 0; i = path.lastIndexOf('/', i - 1)) {
+    const prefix = path.slice(0, i)
+    const name = path.slice(i + 1)
+    if (encoder.encode(prefix).length <= 155 && encoder.encode(name).length <= 100)
+      return { name, prefix }
+  }
+  throw new Error(`${path} is too long to pack`)
+}
+
+function header(path: string, size: number): Uint8Array {
+  const block = new Uint8Array(BLOCK)
+  const put = (bytes: Uint8Array, offset: number) => block.set(bytes, offset)
+  const { name, prefix } = splitPath(path)
+  put(encoder.encode(name), 0)
+  put(octal(0o644, 8), 100)
+  put(octal(0, 8), 108)
+  put(octal(0, 8), 116)
+  put(octal(size, 12), 124)
+  put(octal(NPM_MTIME, 12), 136)
+  // The checksum is computed with its own field read as spaces.
+  put(encoder.encode('        '), 148)
+  put(encoder.encode('0'), 156)
+  put(encoder.encode('ustar\0'), 257)
+  put(encoder.encode('00'), 263)
+  put(encoder.encode(prefix), 345)
+  const sum = block.reduce((total, byte) => total + byte, 0)
+  put(encoder.encode(`${sum.toString(8).padStart(6, '0')}\0 `), 148)
+  return block
+}
+
+/** The entries as an npm package tarball, each under `package/`. */
+export function writeNpmTarball(entries: readonly TarEntry[]): Uint8Array<ArrayBuffer> {
+  const parts: Uint8Array[] = []
+  for (const entry of entries) {
+    if (entry.path.startsWith('/') || entry.path.split('/').includes('..'))
+      throw new Error(`${entry.path} is not a path inside the package`)
+    parts.push(header(`package/${entry.path}`, entry.bytes.length), entry.bytes)
+    const pad = (BLOCK - (entry.bytes.length % BLOCK)) % BLOCK
+    if (pad > 0) parts.push(new Uint8Array(pad))
+  }
+  // Two empty blocks end the archive.
+  parts.push(new Uint8Array(BLOCK * 2))
+  const tar = new Uint8Array(parts.reduce((total, part) => total + part.length, 0))
+  let offset = 0
+  for (const part of parts) {
+    tar.set(part, offset)
+    offset += part.length
+  }
+  return Bun.gzipSync(tar) as Uint8Array<ArrayBuffer>
+}

package/src/zip.ts

@@ -1,6 +1,6 @@
 /**
  * A minimal zip writer, because a created package is a zip and nothing else
- * here needs one (`docs/17-bundles.md`).
+ * here needs one (`docs/bundles.md`).
  *
  * No dependency: `node:zlib` provides `deflateRawSync` and `crc32`, which is
  * everything the format needs. Entries are written in the order given, each