@crombolo/server 0.0.0-stage → 0.9.1

package/src/content-type-udt.ts

@@ -0,0 +1,256 @@
+/**
+ * Content types as Umbraco's `.udt` XML, the format its Export and Import
+ * buttons exchange — so a type exported from an Umbraco site imports here and
+ * the other way round.
+ *
+ * The element names are Umbraco's `EntityXmlSerializer`: `Info`, `Structure`,
+ * `GenericProperties` and `Tabs`, with the root naming the kind. Aliases are
+ * the currency throughout — a composition, an allowed child and a template are
+ * all referenced by alias, because keys differ between sites.
+ */
+import type { ContentTypeAggregate } from '@crombolo/core'
+import type { ContentTypeKind } from '@crombolo/data'
+import { escapeText, parseXml, type XmlElement } from './udt.ts'
+
+const ROOT = {
+  document: 'DocumentType',
+  media: 'MediaType',
+  member: 'MemberType',
+} as const satisfies Record<ContentTypeKind, string>
+
+const KIND_OF: Record<string, ContentTypeKind> = {
+  DocumentType: 'document',
+  MediaType: 'media',
+  MemberType: 'member',
+}
+
+/** Umbraco writes .NET's `True`/`False` and its `ContentVariation` names. */
+const bool = (value: boolean) => (value ? 'True' : 'False')
+
+function variations(aggregate: { variesByCulture: boolean; variesBySegment: boolean }): string {
+  if (aggregate.variesByCulture && aggregate.variesBySegment) return 'CultureAndSegment'
+  if (aggregate.variesByCulture) return 'Culture'
+  if (aggregate.variesBySegment) return 'Segment'
+  return 'Nothing'
+}
+
+const tag = (name: string, value: string | null | undefined, indent: string) =>
+  value === null || value === undefined ? '' : `${indent}<${name}>${escapeText(value)}</${name}>\n`
+
+export interface UdtContext {
+  /** Alias of a content type by key, for compositions and allowed children. */
+  aliasOf(key: string): string | undefined
+  /** Alias of a template by key, for the allowed and default templates. */
+  templateAliasOf(key: string): string | undefined
+}
+
+export function writeContentTypeUdt(
+  aggregate: ContentTypeAggregate,
+  kind: ContentTypeKind,
+  context: UdtContext,
+): string {
+  const root = ROOT[kind]
+  const groupOf = new Map(aggregate.containers.map((container) => [container.key, container]))
+
+  let info = ''
+  info += tag('Name', aggregate.name, '    ')
+  info += tag('Alias', aggregate.alias, '    ')
+  info += tag('Key', aggregate.key, '    ')
+  info += tag('Icon', aggregate.icon, '    ')
+  info += tag('Thumbnail', '', '    ')
+  info += tag('Description', aggregate.description ?? '', '    ')
+  info += tag('AllowAtRoot', bool(aggregate.allowedAsRoot), '    ')
+  info += tag('IsListView', bool(aggregate.collectionKey !== null), '    ')
+  info += tag('IsElement', bool(aggregate.isElement), '    ')
+  info += tag('AllowedInLibrary', bool(aggregate.allowedInLibrary), '    ')
+  info += tag('Variations', variations(aggregate), '    ')
+  if (kind === 'document') {
+    const templates = aggregate.allowedComponentKeys
+      .map((key) => context.templateAliasOf(key))
+      .filter((alias): alias is string => !!alias)
+    info += `    <AllowedTemplates>\n${templates.map((a) => tag('Template', a, '      ')).join('')}    </AllowedTemplates>\n`
+    info += tag(
+      'DefaultTemplate',
+      (aggregate.defaultComponentKey && context.templateAliasOf(aggregate.defaultComponentKey)) ||
+        '',
+      '    ',
+    )
+  }
+
+  const compositions = aggregate.compositions
+    .map((composition) => context.aliasOf(composition.contentTypeKey))
+    .filter((alias): alias is string => !!alias)
+  const structure = aggregate.allowedContentTypes
+    .map((allowed) => context.aliasOf(allowed.contentTypeKey))
+    .filter((alias): alias is string => !!alias)
+
+  const properties = [...aggregate.properties]
+    .sort((a, b) => a.sortOrder - b.sortOrder)
+    .map((property) => {
+      const group = property.containerKey ? groupOf.get(property.containerKey) : undefined
+      let body = ''
+      body += tag('Name', property.name, '      ')
+      body += tag('Alias', property.alias, '      ')
+      body += tag('Key', property.key, '      ')
+      body += tag('Definition', property.dataTypeKey, '      ')
+      if (group)
+        body += `      <Tab Alias="${escapeText(group.alias ?? '')}">${escapeText(group.name ?? '')}</Tab>\n`
+      body += tag('SortOrder', String(property.sortOrder), '      ')
+      body += tag('Mandatory', bool(property.mandatory), '      ')
+      body += tag('LabelOnTop', bool(property.labelOnTop), '      ')
+      if (property.mandatoryMessage)
+        body += tag('MandatoryMessage', property.mandatoryMessage, '      ')
+      if (property.regEx) body += tag('Validation', property.regEx, '      ')
+      if (property.regExMessage)
+        body += tag('ValidationRegExpMessage', property.regExMessage, '      ')
+      if (property.description) body += tag('Description', property.description, '      ')
+      body += tag('Variations', variations(property), '      ')
+      return `    <GenericProperty>\n${body}    </GenericProperty>\n`
+    })
+    .join('')
+
+  const tabs = [...aggregate.containers]
+    .sort((a, b) => a.sortOrder - b.sortOrder)
+    .map((container) => {
+      let body = ''
+      body += tag('Key', container.key, '      ')
+      body += tag('Type', container.type, '      ')
+      body += tag('Caption', container.name ?? '', '      ')
+      body += tag('Alias', container.alias ?? '', '      ')
+      body += tag('SortOrder', String(container.sortOrder), '      ')
+      return `    <Tab>\n${body}    </Tab>\n`
+    })
+    .join('')
+
+  return `<?xml version="1.0" encoding="utf-8"?>
+<${root}>
+  <Info>
+${info}  </Info>
+  <Compositions>
+${compositions.map((a) => tag('Composition', a, '    ')).join('')}  </Compositions>
+  <Structure>
+${structure.map((a) => tag(root, a, '    ')).join('')}  </Structure>
+  <GenericProperties>
+${properties}  </GenericProperties>
+  <Tabs>
+${tabs}  </Tabs>
+</${root}>
+`
+}
+
+const child = (element: XmlElement | undefined, name: string): XmlElement | undefined =>
+  element?.children.find((c) => c.name.toLowerCase() === name.toLowerCase())
+
+const childText = (element: XmlElement | undefined, name: string): string | undefined => {
+  if (!element) return undefined
+  const found = child(element, name)
+  return found ? found.text.trim() : undefined
+}
+
+const childrenNamed = (element: XmlElement | undefined, name: string): XmlElement[] =>
+  element ? element.children.filter((c) => c.name.toLowerCase() === name.toLowerCase()) : []
+
+export interface UdtContentType {
+  kind: ContentTypeKind
+  key: string | undefined
+  alias: string
+  name: string
+  icon: string | null
+  description: string | null
+  allowedAsRoot: boolean
+  isElement: boolean
+  allowedInLibrary: boolean
+  variesByCulture: boolean
+  variesBySegment: boolean
+  compositionAliases: string[]
+  allowedAliases: string[]
+  componentAliases: string[]
+  defaultTemplateAlias: string | null
+  containers: Array<{ key?: string; name: string; alias: string; type: string; sortOrder: number }>
+  properties: Array<{
+    key?: string
+    alias: string
+    name: string
+    dataTypeKey: string | undefined
+    tabAlias: string | null
+    sortOrder: number
+    mandatory: boolean
+    labelOnTop: boolean
+    mandatoryMessage: string | null
+    regEx: string | null
+    regExMessage: string | null
+    description: string | null
+    variesByCulture: boolean
+    variesBySegment: boolean
+  }>
+}
+
+const isTrue = (value: string | undefined) => (value ?? '').toLowerCase() === 'true'
+
+/** Reads a `.udt` document; undefined when it is not one we understand. */
+export function readContentTypeUdt(source: string): UdtContentType | undefined {
+  const document = parseXml(source)
+  // parseXml yields the root element itself; a document wrapper only appears
+  // when something precedes it.
+  const root =
+    document && KIND_OF[document.name] ? document : document?.children.find((c) => KIND_OF[c.name])
+  if (!root) return undefined
+  const kind = KIND_OF[root.name] as ContentTypeKind
+  const info = child(root, 'Info')
+  const alias = childText(info, 'Alias')
+  const name = childText(info, 'Name')
+  if (!alias || !name) return undefined
+  const variation = (element: XmlElement | undefined) => {
+    const value = (childText(element, 'Variations') ?? 'Nothing').toLowerCase()
+    return {
+      variesByCulture: value.includes('culture'),
+      variesBySegment: value.includes('segment'),
+    }
+  }
+  return {
+    kind,
+    key: childText(info, 'Key') || undefined,
+    alias,
+    name,
+    icon: childText(info, 'Icon') || null,
+    description: childText(info, 'Description') || null,
+    allowedAsRoot: isTrue(childText(info, 'AllowAtRoot')),
+    isElement: isTrue(childText(info, 'IsElement')),
+    allowedInLibrary: isTrue(childText(info, 'AllowedInLibrary')),
+    ...variation(info),
+    compositionAliases: childrenNamed(child(root, 'Compositions'), 'Composition')
+      .map((c) => c.text.trim())
+      .filter(Boolean),
+    allowedAliases: childrenNamed(child(root, 'Structure'), ROOT[kind])
+      .map((c) => c.text.trim())
+      .filter(Boolean),
+    componentAliases: childrenNamed(child(info, 'AllowedTemplates'), 'Template')
+      .map((c) => c.text.trim())
+      .filter(Boolean),
+    defaultTemplateAlias: childText(info, 'DefaultTemplate') || null,
+    containers: childrenNamed(child(root, 'Tabs'), 'Tab').map((tab) => ({
+      key: childText(tab, 'Key') || undefined,
+      name: childText(tab, 'Caption') ?? '',
+      alias: childText(tab, 'Alias') ?? '',
+      type: childText(tab, 'Type') ?? 'Tab',
+      sortOrder: Number(childText(tab, 'SortOrder') ?? 0),
+    })),
+    properties: childrenNamed(child(root, 'GenericProperties'), 'GenericProperty').map(
+      (property) => ({
+        key: childText(property, 'Key') || undefined,
+        alias: childText(property, 'Alias') ?? '',
+        name: childText(property, 'Name') ?? '',
+        dataTypeKey: childText(property, 'Definition') || undefined,
+        tabAlias: child(property, 'Tab')?.attributes.Alias ?? null,
+        sortOrder: Number(childText(property, 'SortOrder') ?? 0),
+        mandatory: isTrue(childText(property, 'Mandatory')),
+        labelOnTop: isTrue(childText(property, 'LabelOnTop')),
+        mandatoryMessage: childText(property, 'MandatoryMessage') || null,
+        regEx: childText(property, 'Validation') || null,
+        regExMessage: childText(property, 'ValidationRegExpMessage') || null,
+        description: childText(property, 'Description') || null,
+        ...variation(property),
+      }),
+    ),
+  }
+}

package/src/database.ts

@@ -0,0 +1,109 @@
+/**
+ * Database bootstrap: connect, migrate, seed.
+ *
+ * Migration and seeding run at startup rather than through the install wizard;
+ * the wizard is Phase 6. Both are idempotent, so restarting is safe.
+ */
+import { hashPassword, passwordConfigJson } from '@crombolo/auth'
+import {
+  connect,
+  cromboloPlan,
+  type Db,
+  ensureBuiltInDataTypes,
+  ensureSystemContentTypes,
+  ensureSystemMediaTypes,
+  ensureSystemMemberTypes,
+  INITIAL_STATE,
+  migrate,
+  pendingMigrations,
+  readState,
+  seedContent,
+  seedIdentity,
+} from '@crombolo/data'
+import type { CromboloConfig } from './config.ts'
+
+/** Production never auto-upgrades: an existing database with steps pending refuses to boot. */
+export class UpgradePendingError extends Error {
+  constructor(readonly steps: string[]) {
+    super(
+      `${steps.length} framework migration(s) pending on this database: ${steps.join(', ')}. ` +
+        'Production does not upgrade at boot; run `crombolo upgrade check`, then `crombolo upgrade`.',
+    )
+  }
+}
+
+export interface Bootstrap {
+  db: Db
+  /** Present only when this run created the admin account. */
+  seededAdminPassword: string | undefined
+}
+
+/** A `web` node boots against a database it does not own; `api` has to have been there first. */
+export class DatabaseNotReadyError extends Error {
+  constructor(readonly steps: string[]) {
+    super(
+      `This node runs as 'web' and does not migrate, but the database has ${steps.length} ` +
+        'migration(s) pending. Start the `api` node first, or run `crombolo upgrade`.',
+    )
+  }
+}
+
+/**
+ * Connects without migrating or seeding, for a node whose role is to read.
+ *
+ * The refusal matters more than the connection: a read-only node that quietly
+ * migrated would race the node that owns the schema, and one that quietly
+ * served an unmigrated database would answer queries against missing tables.
+ */
+export async function attachDatabase(config: CromboloConfig): Promise<Bootstrap> {
+  const db = await connect({ file: config.sqliteFile })
+  const pending = await pendingMigrations(db, cromboloPlan)
+  if (pending.length > 0) {
+    await db.close()
+    throw new DatabaseNotReadyError(pending.map((m) => m.name))
+  }
+  return { db, seededAdminPassword: undefined }
+}
+
+export async function bootstrapDatabase(config: CromboloConfig): Promise<Bootstrap> {
+  const db = await connect({ file: config.sqliteFile })
+  // pendingMigrations throws when the database is newer than this code.
+  const pending = await pendingMigrations(db, cromboloPlan)
+  const fresh = (await readState(db)) === INITIAL_STATE
+  if (pending.length > 0 && !fresh && !config.development) {
+    await db.close()
+    throw new UpgradePendingError(pending.map((m) => m.name))
+  }
+  return { db, seededAdminPassword: await installDatabase(db, config) }
+}
+
+/**
+ * Migrates and seeds — idempotent. Boot calls it after the production check
+ * above; the upgrade CLI calls it directly, since applying the framework's
+ * steps is exactly its job. Returns the admin password when this run created
+ * the account and none was configured.
+ */
+export async function installDatabase(db: Db, config: CromboloConfig): Promise<string | undefined> {
+  await migrate(db, cromboloPlan, { by: config.nodeId })
+
+  const password = config.adminPassword ?? crypto.randomUUID().replaceAll('-', '').slice(0, 20)
+  const seeded = await seedIdentity(db, {
+    admin: {
+      name: 'Administrator',
+      login: config.adminLogin,
+      email: config.adminLogin,
+      password,
+    },
+    hashPassword,
+    passwordConfig: passwordConfigJson(),
+  })
+
+  await seedContent(db)
+  // A database from an older version gains the built-ins it lacks.
+  await ensureBuiltInDataTypes(db)
+  await ensureSystemMediaTypes(db)
+  await ensureSystemMemberTypes(db)
+  await ensureSystemContentTypes(db)
+
+  return seeded && !config.adminPassword ? password : undefined
+}

package/src/diagnostics.ts

@@ -0,0 +1,143 @@
+/**
+ * Runtime diagnostics for a live node: a trace of every request, and a regular
+ * report of the heap, memory and in-flight work. Off unless configured; the dev
+ * stack turns both on (compose.yaml).
+ *
+ * The report counts in-flight requests twice, and the difference is the point.
+ * Bun's `pendingRequests` holds a request until its response has been written;
+ * ours releases it when the handler returns a `Response`. Bun's count staying
+ * high while ours is idle means responses are being returned and never finished
+ * — a stall in the transport, not in any handler.
+ */
+import { heapStats } from 'bun:jsc'
+import { logger } from './logging.ts'
+
+export interface DiagnosticsOptions {
+  /** Log every request as it arrives and as its handler returns. */
+  traceRequests: boolean
+  /** Seconds between reports; 0 turns them off. */
+  heapIntervalSeconds: number
+  /** A handler still running after this long is named in the report. */
+  slowRequestMs?: number
+}
+
+/** The part of Bun's server the report reads. */
+export interface ServerCounts {
+  readonly pendingRequests: number
+  readonly pendingWebSockets: number
+}
+
+export interface DiagnosticsSample {
+  /** What the JS heap has reserved. */
+  heapMb: number
+  /** What was live at the last collection, so it can read low just after one. */
+  liveMb: number
+  /** Memory held outside the heap by heap objects: buffers, file bodies. */
+  externalMb: number
+  rssMb: number
+  /** Requests Bun has not finished, response bodies included. */
+  pendingInBun: number | undefined
+  pendingWebSockets: number | undefined
+  /** Requests whose handler has not yet returned. */
+  inHandlers: number
+  slow: { method: string; path: string; ms: number }[]
+}
+
+type Fetch<S> = (request: Request, server: S) => Promise<Response | undefined>
+
+export interface Diagnostics {
+  /** `fetch`, traced and counted. Returns it unchanged when tracing and reports are both off. */
+  wrap<S extends ServerCounts>(fetch: Fetch<S>): Fetch<S>
+  sample(): DiagnosticsSample
+  stop(): void
+}
+
+const mb = (bytes: number) => Math.round((bytes / 1024 / 1024) * 10) / 10
+
+export function createDiagnostics(options: DiagnosticsOptions): Diagnostics {
+  const http = logger('http')
+  const log = logger('diagnostics')
+  const slowMs = options.slowRequestMs ?? 5_000
+  const inFlight = new Map<number, { method: string; path: string; started: number }>()
+  let next = 0
+  let server: ServerCounts | undefined
+
+  const sample = (): DiagnosticsSample => {
+    const heap = heapStats()
+    const now = performance.now()
+    return {
+      heapMb: mb(heap.heapCapacity),
+      liveMb: mb(heap.heapSize),
+      externalMb: mb(heap.extraMemorySize),
+      rssMb: mb(process.memoryUsage().rss),
+      pendingInBun: server?.pendingRequests,
+      pendingWebSockets: server?.pendingWebSockets,
+      inHandlers: inFlight.size,
+      slow: [...inFlight.values()]
+        .map((request) => ({
+          method: request.method,
+          path: request.path,
+          ms: Math.round(now - request.started),
+        }))
+        .filter((request) => request.ms >= slowMs)
+        .sort((a, b) => b.ms - a.ms)
+        .slice(0, 10),
+    }
+  }
+
+  const timer =
+    options.heapIntervalSeconds > 0
+      ? setInterval(() => {
+          const s = sample()
+          log.info(
+            'Heap {heapMb} MB ({liveMb} MB live), external {externalMb} MB, RSS {rssMb} MB; requests pending in Bun {pendingInBun}, in handlers {inHandlers}; websockets {pendingWebSockets}',
+            { ...s, slow: undefined },
+          )
+          for (const request of s.slow) {
+            log.warning('Still in its handler after {ms} ms: {method} {path}', request)
+          }
+        }, options.heapIntervalSeconds * 1_000)
+      : undefined
+  timer?.unref()
+
+  return {
+    wrap(fetch) {
+      if (!options.traceRequests && !timer) return fetch
+      return async (request, bunServer) => {
+        server = bunServer
+        const id = next++
+        const url = new URL(request.url)
+        const path = `${url.pathname}${url.search}`
+        const started = performance.now()
+        inFlight.set(id, { method: request.method, path, started })
+        if (options.traceRequests) http.debug('→ {method} {path}', { method: request.method, path })
+        try {
+          const response = await fetch(request, bunServer)
+          if (options.traceRequests) {
+            http.debug('← {status} {method} {path} {ms} ms', {
+              status: response?.status ?? 'upgrade',
+              method: request.method,
+              path,
+              ms: Math.round(performance.now() - started),
+            })
+          }
+          return response
+        } catch (error) {
+          http.error('✗ {method} {path} threw after {ms} ms', {
+            method: request.method,
+            path,
+            ms: Math.round(performance.now() - started),
+            error,
+          })
+          throw error
+        } finally {
+          inFlight.delete(id)
+        }
+      }
+    },
+    sample,
+    stop() {
+      if (timer) clearInterval(timer)
+    },
+  }
+}

package/src/dictionary-transfer.ts

@@ -0,0 +1,123 @@
+/**
+ * Dictionary items as a `.udt` file, for seeding one environment from another.
+ *
+ * `09-schema-as-code.md` leaves the dictionary in the database rather than in
+ * `schema/`, because translators work in production where the schema files are
+ * read-only — and promises this pair of commands as the way a lower environment
+ * gets the translations.
+ *
+ * The logic was already in the Management API adapter, tied to a temporary
+ * upload. It lives here so the endpoint and the CLI share one implementation:
+ * the same `.udt` format Umbraco uses, the same upsert by key, and the same rule
+ * that translations for languages this site does not have are skipped.
+ */
+import { type Db, DictionaryRepository } from '@crombolo/data'
+import { readUdt, type UdtItem, writeUdtAll } from './udt.ts'
+
+export interface DictionaryImportOutcome {
+  ok: boolean
+  reason?: string
+  /** Items written, counting nested children. */
+  imported: number
+  /** Translations dropped because this environment has no such language. */
+  skipped: string[]
+}
+
+/**
+ * One item and its children as `.udt`, or the whole dictionary when no key is
+ * given. `children()` returns a lighter row than the item itself, so each one
+ * is read back for its translations.
+ */
+export async function dictionaryToUdt(
+  repo: DictionaryRepository,
+  key?: string,
+): Promise<UdtItem[]> {
+  const toUdt = async (itemKey: string): Promise<UdtItem | undefined> => {
+    const item = await repo.get(itemKey)
+    if (!item) return undefined
+    const children: UdtItem[] = []
+    for (const child of await repo.children(item.key)) {
+      const nested = await toUdt(child.key)
+      if (nested) children.push(nested)
+    }
+    return { key: item.key, name: item.name, translations: item.translations, children }
+  }
+  if (key) {
+    const item = await toUdt(key)
+    return item ? [item] : []
+  }
+  const roots: UdtItem[] = []
+  for (const root of await repo.children(null)) {
+    const item = await toUdt(root.key)
+    if (item) roots.push(item)
+  }
+  return roots
+}
+
+export const writeDictionaryUdt = writeUdtAll
+
+/**
+ * Applies a `.udt` file. Items are upserted by key, so running it twice is a
+ * no-op and an item translated here keeps any language the file does not carry.
+ */
+export async function importDictionaryUdt(
+  db: Db,
+  text: string,
+  parentKey: string | null = null,
+): Promise<DictionaryImportOutcome> {
+  const repo = new DictionaryRepository(db)
+  const items = readUdt(text)
+  if (!items) return { ok: false, reason: 'not a dictionary export', imported: 0, skipped: [] }
+  if (parentKey && !(await repo.get(parentKey)))
+    return { ok: false, reason: 'the parent item does not exist', imported: 0, skipped: [] }
+
+  const languages = new Set(
+    (await db.query<{ iso_code: string }>('SELECT iso_code FROM language')).map((row) =>
+      String(row.iso_code).toLowerCase(),
+    ),
+  )
+  const skipped = new Set<string>()
+  // As Umbraco does: a translation for a language this site has not got is
+  // dropped rather than refused, so a partial language set still imports.
+  const known = (item: UdtItem) =>
+    item.translations.filter((t) => {
+      if (languages.has(t.isoCode.toLowerCase())) return true
+      skipped.add(t.isoCode)
+      return false
+    })
+
+  let imported = 0
+  const save = async (item: UdtItem, parent: string | null): Promise<void> => {
+    const existing = await repo.get(item.key)
+    const mine = known(item)
+    const result = existing
+      ? await repo.update(item.key, {
+          name: existing.name,
+          translations: [
+            ...existing.translations.filter(
+              (t) => !mine.some((n) => n.isoCode.toLowerCase() === t.isoCode.toLowerCase()),
+            ),
+            ...mine,
+          ],
+        })
+      : await repo.create({
+          key: item.key,
+          name: item.name,
+          parentKey: parent,
+          translations: mine,
+        })
+    if (result !== 'Success') return
+    imported += 1
+    for (const child of item.children) await save(child, item.key)
+  }
+
+  for (const item of items) await save(item, parentKey)
+  return imported > 0
+    ? { ok: true, imported, skipped: [...skipped] }
+    : {
+        ok: false,
+        reason: 'nothing in the file could be imported: a name is already in use',
+        imported: 0,
+        skipped: [...skipped],
+      }
+}