@crombolo/server 0.11.0 → 0.12.1

package/src/adapters/schema-files.ts

@@ -8,7 +8,7 @@
 import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'
 import { join, relative } from 'node:path'
 import type { PortFailure } from '@crombolo/api-management'
-import type { ContentTypeAggregate, DataTypeModel } from '@crombolo/core'
+import type { ContentTypeAggregate, DataTypeModel, FormTypes } from '@crombolo/core'
 import { currentSchemaState, type Db, updateSchemaStateHash } from '@crombolo/data'
 import {
   type ChangeClass,
@@ -18,6 +18,7 @@ import {
   loadSchemaDirectory,
   nextSchemaVersion,
   type SchemaTypeKind,
+  type SchemaUrls,
   TYPE_KIND_FILES,
   validateSchemaSet,
   writeContentType,
@@ -32,6 +33,8 @@ export interface SchemaFiles {
   schemaDir: string
   writable: boolean
   componentAliases(): ReadonlySet<string>
+  /** The form types the site's wired bundles add. */
+  formTypes?: FormTypes
   /**
    * Called after any write, so a shared schema store outlives this container.
    * Absent when the schema is a directory on disk, which needs no publishing.
@@ -54,6 +57,8 @@ export interface SchemaFileWriter {
   writeDataType(model: DataTypeModel): Promise<void>
   removeDataType(alias: string): Promise<void>
   writeLanguages(): Promise<void>
+  /** `[urls]` in `schema.toml`, the version left as it is. */
+  writeUrlOptions(urls: SchemaUrls): Promise<void>
   /** After a move or a folder change: every file, from the database, canonically. */
   rewriteAll(): Promise<void>
 }
@@ -82,7 +87,7 @@ export function createSchemaFileWriter(
     await updateSchemaStateHash(
       db,
       state.id,
-      hashSchemaSet(loadSchemaDirectory(files.schemaDir).set),
+      hashSchemaSet(loadSchemaDirectory(files.schemaDir, { formTypes: files.formTypes }).set),
     )
     await files.onChanged?.()
   }
@@ -100,10 +105,10 @@ export function createSchemaFileWriter(
 
     async bumpVersion(classification) {
       if (!managed() || !files.writable || classification === 'none') return
-      const loaded = loadSchemaDirectory(files.schemaDir)
+      const loaded = loadSchemaDirectory(files.schemaDir, { formTypes: files.formTypes })
       const next = nextSchemaVersion(loaded.set.version, classification)
       if (next === loaded.set.version) return
-      writeFileSync(join(files.schemaDir, 'schema.toml'), writeSchemaVersion(next))
+      writeFileSync(join(files.schemaDir, 'schema.toml'), writeSchemaVersion(next, loaded.set.urls))
       logger('schema').info('Schema version {from} → {to} ({classification})', {
         from: loaded.set.version,
         to: next,
@@ -113,7 +118,10 @@ export function createSchemaFileWriter(
 
     async write(aggregate, previousAlias) {
       if (!managed() || !files.writable) return
-      const set = await exportSchemaSet(db, loadSchemaDirectory(files.schemaDir).set.version)
+      const set = await exportSchemaSet(
+        db,
+        loadSchemaDirectory(files.schemaDir, { formTypes: files.formTypes }).set.version,
+      )
       const list =
         kind === 'document'
           ? set.contentTypes
@@ -122,7 +130,10 @@ export function createSchemaFileWriter(
             : (set.memberTypes ?? [])
       const type = list.find((t) => t.key === aggregate.key)
       if (!type) return
-      const problems = validateSchemaSet(set, { componentAliases: files.componentAliases() })
+      const problems = validateSchemaSet(set, {
+        componentAliases: files.componentAliases(),
+        types: files.formTypes,
+      })
       if (problems.length > 0) {
         const detail = problems.map((p) => `${p.file}: ${p.message}`).join('; ')
         logger('schema').error('Not writing {file}: {detail}', {
@@ -146,7 +157,10 @@ export function createSchemaFileWriter(
 
     async writeDataType(model) {
       if (!managed() || !files.writable || !model.alias) return
-      const set = await exportSchemaSet(db, loadSchemaDirectory(files.schemaDir).set.version)
+      const set = await exportSchemaSet(
+        db,
+        loadSchemaDirectory(files.schemaDir, { formTypes: files.formTypes }).set.version,
+      )
       const dataType = set.dataTypes.find((d) => d.key === model.key)
       // An unchanged built-in has no file; a changed one does.
       if (!dataType) {
@@ -157,7 +171,10 @@ export function createSchemaFileWriter(
       // The same guard `write` applies to a content type: a set that does not
       // validate never reaches the disk, because the file is what every other
       // node reads and what a deploy boots from.
-      const problems = validateSchemaSet(set, { componentAliases: files.componentAliases() })
+      const problems = validateSchemaSet(set, {
+        componentAliases: files.componentAliases(),
+        types: files.formTypes,
+      })
       if (problems.length > 0) {
         const detail = problems.map((p) => `${p.file}: ${p.message}`).join('; ')
         logger('schema').error('Not writing {file}: {detail}', {
@@ -182,16 +199,29 @@ export function createSchemaFileWriter(
 
     async writeLanguages() {
       if (!managed() || !files.writable) return
-      const set = await exportSchemaSet(db, loadSchemaDirectory(files.schemaDir).set.version)
+      const set = await exportSchemaSet(
+        db,
+        loadSchemaDirectory(files.schemaDir, { formTypes: files.formTypes }).set.version,
+      )
       writeFileSync(join(files.schemaDir, 'languages.toml'), writeLanguages(set.languages))
       await settle()
     },
 
+    async writeUrlOptions(urls) {
+      if (!managed() || !files.writable) return
+      const { version } = loadSchemaDirectory(files.schemaDir, { formTypes: files.formTypes }).set
+      writeFileSync(join(files.schemaDir, 'schema.toml'), writeSchemaVersion(version, urls))
+      await settle()
+    },
+
     async rewriteAll() {
       if (!managed() || !files.writable) return
-      const loaded = loadSchemaDirectory(files.schemaDir)
+      const loaded = loadSchemaDirectory(files.schemaDir, { formTypes: files.formTypes })
       const set = await exportSchemaSet(db, loaded.set.version)
-      const problems = validateSchemaSet(set, { componentAliases: files.componentAliases() })
+      const problems = validateSchemaSet(set, {
+        componentAliases: files.componentAliases(),
+        types: files.formTypes,
+      })
       if (problems.length > 0) {
         const detail = problems.map((p) => `${p.file}: ${p.message}`).join('; ')
         logger('schema').error('Not rewriting the schema files: {detail}', { detail })

package/src/adapters/schema-proposals.ts

@@ -18,6 +18,7 @@ import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'
 import { dirname, join } from 'node:path'
 import type { SchemaProposalPort } from '@crombolo/assistant'
 import type { Db } from '@crombolo/data'
+import type { SyncReport } from '@crombolo/schema'
 import {
   fileNameFor,
   loadSchemaDirectory,
@@ -32,6 +33,7 @@ import {
   writeSchemaVersion,
 } from '@crombolo/schema'
 import type { CromboloConfig } from '../config.ts'
+import { formTypesFor } from '../form-types.ts'
 import { logger } from '../logging.ts'
 import { componentAliasesIn, importSchema } from '../schema.ts'
 
@@ -48,6 +50,8 @@ export interface SchemaProposalOptions {
   config: CromboloConfig
   /** The node's identity and state, refreshed in place by an import. */
   boot: { nodeId: string; nodeState: { version: string; revision: string } }
+  /** After an import applies: what a change to how URLs are built needs. */
+  afterSync?(report: SyncReport | undefined): Promise<void>
 }
 
 export function createSchemaProposals(options: SchemaProposalOptions): SchemaProposalPort {
@@ -70,12 +74,12 @@ export function createSchemaProposals(options: SchemaProposalOptions): SchemaPro
           ? parsed.problems
           : [{ file, path: '', message: 'the file defines no data type' }]
       }
-      const set = loadSchemaDirectory(config.schemaDir).set
+      const set = loadSchemaDirectory(config.schemaDir, { formTypes: formTypesFor(config) }).set
       const dataTypes = [
         ...set.dataTypes.filter((existing) => existing.alias !== dataType.alias),
         dataType,
       ]
-      return validateSchemaSet({ ...set, dataTypes })
+      return validateSchemaSet({ ...set, dataTypes }, { types: formTypesFor(config) })
     }
 
     const parsed = parseContentType(file, toml)
@@ -85,14 +89,14 @@ export function createSchemaProposals(options: SchemaProposalOptions): SchemaPro
         ? parsed.problems
         : [{ file, path: '', message: 'the file defines no content type' }]
     }
-    const set = loadSchemaDirectory(config.schemaDir).set
+    const set = loadSchemaDirectory(config.schemaDir, { formTypes: formTypesFor(config) }).set
     const contentTypes = [
       ...set.contentTypes.filter((existing) => existing.alias !== contentType.alias),
       contentType,
     ]
     return validateSchemaSet(
       { ...set, contentTypes },
-      { componentAliases: componentAliasesIn(config.componentsDir) },
+      { componentAliases: componentAliasesIn(config.componentsDir), types: formTypesFor(config) },
     )
   }
 
@@ -140,9 +144,11 @@ export function createSchemaProposals(options: SchemaProposalOptions): SchemaPro
       // import lands at a version the other nodes already have and they never
       // learn they are behind.
       const ahead = await importSchema(options.db, config, options.boot, { dryRun: true })
-      const version = loadSchemaDirectory(config.schemaDir).set.version
+      const { version, urls } = loadSchemaDirectory(config.schemaDir, {
+        formTypes: formTypesFor(config),
+      }).set
       const next = nextSchemaVersion(version, ahead.classification)
-      if (next !== version) await writeFile(versionFile, writeSchemaVersion(next), 'utf8')
+      if (next !== version) await writeFile(versionFile, writeSchemaVersion(next, urls), 'utf8')
 
       const result = await importSchema(options.db, config, options.boot)
       if (result.report && result.report.action !== 'applied') {
@@ -154,6 +160,7 @@ export function createSchemaProposals(options: SchemaProposalOptions): SchemaPro
           error: result.report.reason ?? `the import was ${result.report.action}`,
         }
       }
+      await options.afterSync?.(result.report)
       log.info('{alias} applied from a proposal: {classification}, schema at {version}', {
         alias,
         classification: ahead.classification,

package/src/adapters/webhooks.ts

@@ -0,0 +1,76 @@
+/** Webhooks, shaped for the Management API: secrets masked, logs as Umbraco's client reads them. */
+import type { WebhookPort, WebhookPortView } from '@crombolo/api-management'
+import { nameBasedUuid, type WebhookLogEntry } from '@crombolo/data'
+import { envReference } from '../webhook-files.ts'
+import { MASKED, type WebhookService, type WebhookView } from '../webhooks.ts'
+
+/** A typed-in secret never leaves the server; an `env:` reference is not a secret. */
+const masked = (headers: Record<string, string>) =>
+  Object.fromEntries(
+    Object.entries(headers).map(([name, value]) => [name, envReference(value) ? value : MASKED]),
+  )
+
+function toView(webhook: WebhookView): WebhookPortView {
+  return {
+    id: webhook.key,
+    name: webhook.name,
+    description: webhook.description,
+    url: webhook.url,
+    enabled: webhook.enabled,
+    events: webhook.events,
+    contentTypeKeys: webhook.contentTypeKeys,
+    headers: masked(webhook.headers),
+  }
+}
+
+/** A bundle's handler has no webhook key, so its log rows take a stable one of their own. */
+const subscriberKey = (subscriber: string) =>
+  subscriber.startsWith('bundle:') ? nameBasedUuid(subscriber) : subscriber
+
+function toLog(entry: WebhookLogEntry) {
+  return {
+    key: entry.key,
+    webhookKey: subscriberKey(entry.subscriber),
+    statusCode: entry.exception
+      ? 'Exception'
+      : entry.statusCode === null
+        ? entry.success
+          ? 'Handled'
+          : 'Failed'
+        : String(entry.statusCode),
+    httpStatusCode: entry.statusCode,
+    isSuccessStatusCode: entry.success,
+    date: entry.date.toISOString(),
+    eventAlias: entry.eventAlias,
+    url: entry.url,
+    retryCount: entry.retryCount,
+    requestHeaders: entry.requestHeaders,
+    requestBody: entry.requestBody,
+    responseHeaders: entry.responseHeaders,
+    responseBody: entry.responseBody,
+    exceptionOccured: entry.exception,
+  }
+}
+
+export function createWebhookPort(service: WebhookService): WebhookPort {
+  return {
+    list: async () => (await service.list()).map(toView),
+    async get(id) {
+      const webhook = await service.get(id)
+      return webhook ? toView(webhook) : undefined
+    },
+    async create(input) {
+      const result = await service.create({ ...input, key: input.id ?? crypto.randomUUID() })
+      return result.ok ? { ok: true, id: result.key } : result
+    },
+    async update(id, input) {
+      const result = await service.update(id, { ...input, key: id })
+      return result.ok ? { ok: true, id: result.key } : result
+    },
+    remove: (id) => service.remove(id),
+    async logs(id, paging) {
+      const result = await service.logs(id?.toLowerCase(), paging)
+      return { total: result.total, items: result.items.map(toLog) }
+    },
+  }
+}

package/src/boot-failure.ts

@@ -0,0 +1,147 @@
+/**
+ * Why a server could not start, said so a person can fix it, and how long to
+ * wait before trying again. Pure: `crombolo start` serves the page and does the
+ * waiting. Nothing here may carry a secret — the database is named by host and
+ * database only, never by its URL.
+ */
+import { MigrationStepError } from '@crombolo/data'
+import { DatabaseNotReadyError, UpgradePendingError } from './database.ts'
+import { SchemaBootError } from './schema.ts'
+
+export interface BootFailure {
+  title: string
+  detail: string
+  /** The setting or command that fixes it. */
+  remedy: string
+}
+
+const sqlState = (error: { code?: unknown; errno?: unknown }) =>
+  typeof error.errno === 'string' ? error.errno : typeof error.code === 'string' ? error.code : ''
+
+export function describeBootFailure(error: unknown, database: string): BootFailure {
+  if (error instanceof UpgradePendingError)
+    return {
+      title: 'An upgrade is waiting',
+      detail: `The database at ${database} needs upgrading before this version can use it.`,
+      remedy: 'Run `crombolo upgrade check`, then `crombolo upgrade`. This page retries by itself.',
+    }
+  if (error instanceof DatabaseNotReadyError)
+    return {
+      title: 'The database has not been prepared yet',
+      detail: `The database at ${database} is behind this version, and this server only reads.`,
+      remedy: 'Start the server that edits the site first, or run `crombolo upgrade`.',
+    }
+  if (error instanceof MigrationStepError)
+    return {
+      title: 'An upgrade step failed',
+      detail: error.message,
+      remedy: 'Fix the cause named above; the next attempt carries on from that step.',
+    }
+  if (error instanceof SchemaBootError)
+    return {
+      title: "The site's content types could not be loaded",
+      detail: error.message,
+      remedy: 'Correct the files named above in `schema/`, then deploy again.',
+    }
+  const e = (error ?? {}) as { code?: unknown; errno?: unknown; message?: unknown }
+  const state = sqlState(e)
+  if (state === 'ERR_POSTGRES_CONNECTION_REFUSED' || state === 'ECONNREFUSED')
+    return {
+      title: "Can't reach the database",
+      detail: `Nothing answered at ${database}.`,
+      remedy:
+        'Check the host and port in `CROMBOLO_POSTGRES_URL`, and that the database server is running and reachable from here.',
+    }
+  if (state === '28P01' || state === '28000')
+    return {
+      title: 'The database refused the sign-in',
+      detail: `${database} did not accept the user name and password it was given.`,
+      remedy: 'Check the user name and password in `CROMBOLO_POSTGRES_URL`.',
+    }
+  if (state === '3D000')
+    return {
+      title: 'The database does not exist',
+      detail: `The server at ${database} has no database by that name.`,
+      remedy: 'Create the database, or correct its name in `CROMBOLO_POSTGRES_URL`.',
+    }
+  if (state === 'SQLITE_CANTOPEN')
+    return {
+      title: "Can't open the database file",
+      detail: `${database} could not be opened or created.`,
+      remedy:
+        'Check `CROMBOLO_SQLITE_FILE`, and that its folder exists and this server may write to it.',
+    }
+  if (/no connection string/i.test(String(e.message ?? '')))
+    return {
+      title: 'No database is configured',
+      detail: 'Postgres was selected, but no connection string was given.',
+      remedy: 'Set `CROMBOLO_POSTGRES_URL`, or unset `CROMBOLO_DB` to use SQLite.',
+    }
+  return {
+    title: 'The site could not start',
+    detail: 'Something unexpected stopped it. The server log has the details.',
+    remedy: 'Read the latest error in the server log.',
+  }
+}
+
+/** Waits between attempts: doubling from `baseMs`, never longer than `capMs`. */
+export function bootRetryDelays(attempts: number, baseMs = 2_000, capMs = 60_000): number[] {
+  return Array.from({ length: Math.max(0, attempts - 1) }, (_, i) =>
+    Math.min(capMs, baseMs * 2 ** i),
+  )
+}
+
+const escapeHtml = (value: string): string =>
+  value.replace(
+    /[&<>"']/g,
+    (c) => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' })[c] as string,
+  )
+
+const withCode = (value: string) =>
+  escapeHtml(value).replace(/`([^`]+)`/g, (_, code: string) => `<code>${code}</code>`)
+
+/** The page a starting server serves while it cannot start. */
+export function bootFailurePage(
+  failure: BootFailure | undefined,
+  progress: { attempt: number; attempts: number; retryInSeconds: number | undefined },
+): string {
+  const title = failure?.title ?? 'The site is starting'
+  const next =
+    progress.retryInSeconds === undefined
+      ? 'It will not try again; the server is stopping.'
+      : `Trying again in ${progress.retryInSeconds} seconds (attempt ${progress.attempt} of ${progress.attempts}).`
+  return `<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<meta name="robots" content="noindex">
+<meta http-equiv="refresh" content="10">
+<title>${escapeHtml(title)}</title>
+<style>
+  :root { color-scheme: light dark; }
+  body { margin: 0; min-height: 100vh; display: grid; place-items: center;
+    font: 16px/1.6 ui-sans-serif, system-ui, -apple-system, "Segoe UI", sans-serif;
+    background: #f6f5f3; color: #1b1b1b; }
+  main { max-width: 34rem; padding: 2.5rem 1rem; }
+  h1 { font-size: 1.6rem; margin: 0 0 .75rem; }
+  p { margin: 0 0 1rem; white-space: pre-line; }
+  code { font-size: .9em; background: #e9e7e2; padding: .1em .35em; border-radius: 3px; }
+  .fine { font-size: .85rem; color: #6b6b63; }
+  @media (prefers-color-scheme: dark) {
+    body { background: #16171a; color: #e8e6e3; }
+    code { background: #26282c; }
+    .fine { color: #9a9a93; }
+  }
+</style>
+</head>
+<body>
+<main>
+  <h1>${escapeHtml(title)}</h1>
+  ${failure ? `<p>${withCode(failure.detail)}</p>\n  <p><strong>To fix it:</strong> ${withCode(failure.remedy)}</p>` : '<p>This takes a few seconds.</p>'}
+  <p class="fine">${escapeHtml(next)}</p>
+</main>
+</body>
+</html>
+`
+}