@crombolo/server 0.0.0-stage → 0.9.1

package/src/schema-classify.ts

@@ -0,0 +1,82 @@
+/**
+ * What a content-type change does to the content already stored.
+ *
+ * `runCheck` answers this for files against a database, which is the right shape
+ * for an import and the wrong one for a save: by the time a save has happened the
+ * database already holds the change, so there is nothing left to compare. This
+ * asks the same questions of the two aggregates a save has in hand — what the
+ * type was, and what it is about to become — so an editor's change can be
+ * classified before it is applied, and the schema version moved accordingly.
+ *
+ * The rules are the ones `packages/schema/src/check.ts` applies, kept in step
+ * with it deliberately rather than by accident:
+ *
+ * - a property's editor changed **and content exists** → breaking, because
+ *   getting there converts data
+ * - a property became mandatory **and content exists** → data-requiring
+ * - anything else that differs → additive
+ *
+ * Removing a property is none of these: the schema system retires properties
+ * rather than deleting them, and their values come back if the property does.
+ */
+import type { ContentTypeAggregate, PropertyTypeModel } from '@crombolo/core'
+import { type Db, DocumentRepository } from '@crombolo/data'
+import { type ChangeClass, typeNodeIdsCarrying } from '@crombolo/schema'
+
+/** Whether anything is stored against this type, compositions included. */
+async function hasContent(db: Db, typeKey: string): Promise<boolean> {
+  const carriers = await typeNodeIdsCarrying(db, typeKey)
+  if (carriers.length === 0) return false
+  const documents = await new DocumentRepository(db).documentsOfTypes(carriers)
+  return documents.length > 0
+}
+
+const byKeyOrAlias = (
+  properties: readonly PropertyTypeModel[],
+  property: PropertyTypeModel,
+): PropertyTypeModel | undefined =>
+  properties.find((candidate) => candidate.key === property.key) ??
+  properties.find((candidate) => candidate.alias === property.alias)
+
+/**
+ * How `after` differs from `before`, in terms of what it costs the content.
+ * `before` absent means the type is new, which can only be additive.
+ */
+export async function classifyTypeChange(
+  db: Db,
+  before: ContentTypeAggregate | undefined,
+  after: ContentTypeAggregate,
+): Promise<ChangeClass> {
+  if (!before) return 'additive'
+
+  let live: boolean | undefined
+  const contentExists = async () => {
+    live ??= await hasContent(db, before.key)
+    return live
+  }
+
+  let dataRequiring = false
+  for (const property of after.properties) {
+    const previous = byKeyOrAlias(before.properties, property)
+    if (!previous) continue
+
+    if (previous.dataTypeKey !== property.dataTypeKey && (await contentExists())) return 'breaking'
+    if (property.mandatory && !previous.mandatory && (await contentExists())) dataRequiring = true
+  }
+  if (dataRequiring) return 'data-requiring'
+
+  const same =
+    before.alias === after.alias &&
+    before.name === after.name &&
+    before.properties.length === after.properties.length &&
+    after.properties.every((property) => {
+      const previous = byKeyOrAlias(before.properties, property)
+      return (
+        previous !== undefined &&
+        previous.alias === property.alias &&
+        previous.dataTypeKey === property.dataTypeKey &&
+        previous.mandatory === property.mandatory
+      )
+    })
+  return same ? 'none' : 'additive'
+}

package/src/schema-store.ts

@@ -0,0 +1,145 @@
+/**
+ * Schema files in shared storage, so a deployed site can own its own schema.
+ *
+ * Without this, `schema/` is a directory in the image: a node cannot change it
+ * durably — the container's disk is thrown away on redeploy — so
+ * `schemaWritable` is off in production and metadata is a developer's job. With a
+ * store configured, the schema lives in S3 or Azure instead, every node reads the
+ * same copy, and an editor can change a content type on a deployed site and have
+ * it survive and reach the other nodes.
+ *
+ * **The store is materialised into a local directory at boot**, and everything
+ * downstream — loading, validation, hashing, sync, the writer — carries on
+ * against files exactly as before. `loadSchemaDirectory` is synchronous and used
+ * by the boot path, the sync, the validator and the CLI; making it async to reach
+ * a network would have rippled through all of it for no gain. The local copy is a
+ * cache, the store is the truth, and "the files are the schema" stays literally
+ * true.
+ *
+ * A store is any `MediaStore`: the same four verbs over the same key space, so
+ * the S3 and Azure implementations serve this with nothing duplicated.
+ */
+import { existsSync } from 'node:fs'
+import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises'
+import { dirname, join, relative, sep } from 'node:path'
+import type { MediaStore } from './media-store.ts'
+import { type AzureMediaStoreOptions, azureMediaStore } from './media-store-azure.ts'
+import { type S3MediaStoreOptions, s3MediaStore } from './media-store-s3.ts'
+
+export interface SchemaStore {
+  /** Named in the boot banner and in logs, e.g. `s3://bucket/schema`. */
+  readonly description: string
+  /** Every schema file, as a path relative to the schema root. */
+  list(): Promise<string[]>
+  read(key: string): Promise<string | undefined>
+  write(key: string, contents: string): Promise<void>
+  remove(key: string): Promise<void>
+}
+
+/** Schema is TOML and nothing else; anything else in the bucket is ignored. */
+const isSchemaFile = (key: string) => key.toLowerCase().endsWith('.toml')
+
+/** Keys are always `/`-separated, whatever the host platform writes. */
+const toKey = (path: string) => path.split(sep).join('/')
+
+/** Any blob store, used for schema. */
+export function schemaStoreOver(store: MediaStore): SchemaStore {
+  return {
+    description: store.description,
+    async list() {
+      return (await store.list('')).filter(isSchemaFile)
+    },
+    async read(key) {
+      const found = await store.get(key)
+      return found ? new TextDecoder().decode(await found.bytes()) : undefined
+    },
+    async write(key, contents) {
+      await store.put(key, new TextEncoder().encode(contents), { contentType: 'text/plain' })
+    },
+    remove: (key) => store.delete(key),
+  }
+}
+
+export interface S3SchemaStoreOptions extends S3MediaStoreOptions {}
+export interface AzureSchemaStoreOptions extends AzureMediaStoreOptions {}
+
+/** Schema in an S3 bucket, or anything that speaks S3. Defaults to a `schema/` folder. */
+export const s3SchemaStore = (options: S3SchemaStoreOptions): SchemaStore =>
+  schemaStoreOver(s3MediaStore({ ...options, prefix: options.prefix ?? 'schema' }))
+
+/** Schema in an Azure Blob container. Defaults to a `schema/` folder. */
+export const azureSchemaStore = (options: AzureSchemaStoreOptions): SchemaStore =>
+  schemaStoreOver(azureMediaStore({ ...options, prefix: options.prefix ?? 'schema' }))
+
+/** Every `.toml` under `dir`, as store keys. */
+async function localFiles(dir: string): Promise<string[]> {
+  if (!existsSync(dir)) return []
+  const found: string[] = []
+  const walk = async (at: string): Promise<void> => {
+    for (const entry of await readdir(at, { withFileTypes: true })) {
+      const path = join(at, entry.name)
+      if (entry.isDirectory()) await walk(path)
+      else if (isSchemaFile(entry.name)) found.push(toKey(relative(dir, path)))
+    }
+  }
+  await walk(dir)
+  return found.sort()
+}
+
+/**
+ * Copies the store into `dir`, which becomes the schema directory for this boot.
+ *
+ * A mirror, not a merge: a file the store no longer has is removed locally, so a
+ * node that restarts with a stale cache cannot resurrect a deleted content type.
+ */
+export async function materialiseSchema(store: SchemaStore, dir: string): Promise<string[]> {
+  const keys = await store.list()
+  await mkdir(dir, { recursive: true })
+
+  for (const key of keys) {
+    const contents = await store.read(key)
+    if (contents === undefined) continue
+    const path = join(dir, ...key.split('/'))
+    await mkdir(dirname(path), { recursive: true })
+    await writeFile(path, contents, 'utf8')
+  }
+
+  const wanted = new Set(keys)
+  for (const key of await localFiles(dir)) {
+    if (!wanted.has(key)) await rm(join(dir, ...key.split('/')), { force: true })
+  }
+  return keys
+}
+
+export interface PublishReport {
+  written: string[]
+  removed: string[]
+}
+
+/**
+ * Sends `dir` to the store, so a schema the backoffice just changed outlives this
+ * container. A mirror in the same way: a file deleted locally is deleted there.
+ *
+ * Unchanged files are skipped, which keeps a `rewriteAll` — every file, after a
+ * move — from rewriting the whole bucket.
+ */
+export async function publishSchema(store: SchemaStore, dir: string): Promise<PublishReport> {
+  const local = await localFiles(dir)
+  const remote = new Set(await store.list())
+  const report: PublishReport = { written: [], removed: [] }
+
+  for (const key of local) {
+    const contents = await readFile(join(dir, ...key.split('/')), 'utf8')
+    if (remote.has(key) && (await store.read(key)) === contents) continue
+    await store.write(key, contents)
+    report.written.push(key)
+  }
+
+  const wanted = new Set(local)
+  for (const key of remote) {
+    if (wanted.has(key)) continue
+    await store.remove(key)
+    report.removed.push(key)
+  }
+  return report
+}

package/src/schema.ts

@@ -0,0 +1,388 @@
+/**
+ * Schema-as-code at runtime: the boot sync, the development watcher that
+ * re-syncs on file changes, and the cache-instruction poll that keeps this
+ * node's caches coherent with the others. docs/09-schema-as-code.md.
+ */
+
+import { existsSync, watch } from 'node:fs'
+import {
+  type CacheInstructionKind,
+  cacheInstructionsAfter,
+  compareStates,
+  contractsLedgeredAfter,
+  currentSchemaState,
+  type Db,
+  ensureSystemContentTypes,
+  ensureSystemMediaTypes,
+  ensureSystemMemberTypes,
+  type Finding,
+  Locks,
+  latestCacheInstructionId,
+  latestLedgerId,
+  type NodeSchemaState,
+  readWritesPaused,
+  touchServer,
+  updateSchemaStateHash,
+  type WritesPaused,
+  writeReport,
+} from '@crombolo/data'
+import {
+  type ChangeClass,
+  type CheckReport,
+  hashSchemaSet,
+  loadSchemaDirectory,
+  loadValueMigrations,
+  runCheck,
+  type SyncOptions,
+  type SyncReport,
+  syncSchema,
+  writeKeysBack,
+} from '@crombolo/schema'
+import { listComponents } from './components.ts'
+import type { CromboloConfig } from './config.ts'
+import { logger } from './logging.ts'
+import { materialiseSchema } from './schema-store.ts'
+
+export interface SchemaBoot {
+  /** The state this node runs at: the files' version, the deploy's revision. */
+  nodeState: NodeSchemaState
+  nodeId: string
+  /** Undefined when there is no schema directory. */
+  report: SyncReport | undefined
+  /** True when the database is ahead of the files: reads only, 409 on writes. */
+  compatibilityMode: boolean
+  /** The pre-upgrade check run at boot (development), written to `change_report`. */
+  check?: CheckReport
+}
+
+/** Production boot found the site diff needs data work: that is `upgrade check`'s job, not boot's. */
+export class UpgradeRequiredError extends Error {
+  constructor(readonly check: CheckReport) {
+    super(
+      `schema/ changes need data work before they can apply (${check.outstanding.length} finding(s), ${check.classification}): ` +
+        'run `crombolo upgrade check`, then `crombolo upgrade check --fix`, then `crombolo upgrade`.',
+    )
+  }
+}
+
+/** The check over the site's diff, with its value migrations loaded. */
+export async function checkSchemaDirectory(
+  db: Db,
+  config: CromboloConfig,
+  node: { nodeId: string; revision: string },
+  options: { set?: Record<string, unknown> } = {},
+): Promise<CheckReport> {
+  const loaded = loadSchemaDirectory(config.schemaDir)
+  return runCheck(
+    db,
+    loaded,
+    { nodeId: node.nodeId, version: loaded.set.version, revision: node.revision },
+    {
+      ...syncOptionsFor(config),
+      set: options.set,
+      migrations: await loadValueMigrations(config.schemaDir),
+      backOfficePath: config.backOfficePath,
+    },
+  )
+}
+
+export class SchemaBootError extends Error {
+  constructor(readonly report: SyncReport) {
+    super(
+      report.problems.length > 0
+        ? `schema/ has problems:\n${report.problems.map((p) => `  ${p.file}: ${p.path ? `${p.path}: ` : ''}${p.message}`).join('\n')}`
+        : `schema sync refused: ${report.reason ?? 'unknown reason'}`,
+    )
+  }
+}
+
+/**
+ * The components a schema may name as templates: every one of them, at any
+ * depth, keyed by the path that is its alias.
+ */
+export function componentAliasesIn(componentsDir: string): Set<string> {
+  return new Set(listComponents(componentsDir).map((component) => component.alias))
+}
+
+export function syncOptionsFor(config: CromboloConfig): SyncOptions {
+  return {
+    mode: config.development ? 'development' : 'production',
+    componentAliases: componentAliasesIn(config.componentsDir),
+  }
+}
+
+/** One sync of the directory, with key write-back in development. */
+export async function syncSchemaDirectory(
+  db: Db,
+  config: CromboloConfig,
+  node: { nodeId: string; version?: string; revision: string },
+  options: Partial<SyncOptions> = {},
+): Promise<SyncReport> {
+  // The files are read under the lock the backoffice's schema writes hold, so a
+  // sync never applies a snapshot that a later backoffice change has overtaken.
+  return db.locks.withLock(Locks.Schema, async () => {
+    const loaded = loadSchemaDirectory(config.schemaDir)
+    const report = await syncSchema(
+      db,
+      loaded,
+      { nodeId: node.nodeId, version: loaded.set.version, revision: node.revision },
+      { ...syncOptionsFor(config), ...options },
+    )
+    if (
+      report.action === 'applied' &&
+      config.schemaWritable &&
+      report.assignedKeys.length > 0 &&
+      report.state
+    ) {
+      writeKeysBack(loaded, report.assignedKeys)
+      await updateSchemaStateHash(db, report.state.id, hashSchemaSet(loaded.set))
+    }
+    return report
+  })
+}
+
+export async function bootSchema(db: Db, config: CromboloConfig): Promise<SchemaBoot> {
+  const nodeId = config.nodeId
+  if (!existsSync(config.schemaDir)) {
+    const nodeState = { version: '0', revision: config.schemaRevision }
+    await touchServer(db, { nodeId, ...nodeState, role: config.role })
+    await ensureSystemMediaTypes(db, { linkFolderChildren: true })
+    await ensureSystemMemberTypes(db)
+    await ensureSystemContentTypes(db)
+    return { nodeState, nodeId, report: undefined, compatibilityMode: false }
+  }
+  const loaded = loadSchemaDirectory(config.schemaDir)
+  const nodeState = { version: loaded.set.version, revision: config.schemaRevision }
+  await touchServer(db, { nodeId, ...nodeState, role: config.role })
+  if (!config.syncSchemaAtBoot)
+    return { nodeState, nodeId, report: undefined, compatibilityMode: false }
+
+  const node = { nodeId, revision: config.schemaRevision }
+  if (!config.development) {
+    // A rolling deploy expands; anything needing data refuses and names the command.
+    const check = await checkSchemaDirectory(db, config, node)
+    if (check.sync.action === 'would-apply' && check.outstanding.length > 0)
+      throw new UpgradeRequiredError(check)
+  }
+  const report = await syncSchemaDirectory(db, config, node)
+  if (report.action === 'refused') throw new SchemaBootError(report)
+  // Folder allows whichever built-in media types the site has — unless a file
+  // defines Folder, and so says itself what it allows.
+  const folderInFiles = (loaded.set.mediaTypes ?? []).some((t) => t.alias === 'Folder')
+  await ensureSystemMediaTypes(db, { linkFolderChildren: !folderInFiles })
+  await ensureSystemMemberTypes(db)
+  await ensureSystemContentTypes(db)
+  const boot: SchemaBoot = {
+    nodeState,
+    nodeId,
+    report,
+    compatibilityMode: report.action === 'skipped-older',
+  }
+  if (config.development) {
+    // pending-allowed: the findings go to the dashboard, and boot continues.
+    boot.check = await checkSchemaDirectory(db, config, node)
+    await writeReport(db, boot.check.findings, { source: 'upgrade' })
+  }
+  return boot
+}
+
+export interface SchemaWatcher {
+  close(): void
+}
+
+/** Re-syncs on every change under `schema/`; development only. */
+export function watchSchema(
+  db: Db,
+  config: CromboloConfig,
+  node: { nodeId: string; revision: string },
+  onReport: (report: SyncReport) => void,
+): SchemaWatcher | undefined {
+  if (!existsSync(config.schemaDir)) return undefined
+  let timer: ReturnType<typeof setTimeout> | undefined
+  let running = false
+  let again = false
+  let closed = false
+  const run = async () => {
+    if (closed) return
+    if (running) {
+      again = true
+      return
+    }
+    running = true
+    try {
+      onReport(await syncSchemaDirectory(db, config, node))
+    } catch (error) {
+      if (!closed) logger('schema').error('Watching the schema files failed', { error })
+    } finally {
+      running = false
+      if (again) {
+        again = false
+        void run()
+      }
+    }
+  }
+  const watcher = watch(config.schemaDir, { recursive: true }, (_event, file) => {
+    if (file && !String(file).endsWith('.toml')) return
+    clearTimeout(timer)
+    timer = setTimeout(() => void run(), 150)
+  })
+  watcher.unref()
+  return {
+    close() {
+      closed = true
+      clearTimeout(timer)
+      watcher.close()
+    },
+  }
+}
+
+export interface CachePollerOptions {
+  db: Db
+  nodeId: string
+  nodeState: NodeSchemaState
+  /** Recorded in `server` on each poll, so the cluster can be read back by role. */
+  role?: string
+  /** Called for instructions other nodes appended. */
+  onInstruction(kind: CacheInstructionKind, payload: Record<string, unknown>): void
+}
+
+/** Polls `cache_instruction` from where this node came in; own instructions are skipped. */
+export class CacheInstructionPoller {
+  #options: CachePollerOptions
+  #cursor = 0
+  #timer: ReturnType<typeof setInterval> | undefined
+  #behind = false
+  #bootLedgerId = 0
+  #readsUnsafe = false
+  #paused: WritesPaused | undefined
+
+  /** True once an upgrade has moved the database past this node: reads only. */
+  get behind(): boolean {
+    return this.#behind
+  }
+
+  /**
+   * True once the node is behind and a contract — a framework contract or a
+   * site purge — has run since it booted. Expand-only changes leave an old
+   * node's reads correct, as-of its own state; a contract removes something it
+   * may still read, so only then must it stop serving.
+   */
+  get readsUnsafe(): boolean {
+    return this.#readsUnsafe
+  }
+
+  /** Set while an operator has paused editing cluster-wide. */
+  get paused(): WritesPaused | undefined {
+    return this.#paused
+  }
+
+  constructor(options: CachePollerOptions) {
+    this.#options = options
+  }
+
+  async start(intervalMs: number): Promise<void> {
+    this.#bootLedgerId = await latestLedgerId(this.#options.db)
+    this.#paused = await readWritesPaused(this.#options.db)
+    this.#cursor = await latestCacheInstructionId(this.#options.db)
+    if (intervalMs > 0) {
+      this.#timer = setInterval(() => void this.poll().catch(() => {}), intervalMs)
+      this.#timer.unref()
+    }
+  }
+
+  /** Returns how many foreign instructions were applied. */
+  async poll(): Promise<number> {
+    const { db, nodeId, nodeState, onInstruction } = this.#options
+    const instructions = await cacheInstructionsAfter(db, this.#cursor)
+    let applied = 0
+    for (const instruction of instructions) {
+      this.#cursor = Math.max(this.#cursor, instruction.id)
+      if (instruction.createdBy === nodeId) continue
+      onInstruction(instruction.kind, instruction.payload)
+      applied += 1
+    }
+    await touchServer(db, { nodeId, ...nodeState, role: this.#options.role })
+    const current = await currentSchemaState(db)
+    const behind = compareStates(nodeState, current) < 0
+    if (behind && !this.#behind) onInstruction('schema', { stateId: current.id })
+    this.#behind = behind
+    // Sticky: once a contract has passed this node, no later poll makes its
+    // reads trustworthy again — only a replacement node does.
+    if (behind && !this.#readsUnsafe)
+      this.#readsUnsafe = (await contractsLedgeredAfter(db, this.#bootLedgerId)) > 0
+    this.#paused = await readWritesPaused(db)
+    return applied
+  }
+
+  stop(): void {
+    if (this.#timer) clearInterval(this.#timer)
+    this.#timer = undefined
+  }
+}
+
+export interface SchemaImportResult {
+  /** Files copied down from the store; undefined when the schema is a directory. */
+  materialised: number | undefined
+  /** How the files compare with the live database, from the same check that gates an upgrade. */
+  classification: ChangeClass
+  /** Findings a person should see before this is applied. */
+  findings: Finding[]
+  /** Undefined for a dry run, which asks what would happen without doing it. */
+  report: SyncReport | undefined
+}
+
+/**
+ * Applies the schema files to the database at runtime, rather than only at boot.
+ *
+ * This is what makes the files the source of truth rather than a projection of
+ * the database: a TOML edited in a commit, published by another node, or written
+ * by the backoffice arrives here and is imported the same way, through one path.
+ *
+ * Deliberately never automatic. It runs when someone asks, or when an operation
+ * needs it — committing an assistant changeset, publishing type changes — because
+ * importing a half-finished schema underneath an editor mid-save is worse than
+ * importing a minute later.
+ *
+ * `nodeState` is refreshed in place on success: the poller, `/health` and the
+ * published-content source all hold that one object, and without it a node that
+ * has just imported its own change would immediately judge itself behind and
+ * drain.
+ */
+export async function importSchema(
+  db: Db,
+  config: CromboloConfig,
+  boot: { nodeId: string; nodeState: NodeSchemaState },
+  options: { dryRun?: boolean } = {},
+): Promise<SchemaImportResult> {
+  let materialised: number | undefined
+  if (config.schemaStore) {
+    materialised = (await materialiseSchema(config.schemaStore, config.schemaCacheDir)).length
+    config.schemaDir = config.schemaCacheDir
+  }
+  if (!existsSync(config.schemaDir)) {
+    return { materialised, classification: 'none', findings: [], report: undefined }
+  }
+
+  const node = { nodeId: boot.nodeId, revision: config.schemaRevision }
+  const check = await checkSchemaDirectory(db, config, node)
+  if (options.dryRun) {
+    return {
+      materialised,
+      classification: check.classification,
+      findings: check.outstanding,
+      report: undefined,
+    }
+  }
+
+  const report = await syncSchemaDirectory(db, config, node)
+  if (report.action === 'applied') {
+    boot.nodeState.version = loadSchemaDirectory(config.schemaDir).set.version
+    boot.nodeState.revision = config.schemaRevision
+  }
+  return {
+    materialised,
+    classification: check.classification,
+    findings: check.outstanding,
+    report,
+  }
+}