@crombolo/server 0.0.0-stage → 0.10.0

package/src/status.ts

@@ -0,0 +1,275 @@
+/**
+ * What an environment actually resolved to, without booting it.
+ *
+ * Everything here is already knowable — which database, which schema state,
+ * what is pending, what is outstanding — but until now only by starting a
+ * server and reading its banner, which is no use when the question is *why* it
+ * will not start.
+ *
+ * Nothing in here writes. `bootstrapDatabase` migrates and seeds, so it is
+ * deliberately not used: asking a production node how it is must never be the
+ * thing that changes it.
+ */
+
+import { existsSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { ObjectTypes } from '@crombolo/core'
+import {
+  connect,
+  cromboloPlan,
+  currentSchemaState,
+  type Db,
+  INITIAL_STATE,
+  pendingMigrations,
+  readLedger,
+  readReport,
+  readState,
+  TransferRunRepository,
+} from '@crombolo/data'
+import { loadSchemaDirectory } from '@crombolo/schema'
+import { BundleConfigurationError, bundleChainsFor, pendingBundleSteps } from './bundle-tables.ts'
+import { type CromboloConfig, describeDatabase, VERSION } from './config.ts'
+import { type EmailAvailability, emailAvailability, resolveEmailPort } from './email.ts'
+import { assertViewRuntime } from './view-runtime.ts'
+
+export interface SiteStatus {
+  site: {
+    name: string
+    dir: string
+    components: string
+    schema: string
+    /** Where views are snapshotted so an edit needs no restart. */
+    componentsCache: string
+  }
+  database: {
+    dialect: string
+    description: string
+    reachable: boolean
+    /** Why it could not be read, when it could not. */
+    problem?: string
+    installed: boolean
+  }
+  schema: {
+    /** The version in `schema/schema.toml`, and the revision this node deploys at. */
+    files?: string
+    revision: string
+    /** What the database is at, when it is installed. */
+    database?: { version: string; revision: string }
+    /** The database is ahead of the files: this node reads and refuses writes. */
+    compatibilityMode: boolean
+    /** The files are ahead: a sync or an upgrade is pending. */
+    pending: boolean
+  }
+  framework: { pending: string[] }
+  /**
+   * Each wired bundle that owns tables, by id: its chain's state and the steps
+   * still to run. Only a node that mounts bundles applies them, so this is what
+   * such a node would refuse to boot over in production.
+   */
+  bundles: Record<string, { name: string; state: string | null; pending: string[] }>
+  content: { documents: number; media: number; elements: number; lastImport?: string }
+  /**
+   * Open findings by kind. `auto` ones are reported, not counted against the
+   * site: "this page is new here" is a note, not a thing to fix.
+   */
+  findings: { blocking: number; person: number; auto: number; resolved: number }
+  versions: { crombolo: string; backoffice?: string; bun: string }
+  /**
+   * Whether e-mail can leave this site, and what is unavailable when it cannot.
+   * Not a problem — no e-mail is a valid configuration — so it is reported
+   * rather than counted against the site.
+   */
+  email: EmailAvailability
+  /** Anything that would stop this site working, in the order worth fixing. */
+  problems: string[]
+}
+
+/** `1.1.0` against `1.0.0`: which way round the two versions are. */
+function compareVersions(a: string, b: string): number {
+  const parts = (value: string) => value.split('.').map((part) => Number(part) || 0)
+  const [left, right] = [parts(a), parts(b)]
+  for (let i = 0; i < Math.max(left.length, right.length); i++) {
+    const difference = (left[i] ?? 0) - (right[i] ?? 0)
+    if (difference !== 0) return difference
+  }
+  return 0
+}
+
+async function contentCounts(db: Db): Promise<SiteStatus['content']> {
+  const rows = await db.query<{ node_object_type: string; n: number }>(
+    `SELECT node_object_type, COUNT(*) AS n FROM node
+     WHERE trashed = ? GROUP BY node_object_type`,
+    [db.dialect.boolValue(false)],
+  )
+  const of = (objectType: string) =>
+    Number(rows.find((row) => row.node_object_type === objectType)?.n ?? 0)
+  return {
+    documents: of(ObjectTypes.Document),
+    media: of(ObjectTypes.Media),
+    elements: of(ObjectTypes.Element),
+  }
+}
+
+/** The vendored client's version, which is a fact about this deployment too. */
+function backofficeVersion(): string | undefined {
+  for (const base of [join(import.meta.dir, '..', '..'), process.cwd()]) {
+    const file = join(base, 'backoffice-dist', 'package.json')
+    if (!existsSync(file)) continue
+    try {
+      return (JSON.parse(readFileSync(file, 'utf8')) as { version?: string }).version
+    } catch {
+      return undefined
+    }
+  }
+  return undefined
+}
+
+export async function siteStatus(config: CromboloConfig): Promise<SiteStatus> {
+  const loaded = existsSync(config.schemaDir) ? loadSchemaDirectory(config.schemaDir) : undefined
+  const status: SiteStatus = {
+    site: {
+      name: config.siteName,
+      dir: config.siteDir,
+      components: config.componentsDir,
+      schema: config.schemaDir,
+      componentsCache: config.componentsCacheDir,
+    },
+    database: {
+      dialect: config.dialect,
+      description: describeDatabase(config),
+      reachable: false,
+      installed: false,
+    },
+    schema: {
+      files: loaded?.set.version,
+      revision: config.schemaRevision,
+      compatibilityMode: false,
+      pending: false,
+    },
+    framework: { pending: [] },
+    bundles: {},
+    content: { documents: 0, media: 0, elements: 0 },
+    findings: { blocking: 0, person: 0, auto: 0, resolved: 0 },
+    versions: { crombolo: VERSION, backoffice: backofficeVersion(), bun: Bun.version },
+    email: emailAvailability(resolveEmailPort(config)),
+    problems: [],
+  }
+
+  // Opening a SQLite file creates it, which is a change — and the question
+  // "what is this environment?" must not be the thing that makes one.
+  // Checked here as well as at boot, so a deploy learns before it serves: a
+  // views cache the JSX runtime does not resolve from fails every render.
+  try {
+    assertViewRuntime(config.componentsCacheDir)
+  } catch (error) {
+    status.problems.push((error as Error).message.split('\n')[0] as string)
+  }
+
+  if (
+    config.dialect === 'sqlite' &&
+    config.sqliteFile !== ':memory:' &&
+    !existsSync(config.sqliteFile)
+  ) {
+    status.database.reachable = true
+    status.problems.push('there is no database yet; `crombolo start` will create one')
+    return status
+  }
+
+  let db: Db | undefined
+  try {
+    db = await connect({ file: config.sqliteFile })
+    status.database.reachable = true
+    try {
+      status.database.installed = (await readState(db)) !== INITIAL_STATE
+    } catch {
+      // Reachable, with no tables in it at all: not installed, rather than
+      // unreachable. A half-finished install looks like this.
+      status.database.installed = false
+    }
+
+    status.framework.pending = (await pendingMigrations(db, cromboloPlan)).map((m) => m.name)
+    if (!status.database.installed) {
+      status.problems.push('the database is not installed yet; `crombolo start` will install it')
+      return status
+    }
+    if (status.framework.pending.length > 0 && !config.development)
+      status.problems.push(
+        `${status.framework.pending.length} framework migration(s) pending: run \`crombolo upgrade\``,
+      )
+
+    try {
+      const chains = bundleChainsFor(config)
+      for (const chain of chains)
+        status.bundles[chain.id] = { name: chain.name, state: null, pending: [] }
+      for (const bundle of await pendingBundleSteps(db, chains)) {
+        status.bundles[bundle.id] = {
+          name: bundle.name,
+          state: bundle.state,
+          pending: bundle.pending.map((step) => step.name),
+        }
+        if (bundle.pending.length > 0 && !config.development)
+          status.problems.push(
+            `the bundle "${bundle.id}" has ${bundle.pending.length} step(s) pending: run \`crombolo upgrade\``,
+          )
+      }
+    } catch (error) {
+      if (error instanceof BundleConfigurationError)
+        for (const problem of error.problems) status.problems.push(problem)
+      else status.problems.push((error as Error).message)
+    }
+
+    const state = await currentSchemaState(db)
+    status.schema.database = { version: state.version, revision: state.revision }
+    if (loaded) {
+      const difference = compareVersions(loaded.set.version, state.version)
+      status.schema.compatibilityMode = difference < 0
+      status.schema.pending =
+        difference > 0 || (difference === 0 && config.schemaRevision > state.revision)
+      if (status.schema.compatibilityMode)
+        status.problems.push(
+          `the database is at ${state.version} and these files are ${loaded.set.version}: reads only, writes refused with 409`,
+        )
+      if (status.schema.pending)
+        status.problems.push(
+          'schema/ is ahead of the database: run `crombolo upgrade check` to see what it would do',
+        )
+      for (const problem of loaded.problems)
+        status.problems.push(`${problem.file}: ${problem.message}`)
+    }
+
+    status.content = await contentCounts(db)
+    const [run] = await new TransferRunRepository(db).recent(1)
+    if (run) status.content.lastImport = run.startedAt.toISOString()
+
+    const open = await readReport(db)
+    const all = await readReport(db, { includeResolved: true })
+    const of = (kind: string) => open.filter((finding) => finding.kind === kind).length
+    status.findings = {
+      blocking: of('blocking'),
+      person: of('person'),
+      auto: of('auto'),
+      resolved: all.length - open.length,
+    }
+    const wanting = status.findings.blocking + status.findings.person
+    if (wanting > 0)
+      status.problems.push(
+        `${wanting} finding(s) need somebody: \`crombolo upgrade check\` or the Changes dashboard`,
+      )
+    // Read so that a database whose ledger cannot be read is noticed here
+    // rather than at the next upgrade.
+    await readLedger(db)
+  } catch (error) {
+    status.database.problem = (error as Error).message
+    status.problems.push(
+      status.database.reachable
+        ? `the database could not be read: ${(error as Error).message}`
+        : `the database cannot be reached: ${(error as Error).message}`,
+    )
+  } finally {
+    // One exit, so a connection is never left open: on Postgres a leaked
+    // session blocks the next `DROP SCHEMA`, and the symptom is a test run
+    // that slows to a crawl rather than anything that looks like this.
+    await db?.close()
+  }
+  return status
+}

package/src/tar.ts

@@ -0,0 +1,85 @@
+/**
+ * A minimal npm package tarball writer: a gzipped ustar archive with every entry
+ * under `package/`, which is the layout `npm pack` produces and `bun add
+ * ./file.tgz` and `bun publish` read (`docs/bundles.md`).
+ *
+ * No dependency: the format is 512-byte headers and padded file bodies, and
+ * `Bun.gzipSync` does the compression. Timestamps are fixed, as npm fixes them,
+ * so the same selection packs to the same bytes.
+ */
+
+export interface TarEntry {
+  /** The path inside the package, `/`-separated and never absolute. */
+  path: string
+  bytes: Uint8Array
+}
+
+const BLOCK = 512
+
+/** 1985-10-26T08:15:00Z, the mtime npm writes into every packed file. */
+const NPM_MTIME = 499162500
+
+const encoder = new TextEncoder()
+
+/** An octal field, NUL-terminated and zero-padded to its width. */
+function octal(value: number, width: number): Uint8Array {
+  return encoder.encode(`${value.toString(8).padStart(width - 1, '0')}\0`)
+}
+
+/**
+ * Splits a path across ustar's `prefix` (155 bytes) and `name` (100 bytes) at a
+ * slash, which is how a path longer than 100 bytes is written without an
+ * extension header.
+ */
+function splitPath(path: string): { name: string; prefix: string } {
+  if (encoder.encode(path).length <= 100) return { name: path, prefix: '' }
+  for (let i = path.lastIndexOf('/'); i > 0; i = path.lastIndexOf('/', i - 1)) {
+    const prefix = path.slice(0, i)
+    const name = path.slice(i + 1)
+    if (encoder.encode(prefix).length <= 155 && encoder.encode(name).length <= 100)
+      return { name, prefix }
+  }
+  throw new Error(`${path} is too long to pack`)
+}
+
+function header(path: string, size: number): Uint8Array {
+  const block = new Uint8Array(BLOCK)
+  const put = (bytes: Uint8Array, offset: number) => block.set(bytes, offset)
+  const { name, prefix } = splitPath(path)
+  put(encoder.encode(name), 0)
+  put(octal(0o644, 8), 100)
+  put(octal(0, 8), 108)
+  put(octal(0, 8), 116)
+  put(octal(size, 12), 124)
+  put(octal(NPM_MTIME, 12), 136)
+  // The checksum is computed with its own field read as spaces.
+  put(encoder.encode('        '), 148)
+  put(encoder.encode('0'), 156)
+  put(encoder.encode('ustar\0'), 257)
+  put(encoder.encode('00'), 263)
+  put(encoder.encode(prefix), 345)
+  const sum = block.reduce((total, byte) => total + byte, 0)
+  put(encoder.encode(`${sum.toString(8).padStart(6, '0')}\0 `), 148)
+  return block
+}
+
+/** The entries as an npm package tarball, each under `package/`. */
+export function writeNpmTarball(entries: readonly TarEntry[]): Uint8Array<ArrayBuffer> {
+  const parts: Uint8Array[] = []
+  for (const entry of entries) {
+    if (entry.path.startsWith('/') || entry.path.split('/').includes('..'))
+      throw new Error(`${entry.path} is not a path inside the package`)
+    parts.push(header(`package/${entry.path}`, entry.bytes.length), entry.bytes)
+    const pad = (BLOCK - (entry.bytes.length % BLOCK)) % BLOCK
+    if (pad > 0) parts.push(new Uint8Array(pad))
+  }
+  // Two empty blocks end the archive.
+  parts.push(new Uint8Array(BLOCK * 2))
+  const tar = new Uint8Array(parts.reduce((total, part) => total + part.length, 0))
+  let offset = 0
+  for (const part of parts) {
+    tar.set(part, offset)
+    offset += part.length
+  }
+  return Bun.gzipSync(tar) as Uint8Array<ArrayBuffer>
+}

package/src/udt.ts

@@ -0,0 +1,168 @@
+/**
+ * Umbraco's dictionary export format (`.udt`): nested `<DictionaryItem Key Name>`
+ * elements, each with a `<Value LanguageCultureAlias>` per translation in CDATA.
+ * The reader understands only the XML this format uses: elements, attributes,
+ * text, CDATA, comments and the prolog.
+ */
+
+export interface UdtItem {
+  key: string
+  name: string
+  translations: Array<{ isoCode: string; translation: string }>
+  children: UdtItem[]
+}
+
+export interface XmlElement {
+  name: string
+  attributes: Record<string, string>
+  children: XmlElement[]
+  text: string
+}
+
+/** XML text content: the three characters that cannot appear raw. */
+export const escapeText = (value: string) =>
+  value.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;')
+
+const escapeAttribute = (value: string) =>
+  value
+    .replaceAll('&', '&amp;')
+    .replaceAll('<', '&lt;')
+    .replaceAll('>', '&gt;')
+    .replaceAll('"', '&quot;')
+
+const cdata = (value: string) => `<![CDATA[${value.replaceAll(']]>', ']]]]><![CDATA[>')}]]>`
+
+function writeItem(item: UdtItem, indent: string): string {
+  const inner = [
+    ...item.translations.map(
+      (t) =>
+        `${indent}  <Value LanguageCultureAlias="${escapeAttribute(t.isoCode)}">${cdata(t.translation)}</Value>`,
+    ),
+    ...item.children.map((child) => writeItem(child, `${indent}  `)),
+  ]
+  const open = `${indent}<DictionaryItem Key="${escapeAttribute(item.key)}" Name="${escapeAttribute(item.name)}"`
+  return inner.length === 0
+    ? `${open} />`
+    : `${open}>\n${inner.join('\n')}\n${indent}</DictionaryItem>`
+}
+
+/** One item, and its descendants when they are included, as a `.udt` document. */
+export function writeUdt(item: UdtItem): string {
+  return `<?xml version="1.0" encoding="utf-8"?>\n${writeItem(item, '')}`
+}
+
+/**
+ * Several items as one `.udt` document, under the `<DictionaryItems>` wrapper
+ * `readUdt` already accepts — which is what a whole-dictionary export needs,
+ * since the tree has more than one root.
+ */
+export function writeUdtAll(items: readonly UdtItem[]): string {
+  if (items.length === 1) return writeUdt(items[0] as UdtItem)
+  const inner = items.map((item) => writeItem(item, '  ')).join('\n')
+  return `<?xml version="1.0" encoding="utf-8"?>\n<DictionaryItems>\n${inner}\n</DictionaryItems>`
+}
+
+const ENTITIES: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }
+
+const decode = (value: string) =>
+  value.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (whole, entity: string) => {
+    if (entity.startsWith('#x') || entity.startsWith('#X'))
+      return String.fromCodePoint(Number.parseInt(entity.slice(2), 16))
+    if (entity.startsWith('#')) return String.fromCodePoint(Number.parseInt(entity.slice(1), 10))
+    return ENTITIES[entity.toLowerCase()] ?? whole
+  })
+
+/** Parses a document into its root element; undefined when it is not well formed. */
+export function parseXml(source: string): XmlElement | undefined {
+  const root: XmlElement = { name: '#document', attributes: {}, children: [], text: '' }
+  const stack: XmlElement[] = [root]
+  let i = 0
+  const top = () => stack[stack.length - 1] as XmlElement
+  while (i < source.length) {
+    const lt = source.indexOf('<', i)
+    if (lt === -1) {
+      top().text += decode(source.slice(i))
+      break
+    }
+    top().text += decode(source.slice(i, lt))
+    if (source.startsWith('<![CDATA[', lt)) {
+      const end = source.indexOf(']]>', lt)
+      if (end === -1) return undefined
+      top().text += source.slice(lt + 9, end)
+      i = end + 3
+    } else if (source.startsWith('<!--', lt)) {
+      const end = source.indexOf('-->', lt)
+      if (end === -1) return undefined
+      i = end + 3
+    } else if (source.startsWith('<?', lt) || source.startsWith('<!', lt)) {
+      const end = source.indexOf('>', lt)
+      if (end === -1) return undefined
+      i = end + 1
+    } else if (source.startsWith('</', lt)) {
+      const end = source.indexOf('>', lt)
+      if (end === -1) return undefined
+      const name = source.slice(lt + 2, end).trim()
+      const closing = stack.pop()
+      if (!closing || closing.name !== name) return undefined
+      i = end + 1
+    } else {
+      const tag = /^<([A-Za-z_][\w.:-]*)((?:\s+[\w.:-]+\s*=\s*(?:"[^"]*"|'[^']*'))*)\s*(\/?)>/.exec(
+        source.slice(lt),
+      )
+      if (!tag) return undefined
+      const element: XmlElement = {
+        name: tag[1] as string,
+        attributes: {},
+        children: [],
+        text: '',
+      }
+      for (const a of (tag[2] ?? '').matchAll(/([\w.:-]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/g))
+        element.attributes[a[1] as string] = decode(a[2] ?? a[3] ?? '')
+      top().children.push(element)
+      if (tag[3] !== '/') stack.push(element)
+      i = lt + tag[0].length
+    }
+  }
+  if (stack.length !== 1) return undefined
+  return root.children[0]
+}
+
+function readItem(element: XmlElement): UdtItem | undefined {
+  const key = element.attributes.Key
+  const name = element.attributes.Name
+  if (!key || !name) return undefined
+  const children: UdtItem[] = []
+  for (const child of element.children.filter((c) => c.name === 'DictionaryItem')) {
+    const item = readItem(child)
+    if (!item) return undefined
+    children.push(item)
+  }
+  return {
+    key,
+    name,
+    translations: element.children
+      .filter((c) => c.name === 'Value' && c.attributes.LanguageCultureAlias)
+      .map((c) => ({ isoCode: c.attributes.LanguageCultureAlias as string, translation: c.text })),
+    children,
+  }
+}
+
+/** The items a `.udt` holds: one root item, or several under `<DictionaryItems>`. */
+export function readUdt(source: string): UdtItem[] | undefined {
+  const root = parseXml(source)
+  if (!root) return undefined
+  const elements =
+    root.name === 'DictionaryItems'
+      ? root.children.filter((c) => c.name === 'DictionaryItem')
+      : root.name === 'DictionaryItem'
+        ? [root]
+        : []
+  if (elements.length === 0) return undefined
+  const items: UdtItem[] = []
+  for (const element of elements) {
+    const item = readItem(element)
+    if (!item) return undefined
+    items.push(item)
+  }
+  return items
+}

package/src/value-schemas.ts

@@ -0,0 +1,64 @@
+/**
+ * JSON Schema for what a property or a whole content type holds.
+ *
+ * Umbraco derives these from the .NET type a property editor stores, so the
+ * names it reports are .NET's. The shape is what a caller needs to validate or
+ * generate a value, which for us follows the data type's storage column.
+ */
+import type { ValueStorageTypeName } from '@crombolo/core'
+
+/** The .NET name Umbraco reports for each storage type. */
+export const VALUE_TYPE_NAMES: Record<ValueStorageTypeName, string> = {
+  Ntext: 'System.String',
+  Nvarchar: 'System.String',
+  Integer: 'System.Int32',
+  Decimal: 'System.Decimal',
+  Date: 'System.DateTime',
+}
+
+export function schemaForStorage(storage: string): Record<string, unknown> {
+  switch (storage) {
+    case 'Integer':
+      return { type: 'integer' }
+    case 'Decimal':
+      return { type: 'number' }
+    case 'Date':
+      return { type: 'string', format: 'date-time' }
+    default:
+      return { type: 'string' }
+  }
+}
+
+export const valueTypeName = (storage: string): string =>
+  VALUE_TYPE_NAMES[storage as ValueStorageTypeName] ?? 'System.String'
+
+export interface SchemaProperty {
+  alias: string
+  name: string
+  description: string | null
+  mandatory: boolean
+  storage: string
+}
+
+/** A content type as a JSON Schema object: one property per alias. */
+export function contentTypeJsonSchema(
+  type: { alias: string; name: string; description: string | null },
+  properties: readonly SchemaProperty[],
+): Record<string, unknown> {
+  const shape: Record<string, unknown> = {}
+  for (const property of properties) {
+    const schema = schemaForStorage(property.storage) as Record<string, unknown>
+    shape[property.alias] = property.description
+      ? { ...schema, title: property.name, description: property.description }
+      : { ...schema, title: property.name }
+  }
+  const required = properties.filter((p) => p.mandatory).map((p) => p.alias)
+  return {
+    $schema: 'https://json-schema.org/draft/2020-12/schema',
+    title: type.name,
+    description: type.description ?? undefined,
+    type: 'object',
+    properties: shape,
+    ...(required.length > 0 ? { required } : {}),
+  }
+}

package/src/view-runtime.ts

@@ -0,0 +1,42 @@
+/**
+ * Proving a directory can import a view, before one has to.
+ *
+ * JSX compiles to an import of `@crombolo/crombolo/jsx-runtime`, which resolves by walking
+ * up from the file that needs it. So a views directory — or a snapshot of one —
+ * outside the site's own tree cannot load at all:
+ *
+ *   Cannot find package '@crombolo/crombolo' imported from /mnt/views/homePage.tsx
+ *
+ * Checked at boot because the alternative is every render failing with a message
+ * that names the runtime rather than the configuration that moved it.
+ */
+import { mkdirSync } from 'node:fs'
+
+export const JSX_RUNTIME = '@crombolo/crombolo/jsx-runtime'
+
+export class ViewRuntimeUnreachableError extends Error {
+  constructor(
+    readonly dir: string,
+    readonly reason: string,
+  ) {
+    super(
+      `Views cannot be imported from ${dir}: '${JSX_RUNTIME}' does not resolve from there (${reason}).\n` +
+        '  JSX compiles to an import of it, resolved by walking up from the file, so the directory\n' +
+        '  has to sit inside a tree whose node_modules carries @crombolo/crombolo. Move it under the site, or\n' +
+        '  set CROMBOLO_COMPONENTS_CACHE_DIR to somewhere that is.',
+    )
+  }
+}
+
+/** Throws unless a `.tsx` placed in `dir` could resolve the JSX runtime. */
+export function assertViewRuntime(dir: string): void {
+  mkdirSync(dir, { recursive: true })
+  try {
+    Bun.resolveSync(JSX_RUNTIME, dir)
+  } catch (error) {
+    throw new ViewRuntimeUnreachableError(
+      dir,
+      error instanceof Error ? error.message : String(error),
+    )
+  }
+}