@crombolo/data 0.0.0-stage → 0.9.1

package/src/index.ts

@@ -0,0 +1,60 @@
+export * from './change-report.ts'
+export * from './cluster.ts'
+export * from './database.ts'
+export * from './dialect.ts'
+export * from './locks.ts'
+export * from './maintenance.ts'
+export * from './migrations.ts'
+export * from './node-ref.ts'
+export * from './pending.ts'
+export * from './repositories/bundles.ts'
+export * from './repositories/content-types.ts'
+export * from './repositories/data-types.ts'
+export * from './repositories/dictionary.ts'
+export * from './repositories/documents.ts'
+export * from './repositories/domains.ts'
+export * from './repositories/folders.ts'
+export * from './repositories/form-entries.ts'
+export * from './repositories/form-workflows.ts'
+export * from './repositories/languages.ts'
+export * from './repositories/log-searches.ts'
+export * from './repositories/members.ts'
+export * from './repositories/nodes.ts'
+export * from './repositories/notifications.ts'
+export * from './repositories/public-access.ts'
+export * from './repositories/redirects.ts'
+export * from './repositories/references.ts'
+export * from './repositories/tags.ts'
+export * from './repositories/templates.ts'
+export * from './repositories/transfer-runs.ts'
+export * from './repositories/user-data.ts'
+export * from './repositories/users.ts'
+export * from './schema/assistant.ts'
+export * from './schema/built-in-content-types.ts'
+export * from './schema/built-in-media-types.ts'
+export * from './schema/built-in-member-types.ts'
+export * from './schema/bundles-rename.ts'
+export * from './schema/change-report.ts'
+export * from './schema/client-id-prefix.ts'
+export * from './schema/content.ts'
+export * from './schema/content-seed.ts'
+export * from './schema/created-packages.ts'
+export * from './schema/elements.ts'
+export * from './schema/form-permission-prefix.ts'
+export * from './schema/group-permissions.ts'
+export * from './schema/identity.ts'
+export * from './schema/log-viewer.ts'
+export * from './schema/member-property-type.ts'
+export * from './schema/members.ts'
+export * from './schema/plan.ts'
+export * from './schema/public-access.ts'
+export * from './schema/recycle-bin.ts'
+export * from './schema/rich-text-ui.ts'
+export * from './schema/seed.ts'
+export * from './schema/since-version.ts'
+export * from './schema/transfer-runs.ts'
+export * from './schema/upgrade-report.ts'
+export * from './schema/users-localisation.ts'
+export * from './schema/values.ts'
+export * from './schema-ops.ts'
+export * from './schema-state.ts'

package/src/locks.ts

@@ -0,0 +1,103 @@
+/**
+ * Named write locks around content-tree mutation. Umbraco takes a row lock per
+ * lock id; SQLite has a single writer, so an in-process queue is both sufficient
+ * and necessary, while Postgres uses advisory locks.
+ */
+import { AsyncLocalStorage } from 'node:async_hooks'
+import type { Db } from './database.ts'
+
+/** Mirrors Umbraco's Constants.Locks. */
+export const Locks = {
+  Servers: -331,
+  ContentTypes: -332,
+  ContentTree: -333,
+  MediaTree: -334,
+  MemberTree: -335,
+  MediaTypes: -336,
+  MemberTypes: -337,
+  Domains: -338,
+  KeyValues: -339,
+  Languages: -340,
+  ScheduledPublishing: -341,
+  /** Ours: schema sync and schema-state transitions. */
+  Schema: -342,
+} as const
+
+export type LockId = (typeof Locks)[keyof typeof Locks]
+
+export interface LockManager {
+  /** Runs `fn` while holding the named lock. Re-entrant within one chain. */
+  withLock<T>(id: LockId, fn: () => Promise<T>): Promise<T>
+}
+
+/**
+ * The locks the current async chain holds. A holder is emptied on release, so
+ * work the holder left running in the background does not inherit the lock.
+ */
+const held = new AsyncLocalStorage<Set<LockId>>()
+
+/** Runs `fn` directly when this chain already holds `id`; otherwise takes it. */
+async function reentrant<T>(
+  id: LockId,
+  fn: () => Promise<T>,
+  acquire: (run: () => Promise<T>) => Promise<T>,
+): Promise<T> {
+  const current = held.getStore()
+  if (current?.has(id)) return fn()
+  return acquire(async () => {
+    const holding = new Set(current)
+    holding.add(id)
+    try {
+      return await held.run(holding, fn)
+    } finally {
+      holding.clear()
+    }
+  })
+}
+
+class SqliteLockManager implements LockManager {
+  #queues = new Map<number, Promise<unknown>>()
+
+  withLock<T>(id: LockId, fn: () => Promise<T>): Promise<T> {
+    return reentrant(id, fn, (run) => this.#acquire(id, run))
+  }
+
+  async #acquire<T>(id: LockId, fn: () => Promise<T>): Promise<T> {
+    const previous = this.#queues.get(id) ?? Promise.resolve()
+    let release!: () => void
+    const gate = new Promise<void>((resolve) => {
+      release = resolve
+    })
+    this.#queues.set(
+      id,
+      previous.then(() => gate),
+    )
+    await previous
+    try {
+      return await fn()
+    } finally {
+      release()
+      if (this.#queues.get(id) === gate) this.#queues.delete(id)
+    }
+  }
+}
+
+class PostgresLockManager implements LockManager {
+  #db: Db
+  constructor(db: Db) {
+    this.#db = db
+  }
+
+  withLock<T>(id: LockId, fn: () => Promise<T>): Promise<T> {
+    return reentrant(id, fn, (run) =>
+      this.#db.transaction(async (tx) => {
+        await tx.exec('SELECT pg_advisory_xact_lock(?)', [id])
+        return run()
+      }),
+    )
+  }
+}
+
+export function createLockManager(db: Db): LockManager {
+  return db.dialect.name === 'postgres' ? new PostgresLockManager(db) : new SqliteLockManager()
+}

package/src/maintenance.ts

@@ -0,0 +1,51 @@
+/**
+ * A cluster-wide pause on editing, held in `key_value` so every node sees it on
+ * its next write rather than its next poll. Readers are never affected; the
+ * write gate (`assertNodeMayWrite`) refuses editor writes while it is set.
+ */
+import type { Db } from './database.ts'
+import { readKeyValue, writeKeyValue } from './migrations.ts'
+
+export const WRITES_PAUSED_KEY = 'Crombolo.Operations.WritesPaused'
+
+export interface WritesPaused {
+  reason: string
+  by: string | null
+  at: string
+}
+
+export function parseWritesPaused(value: string | null | undefined): WritesPaused | undefined {
+  if (!value) return undefined
+  try {
+    const parsed = JSON.parse(value) as Partial<WritesPaused>
+    return {
+      reason: String(parsed.reason ?? ''),
+      by: parsed.by ?? null,
+      at: String(parsed.at ?? ''),
+    }
+  } catch {
+    return { reason: '', by: null, at: '' }
+  }
+}
+
+export async function readWritesPaused(db: Db): Promise<WritesPaused | undefined> {
+  return parseWritesPaused(await readKeyValue(db, WRITES_PAUSED_KEY))
+}
+
+export async function pauseWrites(
+  db: Db,
+  options: { reason: string; by?: string },
+): Promise<WritesPaused> {
+  const paused: WritesPaused = {
+    reason: options.reason,
+    by: options.by ?? null,
+    at: new Date().toISOString(),
+  }
+  await writeKeyValue(db, WRITES_PAUSED_KEY, JSON.stringify(paused))
+  return paused
+}
+
+/** An empty value rather than a deleted row, so the key keeps its history in `update_date`. */
+export async function resumeWrites(db: Db): Promise<void> {
+  await writeKeyValue(db, WRITES_PAUSED_KEY, '')
+}

package/src/migrations.ts

@@ -0,0 +1,431 @@
+/**
+ * Umbraco tracks upgrade state as a single state token in `key_value`, not a
+ * version number and not a list of applied migrations: the plan is a chain of
+ * transitions walked forward to its final state. That design handles concurrent
+ * development branches far better than sequence numbers, so we keep it — and
+ * harden it (docs/10-packaging-and-upgrades.md): a ledger of every step that
+ * ran, expand/contract declared and linted, a recorded plan, and an explicit
+ * refusal when the database is newer than the code.
+ */
+import type { Db } from './database.ts'
+import type { Dialect } from './dialect.ts'
+import type { LockManager } from './locks.ts'
+import { Locks } from './locks.ts'
+
+export const MIGRATION_STATE_KEY = 'Crombolo.Core.Upgrader.State'
+export const INITIAL_STATE = '{00000000-0000-0000-0000-000000000000}'
+
+export type MigrationKind = 'expand' | 'contract'
+
+export interface Migration {
+  /** State the database must be in for this step to apply. */
+  from: string
+  /** State the database is in once this step has run. */
+  to: string
+  name: string
+  /** Expand adds and never removes; contract removes what an earlier release's expand added. Default expand. */
+  kind?: MigrationKind
+  /** The framework release that ships this step. Required for a contract and for what it contracts. */
+  release?: string
+  /** For a contract: the name of the expand migration it removes. */
+  contracts?: string
+  up(db: Db): Promise<void>
+}
+
+function compareRelease(a: string, b: string): number {
+  const pa = a.split(/[.+-]/).map(Number)
+  const pb = b.split(/[.+-]/).map(Number)
+  for (let i = 0; i < Math.max(pa.length, pb.length); i++) {
+    const da = pa[i] ?? 0
+    const db = pb[i] ?? 0
+    if (da !== db) return da < db ? -1 : 1
+  }
+  return 0
+}
+
+/**
+ * What a release promises a running cluster. `compatible`: every migration it
+ * ships is an expand, so the previous release keeps working against the
+ * migrated database and nodes can be rolled one at a time. `breaking`: it ships
+ * a contract, which removes something an older node may still read.
+ */
+export type MigrationCompatibility = 'compatible' | 'breaking'
+
+/** Derived, not asserted: `breaking` exactly when the release ships a contract. */
+export function migrationCompatibility(
+  plan: MigrationPlan,
+  release: string,
+): MigrationCompatibility {
+  const contracts = plan.migrations.some(
+    (m) =>
+      m.kind === 'contract' && m.release !== undefined && compareRelease(m.release, release) === 0,
+  )
+  return contracts ? 'breaking' : 'compatible'
+}
+
+/** `x.0.0` with x ≥ 1. Pre-1.0 no release is major, so nothing may contract before 1.0.0. */
+export function isMajorRelease(release: string): boolean {
+  const [major = 0, minor = 0, patch = 0] = release.split(/[.+-]/).map(Number)
+  return major >= 1 && minor === 0 && patch === 0
+}
+
+export class MigrationPlan {
+  readonly migrations: readonly Migration[]
+
+  constructor(migrations: readonly Migration[]) {
+    this.migrations = migrations
+    let expected = INITIAL_STATE
+    const seen = new Map<string, Migration>()
+    for (const migration of migrations) {
+      if (migration.from !== expected) {
+        throw new Error(
+          `Migration '${migration.name}' expects state ${migration.from} but the chain is at ${expected}.`,
+        )
+      }
+      if (migration.kind === 'contract') {
+        const expand = migration.contracts ? seen.get(migration.contracts) : undefined
+        if (!migration.contracts || !expand)
+          throw new Error(
+            `Contract migration '${migration.name}' must name the expand migration it removes.`,
+          )
+        if (expand.kind === 'contract')
+          throw new Error(
+            `Contract migration '${migration.name}' cannot contract another contract.`,
+          )
+        if (!migration.release || !expand.release)
+          throw new Error(
+            `Contract migration '${migration.name}' and '${expand.name}' must both declare a release.`,
+          )
+        if (compareRelease(expand.release, migration.release) >= 0)
+          throw new Error(
+            `Contract migration '${migration.name}' (${migration.release}) may only remove what an earlier release added; '${expand.name}' is ${expand.release}.`,
+          )
+        if (!isMajorRelease(migration.release))
+          throw new Error(
+            `Contract migration '${migration.name}' ships in ${migration.release}, but a contract may only ship in a major release (x.0.0): minors and patches are rolled under live nodes and must only add.`,
+          )
+      }
+      seen.set(migration.name, migration)
+      expected = migration.to
+    }
+  }
+
+  get finalState(): string {
+    return this.migrations.at(-1)?.to ?? INITIAL_STATE
+  }
+
+  /** The steps still to run from a state; empty when current. Undefined when the state is not in the plan. */
+  pendingFrom(state: string): Migration[] | undefined {
+    if (state === this.finalState) return []
+    const index = this.migrations.findIndex((m) => m.from === state)
+    if (index < 0) return undefined
+    return this.migrations.slice(index)
+  }
+}
+
+async function ensureTables(db: Db): Promise<void> {
+  const t = db.dialect.types
+  await db.exec(
+    `CREATE TABLE IF NOT EXISTS key_value (
+       key ${t.text} PRIMARY KEY,
+       value ${t.text},
+       update_date ${t.timestamp} NOT NULL
+     )`,
+  )
+  await db.exec(
+    `CREATE TABLE IF NOT EXISTS migration_history (
+       id ${t.identity},
+       name ${t.varchar(255)} NOT NULL,
+       kind ${t.varchar(20)} NOT NULL,
+       from_state ${t.varchar(255)},
+       to_state ${t.varchar(255)},
+       applied_at ${t.timestamp} NOT NULL,
+       duration_ms ${t.integer} NOT NULL,
+       checksum ${t.varchar(64)},
+       applied_by ${t.varchar(255)},
+       note ${t.text}
+     )`,
+  )
+}
+
+export async function readState(db: Db): Promise<string> {
+  await ensureTables(db)
+  const rows = await db.query<{ value: string }>('SELECT value FROM key_value WHERE key = ?', [
+    MIGRATION_STATE_KEY,
+  ])
+  return rows[0]?.value ?? INITIAL_STATE
+}
+
+async function writeState(db: Db, state: string): Promise<void> {
+  const now = db.dialect.now()
+  await db.query(`UPDATE key_value SET value = ?, update_date = ${now} WHERE key = ?`, [
+    state,
+    MIGRATION_STATE_KEY,
+  ])
+  const rows = await db.query<{ value: string }>('SELECT value FROM key_value WHERE key = ?', [
+    MIGRATION_STATE_KEY,
+  ])
+  if (rows.length === 0) {
+    await db.exec(
+      `INSERT INTO key_value (key, value, update_date) VALUES (${db.dialect.placeholder(1)}, ${db.dialect.placeholder(2)}, ${now})`,
+      [MIGRATION_STATE_KEY, state],
+    )
+  }
+}
+
+export interface LedgerEntry {
+  name: string
+  /** 'expand' | 'contract' for framework steps; 'value' for a value migration; 'upgrade' for a cut-over. */
+  kind: string
+  fromState?: string | null
+  toState?: string | null
+  durationMs: number
+  checksum?: string | null
+  appliedBy?: string | null
+  note?: string | null
+}
+
+/**
+ * A setting in `key_value`, Umbraco's own place for one, read and written as a
+ * string. Used for facts a node must agree on across restarts and across a load
+ * balanced set — the key member session cookies are signed with, for one.
+ */
+export async function readKeyValue(db: Db, key: string): Promise<string | undefined> {
+  await ensureTables(db)
+  const rows = await db.query<{ value: string | null }>(
+    'SELECT value FROM key_value WHERE key = ?',
+    [key],
+  )
+  return rows[0]?.value ?? undefined
+}
+
+export async function writeKeyValue(db: Db, key: string, value: string): Promise<void> {
+  await ensureTables(db)
+  const now = db.dialect.now()
+  await db.exec(`UPDATE key_value SET value = ?, update_date = ${now} WHERE key = ?`, [value, key])
+  const rows = await db.query('SELECT value FROM key_value WHERE key = ?', [key])
+  if (rows.length === 0)
+    await db.exec(`INSERT INTO key_value (key, value, update_date) VALUES (?, ?, ${now})`, [
+      key,
+      value,
+    ])
+}
+
+/**
+ * Reads a setting, generating and storing one the first time it is asked for.
+ * Two nodes racing settle on whichever row landed first.
+ */
+export async function ensureKeyValue(db: Db, key: string, create: () => string): Promise<string> {
+  const existing = await readKeyValue(db, key)
+  if (existing) return existing
+  await writeKeyValue(db, key, create())
+  return (await readKeyValue(db, key)) ?? ''
+}
+
+/** Appends a row to `migration_history`. */
+export async function recordInLedger(db: Db, entry: LedgerEntry): Promise<void> {
+  await ensureTables(db)
+  await db.exec(
+    `INSERT INTO migration_history (name, kind, from_state, to_state, applied_at, duration_ms, checksum, applied_by, note)
+     VALUES (?, ?, ?, ?, ${db.dialect.now()}, ?, ?, ?, ?)`,
+    [
+      entry.name,
+      entry.kind,
+      entry.fromState ?? null,
+      entry.toState ?? null,
+      entry.durationMs,
+      entry.checksum ?? null,
+      entry.appliedBy ?? null,
+      entry.note ?? null,
+    ],
+  )
+}
+
+/** The newest ledger row's id, so a node can later ask what was ledgered after it booted. */
+export async function latestLedgerId(db: Db): Promise<number> {
+  await ensureTables(db)
+  const rows = await db.query<{ n: number | null }>('SELECT MAX(id) AS n FROM migration_history')
+  return Number(rows[0]?.n ?? 0)
+}
+
+/** How many contract steps — framework contracts and site purges — were ledgered after a row id. */
+export async function contractsLedgeredAfter(db: Db, afterId: number): Promise<number> {
+  const rows = await db.query<{ n: number }>(
+    "SELECT COUNT(*) AS n FROM migration_history WHERE id > ? AND kind = 'contract'",
+    [afterId],
+  )
+  return Number(rows[0]?.n ?? 0)
+}
+
+export interface LedgerRow extends LedgerEntry {
+  id: number
+  appliedAt: string
+}
+
+export async function readLedger(db: Db): Promise<LedgerRow[]> {
+  await ensureTables(db)
+  const rows = await db.query(
+    'SELECT id, name, kind, from_state, to_state, applied_at, duration_ms, checksum, applied_by, note FROM migration_history ORDER BY id',
+  )
+  return rows.map((row) => ({
+    id: Number(row.id),
+    name: String(row.name),
+    kind: String(row.kind),
+    fromState: (row.from_state as string | null) ?? null,
+    toState: (row.to_state as string | null) ?? null,
+    appliedAt: String(row.applied_at),
+    durationMs: Number(row.duration_ms),
+    checksum: (row.checksum as string | null) ?? null,
+    appliedBy: (row.applied_by as string | null) ?? null,
+    note: (row.note as string | null) ?? null,
+  }))
+}
+
+/** The database is at a state this code's plan does not know: newer code wrote it. */
+export class UpgradeStateError extends Error {
+  constructor(
+    readonly databaseState: string,
+    readonly codeState: string,
+  ) {
+    super(
+      `The database is at migration state ${databaseState}, which this version of crombolo (plan ends at ${codeState}) does not know. ` +
+        'It was upgraded by a newer version; run that version, or restore the backup taken before the upgrade.',
+    )
+  }
+}
+
+/** Steps pending on this database, in order. Throws when the database is ahead of the code. */
+export async function pendingMigrations(db: Db, plan: MigrationPlan): Promise<Migration[]> {
+  const state = await readState(db)
+  const pending = plan.pendingFrom(state)
+  if (!pending) throw new UpgradeStateError(state, plan.finalState)
+  return pending
+}
+
+export interface MigrateResult {
+  from: string
+  to: string
+  applied: string[]
+}
+
+export interface MigrateOptions {
+  /** Recorded as `applied_by` in the ledger. */
+  by?: string
+}
+
+/** Walks the plan forward from the database's current state, one transaction per step, ledgering each. */
+export async function migrate(
+  db: Db,
+  plan: MigrationPlan,
+  options: MigrateOptions = {},
+): Promise<MigrateResult> {
+  return db.locks.withLock(Locks.KeyValues, async () => {
+    const from = await readState(db)
+    let state = from
+    const applied: string[] = []
+    const pending = plan.pendingFrom(state)
+    if (!pending) throw new UpgradeStateError(state, plan.finalState)
+    for (const next of pending) {
+      const started = performance.now()
+      await db.transaction(async (tx) => {
+        await next.up(tx)
+      })
+      state = next.to
+      await writeState(db, state)
+      await recordInLedger(db, {
+        name: next.name,
+        kind: next.kind ?? 'expand',
+        fromState: next.from,
+        toState: next.to,
+        durationMs: Math.round(performance.now() - started),
+        checksum: next.name,
+        appliedBy: options.by ?? null,
+        note: next.release ? `release ${next.release}` : null,
+      })
+      applied.push(next.name)
+    }
+    return { from, to: state, applied }
+  })
+}
+
+/** A fresh install stamps the final state so no migrations then run. */
+export async function stampFinalState(db: Db, plan: MigrationPlan): Promise<void> {
+  await ensureTables(db)
+  await writeState(db, plan.finalState)
+}
+
+// ---------------------------------------------------------------- the plan
+
+/** Thrown by the recording database when a migration reads: its writes cannot be predicted. */
+export class UnplannableError extends Error {
+  constructor(readonly sql: string) {
+    super(`reads data to decide what to write: ${sql.trim().split('\n')[0]}`)
+  }
+}
+
+/** A `Db` that captures statements instead of executing them. */
+class RecordingDb implements Db {
+  readonly dialect: Dialect
+  readonly locks: LockManager
+  readonly statements: string[]
+
+  constructor(dialect: Dialect, locks: LockManager, statements: string[] = []) {
+    this.dialect = dialect
+    this.locks = locks
+    this.statements = statements
+  }
+
+  async query<T = Record<string, unknown>>(sql: string): Promise<T[]> {
+    throw new UnplannableError(sql)
+  }
+
+  async exec(sql: string, params: unknown[] = []): Promise<void> {
+    const rendered = this.dialect.render(sql)
+    this.statements.push(
+      params.length > 0 ? `${rendered}  -- params: ${JSON.stringify(params)}` : rendered,
+    )
+  }
+
+  async transaction<T>(fn: (tx: Db) => Promise<T>): Promise<T> {
+    return fn(this)
+  }
+
+  async close(): Promise<void> {}
+}
+
+export interface PlannedStep {
+  name: string
+  kind: MigrationKind
+  release: string | null
+  from: string
+  to: string
+  /** The DDL this step would run, or undefined when it reads data to decide. */
+  statements: string[] | undefined
+  unplannable?: string
+}
+
+/** What `migrate()` would do, without doing it. */
+export async function planUpgrade(db: Db, plan: MigrationPlan): Promise<PlannedStep[]> {
+  const pending = await pendingMigrations(db, plan)
+  const steps: PlannedStep[] = []
+  for (const migration of pending) {
+    const recorder = new RecordingDb(db.dialect, db.locks)
+    const step: PlannedStep = {
+      name: migration.name,
+      kind: migration.kind ?? 'expand',
+      release: migration.release ?? null,
+      from: migration.from,
+      to: migration.to,
+      statements: undefined,
+    }
+    try {
+      await migration.up(recorder)
+      step.statements = recorder.statements
+    } catch (error) {
+      if (error instanceof UnplannableError) step.unplannable = error.message
+      else throw error
+    }
+    steps.push(step)
+  }
+  return steps
+}