@crombolo/data 0.9.1 → 0.11.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crombolo/data",
-  "version": "0.9.1",
+  "version": "0.11.0",
   "description": "SQLite and Postgres storage for crombolo: schema, migrations and repositories.",
   "license": "MIT",
   "author": "Crombolo",
@@ -28,6 +28,6 @@
     "access": "public"
   },
   "dependencies": {
-    "@crombolo/core": "0.9.1"
+    "@crombolo/core": "0.11.0"
   }
 }

package/src/audit.ts

@@ -0,0 +1,291 @@
+/**
+ * The audit log: who did what, when and from where. Entries are only ever
+ * added, read and aged out; nothing edits one. docs/audit-log.md.
+ */
+import { normaliseUuid } from '@crombolo/core'
+import type { Db } from './database.ts'
+import { DbDate } from './dialect.ts'
+import { STATE_AUDIT_LOG } from './schema/audit-log.ts'
+
+export type AuditCategory =
+  | 'content'
+  | 'media'
+  | 'member'
+  | 'schema'
+  | 'settings'
+  | 'user'
+  | 'permission'
+  | 'sign-in'
+  | 'system'
+
+export const AUDIT_CATEGORIES: readonly AuditCategory[] = [
+  'content',
+  'media',
+  'member',
+  'schema',
+  'settings',
+  'user',
+  'permission',
+  'sign-in',
+  'system',
+]
+
+/** Where an entry came from: a request (with its address), the command line, or a background job. */
+export type AuditSource = 'request' | 'cli' | 'job'
+
+export interface AuditEntryInput {
+  category: AuditCategory
+  action: string
+  source: AuditSource
+  userKey?: string | null
+  /** The user's name when the entry was written; for a failed sign-in, the login tried. */
+  userName?: string | null
+  entityType?: string | null
+  entityKey?: string | null
+  entityName?: string | null
+  ipAddress?: string | null
+  forwardedFor?: string | null
+  detail?: string | null
+  occurredAt?: Date
+}
+
+export interface AuditEntry {
+  id: number
+  occurredAt: Date
+  category: AuditCategory
+  action: string
+  source: AuditSource
+  userKey: string | null
+  userName: string | null
+  entityType: string | null
+  entityKey: string | null
+  entityName: string | null
+  ipAddress: string | null
+  forwardedFor: string | null
+  detail: string | null
+}
+
+export interface AuditFilter {
+  userKey?: string
+  entityKey?: string
+  category?: AuditCategory
+  action?: string
+  since?: Date
+  until?: Date
+  /** Matched anywhere in the item's name or key, or the user's name. */
+  text?: string
+  ascending?: boolean
+}
+
+/** What a key names, for an entry to carry once the thing itself has gone. */
+export interface AuditSubject {
+  entityType: string
+  entityKey: string
+  entityName: string | null
+}
+
+const UUID = /^\{?[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\}?$/i
+
+const LIMITS = {
+  action: 64,
+  userName: 255,
+  entityType: 64,
+  entityKey: 255,
+  entityName: 255,
+  ipAddress: 64,
+  forwardedFor: 255,
+  detail: 4000,
+} as const
+
+function clip(value: string | null | undefined, limit: number): string | null {
+  if (value === null || value === undefined || value === '') return null
+  return value.length > limit ? value.slice(0, limit) : value
+}
+
+function text(value: unknown): string | null {
+  return value === null || value === undefined ? null : String(value)
+}
+
+function toEntry(row: Record<string, unknown>): AuditEntry {
+  return {
+    id: Number(row.id),
+    occurredAt: DbDate.fromDb(row.occurred_at) ?? new Date(0),
+    category: String(row.category) as AuditCategory,
+    action: String(row.action),
+    source: String(row.source) as AuditSource,
+    userKey: row.user_key ? normaliseUuid(String(row.user_key)) : null,
+    userName: text(row.user_name),
+    entityType: text(row.entity_type),
+    entityKey: text(row.entity_key),
+    entityName: text(row.entity_name),
+    ipAddress: text(row.ip_address),
+    forwardedFor: text(row.forwarded_for),
+    detail: text(row.detail),
+  }
+}
+
+export class AuditLog {
+  #db: Db
+
+  constructor(db: Db) {
+    this.#db = db
+  }
+
+  async record(entries: AuditEntryInput | readonly AuditEntryInput[]): Promise<void> {
+    const list = Array.isArray(entries) ? entries : [entries as AuditEntryInput]
+    for (const entry of list) {
+      const userKey =
+        entry.userKey && UUID.test(entry.userKey) ? normaliseUuid(entry.userKey) : null
+      const entityKey =
+        entry.entityKey && UUID.test(entry.entityKey)
+          ? normaliseUuid(entry.entityKey)
+          : entry.entityKey
+      await this.#db.exec(
+        `INSERT INTO audit_entry (occurred_at, category, action, user_key, user_name, entity_type,
+           entity_key, entity_name, source, ip_address, forwarded_for, detail)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+        [
+          DbDate.toDb(entry.occurredAt ?? new Date()),
+          entry.category,
+          clip(entry.action, LIMITS.action),
+          userKey,
+          clip(entry.userName, LIMITS.userName),
+          clip(entry.entityType, LIMITS.entityType),
+          clip(entityKey, LIMITS.entityKey),
+          clip(entry.entityName, LIMITS.entityName),
+          entry.source,
+          clip(entry.ipAddress, LIMITS.ipAddress),
+          clip(entry.forwardedFor, LIMITS.forwardedFor),
+          clip(entry.detail, LIMITS.detail),
+        ],
+      )
+    }
+  }
+
+  async query(
+    filter: AuditFilter = {},
+    page: { skip: number; take: number } = { skip: 0, take: 100 },
+  ): Promise<{ total: number; items: AuditEntry[] }> {
+    const where: string[] = []
+    const params: unknown[] = []
+    if (filter.userKey !== undefined) {
+      if (!UUID.test(filter.userKey)) return { total: 0, items: [] }
+      where.push('user_key = ?')
+      params.push(normaliseUuid(filter.userKey))
+    }
+    if (filter.entityKey !== undefined) {
+      where.push('entity_key = ?')
+      params.push(UUID.test(filter.entityKey) ? normaliseUuid(filter.entityKey) : filter.entityKey)
+    }
+    if (filter.category !== undefined) {
+      where.push('category = ?')
+      params.push(filter.category)
+    }
+    if (filter.action !== undefined) {
+      where.push('action = ?')
+      params.push(filter.action)
+    }
+    if (filter.since !== undefined) {
+      where.push('occurred_at >= ?')
+      params.push(DbDate.toDb(filter.since))
+    }
+    if (filter.until !== undefined) {
+      where.push('occurred_at < ?')
+      params.push(DbDate.toDb(filter.until))
+    }
+    const needle = filter.text?.trim().toLowerCase()
+    if (needle) {
+      const like = `%${needle.replace(/[\\%_]/g, (c) => `\\${c}`)}%`
+      where.push(
+        `(LOWER(entity_name) LIKE ? ESCAPE '\\' OR LOWER(entity_key) LIKE ? ESCAPE '\\' OR LOWER(user_name) LIKE ? ESCAPE '\\')`,
+      )
+      params.push(like, like, like)
+    }
+    const clause = where.length > 0 ? `WHERE ${where.join(' AND ')}` : ''
+    const counted = await this.#db.query<{ total: number }>(
+      `SELECT COUNT(*) AS total FROM audit_entry ${clause}`,
+      params,
+    )
+    const order = filter.ascending ? 'ASC' : 'DESC'
+    const rows = await this.#db.query(
+      `SELECT * FROM audit_entry ${clause} ORDER BY occurred_at ${order}, id ${order} LIMIT ? OFFSET ?`,
+      [...params, page.take, page.skip],
+    )
+    return { total: Number(counted[0]?.total ?? 0), items: rows.map(toEntry) }
+  }
+
+  /** Everyone the log names, by the name they had most recently. */
+  async users(): Promise<Array<{ key: string; name: string }>> {
+    const rows = await this.#db.query<{ user_key: string; user_name: string | null }>(
+      `SELECT e.user_key, e.user_name FROM audit_entry e
+       JOIN (SELECT user_key, MAX(id) AS id FROM audit_entry WHERE user_key IS NOT NULL GROUP BY user_key) latest
+         ON latest.id = e.id
+       ORDER BY e.user_name`,
+    )
+    return rows.map((row) => ({
+      key: normaliseUuid(String(row.user_key)),
+      name: row.user_name ? String(row.user_name) : '',
+    }))
+  }
+
+  /** Deletes entries older than `before`, and returns how many went. */
+  async prune(before: Date): Promise<number> {
+    const counted = await this.#db.query<{ total: number }>(
+      'SELECT COUNT(*) AS total FROM audit_entry WHERE occurred_at < ?',
+      [DbDate.toDb(before)],
+    )
+    const total = Number(counted[0]?.total ?? 0)
+    if (total > 0)
+      await this.#db.exec('DELETE FROM audit_entry WHERE occurred_at < ?', [DbDate.toDb(before)])
+    return total
+  }
+
+  /** When this database began keeping the log; history before it is in the content versions. */
+  async startedAt(): Promise<Date | undefined> {
+    const rows = await this.#db.query<{ applied_at: unknown }>(
+      'SELECT MIN(applied_at) AS applied_at FROM migration_history WHERE to_state = ?',
+      [STATE_AUDIT_LOG],
+    )
+    return DbDate.fromDb(rows[0]?.applied_at)
+  }
+
+  /**
+   * What a key names. A version answers for the item it belongs to, so a
+   * rollback is filed under the page it rolled back.
+   */
+  async describe(entityType: string, key: string): Promise<AuditSubject> {
+    const named = (entityName: unknown, type = entityType, entityKey = key): AuditSubject => ({
+      entityType: type,
+      entityKey,
+      entityName: entityName === null || entityName === undefined ? null : String(entityName),
+    })
+    if (entityType.endsWith('Version') && /^\d+$/.test(key)) {
+      const rows = await this.#db.query<{ unique_id: string; text: string | null }>(
+        'SELECT n.unique_id, n.text FROM content_version cv JOIN node n ON n.id = cv.node_id WHERE cv.id = ?',
+        [Number(key)],
+      )
+      const row = rows[0]
+      const owner = entityType.slice(0, -'Version'.length)
+      return row ? named(row.text, owner, normaliseUuid(String(row.unique_id))) : named(null)
+    }
+    if (entityType === 'Language') {
+      const rows = await this.#db.query<{ culture_name: string }>(
+        'SELECT culture_name FROM language WHERE iso_code = ?',
+        [key],
+      )
+      return named(rows[0]?.culture_name ?? null)
+    }
+    if (!UUID.test(key)) return named(null)
+    const uuid = normaliseUuid(key)
+    const sql =
+      entityType === 'User'
+        ? 'SELECT user_name AS name FROM user_account WHERE key = ?'
+        : entityType === 'UserGroup'
+          ? 'SELECT name FROM user_group WHERE key = ?'
+          : entityType === 'DictionaryItem'
+            ? 'SELECT item_key AS name FROM dictionary_item WHERE key = ?'
+            : 'SELECT text AS name FROM node WHERE unique_id = ?'
+    const rows = await this.#db.query<{ name: string | null }>(sql, [uuid])
+    return named(rows[0]?.name ?? null, entityType, uuid)
+  }
+}

package/src/bundle-tables/builder.ts

@@ -0,0 +1,264 @@
+/**
+ * The declarative builder a bundle writes its tables with. `docs/bundles.md`,
+ * Part 4.
+ *
+ * It is the only way a bundle writes schema: there is no SQL in it, and no
+ * escape hatch to some. A step's `up` returns the operations it wants, as plain
+ * data, which is what makes a step checkable before it runs — linted for
+ * expand and contract, hashed for its checksum, rendered for `upgrade --plan` —
+ * and what lets the final shape of every table be worked out from the
+ * declaration's types alone.
+ */
+
+export type ColumnType =
+  | 'uuid'
+  | 'varchar'
+  | 'text'
+  | 'integer'
+  | 'bigint'
+  | 'boolean'
+  | 'timestamp'
+  | 'json'
+
+/** A scalar a default may be: what DDL can carry as a literal. */
+export type DefaultValue = string | number | boolean
+
+export interface ColumnSpec {
+  type: ColumnType
+  /** For `varchar`. */
+  length?: number
+  nullable: boolean
+  default?: DefaultValue
+  primaryKey: boolean
+  indexed: boolean
+}
+
+/** What a column holds, as the capability reads and writes it. */
+export type JsonValue =
+  | string
+  | number
+  | boolean
+  | null
+  | JsonValue[]
+  | { [key: string]: JsonValue }
+
+interface ValueOf {
+  uuid: string
+  varchar: string
+  text: string
+  integer: number
+  bigint: number
+  boolean: boolean
+  timestamp: Date
+  json: JsonValue
+}
+
+/**
+ * A column being described. Its type parameters carry what the declaration
+ * says, so a table's row type can be read off it.
+ */
+export interface Column<
+  T extends ColumnType = ColumnType,
+  N extends boolean = false,
+  D extends boolean = false,
+> {
+  readonly spec: ColumnSpec
+  /** Allows null. */
+  nullable(): Column<T, true, D>
+  /** What a row gets when an insert leaves it out. */
+  default(
+    value: T extends 'json' ? never : ValueOf[T] extends Date ? never : ValueOf[T],
+  ): Column<T, N, true>
+  primaryKey(): Column<T, N, D>
+  /** Gives the column an index of its own. */
+  indexed(): Column<T, N, D>
+  /** Type-level only: what the column holds. */
+  readonly __value?: ValueOf[T]
+  readonly __nullable?: N
+  readonly __defaulted?: D
+}
+
+function column<T extends ColumnType>(type: T, spec: Partial<ColumnSpec> = {}): Column<T> {
+  const make = (s: ColumnSpec): Column<T, boolean, boolean> =>
+    Object.freeze({
+      spec: Object.freeze(s),
+      nullable: () => make({ ...s, nullable: true }),
+      default: (value: DefaultValue) => make({ ...s, default: value }),
+      primaryKey: () => make({ ...s, primaryKey: true }),
+      indexed: () => make({ ...s, indexed: true }),
+    }) as Column<T, boolean, boolean>
+  return make({ type, nullable: false, primaryKey: false, indexed: false, ...spec }) as Column<T>
+}
+
+export type Columns = Record<string, Column<ColumnType, boolean, boolean>>
+
+export type Operation =
+  | { op: 'createTable'; table: string; columns: Record<string, ColumnSpec> }
+  | { op: 'addColumn'; table: string; column: string; spec: ColumnSpec }
+  | { op: 'addIndex'; table: string; columns: string[]; unique: boolean }
+  | { op: 'widenColumn'; table: string; column: string; spec: ColumnSpec }
+  | { op: 'dropTable'; table: string }
+  | { op: 'dropColumn'; table: string; column: string }
+  | { op: 'dropIndex'; table: string; columns: string[] }
+
+/** An operation, carrying at the type level what it does to the schema. */
+export type Op<Effect = never> = Operation & { readonly __effect?: Effect }
+
+export interface CreateTable<T extends string, C extends Columns> {
+  kind: 'create'
+  table: T
+  columns: C
+}
+export interface AddColumn<
+  T extends string,
+  K extends string,
+  C extends Column<ColumnType, boolean, boolean>,
+> {
+  kind: 'add'
+  table: T
+  column: K
+  spec: C
+}
+export interface DropTable<T extends string> {
+  kind: 'dropTable'
+  table: T
+}
+export interface DropColumn<T extends string, K extends string> {
+  kind: 'dropColumn'
+  table: T
+  column: K
+}
+
+/** What a step's `up` is handed. */
+export interface SchemaBuilder {
+  uuid(): Column<'uuid'>
+  varchar(length: number): Column<'varchar'>
+  text(): Column<'text'>
+  integer(): Column<'integer'>
+  bigint(): Column<'bigint'>
+  boolean(): Column<'boolean'>
+  timestamp(): Column<'timestamp'>
+  json(): Column<'json'>
+
+  createTable<T extends string, C extends Columns>(table: T, columns: C): Op<CreateTable<T, C>>
+  /** A new column must be nullable or have a default, so it adds under live rows. */
+  addColumn<T extends string, K extends string, C extends Column<ColumnType, boolean, boolean>>(
+    table: T,
+    column: K,
+    spec: C,
+  ): Op<AddColumn<T, K, C>>
+  addIndex(table: string, columns: readonly string[], options?: { unique?: boolean }): Op
+  /** Longer text, or a wider integer; never narrower, never another kind. */
+  widenColumn(table: string, column: string, spec: Column<ColumnType, boolean, boolean>): Op
+  dropTable<T extends string>(table: T): Op<DropTable<T>>
+  dropColumn<T extends string, K extends string>(table: T, column: K): Op<DropColumn<T, K>>
+  dropIndex(table: string, columns: readonly string[]): Op
+}
+
+const specs = (columns: Columns): Record<string, ColumnSpec> => {
+  const out: Record<string, ColumnSpec> = {}
+  for (const [name, col] of Object.entries(columns)) out[name] = col.spec
+  return out
+}
+
+/** The builder: every method returns plain data. */
+export const schemaBuilder: SchemaBuilder = Object.freeze<SchemaBuilder>({
+  uuid: () => column('uuid'),
+  varchar: (length: number) => column('varchar', { length }),
+  text: () => column('text'),
+  integer: () => column('integer'),
+  bigint: () => column('bigint'),
+  boolean: () => column('boolean'),
+  timestamp: () => column('timestamp'),
+  json: () => column('json'),
+  createTable: (table, columns) => ({ op: 'createTable', table, columns: specs(columns) }) as never,
+  addColumn: (table, name, spec) =>
+    ({ op: 'addColumn', table, column: name, spec: spec.spec }) as never,
+  addIndex: (table, columns, options = {}) => ({
+    op: 'addIndex',
+    table,
+    columns: [...columns],
+    unique: options.unique ?? false,
+  }),
+  widenColumn: (table, name, spec) => ({ op: 'widenColumn', table, column: name, spec: spec.spec }),
+  dropTable: (table) => ({ op: 'dropTable', table }) as never,
+  dropColumn: (table, name) => ({ op: 'dropColumn', table, column: name }) as never,
+  dropIndex: (table, columns) => ({ op: 'dropIndex', table, columns: [...columns] }),
+})
+
+// ------------------------------------------------------------ the row types
+
+type ColumnValue<C> =
+  C extends Column<infer T, infer N, boolean>
+    ? N extends true
+      ? ValueOf[T] | null
+      : ValueOf[T]
+    : never
+
+/** A table's row, from its columns. */
+export type RowOf<C extends Columns> = { [K in keyof C]: ColumnValue<C[K]> }
+
+type Optional<C extends Columns> = {
+  [K in keyof C]: C[K] extends Column<ColumnType, infer N, infer D>
+    ? N extends true
+      ? K
+      : D extends true
+        ? K
+        : never
+    : never
+}[keyof C]
+
+/** What an insert must and may supply: a nullable or defaulted column may be left out. */
+export type InsertOf<C extends Columns> = Omit<RowOf<C>, Optional<C>> &
+  Partial<Pick<RowOf<C>, Optional<C>>>
+
+/** Every table's columns, after a run of operations. */
+export type TablesShape = Record<string, Columns>
+
+type Effects<R> = R extends readonly unknown[] ? R[number] : R
+type EffectOf<O> = O extends Op<infer E> ? E : never
+
+type Apply<S extends TablesShape, E> =
+  E extends CreateTable<infer T, infer C>
+    ? Omit<S, T> & { [K in T]: C }
+    : E extends AddColumn<infer T, infer K, infer C>
+      ? T extends keyof S
+        ? Omit<S, T> & { [P in T]: S[T] & { [Q in K]: C } }
+        : S
+      : E extends DropTable<infer T>
+        ? Omit<S, T>
+        : E extends DropColumn<infer T, infer K>
+          ? T extends keyof S
+            ? Omit<S, T> & { [P in T]: Omit<S[T], K> }
+            : S
+          : S
+
+type ApplyAll<S extends TablesShape, E> = [E] extends [never]
+  ? S
+  : // A union of effects within one step is applied one member at a time.
+    UnionToTuple<E> extends infer L
+    ? L extends readonly [infer H, ...infer Rest]
+      ? ApplyAll<Flatten<Apply<S, H>>, Rest[number]>
+      : S
+    : S
+
+type Flatten<T> = { [K in keyof T]: T[K] } & {}
+
+type UnionToIntersection<U> = (U extends unknown ? (k: U) => void : never) extends (
+  k: infer I,
+) => void
+  ? I
+  : never
+type LastOf<U> =
+  UnionToIntersection<U extends unknown ? () => U : never> extends () => infer R ? R : never
+type UnionToTuple<U, L = LastOf<U>> = [U] extends [never] ? [] : [...UnionToTuple<Exclude<U, L>>, L]
+
+/** The shape after every step, in order: what `context.host.tables` is typed from. */
+export type ShapeOf<
+  Steps extends readonly unknown[],
+  S extends TablesShape = Record<never, never>,
+> = Steps extends readonly [infer First, ...infer Rest]
+  ? First extends { up: (s: SchemaBuilder) => infer R }
+    ? ShapeOf<Rest, Flatten<ApplyAll<S, EffectOf<Effects<R>>>>>
+    : ShapeOf<Rest, S>
+  : S