@crombolo/data 0.10.0 → 0.11.0

package/src/repositories/nodes.ts

@@ -7,6 +7,7 @@
 import { normaliseUuid, ObjectTypes, type Page, SystemNodes } from '@crombolo/core'
 import type { Db } from '../database.ts'
 import { DbDate, fromDbBool } from '../dialect.ts'
+import { searchTerms } from '../search-text.ts'
 
 export interface NodeRow {
   id: number
@@ -231,30 +232,54 @@ export class NodeRepository {
     return { items: rows.slice(start, end), totalBefore: start, totalAfter: rows.length - end }
   }
 
-  /** Name search, case-insensitive, optionally under a parent. */
+  /**
+   * Name search, case-insensitive, optionally under a parent. With `fullText`,
+   * an item also matches when the search index holds every word of the query by
+   * prefix; items whose name matches come first, then the best index matches.
+   */
   async search(
     objectType: string,
     query: string,
-    options: { trashed?: boolean; parentId?: number; skip: number; take: number },
+    options: {
+      trashed?: boolean
+      parentId?: number
+      fullText?: boolean
+      skip: number
+      take: number
+    },
   ): Promise<Page<NodeRow>> {
-    const conditions = ['node_object_type = ?', 'trashed = ?', 'LOWER(text) LIKE ?']
+    const like = `%${query.toLowerCase().replaceAll('%', '').replaceAll('_', '')}%`
+    const terms = options.fullText ? searchTerms(query) : []
+    const indexed = terms.length > 0
+    const conditions = [
+      'node_object_type = ?',
+      'trashed = ?',
+      indexed ? '(LOWER(text) LIKE ? OR h.node_id IS NOT NULL)' : 'LOWER(text) LIKE ?',
+    ]
     const params: unknown[] = [
+      ...(indexed ? [this.#db.dialect.fullTextQuery(terms)] : []),
       objectType,
       this.#db.dialect.boolValue(options.trashed ?? false),
-      `%${query.toLowerCase().replaceAll('%', '').replaceAll('_', '')}%`,
+      like,
     ]
     if (options.parentId !== undefined) {
       conditions.push('parent_id = ?')
       params.push(options.parentId)
     }
+    const from = indexed
+      ? `node LEFT JOIN (${this.#db.dialect.fullTextHits}) h ON h.node_id = node.id`
+      : 'node'
     const where = conditions.join(' AND ')
     const totals = await this.#db.query<{ n: number }>(
-      `SELECT COUNT(*) AS n FROM node WHERE ${where}`,
+      `SELECT COUNT(*) AS n FROM ${from} WHERE ${where}`,
       params,
     )
+    const order = indexed
+      ? 'CASE WHEN LOWER(text) LIKE ? THEN 0 ELSE 1 END, COALESCE(h.score, 0) DESC, text, id'
+      : 'text, id'
     const rows = await this.#db.query(
-      `SELECT ${NODE_COLUMNS} FROM node WHERE ${where} ORDER BY text, id LIMIT ? OFFSET ?`,
-      [...params, options.take, options.skip],
+      `SELECT ${NODE_COLUMNS} FROM ${from} WHERE ${where} ORDER BY ${order} LIMIT ? OFFSET ?`,
+      [...params, ...(indexed ? [like] : []), options.take, options.skip],
     )
     return { total: Number(totals[0]?.n ?? 0), items: rows.map(mapNode) }
   }

package/src/schema/audit-log.ts

@@ -0,0 +1,45 @@
+/**
+ * Migration 033 — the audit log. docs/audit-log.md.
+ *
+ * `audit_entry` records who did what, when and from where: every backoffice
+ * change, every sign-in and failed sign-in, and what the command line and the
+ * background jobs changed. Names are copied in when an entry is written, so an
+ * entry still reads after its user or item is gone. It starts empty: history from
+ * before this migration stays in the content versions it came from.
+ */
+import type { Db } from '../database.ts'
+import type { Migration } from '../migrations.ts'
+import { STATE_PROPERTY_SEARCHABLE } from './property-searchable.ts'
+
+export const STATE_AUDIT_LOG = '{6f1d4a8e-0033-4c1a-9e7d-3a5b8c2d4e33}'
+
+export const auditLogMigration: Migration = {
+  from: STATE_PROPERTY_SEARCHABLE,
+  to: STATE_AUDIT_LOG,
+  name: 'AddAuditLog',
+  kind: 'expand',
+  release: '0.10.0',
+  async up(db: Db) {
+    const t = db.dialect.types
+    await db.exec(
+      `CREATE TABLE audit_entry (
+         id ${t.identity},
+         occurred_at ${t.timestamp} NOT NULL,
+         category ${t.varchar(32)} NOT NULL,
+         action ${t.varchar(64)} NOT NULL,
+         user_key ${t.uuid},
+         user_name ${t.varchar(255)},
+         entity_type ${t.varchar(64)},
+         entity_key ${t.varchar(255)},
+         entity_name ${t.varchar(255)},
+         source ${t.varchar(16)} NOT NULL,
+         ip_address ${t.varchar(64)},
+         forwarded_for ${t.varchar(255)},
+         detail ${t.text}
+       )`,
+    )
+    await db.exec('CREATE INDEX ix_audit_entry_occurred ON audit_entry (occurred_at)')
+    await db.exec('CREATE INDEX ix_audit_entry_entity ON audit_entry (entity_key, occurred_at)')
+    await db.exec('CREATE INDEX ix_audit_entry_user ON audit_entry (user_key, occurred_at)')
+  },
+}

package/src/schema/plan.ts

@@ -13,6 +13,7 @@
  */
 import { MigrationPlan } from '../migrations.ts'
 import { assistantMigration } from './assistant.ts'
+import { auditLogMigration } from './audit-log.ts'
 import { bundleComponentsMigration } from './bundle-components.ts'
 import { bundlePackageMigration } from './bundle-package.ts'
 import { bundlesRenameMigration } from './bundles-rename.ts'
@@ -31,10 +32,12 @@ import { identityMigration } from './identity.ts'
 import { logViewerMigration } from './log-viewer.ts'
 import { memberPropertyTypeMigration } from './member-property-type.ts'
 import { membersMigration } from './members.ts'
+import { propertySearchableMigration } from './property-searchable.ts'
 import { publicAccessMigration } from './public-access.ts'
 import { recycleBinMigration } from './recycle-bin.ts'
 import { redirectsMigration } from './redirects.ts'
 import { richTextUiMigration } from './rich-text-ui.ts'
+import { searchIndexMigration } from './search-index.ts'
 import { serverRoleMigration } from './server-role.ts'
 import { sinceVersionMigration } from './since-version.ts'
 import { transferRunPackageMigration } from './transfer-run-package.ts'
@@ -74,4 +77,7 @@ export const cromboloPlan = new MigrationPlan([
   bundleComponentsMigration,
   bundlePackageMigration,
   transferRunPackageMigration,
+  searchIndexMigration,
+  propertySearchableMigration,
+  auditLogMigration,
 ])

package/src/schema/property-searchable.ts

@@ -0,0 +1,29 @@
+/**
+ * Migration 032 — a property can be kept out of backoffice search.
+ *
+ * `searchable = false` on a property in the schema files. Stored on the
+ * property type, because the backoffice writes those files back from the
+ * database and its save carries no such flag, so the column is what a save
+ * preserves. Every existing property stays searchable. docs/search.md.
+ */
+import type { Db } from '../database.ts'
+import type { Migration } from '../migrations.ts'
+import { schemaOps } from '../schema-ops.ts'
+import { STATE_SEARCH_INDEX } from './search-index.ts'
+
+export const STATE_PROPERTY_SEARCHABLE = '{6f1d4a8e-0032-4b7e-8a2c-5d9f1e3b6c32}'
+
+export const propertySearchableMigration: Migration = {
+  from: STATE_SEARCH_INDEX,
+  to: STATE_PROPERTY_SEARCHABLE,
+  name: 'AddPropertySearchable',
+  kind: 'expand',
+  release: '0.10.0',
+  async up(db: Db) {
+    await schemaOps(db).addColumn(
+      'property_type',
+      'searchable',
+      `${db.dialect.types.boolean} NOT NULL DEFAULT ${db.dialect.name === 'postgres' ? 'TRUE' : '1'}`,
+    )
+  },
+}

package/src/schema/search-index.ts

@@ -0,0 +1,92 @@
+/**
+ * Migration 031 — the backoffice search index. docs/search.md.
+ *
+ * `search_document` holds one row per item and culture: its name and the text of
+ * its current values, folded to lowercase without diacritics. SQLite matches it
+ * through an FTS5 table kept in step by triggers; Postgres through a generated
+ * `tsvector` column. Both use language-neutral word splitting, so the two
+ * dialects find the same things.
+ *
+ * `search_index_queue` is what a write leaves for the indexer: a node id and a
+ * generation, bumped each time the node is queued again, so a row re-queued
+ * while it was being indexed survives the indexer's delete. Every existing
+ * page, media item and member is queued here, which is the backfill.
+ */
+import { ObjectTypes } from '@crombolo/core'
+import type { Db } from '../database.ts'
+import type { Migration } from '../migrations.ts'
+import { STATE_TRANSFER_RUN_PACKAGE } from './transfer-run-package.ts'
+
+export const STATE_SEARCH_INDEX = '{6f1d4a8e-0031-4f3a-9c6d-2b8e5a1f7d31}'
+
+export const searchIndexMigration: Migration = {
+  from: STATE_TRANSFER_RUN_PACKAGE,
+  to: STATE_SEARCH_INDEX,
+  name: 'AddSearchIndex',
+  kind: 'expand',
+  release: '0.10.0',
+  async up(db: Db) {
+    const t = db.dialect.types
+    const postgres = db.dialect.name === 'postgres'
+    await db.exec(
+      `CREATE TABLE search_document (
+         id ${t.identity},
+         node_id ${t.integer} NOT NULL REFERENCES node (id) ON DELETE CASCADE,
+         culture ${t.varchar(14)} NOT NULL,
+         name ${t.text} NOT NULL,
+         body ${t.text} NOT NULL,
+         fields ${t.json} NOT NULL,
+         field_count ${t.integer} NOT NULL,
+         indexed_at ${t.timestamp} NOT NULL${
+           postgres
+             ? `,
+         terms tsvector GENERATED ALWAYS AS (
+           setweight(to_tsvector('simple', name), 'A') || setweight(to_tsvector('simple', body), 'B')
+         ) STORED`
+             : ''
+         },
+         UNIQUE (node_id, culture)
+       )`,
+    )
+    if (postgres) {
+      await db.exec('CREATE INDEX ix_search_document_terms ON search_document USING GIN (terms)')
+    } else {
+      await db.exec(
+        `CREATE VIRTUAL TABLE search_document_fts USING fts5(
+           name, body, content='search_document', content_rowid='id',
+           tokenize='unicode61 remove_diacritics 2'
+         )`,
+      )
+      await db.exec(
+        `CREATE TRIGGER search_document_ai AFTER INSERT ON search_document BEGIN
+           INSERT INTO search_document_fts (rowid, name, body) VALUES (new.id, new.name, new.body);
+         END`,
+      )
+      await db.exec(
+        `CREATE TRIGGER search_document_ad AFTER DELETE ON search_document BEGIN
+           INSERT INTO search_document_fts (search_document_fts, rowid, name, body)
+           VALUES ('delete', old.id, old.name, old.body);
+         END`,
+      )
+      await db.exec(
+        `CREATE TRIGGER search_document_au AFTER UPDATE ON search_document BEGIN
+           INSERT INTO search_document_fts (search_document_fts, rowid, name, body)
+           VALUES ('delete', old.id, old.name, old.body);
+           INSERT INTO search_document_fts (rowid, name, body) VALUES (new.id, new.name, new.body);
+         END`,
+      )
+    }
+    await db.exec(
+      `CREATE TABLE search_index_queue (
+         node_id ${t.integer} PRIMARY KEY REFERENCES node (id) ON DELETE CASCADE,
+         generation ${t.integer} NOT NULL,
+         queued_at ${t.timestamp} NOT NULL
+       )`,
+    )
+    await db.exec(
+      `INSERT INTO search_index_queue (node_id, generation, queued_at)
+       SELECT id, 1, ${db.dialect.now()} FROM node WHERE node_object_type IN (?, ?, ?)`,
+      [ObjectTypes.Document, ObjectTypes.Media, ObjectTypes.Member],
+    )
+  },
+}

package/src/search-text.ts

@@ -0,0 +1,133 @@
+/**
+ * The words a stored value contributes to the search index, and the folding
+ * both the index and a query go through so they meet. docs/search.md.
+ */
+
+/**
+ * Lowercase words without diacritics: letters and digits only, everything else
+ * a separator. The index stores text in this form, so the full-text engines on
+ * both dialects only ever split on spaces, and match the same way.
+ */
+export function searchWords(text: string): string[] {
+  return text
+    .normalize('NFKD')
+    .replace(/\p{M}+/gu, '')
+    .toLowerCase()
+    .split(/[^\p{L}\p{N}]+/u)
+    .filter((word) => word.length > 0)
+}
+
+/** The distinct words of a query, at most a dozen: what every result must hold by prefix. */
+export function searchTerms(query: string): string[] {
+  return [...new Set(searchWords(query))].slice(0, 12)
+}
+
+const IDENTIFIER =
+  /^(?:umb:\/\/\S+|\{?[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\}?)$/i
+
+/**
+ * Keys inside an editor's JSON that hold structure rather than words: block
+ * and media references, editor aliases, crops. Their values are never text an
+ * editor wrote.
+ */
+const STRUCTURAL_KEYS = new Set([
+  'alias',
+  'contentKey',
+  'contentTypeKey',
+  'crops',
+  'culture',
+  'editorAlias',
+  'expose',
+  'focalPoint',
+  'icon',
+  'key',
+  'layout',
+  'mediaKey',
+  'mediaTypeAlias',
+  'segment',
+  'settingsKey',
+  'settingsTypeKey',
+  'udi',
+  'unique',
+])
+
+const ENTITIES: Record<string, string> = {
+  amp: '&',
+  lt: '<',
+  gt: '>',
+  quot: '"',
+  apos: "'",
+  nbsp: ' ',
+}
+
+function stripMarkup(text: string): string {
+  if (!text.includes('<') && !text.includes('&')) return text
+  return text
+    .replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi, ' ')
+    .replace(/<[^>]*>/g, ' ')
+    .replace(/&(#\d+|#x[0-9a-f]+|[a-z]+);/gi, (entity, name: string) => {
+      if (name.startsWith('#x') || name.startsWith('#X'))
+        return String.fromCodePoint(Number.parseInt(name.slice(2), 16))
+      if (name.startsWith('#')) return String.fromCodePoint(Number(name.slice(1)))
+      return ENTITIES[name.toLowerCase()] ?? entity
+    })
+}
+
+/** Whether an element type's property is kept out of search: `searchable = false`. */
+export type UnsearchableProperty = (contentTypeKey: string, alias: string) => boolean
+
+const NOTHING_UNSEARCHABLE: UnsearchableProperty = () => false
+
+function collect(value: unknown, out: string[], unsearchable: UnsearchableProperty): void {
+  if (typeof value === 'string') {
+    const trimmed = value.trim()
+    if (trimmed.length === 0 || IDENTIFIER.test(trimmed)) return
+    if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
+      try {
+        collect(JSON.parse(trimmed), out, unsearchable)
+        return
+      } catch {
+        // Not JSON after all; it is text that happens to start with a bracket.
+      }
+    }
+    const text = stripMarkup(trimmed).replace(/\s+/g, ' ').trim()
+    if (text.length > 0) out.push(text)
+    return
+  }
+  if (Array.isArray(value)) {
+    for (const item of value) collect(item, out, unsearchable)
+    return
+  }
+  if (!value || typeof value !== 'object') return
+  const record = value as Record<string, unknown>
+  // A block's element: its values are listed by alias, or (older format) keyed by it.
+  const elementType = typeof record.contentTypeKey === 'string' ? record.contentTypeKey : undefined
+  for (const [key, item] of Object.entries(record)) {
+    if (STRUCTURAL_KEYS.has(key)) continue
+    if (elementType && unsearchable(elementType, key)) continue
+    if (elementType && key === 'values' && Array.isArray(item)) {
+      for (const entry of item) {
+        const alias = (entry as { alias?: unknown } | null)?.alias
+        if (typeof alias === 'string' && unsearchable(elementType, alias)) continue
+        collect(entry, out, unsearchable)
+      }
+      continue
+    }
+    collect(item, out, unsearchable)
+  }
+}
+
+/**
+ * The readable text in a stored value: plain strings as they are, markup without
+ * its tags, and the strings inside an editor's JSON (blocks, rich text, pickers
+ * with captions) without the keys and references that hold it together, or
+ * the values of a block's properties marked `searchable = false`.
+ */
+export function valueText(
+  raw: unknown,
+  unsearchable: UnsearchableProperty = NOTHING_UNSEARCHABLE,
+): string[] {
+  const out: string[] = []
+  collect(raw, out, unsearchable)
+  return out
+}