@crombolo/data 0.13.2 → 0.14.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crombolo/data",
-  "version": "0.13.2",
+  "version": "0.14.1",
   "description": "SQLite and Postgres storage for crombolo: schema, migrations and repositories.",
   "license": "MIT",
   "author": "Crombolo",
@@ -28,6 +28,6 @@
     "access": "public"
   },
   "dependencies": {
-    "@crombolo/core": "0.13.2"
+    "@crombolo/core": "0.14.1"
   }
 }

package/src/dialect.ts

@@ -44,6 +44,12 @@ export interface Dialect {
    * prefix. Its one parameter is `fullTextQuery(terms)`. docs/search.md.
    */
   readonly fullTextHits: string
+  /**
+   * Site search's matching rows of `site_search_document`, as `row_id` and
+   * `score`, weighted name 10, high 5, normal 2, low 1 on both engines. Its one
+   * parameter is `fullTextQuery(terms)`.
+   */
+  readonly siteSearchHits: string
   /** Terms are already folded words: letters and digits only. */
   fullTextQuery(terms: readonly string[]): string
   /**
@@ -90,6 +96,8 @@ export const sqliteDialect: Dialect = {
           FROM search_document_fts WHERE search_document_fts MATCH ? LIMIT -1) f
     JOIN search_document sd ON sd.id = f.id GROUP BY sd.node_id`,
   fullTextQuery: (terms) => terms.map((term) => `"${term}"*`).join(' AND '),
+  siteSearchHits: `SELECT rowid AS row_id, -bm25(site_search_document_fts, 10.0, 5.0, 2.0, 1.0) AS score
+    FROM site_search_document_fts WHERE site_search_document_fts MATCH ? LIMIT -1`,
   syncIdentity: () => undefined,
 }
 
@@ -126,6 +134,9 @@ export const postgresDialect: Dialect = {
     FROM search_document sd, to_tsquery('simple', ?) q
     WHERE sd.terms @@ q GROUP BY sd.node_id`,
   fullTextQuery: (terms) => terms.map((term) => `${term}:*`).join(' & '),
+  // ts_rank's weights run D, C, B, A: the same 1, 2, 5, 10 as SQLite's bm25 weights.
+  siteSearchHits: `SELECT s.id AS row_id, ts_rank('{0.1, 0.2, 0.5, 1.0}', s.terms, q) AS score
+    FROM site_search_document s, to_tsquery('simple', ?) q WHERE s.terms @@ q`,
   syncIdentity: (table) =>
     `SELECT setval(pg_get_serial_sequence('${table}', 'id'), (SELECT COALESCE(MAX(id), 1) FROM ${table}))`,
 }

package/src/index.ts

@@ -21,6 +21,7 @@ export * from './repositories/dictionary.ts'
 export * from './repositories/documents.ts'
 export * from './repositories/domains.ts'
 export * from './repositories/dynamic-roots.ts'
+export * from './repositories/external-logins.ts'
 export * from './repositories/folders.ts'
 export * from './repositories/form-entries.ts'
 export * from './repositories/form-workflows.ts'
@@ -36,6 +37,7 @@ export * from './repositories/tags.ts'
 export * from './repositories/templates.ts'
 export * from './repositories/transfer-answers.ts'
 export * from './repositories/transfer-runs.ts'
+export * from './repositories/two-factor.ts'
 export * from './repositories/user-data.ts'
 export * from './repositories/users.ts'
 export * from './repositories/webhooks.ts'
@@ -69,9 +71,11 @@ export * from './schema/rich-text-ui.ts'
 export * from './schema/search-index.ts'
 export * from './schema/seed.ts'
 export * from './schema/since-version.ts'
+export * from './schema/site-search.ts'
 export * from './schema/transfer-answers.ts'
 export * from './schema/transfer-run-package.ts'
 export * from './schema/transfer-runs.ts'
+export * from './schema/two-factor.ts'
 export * from './schema/upgrade-report.ts'
 export * from './schema/users-localisation.ts'
 export * from './schema/values.ts'
@@ -81,6 +85,7 @@ export * from './schema-state.ts'
 export * from './search.ts'
 export * from './search-text.ts'
 export * from './setup.ts'
+export * from './site-search.ts'
 export * from './tree-invariants.ts'
 export * from './url-options.ts'
 export * from './workflow-store.ts'

package/src/repositories/content-types.ts

@@ -18,6 +18,7 @@ import {
   type PropertyGroupModel,
   PropertyGroupType,
   type PropertyTypeModel,
+  type SiteSearchLevel,
   SystemNodes,
   variationFlags,
   variesByCulture,
@@ -32,19 +33,36 @@ import { NodeRepository } from './nodes.ts'
 
 const GROUP_TYPE_NAMES = ['Group', 'Tab'] as const
 
-/** The live properties of a type, by alias, variation and storage: what its items are indexed by. */
+/**
+ * The live properties of a type, by alias, variation, storage and search
+ * settings, and whether the type is in site search: what its items are indexed by.
+ */
 async function propertySignature(db: Db, contentTypeNodeId: number): Promise<string> {
   const rows = await db.query<Record<string, unknown>>(
-    `SELECT p.id, p.alias, p.variations, p.data_type_id, p.searchable FROM property_type p
+    `SELECT p.id, p.alias, p.variations, p.data_type_id, p.searchable, p.site_search FROM property_type p
      WHERE p.content_type_id = ? AND p.retired_at IS NULL ORDER BY p.id`,
     [contentTypeNodeId],
   )
-  return rows
-    .map(
+  const [type] = await db.query<{ site_search: unknown }>(
+    'SELECT site_search FROM content_type WHERE node_id = ?',
+    [contentTypeNodeId],
+  )
+  return [
+    `site:${type ? fromDbBool(type.site_search) : true}`,
+    ...rows.map(
       (row) =>
-        `${row.id}:${row.alias}:${row.variations}:${row.data_type_id}:${fromDbBool(row.searchable)}`,
-    )
-    .join('|')
+        `${row.id}:${row.alias}:${row.variations}:${row.data_type_id}:${fromDbBool(row.searchable)}:${row.site_search}`,
+    ),
+  ].join('|')
+}
+
+/** How a property's site-search setting is stored: its level, or `off`. */
+function siteSearchColumn(value: SiteSearchLevel | false | undefined): string {
+  return value === false ? 'off' : (value ?? 'normal')
+}
+
+function siteSearchFromColumn(value: unknown): SiteSearchLevel | false {
+  return value === 'off' ? false : value === 'high' || value === 'low' ? value : 'normal'
 }
 
 export interface SaveContentTypeOptions {
@@ -295,6 +313,12 @@ export class ContentTypeRepository {
           [nodeId, ...typeValues, options.sinceStateId ?? null],
         )
       }
+      // As for a property's flags: only a schema sync says.
+      if (aggregate.siteSearch !== undefined)
+        await tx.exec('UPDATE content_type SET site_search = ? WHERE node_id = ?', [
+          bool(aggregate.siteSearch),
+          nodeId,
+        ])
 
       // ---- groups: upsert by key
       const groupIdByKey = new Map<string, number>()
@@ -386,18 +410,24 @@ export class ContentTypeRepository {
               bool(property.searchable),
               Number(found[0].id),
             ])
+          if (property.siteSearch !== undefined)
+            await tx.exec('UPDATE property_type SET site_search = ? WHERE id = ?', [
+              siteSearchColumn(property.siteSearch),
+              Number(found[0].id),
+            ])
         } else {
           await tx.exec(
             `INSERT INTO property_type
                (unique_id, content_type_id, data_type_id, property_type_group_id, alias, name, description, sort_order,
-                mandatory, mandatory_message, validation_reg_exp, validation_reg_exp_message, label_on_top, variations, since_version, since_state_id, searchable)
-             VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+                mandatory, mandatory_message, validation_reg_exp, validation_reg_exp_message, label_on_top, variations, since_version, since_state_id, searchable, site_search)
+             VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
             [
               key,
               nodeId,
               ...columns,
               options.sinceStateId ?? null,
               bool(property.searchable !== false),
+              siteSearchColumn(property.siteSearch),
             ],
           )
         }
@@ -747,7 +777,7 @@ export class ContentTypeRepository {
     const typeRows = await this.#db.query(
       `SELECT ct.alias, ct.icon, ct.description, ct.list_view, ct.is_element, ct.allow_in_library, ct.allow_at_root, ct.variations,
               ct.prevent_cleanup, ct.keep_all_versions_newer_than_days, ct.keep_latest_version_per_day_for_days,
-              ct.since_version, ss.version AS since_v, ss.revision AS since_r
+              ct.since_version, ct.site_search, ss.version AS since_v, ss.revision AS since_r
        FROM content_type ct LEFT JOIN schema_state ss ON ss.id = ct.since_state_id WHERE ct.node_id = ?`,
       [nodeId],
     )
@@ -775,7 +805,7 @@ export class ContentTypeRepository {
               p.validation_reg_exp, p.validation_reg_exp_message, p.label_on_top, p.variations,
               dtn.unique_id AS data_type_key, g.unique_id AS container_key,
               p.since_version, ss.version AS since_v, ss.revision AS since_r,
-              m.member_can_edit, m.member_can_view, m.is_sensitive, p.searchable
+              m.member_can_edit, m.member_can_view, m.is_sensitive, p.searchable, p.site_search
        FROM property_type p
        JOIN node dtn ON dtn.id = p.data_type_id
        LEFT JOIN property_type_group g ON g.id = p.property_type_group_id
@@ -803,6 +833,7 @@ export class ContentTypeRepository {
       sinceVersion: (row.since_version as string | null) ?? null,
       pending: this.#pending(row),
       ...(fromDbBool(row.searchable) ? {} : { searchable: false }),
+      siteSearch: siteSearchFromColumn(row.site_search),
       ...(this.#kind === 'member'
         ? {
             memberCanEdit: fromDbBool(row.member_can_edit),
@@ -839,6 +870,7 @@ export class ContentTypeRepository {
       variesBySegment: variesBySegment(flags),
       isElement: fromDbBool(type.is_element),
       allowedInLibrary: fromDbBool(type.allow_in_library),
+      siteSearch: fromDbBool(type.site_search),
       collectionKey: type.list_view ? normaliseUuid(String(type.list_view)) : null,
       cleanup: {
         preventCleanup: fromDbBool(type.prevent_cleanup),

package/src/repositories/documents.ts

@@ -967,6 +967,8 @@ export class DocumentRepository {
           node.id,
         ])
       }
+      // Site search holds published values only; what this took offline leaves it.
+      await queueForSearch(tx, [node.id])
       await repo.#instructContent(node.key)
       return repo.#load(node)
     })
@@ -1060,6 +1062,13 @@ export class DocumentRepository {
         const node = await repo.nodes.byKey(key)
         if (!node) return false
         if (node.parentId === bin) return true
+        // Read before the move, while the branch's paths are still under the node's own.
+        const branch = (
+          await tx.query<{ id: number }>('SELECT id FROM node WHERE id = ? OR path LIKE ?', [
+            node.id,
+            `${node.path},%`,
+          ])
+        ).map((row) => Number(row.id))
         const t = tx.dialect.boolValue(true)
         const f = tx.dialect.boolValue(false)
         await tx.exec('UPDATE document SET original_parent_id = ? WHERE node_id = ?', [
@@ -1081,6 +1090,9 @@ export class DocumentRepository {
           await repo.#nextSortOrder(bin, node.id),
           node.id,
         ])
+        // The branch is no longer published: other nodes drop it, and so does site search.
+        await queueForSearch(tx, branch)
+        await repo.#instructContent(node.key)
         return true
       }),
     )

package/src/repositories/external-logins.ts

@@ -0,0 +1,89 @@
+/**
+ * The accounts at external identity providers linked to local members and
+ * users (story 031). `external_login` is polymorphic on `user_or_member_key`, as
+ * Umbraco's is; the login provider says which audience it belongs to, as
+ * Umbraco's scheme names do — `UmbracoMembers.{id}` for members, `Umbraco.{id}`
+ * for users — so one provider configured for both never mixes them up.
+ */
+import { normaliseUuid } from '@crombolo/core'
+import type { Db } from '../database.ts'
+import { DbDate } from '../dialect.ts'
+
+export type ExternalLoginAudience = 'member' | 'user'
+
+/** The login provider a provider id is recorded under, for an audience. */
+export function externalLoginProvider(audience: ExternalLoginAudience, providerId: string): string {
+  return `${audience === 'member' ? 'UmbracoMembers' : 'Umbraco'}.${providerId}`
+}
+
+export interface ExternalLogin {
+  /** The member's or user's key. */
+  ownerKey: string
+  loginProvider: string
+  /** The provider's id for the account: its `sub`. */
+  providerKey: string
+  createDate: Date
+}
+
+const toLogin = (row: Record<string, unknown>): ExternalLogin => ({
+  ownerKey: normaliseUuid(String(row.user_or_member_key)),
+  loginProvider: String(row.login_provider),
+  providerKey: String(row.provider_key),
+  createDate: DbDate.fromDb(row.create_date) ?? new Date(0),
+})
+
+export class ExternalLoginRepository {
+  #db: Db
+
+  constructor(db: Db) {
+    this.#db = db
+  }
+
+  /** Who an account at a provider is linked to, if anyone. */
+  async find(loginProvider: string, providerKey: string): Promise<ExternalLogin | undefined> {
+    const rows = await this.#db.query(
+      'SELECT * FROM external_login WHERE login_provider = ? AND provider_key = ?',
+      [loginProvider, providerKey],
+    )
+    return rows[0] ? toLogin(rows[0]) : undefined
+  }
+
+  /** Every provider account linked to a member or user. */
+  async forOwner(ownerKey: string): Promise<ExternalLogin[]> {
+    const rows = await this.#db.query(
+      'SELECT * FROM external_login WHERE user_or_member_key = ? ORDER BY login_provider',
+      [normaliseUuid(ownerKey)],
+    )
+    return rows.map(toLogin)
+  }
+
+  /** Links a provider account to a member or user, replacing an earlier link to that provider. */
+  async link(ownerKey: string, loginProvider: string, providerKey: string): Promise<void> {
+    await this.#db.transaction(async (tx) => {
+      await removeLink(tx, ownerKey, loginProvider)
+      await tx.exec(
+        `INSERT INTO external_login (user_or_member_key, login_provider, provider_key, create_date)
+         VALUES (?, ?, ?, ?)`,
+        [normaliseUuid(ownerKey), loginProvider, providerKey, DbDate.toDb(new Date())],
+      )
+    })
+  }
+
+  async unlink(ownerKey: string, loginProvider: string): Promise<void> {
+    await this.#db.transaction((tx) => removeLink(tx, ownerKey, loginProvider))
+  }
+}
+
+/** A link and the tokens kept for it, which reference it. */
+async function removeLink(tx: Db, ownerKey: string, loginProvider: string): Promise<void> {
+  const params = [loginProvider, normaliseUuid(ownerKey)]
+  await tx.exec(
+    `DELETE FROM external_login_token WHERE external_login_id IN
+       (SELECT id FROM external_login WHERE login_provider = ? AND user_or_member_key = ?)`,
+    params,
+  )
+  await tx.exec(
+    'DELETE FROM external_login WHERE login_provider = ? AND user_or_member_key = ?',
+    params,
+  )
+}

package/src/repositories/members.ts

@@ -204,6 +204,13 @@ export class MemberRepository {
     )
   }
 
+  /** Replaces a member's groups, by group key; what an identity provider's group sync writes. */
+  async setGroups(key: string, groupKeys: readonly string[]): Promise<void> {
+    const node = await this.#nodes.byKey(key)
+    if (!node || node.objectType !== ObjectTypes.Member) return
+    await this.#db.transaction(() => this.#writeGroups(node.id, groupKeys))
+  }
+
   async #writeGroups(nodeId: number, groupKeys: readonly string[]): Promise<void> {
     await this.#db.exec('DELETE FROM member_group_member WHERE member_id = ?', [nodeId])
     for (const key of groupKeys) {
@@ -352,6 +359,8 @@ export class MemberRepository {
       [node.key],
     )
     await this.#db.exec('DELETE FROM external_login WHERE user_or_member_key = ?', [node.key])
+    await this.#db.exec('DELETE FROM two_factor_login WHERE owner_key = ?', [node.key])
+    await this.#db.exec('DELETE FROM two_factor_recovery_code WHERE owner_key = ?', [node.key])
     await this.#db.exec('DELETE FROM member WHERE node_id = ?', [node.id])
     return this.content.delete(key)
   }

package/src/repositories/two-factor.ts

@@ -0,0 +1,137 @@
+/**
+ * The authenticators and recovery codes of users and members (story 032). The
+ * secret arrives here already sealed and leaves still sealed: sealing is the
+ * server's, with `CROMBOLO_SECRET_KEY`, so this layer never holds a key.
+ */
+import { normaliseUuid } from '@crombolo/core'
+import type { Db } from '../database.ts'
+import { DbDate } from '../dialect.ts'
+
+export interface TwoFactorLogin {
+  ownerKey: string
+  providerName: string
+  sealedSecret: string
+  lastUsedStep: number | null
+}
+
+export class TwoFactorRepository {
+  #db: Db
+
+  constructor(db: Db) {
+    this.#db = db
+  }
+
+  /** The providers an owner has turned on. */
+  async providers(ownerKey: string): Promise<string[]> {
+    const rows = await this.#db.query<{ provider_name: string }>(
+      'SELECT provider_name FROM two_factor_login WHERE owner_key = ? ORDER BY provider_name',
+      [normaliseUuid(ownerKey)],
+    )
+    return rows.map((row) => String(row.provider_name))
+  }
+
+  /** Which of these owners have any provider on, for list views. */
+  async enabledAmong(ownerKeys: readonly string[]): Promise<Set<string>> {
+    if (ownerKeys.length === 0) return new Set()
+    const keys = ownerKeys.map(normaliseUuid)
+    const rows = await this.#db.query<{ owner_key: string }>(
+      `SELECT DISTINCT owner_key FROM two_factor_login WHERE owner_key IN (${keys.map(() => '?').join(', ')})`,
+      keys,
+    )
+    return new Set(rows.map((row) => normaliseUuid(String(row.owner_key))))
+  }
+
+  async get(ownerKey: string, providerName: string): Promise<TwoFactorLogin | undefined> {
+    const rows = await this.#db.query<Record<string, unknown>>(
+      'SELECT * FROM two_factor_login WHERE owner_key = ? AND provider_name = ?',
+      [normaliseUuid(ownerKey), providerName],
+    )
+    const row = rows[0]
+    if (!row) return undefined
+    return {
+      ownerKey: normaliseUuid(String(row.owner_key)),
+      providerName: String(row.provider_name),
+      sealedSecret: String(row.secret),
+      lastUsedStep:
+        row.last_used_step === null || row.last_used_step === undefined
+          ? null
+          : Number(row.last_used_step),
+    }
+  }
+
+  /** Turns a provider on, with its recovery codes' hashes; replaces any earlier setup. */
+  async enable(
+    ownerKey: string,
+    providerName: string,
+    sealedSecret: string,
+    acceptedStep: number,
+    recoveryHashes: readonly string[],
+  ): Promise<void> {
+    const owner = normaliseUuid(ownerKey)
+    await this.#db.transaction(async (tx) => {
+      await tx.exec('DELETE FROM two_factor_login WHERE owner_key = ? AND provider_name = ?', [
+        owner,
+        providerName,
+      ])
+      await tx.exec(
+        `INSERT INTO two_factor_login (owner_key, provider_name, secret, last_used_step, create_date)
+         VALUES (?, ?, ?, ?, ?)`,
+        [owner, providerName, sealedSecret, acceptedStep, DbDate.toDb(new Date())],
+      )
+      await tx.exec('DELETE FROM two_factor_recovery_code WHERE owner_key = ?', [owner])
+      for (const hash of recoveryHashes)
+        await tx.exec('INSERT INTO two_factor_recovery_code (owner_key, code_hash) VALUES (?, ?)', [
+          owner,
+          hash,
+        ])
+    })
+  }
+
+  /** Records a step a code was accepted for, refusing one at or before the last; true when recorded. */
+  async useStep(ownerKey: string, providerName: string, step: number): Promise<boolean> {
+    const rows = await this.#db.query<{ owner_key: string }>(
+      `UPDATE two_factor_login SET last_used_step = ?
+       WHERE owner_key = ? AND provider_name = ? AND (last_used_step IS NULL OR last_used_step < ?)
+       RETURNING owner_key`,
+      [step, normaliseUuid(ownerKey), providerName, step],
+    )
+    return rows.length > 0
+  }
+
+  /** Spends a recovery code by its hash; true when it was there and unused. */
+  async useRecoveryCode(ownerKey: string, hash: string): Promise<boolean> {
+    const rows = await this.#db.query<{ id: number }>(
+      `UPDATE two_factor_recovery_code SET used_date = ?
+       WHERE owner_key = ? AND code_hash = ? AND used_date IS NULL RETURNING id`,
+      [DbDate.toDb(new Date()), normaliseUuid(ownerKey), hash],
+    )
+    return rows.length > 0
+  }
+
+  async recoveryCodesLeft(ownerKey: string): Promise<number> {
+    const rows = await this.#db.query<{ n: number }>(
+      'SELECT COUNT(*) AS n FROM two_factor_recovery_code WHERE owner_key = ? AND used_date IS NULL',
+      [normaliseUuid(ownerKey)],
+    )
+    return Number(rows[0]?.n ?? 0)
+  }
+
+  /** Turns a provider off, or every provider when none is named, with the recovery codes once none is left. */
+  async disable(ownerKey: string, providerName?: string): Promise<void> {
+    const owner = normaliseUuid(ownerKey)
+    await this.#db.transaction(async (tx) => {
+      if (providerName)
+        await tx.exec('DELETE FROM two_factor_login WHERE owner_key = ? AND provider_name = ?', [
+          owner,
+          providerName,
+        ])
+      else await tx.exec('DELETE FROM two_factor_login WHERE owner_key = ?', [owner])
+      const left = await tx.query<{ n: number }>(
+        'SELECT COUNT(*) AS n FROM two_factor_login WHERE owner_key = ?',
+        [owner],
+      )
+      if (Number(left[0]?.n ?? 0) === 0)
+        await tx.exec('DELETE FROM two_factor_recovery_code WHERE owner_key = ?', [owner])
+    })
+  }
+}

package/src/repositories/users.ts

@@ -464,6 +464,14 @@ export class UserRepository {
         'user_login_session',
       ])
         await tx.exec(`DELETE FROM ${table} WHERE user_id = ?`, [user.id])
+      // Their provider accounts too, which are keyed by the user rather than the row.
+      await tx.exec(
+        'DELETE FROM external_login_token WHERE external_login_id IN (SELECT id FROM external_login WHERE user_or_member_key = ?)',
+        [user.key],
+      )
+      await tx.exec('DELETE FROM external_login WHERE user_or_member_key = ?', [user.key])
+      await tx.exec('DELETE FROM two_factor_login WHERE owner_key = ?', [user.key])
+      await tx.exec('DELETE FROM two_factor_recovery_code WHERE owner_key = ?', [user.key])
       await tx.exec('DELETE FROM user_account WHERE id = ?', [user.id])
     })
   }

package/src/schema/plan.ts

@@ -44,9 +44,11 @@ import { richTextUiMigration } from './rich-text-ui.ts'
 import { searchIndexMigration } from './search-index.ts'
 import { serverRoleMigration } from './server-role.ts'
 import { sinceVersionMigration } from './since-version.ts'
+import { siteSearchMigration } from './site-search.ts'
 import { transferAnswersMigration } from './transfer-answers.ts'
 import { transferRunPackageMigration } from './transfer-run-package.ts'
 import { transferRunsMigration } from './transfer-runs.ts'
+import { twoFactorMigration } from './two-factor.ts'
 import { upgradeReportMigration } from './upgrade-report.ts'
 import { usersLocalisationMigration } from './users-localisation.ts'
 import { valuesMigration } from './values.ts'
@@ -91,4 +93,6 @@ export const cromboloPlan = new MigrationPlan([
   transferAnswersMigration,
   deliveryMemberTokensMigration,
   approvalsMigration,
+  siteSearchMigration,
+  twoFactorMigration,
 ])

package/src/schema/site-search.ts

@@ -0,0 +1,103 @@
+/**
+ * Migration 039 — site search for visitors (story 140). docs/search.md.
+ *
+ * `site_search_document` holds one row per page and culture, from its
+ * published values only: the name, and the text of its properties in three
+ * columns by weight, each folded as backoffice search folds. `text` keeps the
+ * same text as written, for excerpts. SQLite matches through an FTS5 table kept
+ * in step by triggers, Postgres through a generated `tsvector` whose four
+ * weights are the four columns.
+ *
+ * `site_search` on a property says which column its text goes in, or `off`;
+ * on a content type, false leaves its pages out. Both default to searchable.
+ * Every page is queued, which is the backfill.
+ */
+import { ObjectTypes } from '@crombolo/core'
+import type { Db } from '../database.ts'
+import type { Migration } from '../migrations.ts'
+import { schemaOps } from '../schema-ops.ts'
+import { STATE_APPROVALS } from './approvals.ts'
+
+export const STATE_SITE_SEARCH = '{9a5d1b4e-0039-4c73-8d18-4e0a3c1f7d59}'
+
+const COLUMNS = ['name', 'high', 'normal', 'low'] as const
+
+export const siteSearchMigration: Migration = {
+  from: STATE_APPROVALS,
+  to: STATE_SITE_SEARCH,
+  name: 'AddSiteSearch',
+  kind: 'expand',
+  release: '0.14.0',
+  async up(db: Db) {
+    const t = db.dialect.types
+    const postgres = db.dialect.name === 'postgres'
+    const ops = schemaOps(db)
+    await ops.addColumn('property_type', 'site_search', `${t.varchar(8)} NOT NULL DEFAULT 'normal'`)
+    await ops.addColumn(
+      'content_type',
+      'site_search',
+      `${t.boolean} NOT NULL DEFAULT ${postgres ? 'TRUE' : '1'}`,
+    )
+    await db.exec(
+      `CREATE TABLE site_search_document (
+         id ${t.identity},
+         node_id ${t.integer} NOT NULL REFERENCES node (id) ON DELETE CASCADE,
+         culture ${t.varchar(14)} NOT NULL,
+         name ${t.text} NOT NULL,
+         high ${t.text} NOT NULL,
+         normal ${t.text} NOT NULL,
+         low ${t.text} NOT NULL,
+         text ${t.json} NOT NULL,
+         indexed_at ${t.timestamp} NOT NULL${
+           postgres
+             ? `,
+         terms tsvector GENERATED ALWAYS AS (
+           setweight(to_tsvector('simple', name), 'A') || setweight(to_tsvector('simple', high), 'B') ||
+           setweight(to_tsvector('simple', normal), 'C') || setweight(to_tsvector('simple', low), 'D')
+         ) STORED`
+             : ''
+         },
+         UNIQUE (node_id, culture)
+       )`,
+    )
+    if (postgres) {
+      await db.exec(
+        'CREATE INDEX ix_site_search_document_terms ON site_search_document USING GIN (terms)',
+      )
+    } else {
+      const columns = COLUMNS.join(', ')
+      const fromNew = COLUMNS.map((c) => `new.${c}`).join(', ')
+      const fromOld = COLUMNS.map((c) => `old.${c}`).join(', ')
+      await db.exec(
+        `CREATE VIRTUAL TABLE site_search_document_fts USING fts5(
+           ${columns}, content='site_search_document', content_rowid='id',
+           tokenize='unicode61 remove_diacritics 2'
+         )`,
+      )
+      await db.exec(
+        `CREATE TRIGGER site_search_document_ai AFTER INSERT ON site_search_document BEGIN
+           INSERT INTO site_search_document_fts (rowid, ${columns}) VALUES (new.id, ${fromNew});
+         END`,
+      )
+      await db.exec(
+        `CREATE TRIGGER site_search_document_ad AFTER DELETE ON site_search_document BEGIN
+           INSERT INTO site_search_document_fts (site_search_document_fts, rowid, ${columns})
+           VALUES ('delete', old.id, ${fromOld});
+         END`,
+      )
+      await db.exec(
+        `CREATE TRIGGER site_search_document_au AFTER UPDATE ON site_search_document BEGIN
+           INSERT INTO site_search_document_fts (site_search_document_fts, rowid, ${columns})
+           VALUES ('delete', old.id, ${fromOld});
+           INSERT INTO site_search_document_fts (rowid, ${columns}) VALUES (new.id, ${fromNew});
+         END`,
+      )
+    }
+    await db.exec(
+      `INSERT INTO search_index_queue (node_id, generation, queued_at)
+       SELECT id, 1, ${db.dialect.now()} FROM node WHERE node_object_type = ?
+       ON CONFLICT (node_id) DO NOTHING`,
+      [ObjectTypes.Document],
+    )
+  },
+}

package/src/schema/two-factor.ts

@@ -0,0 +1,47 @@
+/**
+ * Migration 040 — two-factor sign-in (story 032).
+ *
+ * `two_factor_login` is a user's or member's authenticator, polymorphic on its
+ * owner's key as Umbraco's `umbracoTwoFactorLogin` is: the provider, the secret
+ * sealed with `CROMBOLO_SECRET_KEY`, and the last step a code was accepted for,
+ * so a code cannot be used twice. `two_factor_recovery_code` holds the hashes of
+ * the one-time recovery codes, each marked when used. docs/operations.md.
+ */
+import type { Db } from '../database.ts'
+import type { Migration } from '../migrations.ts'
+import { STATE_SITE_SEARCH } from './site-search.ts'
+
+export const STATE_TWO_FACTOR = '{4c2e7a90-0040-4d1b-9f63-8a5e2c7b1d40}'
+
+export const twoFactorMigration: Migration = {
+  from: STATE_SITE_SEARCH,
+  to: STATE_TWO_FACTOR,
+  name: 'AddTwoFactorLogin',
+  kind: 'expand',
+  release: '0.14.0',
+  async up(db: Db) {
+    const t = db.dialect.types
+    await db.exec(
+      `CREATE TABLE two_factor_login (
+         id ${t.identity},
+         owner_key ${t.varchar(36)} NOT NULL,
+         provider_name ${t.varchar(100)} NOT NULL,
+         secret ${t.text} NOT NULL,
+         last_used_step ${t.bigint},
+         create_date ${t.timestamp} NOT NULL,
+         UNIQUE (owner_key, provider_name)
+       )`,
+    )
+    await db.exec(
+      `CREATE TABLE two_factor_recovery_code (
+         id ${t.identity},
+         owner_key ${t.varchar(36)} NOT NULL,
+         code_hash ${t.varchar(64)} NOT NULL,
+         used_date ${t.timestamp}
+       )`,
+    )
+    await db.exec(
+      'CREATE INDEX ix_two_factor_recovery_code_owner ON two_factor_recovery_code (owner_key)',
+    )
+  },
+}

package/src/search.ts

@@ -1,12 +1,13 @@
 /**
  * The backoffice search index: what a write queues, how the indexer turns a
- * queued item into rows, and the two indexes the Examine dashboard shows.
- * docs/search.md.
+ * queued item into rows, and the two indexes the Examine dashboard shows. The
+ * same indexer writes a page's site-search rows (`site-search.ts`). docs/search.md.
  */
 import { ObjectTypes } from '@crombolo/core'
 import type { Db } from './database.ts'
 import { DbDate, fromDbBool } from './dialect.ts'
 import { searchTerms, searchWords, type UnsearchableProperty, valueText } from './search-text.ts'
+import { siteEntriesFor, writeSiteEntries } from './site-search.ts'
 
 /** Umbraco's names for the two backoffice indexes, which the dashboard shows. */
 export type SearchIndexName = 'InternalIndex' | 'MembersIndex'
@@ -156,6 +157,7 @@ export class SearchIndex {
         )
         if (claimed.length === 0) return false
         await writeEntries(tx, nodeId, await entriesFor(tx, nodeId))
+        await writeSiteEntries(tx, nodeId, await siteEntriesFor(tx, nodeId))
         await tx.exec('DELETE FROM search_index_queue WHERE node_id = ? AND generation = ?', [
           nodeId,
           generation,
@@ -272,11 +274,23 @@ const folded = (texts: readonly string[]) => texts.flatMap(searchWords).join(' '
  * alias, for the blocks inside a value. A property a composition contributes
  * counts on every type composing it, which is the key a block carries.
  */
-async function unsearchableElementProperties(db: Db): Promise<UnsearchableProperty> {
+function unsearchableElementProperties(db: Db): Promise<UnsearchableProperty> {
+  return hiddenElementProperties(db, 'p.searchable = ?', [db.dialect.boolValue(false)])
+}
+
+/**
+ * The properties a condition on `property_type p` picks out, as a test by type
+ * key and alias: what a block's text is checked against. Shared with site search.
+ */
+export async function hiddenElementProperties(
+  db: Db,
+  condition: string,
+  params: readonly unknown[],
+): Promise<UnsearchableProperty> {
   const rows = await db.query<{ type_id: number; alias: string }>(
     `SELECT p.content_type_id AS type_id, p.alias FROM property_type p
-     WHERE p.searchable = ? AND p.retired_at IS NULL`,
-    [db.dialect.boolValue(false)],
+     WHERE (${condition}) AND p.retired_at IS NULL`,
+    [...params],
   )
   if (rows.length === 0) return () => false
   const composedBy = new Map<number, number[]>()

package/src/site-search.ts

@@ -0,0 +1,237 @@
+/**
+ * Site search for visitors (story 140): what a page puts in the index, from
+ * its published values only, and the query a results page runs. The indexer
+ * that writes backoffice search rows writes these too (`search.ts`), from the
+ * same queue. docs/search.md.
+ */
+import { ObjectTypes, type SiteSearchLevel } from '@crombolo/core'
+import type { Db } from './database.ts'
+import { DbDate, fromDbBool } from './dialect.ts'
+import { publishedValueRows } from './repositories/documents.ts'
+import { hiddenElementProperties } from './search.ts'
+import { searchTerms, searchWords, valueText } from './search-text.ts'
+
+/** A page's text as written, by weight: what an excerpt is cut from. */
+export interface SiteSearchText {
+  name: string
+  high: string[]
+  normal: string[]
+  low: string[]
+}
+
+/** One row of `site_search_document`: a page in one culture, '' when invariant. */
+export interface SiteSearchEntry {
+  culture: string
+  text: SiteSearchText
+}
+
+export interface SiteSearchOptions {
+  /** The culture searched, lowercase; an invariant page matches every culture. */
+  culture: string
+  /** Content type aliases; every searchable type when empty or absent. */
+  types?: readonly string[]
+  skip: number
+  take: number
+  /**
+   * Protected nodes, by id, that the person searching may not open and may:
+   * `branchAccess` in `@crombolo/core`. A page is left out when the nearest
+   * protected node on its path is denied.
+   */
+  access?: { denied: readonly number[]; allowed: readonly number[] }
+}
+
+export interface SiteSearchHit {
+  key: string
+  culture: string
+  score: number
+  text: SiteSearchText
+}
+
+const LEVELS: readonly SiteSearchLevel[] = ['high', 'normal', 'low']
+
+const folded = (texts: readonly string[]) => texts.flatMap(searchWords).join(' ')
+
+const marks = (n: number) => Array.from({ length: n }, () => '?').join(', ')
+
+/**
+ * What the site-search index holds for a page: a row per published culture,
+ * or one invariant row, from the values that culture shows visitors. Nothing
+ * for a page that is unpublished or trashed, whose type has no component (it
+ * has no address) or is left out of site search, or for anything not a page.
+ * A property marked `site-search = false` or `searchable = false` is left out,
+ * here and inside blocks.
+ */
+export async function siteEntriesFor(db: Db, nodeId: number): Promise<SiteSearchEntry[]> {
+  const [node] = await db.query<Record<string, unknown>>(
+    `SELECT n.node_object_type, n.trashed, c.content_type_id, ct.site_search, ct.is_element, d.published,
+            (SELECT COUNT(*) FROM content_type_template t WHERE t.content_type_node_id = ct.node_id) AS components
+     FROM node n JOIN content c ON c.node_id = n.id JOIN content_type ct ON ct.node_id = c.content_type_id
+     JOIN document d ON d.node_id = n.id
+     WHERE n.id = ?`,
+    [nodeId],
+  )
+  if (
+    !node ||
+    String(node.node_object_type) !== ObjectTypes.Document ||
+    fromDbBool(node.trashed) ||
+    !fromDbBool(node.published) ||
+    !fromDbBool(node.site_search) ||
+    fromDbBool(node.is_element) ||
+    Number(node.components) === 0
+  )
+    return []
+
+  const [invariant] = await db.query<{ id: number; text: string | null }>(
+    `SELECT cv.id, cv.text FROM content_version cv JOIN document_version dv ON dv.id = cv.id
+     WHERE cv.node_id = ? AND dv.published = ? ORDER BY cv.id DESC LIMIT 1`,
+    [nodeId, db.dialect.boolValue(true)],
+  )
+  if (!invariant) return []
+  const cultures = (
+    await db.query<Record<string, unknown>>(
+      `SELECT dcv.language_id, l.iso_code, dcv.published_event_id, dcv.published_name, dcv.name
+       FROM document_culture_variation dcv JOIN language l ON l.id = dcv.language_id
+       WHERE dcv.node_id = ? AND dcv.published = ?`,
+      [nodeId, db.dialect.boolValue(true)],
+    )
+  ).map((row) => ({
+    languageId: Number(row.language_id),
+    culture: String(row.iso_code).toLowerCase(),
+    eventId: Number(row.published_event_id ?? 0),
+    name: String(row.published_name ?? row.name ?? ''),
+  }))
+
+  const latest = Math.max(Number(invariant.id), ...cultures.map((c) => c.eventId))
+  const history = (
+    await db.query<Record<string, unknown>>(
+      `SELECT pv.id, pv.event_id, pv.property_type_id, pv.language_id, pv.segment,
+              pv.varchar_value, pv.text_value, p.alias, p.site_search
+       FROM property_value pv JOIN property_type p ON p.id = pv.property_type_id
+       WHERE pv.node_id = ? AND pv.event_id <= ? AND p.retired_at IS NULL
+         AND p.searchable = ? AND p.site_search <> 'off'
+       ORDER BY pv.id`,
+      [nodeId, latest, db.dialect.boolValue(true)],
+    )
+  ).map((row) => ({
+    eventId: Number(row.event_id),
+    propertyTypeId: Number(row.property_type_id),
+    languageId: row.language_id === null ? null : Number(row.language_id),
+    segment: (row.segment as string | null) ?? null,
+    raw: row.text_value ?? row.varchar_value,
+    level: (LEVELS as readonly string[]).includes(String(row.site_search))
+      ? (String(row.site_search) as SiteSearchLevel)
+      : 'normal',
+  }))
+  const shown = publishedValueRows(history, Number(invariant.id), cultures).filter(
+    (row) => row.segment === null,
+  )
+  const hidden = await hiddenElementProperties(db, "p.searchable = ? OR p.site_search = 'off'", [
+    db.dialect.boolValue(false),
+  ])
+
+  const entry = (culture: string, languageId: number | null, name: string): SiteSearchEntry => {
+    const text: SiteSearchText = { name, high: [], normal: [], low: [] }
+    for (const row of shown) {
+      if (row.languageId !== null && row.languageId !== languageId) continue
+      text[row.level].push(...valueText(row.raw, hidden))
+    }
+    return { culture, text }
+  }
+  if (cultures.length === 0) return [entry('', null, String(invariant.text ?? ''))]
+  return cultures.map((c) => entry(c.culture, c.languageId, c.name))
+}
+
+/** Replaces a page's site-search rows. */
+export async function writeSiteEntries(
+  tx: Db,
+  nodeId: number,
+  entries: readonly SiteSearchEntry[],
+): Promise<void> {
+  await tx.exec('DELETE FROM site_search_document WHERE node_id = ?', [nodeId])
+  const now = DbDate.toDb(new Date())
+  for (const entry of entries)
+    await tx.exec(
+      `INSERT INTO site_search_document (node_id, culture, name, high, normal, low, text, indexed_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+      [
+        nodeId,
+        entry.culture,
+        folded([entry.text.name]),
+        folded(entry.text.high),
+        folded(entry.text.normal),
+        folded(entry.text.low),
+        JSON.stringify(entry.text),
+        now,
+      ],
+    )
+}
+
+/**
+ * The visitor search over published pages. Every word of the query must be in
+ * a page as a word prefix; the name counts most, then the high, normal and low
+ * properties. Pages the person searching may not open are left out in the query
+ * itself, so `total` counts only what they may see.
+ */
+export class SiteSearch {
+  #db: Db
+
+  constructor(db: Db) {
+    this.#db = db
+  }
+
+  async query(
+    term: string,
+    options: SiteSearchOptions,
+  ): Promise<{ total: number; items: SiteSearchHit[] }> {
+    const terms = searchTerms(term)
+    if (terms.length === 0) return { total: 0, items: [] }
+    const where = ['s.culture IN (?, ?)', 'n.trashed = ?']
+    const params: unknown[] = [
+      this.#db.dialect.fullTextQuery(terms),
+      options.culture.toLowerCase(),
+      '',
+      this.#db.dialect.boolValue(false),
+    ]
+    const types = options.types?.filter(Boolean) ?? []
+    if (types.length > 0) {
+      where.push(`ct.alias IN (${marks(types.length)})`)
+      params.push(...types)
+    }
+    const denied = options.access?.denied ?? []
+    const allowed = options.access?.allowed ?? []
+    if (denied.length > 0) {
+      // Closed: under a denied node, with no allowed node between it and the page.
+      const opened =
+        allowed.length === 0
+          ? ''
+          : `AND NOT EXISTS (SELECT 1 FROM node q WHERE q.id IN (${marks(allowed.length)})
+               AND (q.id = n.id OR n.path LIKE q.path || ',%') AND q.path LIKE p.path || ',%')`
+      where.push(
+        `NOT EXISTS (SELECT 1 FROM node p WHERE p.id IN (${marks(denied.length)})
+           AND (p.id = n.id OR n.path LIKE p.path || ',%') ${opened})`,
+      )
+      params.push(...denied, ...allowed)
+    }
+    const from = `FROM (${this.#db.dialect.siteSearchHits}) h
+      JOIN site_search_document s ON s.id = h.row_id
+      JOIN node n ON n.id = s.node_id
+      JOIN content c ON c.node_id = n.id
+      JOIN content_type ct ON ct.node_id = c.content_type_id
+      WHERE ${where.join(' AND ')}`
+    const [count] = await this.#db.query<{ n: number }>(`SELECT COUNT(*) AS n ${from}`, params)
+    const rows = await this.#db.query<Record<string, unknown>>(
+      `SELECT n.unique_id AS key, s.culture, s.text, h.score ${from}
+       ORDER BY h.score DESC, n.id LIMIT ? OFFSET ?`,
+      [...params, options.take, options.skip],
+    )
+    return {
+      total: Number(count?.n ?? 0),
+      items: rows.map((row) => ({
+        key: String(row.key).toLowerCase(),
+        culture: String(row.culture),
+        score: Number(row.score),
+        text: JSON.parse(String(row.text)) as SiteSearchText,
+      })),
+    }
+  }
+}