@crombolo/data 0.15.0 → 0.17.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crombolo/data",
-  "version": "0.15.0",
+  "version": "0.17.0",
   "description": "SQLite and Postgres storage for crombolo: schema, migrations and repositories.",
   "license": "MIT",
   "author": "Crombolo",
@@ -28,6 +28,6 @@
     "access": "public"
   },
   "dependencies": {
-    "@crombolo/core": "0.15.0"
+    "@crombolo/core": "0.17.0"
   }
 }

package/src/index.ts

@@ -14,6 +14,7 @@ export * from './pending.ts'
 export * from './relations.ts'
 export * from './repositories/approvals.ts'
 export * from './repositories/bundles.ts'
+export * from './repositories/content-notifications.ts'
 export * from './repositories/content-types.ts'
 export * from './repositories/data-types.ts'
 export * from './repositories/delivery-member-tokens.ts'
@@ -53,6 +54,7 @@ export * from './schema/bundles-rename.ts'
 export * from './schema/change-report.ts'
 export * from './schema/client-id-prefix.ts'
 export * from './schema/content.ts'
+export * from './schema/content-notifications.ts'
 export * from './schema/content-seed.ts'
 export * from './schema/created-packages.ts'
 export * from './schema/delivery-member-tokens.ts'
@@ -77,6 +79,7 @@ export * from './schema/site-search.ts'
 export * from './schema/transfer-answers.ts'
 export * from './schema/transfer-run-package.ts'
 export * from './schema/transfer-runs.ts'
+export * from './schema/trashed-date.ts'
 export * from './schema/two-factor.ts'
 export * from './schema/upgrade-report.ts'
 export * from './schema/users-localisation.ts'

package/src/repositories/content-notifications.ts

@@ -0,0 +1,187 @@
+/**
+ * The content notification outbox (story 045): who is to hear of which change
+ * to which page, and whether they have. A row is claimed by one node, the same
+ * way a form workflow run is: the `state = 'pending'` in the UPDATE is the
+ * whole mechanism.
+ */
+import type { Db } from '../database.ts'
+import { DbDate } from '../dialect.ts'
+
+export type ContentNotificationState = 'pending' | 'running' | 'done' | 'failed' | 'skipped'
+
+export interface ContentNotification {
+  id: string
+  recipientUserKey: string
+  actionId: string
+  nodeKey: string
+  nodeName: string
+  actorUserKey: string | null
+  actorName: string | null
+  detail: string | null
+  occurredAt: Date
+  state: ContentNotificationState
+  attempts: number
+  lastError: string | null
+  runAfter: Date
+}
+
+export interface NewContentNotification {
+  recipientUserKey: string
+  actionId: string
+  nodeKey: string
+  nodeName: string
+  actorUserKey: string | null
+  actorName: string | null
+  detail: string | null
+  occurredAt: Date
+  runAfter: Date
+}
+
+const COLUMNS = `id, recipient_user_key, action_id, node_key, node_name, actor_user_key, actor_name,
+  detail, occurred_at, state, attempts, last_error, run_after`
+
+function hydrate(row: Record<string, unknown>): ContentNotification {
+  const text = (value: unknown) => (value === null || value === undefined ? null : String(value))
+  return {
+    id: String(row.id),
+    recipientUserKey: String(row.recipient_user_key),
+    actionId: String(row.action_id),
+    nodeKey: String(row.node_key),
+    nodeName: String(row.node_name),
+    actorUserKey: text(row.actor_user_key),
+    actorName: text(row.actor_name),
+    detail: text(row.detail),
+    occurredAt: DbDate.fromDb(row.occurred_at) ?? new Date(0),
+    state: String(row.state) as ContentNotificationState,
+    attempts: Number(row.attempts),
+    lastError: text(row.last_error),
+    runAfter: DbDate.fromDb(row.run_after) ?? new Date(0),
+  }
+}
+
+export class ContentNotificationRepository {
+  #db: Db
+
+  constructor(db: Db) {
+    this.#db = db
+  }
+
+  async enqueue(rows: readonly NewContentNotification[]): Promise<void> {
+    if (rows.length === 0) return
+    const now = DbDate.toDb(new Date())
+    await this.#db.transaction(async (tx) => {
+      for (const row of rows)
+        await tx.exec(
+          `INSERT INTO content_notification (id, recipient_user_key, action_id, node_key, node_name,
+             actor_user_key, actor_name, detail, occurred_at, state, attempts, last_error, run_after,
+             create_date, update_date)
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', 0, NULL, ?, ?, ?)`,
+          [
+            crypto.randomUUID(),
+            row.recipientUserKey,
+            row.actionId,
+            row.nodeKey,
+            row.nodeName,
+            row.actorUserKey,
+            row.actorName,
+            row.detail,
+            DbDate.toDb(row.occurredAt),
+            DbDate.toDb(row.runAfter),
+            now,
+            now,
+          ],
+        )
+    })
+  }
+
+  /** The people with something due to them: each one is a single e-mail. */
+  async dueRecipients(now = new Date(), limit = 25): Promise<string[]> {
+    const rows = await this.#db.query<{ recipient_user_key: string }>(
+      `SELECT recipient_user_key FROM content_notification
+        WHERE state = 'pending' AND run_after <= ?
+        GROUP BY recipient_user_key ORDER BY MIN(run_after) LIMIT ?`,
+      [DbDate.toDb(now), limit],
+    )
+    return rows.map((row) => String(row.recipient_user_key))
+  }
+
+  /** Takes everything due to one person, or nothing when another node got there first. */
+  async claimFor(recipientUserKey: string, now = new Date()): Promise<ContentNotification[]> {
+    const rows = await this.#db.query(
+      `UPDATE content_notification
+          SET state = 'running', attempts = attempts + 1, update_date = ?
+        WHERE recipient_user_key = ? AND state = 'pending' AND run_after <= ?
+        RETURNING ${COLUMNS}`,
+      [DbDate.toDb(now), recipientUserKey, DbDate.toDb(now)],
+    )
+    return rows.map(hydrate).sort((a, b) => a.occurredAt.getTime() - b.occurredAt.getTime())
+  }
+
+  async #finish(
+    ids: readonly string[],
+    state: ContentNotificationState,
+    error: string | null,
+    runAfter?: Date,
+  ) {
+    if (ids.length === 0) return
+    const marks = ids.map(() => '?').join(', ')
+    await this.#db.exec(
+      `UPDATE content_notification SET state = ?, last_error = ?, update_date = ?${runAfter ? ', run_after = ?' : ''}
+        WHERE id IN (${marks})`,
+      [
+        state,
+        error?.slice(0, 2000) ?? null,
+        DbDate.toDb(new Date()),
+        ...(runAfter ? [DbDate.toDb(runAfter)] : []),
+        ...ids,
+      ],
+    )
+  }
+
+  succeed(ids: readonly string[]): Promise<void> {
+    return this.#finish(ids, 'done', null)
+  }
+
+  /** Not sent, and not to be: the reason is kept for whoever looks. */
+  skip(ids: readonly string[], reason: string): Promise<void> {
+    return this.#finish(ids, 'skipped', reason)
+  }
+
+  /** A failure: back in the queue at `retryAt`, or finished when there is none. */
+  fail(ids: readonly string[], error: string, retryAt?: Date): Promise<void> {
+    return this.#finish(ids, retryAt ? 'pending' : 'failed', error, retryAt)
+  }
+
+  /** Rows a node took and never finished — it died — go back in the queue. */
+  async releaseStale(olderThan: Date): Promise<number> {
+    const rows = await this.#db.query(
+      `UPDATE content_notification SET state = 'pending', update_date = ?
+        WHERE state = 'running' AND update_date < ? RETURNING id`,
+      [DbDate.toDb(new Date()), DbDate.toDb(olderThan)],
+    )
+    return rows.length
+  }
+
+  async prune(before: Date): Promise<number> {
+    const rows = await this.#db.query(
+      `DELETE FROM content_notification WHERE state IN ('done', 'skipped') AND update_date < ? RETURNING id`,
+      [DbDate.toDb(before)],
+    )
+    return rows.length
+  }
+
+  async forRecipient(recipientUserKey: string): Promise<ContentNotification[]> {
+    const rows = await this.#db.query(
+      `SELECT ${COLUMNS} FROM content_notification WHERE recipient_user_key = ? ORDER BY occurred_at`,
+      [recipientUserKey],
+    )
+    return rows.map(hydrate)
+  }
+
+  async all(): Promise<ContentNotification[]> {
+    const rows = await this.#db.query(
+      `SELECT ${COLUMNS} FROM content_notification ORDER BY occurred_at`,
+    )
+    return rows.map(hydrate)
+  }
+}

package/src/repositories/documents.ts

@@ -1071,10 +1071,14 @@ export class DocumentRepository {
         ).map((row) => Number(row.id))
         const t = tx.dialect.boolValue(true)
         const f = tx.dialect.boolValue(false)
-        await tx.exec('UPDATE document SET original_parent_id = ? WHERE node_id = ?', [
-          node.parentId === SystemNodes.Root ? null : node.parentId,
-          node.id,
-        ])
+        await tx.exec(
+          'UPDATE document SET original_parent_id = ?, trashed_date = ? WHERE node_id = ?',
+          [
+            node.parentId === SystemNodes.Root ? null : node.parentId,
+            DbDate.toDb(new Date()),
+            node.id,
+          ],
+        )
         // The whole branch goes with it, and stays unpublished while trashed.
         await tx.exec('UPDATE node SET trashed = ? WHERE id = ? OR path LIKE ?', [
           t,
@@ -1162,9 +1166,10 @@ export class DocumentRepository {
             node.id,
             `${node.path},%`,
           ])
-          await tx.exec('UPDATE document SET original_parent_id = NULL WHERE node_id = ?', [
-            node.id,
-          ])
+          await tx.exec(
+            'UPDATE document SET original_parent_id = NULL, trashed_date = NULL WHERE node_id = ?',
+            [node.id],
+          )
         }
         await repo.nodes.move(node.id, parentId)
         await tx.exec('UPDATE node SET sort_order = ? WHERE id = ?', [
@@ -1313,7 +1318,10 @@ export class DocumentRepository {
           await repo.#nextSortOrder(parentId, node.id),
           node.id,
         ])
-        await tx.exec('UPDATE document SET original_parent_id = NULL WHERE node_id = ?', [node.id])
+        await tx.exec(
+          'UPDATE document SET original_parent_id = NULL, trashed_date = NULL WHERE node_id = ?',
+          [node.id],
+        )
         return 'restored' as const
       }),
     )
@@ -1643,6 +1651,21 @@ export class DocumentRepository {
     return (await this.trashed(null, 0, Number.MAX_SAFE_INTEGER)).items
   }
 
+  /** When each item went into the recycle bin; absent for one trashed before that was recorded. */
+  async trashedDates(nodeIds: readonly number[]): Promise<Map<number, Date>> {
+    if (nodeIds.length === 0) return new Map()
+    const rows = await this.#db.query<{ node_id: number; trashed_date: unknown }>(
+      `SELECT node_id, trashed_date FROM document WHERE trashed_date IS NOT NULL AND node_id IN (${nodeIds.map(() => '?').join(', ')})`,
+      [...nodeIds],
+    )
+    const out = new Map<number, Date>()
+    for (const row of rows) {
+      const date = DbDate.fromDb(row.trashed_date)
+      if (date) out.set(Number(row.node_id), date)
+    }
+    return out
+  }
+
   async #nextSortOrder(parentId: number, excludingId: number): Promise<number> {
     const rows = await this.#db.query<{ next: number | null }>(
       'SELECT MAX(sort_order) AS next FROM node WHERE parent_id = ? AND node_object_type = ? AND id <> ?',

package/src/repositories/notifications.ts

@@ -1,7 +1,7 @@
 /**
  * Which actions on a node a user asked to be notified of (Umbraco's
- * `umbracoUser2NodeNotify`). Sending the notification e-mails needs e-mail,
- * which the site does not have yet; the subscriptions are kept regardless.
+ * `umbracoUser2NodeNotify`). Delivered by e-mail through the content
+ * notification outbox (story 045); kept regardless while a site has no e-mail.
  */
 import type { Db } from '../database.ts'
 
@@ -51,4 +51,18 @@ export class NotificationRepository {
           )
     })
   }
+
+  /**
+   * Who asked to hear of an action on any node of a path: a subscription on a
+   * page covers the pages under it, as Umbraco's does.
+   */
+  async subscribersFor(actionId: string, pathNodeIds: readonly number[]): Promise<number[]> {
+    if (pathNodeIds.length === 0) return []
+    const rows = await this.#db.query<{ user_id: number }>(
+      `SELECT DISTINCT user_id FROM user_notification
+        WHERE action = ? AND node_id IN (${pathNodeIds.map(() => '?').join(', ')})`,
+      [actionId, ...pathNodeIds],
+    )
+    return rows.map((row) => Number(row.user_id))
+  }
 }

package/src/schema/content-notifications.ts

@@ -0,0 +1,53 @@
+/**
+ * Migration 043 — content notifications delivered (story 045).
+ *
+ * `content_notification` is the outbox: one row per person per change to a
+ * page they asked to hear about, sent by a background job and claimed by one
+ * node. It holds names as they were, and no foreign keys, so deleting a page or
+ * a user afterwards cannot break a row. `user_notification` gains the index a
+ * lookup by page needs.
+ */
+import type { Db } from '../database.ts'
+import type { Migration } from '../migrations.ts'
+import { STATE_TRASHED_DATE } from './trashed-date.ts'
+
+export const STATE_CONTENT_NOTIFICATIONS = '{5a7c2e19-0043-4d8f-b361-0f4e9a2c7d43}'
+
+export const contentNotificationsMigration: Migration = {
+  from: STATE_TRASHED_DATE,
+  to: STATE_CONTENT_NOTIFICATIONS,
+  name: 'AddContentNotifications',
+  kind: 'expand',
+  release: '0.17.0',
+  async up(db: Db) {
+    const t = db.dialect.types
+    await db.exec(
+      `CREATE TABLE content_notification (
+         id ${t.varchar(64)} PRIMARY KEY,
+         recipient_user_key ${t.uuid} NOT NULL,
+         action_id ${t.varchar(64)} NOT NULL,
+         node_key ${t.uuid} NOT NULL,
+         node_name ${t.text} NOT NULL,
+         actor_user_key ${t.uuid},
+         actor_name ${t.text},
+         detail ${t.text},
+         occurred_at ${t.timestamp} NOT NULL,
+         state ${t.varchar(16)} NOT NULL,
+         attempts ${t.integer} NOT NULL,
+         last_error ${t.text},
+         run_after ${t.timestamp} NOT NULL,
+         create_date ${t.timestamp} NOT NULL,
+         update_date ${t.timestamp} NOT NULL
+       )`,
+    )
+    await db.exec(
+      'CREATE INDEX ix_content_notification_due ON content_notification (state, run_after)',
+    )
+    await db.exec(
+      'CREATE INDEX ix_content_notification_recipient ON content_notification (recipient_user_key, state)',
+    )
+    await db.exec(
+      'CREATE INDEX ix_user_notification_node_action ON user_notification (node_id, action)',
+    )
+  },
+}

package/src/schema/plan.ts

@@ -23,6 +23,7 @@ import { changeReportMigration } from './change-report.ts'
 import { clientIdPrefixMigration } from './client-id-prefix.ts'
 import { contentMigration } from './content.ts'
 import { contentEditingMigration } from './content-editing.ts'
+import { contentNotificationsMigration } from './content-notifications.ts'
 import { createdPackagesMigration } from './created-packages.ts'
 import { deliveryMemberTokensMigration } from './delivery-member-tokens.ts'
 import { diagnosticsMigration } from './diagnostics.ts'
@@ -49,6 +50,7 @@ import { siteSearchMigration } from './site-search.ts'
 import { transferAnswersMigration } from './transfer-answers.ts'
 import { transferRunPackageMigration } from './transfer-run-package.ts'
 import { transferRunsMigration } from './transfer-runs.ts'
+import { trashedDateMigration } from './trashed-date.ts'
 import { twoFactorMigration } from './two-factor.ts'
 import { upgradeReportMigration } from './upgrade-report.ts'
 import { usersLocalisationMigration } from './users-localisation.ts'
@@ -97,4 +99,6 @@ export const cromboloPlan = new MigrationPlan([
   siteSearchMigration,
   twoFactorMigration,
   remoteMigration,
+  trashedDateMigration,
+  contentNotificationsMigration,
 ])

package/src/schema/trashed-date.ts

@@ -0,0 +1,23 @@
+/**
+ * Migration 042 — `document.trashed_date`: when an item went into the recycle
+ * bin, so `crombolo recycle-bin empty --older-than` can leave what was trashed
+ * recently (story 152). Items trashed before this migration have none, and are
+ * treated as of unknown age.
+ */
+import type { Db } from '../database.ts'
+import type { Migration } from '../migrations.ts'
+import { schemaOps } from '../schema-ops.ts'
+import { STATE_REMOTE } from './remote.ts'
+
+export const STATE_TRASHED_DATE = '{8d3f1a62-0042-4b7e-9c15-2e6a4f0b8d43}'
+
+export const trashedDateMigration: Migration = {
+  from: STATE_REMOTE,
+  to: STATE_TRASHED_DATE,
+  name: 'AddTrashedDate',
+  kind: 'expand',
+  release: '0.16.0',
+  async up(db: Db) {
+    await schemaOps(db).addColumn('document', 'trashed_date', db.dialect.types.timestamp)
+  },
+}

package/src/search-text.ts

@@ -60,17 +60,24 @@ const ENTITIES: Record<string, string> = {
   nbsp: ' ',
 }
 
+/** Decodes numeric entities and the few named ones markup actually carries. */
+export function decodeEntities(text: string): string {
+  return text.replace(/&(#\d+|#x[0-9a-f]+|[a-z]+);/gi, (entity, name: string) => {
+    if (!name.startsWith('#')) return ENTITIES[name.toLowerCase()] ?? entity
+    const code =
+      name[1] === 'x' || name[1] === 'X'
+        ? Number.parseInt(name.slice(2), 16)
+        : Number(name.slice(1))
+    // Past the last code point is not a character; left as written.
+    return code <= 0x10ffff ? String.fromCodePoint(code) : entity
+  })
+}
+
 function stripMarkup(text: string): string {
   if (!text.includes('<') && !text.includes('&')) return text
-  return text
-    .replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi, ' ')
-    .replace(/<[^>]*>/g, ' ')
-    .replace(/&(#\d+|#x[0-9a-f]+|[a-z]+);/gi, (entity, name: string) => {
-      if (name.startsWith('#x') || name.startsWith('#X'))
-        return String.fromCodePoint(Number.parseInt(name.slice(2), 16))
-      if (name.startsWith('#')) return String.fromCodePoint(Number(name.slice(1)))
-      return ENTITIES[name.toLowerCase()] ?? entity
-    })
+  return decodeEntities(
+    text.replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi, ' ').replace(/<[^>]*>/g, ' '),
+  )
 }
 
 /** Whether an element type's property is kept out of search: `searchable = false`. */