@ultimat3/entity 30.9.0 → 31.1.0

package/src/personal-data.ts

@@ -0,0 +1,66 @@
+// The readers a GDPR export or erase job walks by: every column `.personalData(kind)` marked, per
+// entity and across the registry. The mark is `$meta.personalData`, never on `$describe()` — being
+// personal data changes no column, so it must not move the drift hash.
+
+import { columnName } from './column';
+import type { EntityCore } from './entity';
+import type { PersonalDataKind } from './personal-data-kind';
+import { registeredEntities } from './registry';
+import type { AnyColumn, ColumnKind } from './types';
+
+// The kind set's home is `personal-data-kind.ts` (the chain reads it); the barrel takes it from here.
+export type { PersonalDataKind } from './personal-data-kind';
+export { PERSONAL_DATA_KINDS } from './personal-data-kind';
+
+/** The kind a column was marked with, or `undefined` for a column that holds no personal data. */
+export const personalDataKindOf = (column: AnyColumn): PersonalDataKind | undefined =>
+  column.$meta.personalData;
+
+/** One marked column of one entity, resolved to where it landed. */
+export interface PersonalDataColumn {
+  /** The row property — what a repository write names. */
+  readonly property: string;
+  /** The physical column (the base name, for a money column's parts). */
+  readonly column: string;
+  readonly kind: PersonalDataKind;
+  readonly columnKind: ColumnKind;
+  /** Whether an erase may write `null`; otherwise it must write a replacement value. */
+  readonly nullable: boolean;
+}
+
+/** Every marked column of `entity`, in declaration order. */
+export const personalDataOf = (
+  entity: Pick<EntityCore<unknown>, '$columns'>,
+): readonly PersonalDataColumn[] =>
+  Object.entries(entity.$columns).flatMap(([property, column]) => {
+    const kind = column.$meta.personalData;
+    if (kind === undefined) return [];
+    return [
+      {
+        property,
+        column: columnName(property, column.$meta),
+        kind,
+        columnKind: column.$meta.kind,
+        nullable: !column.$meta.notNull,
+      },
+    ];
+  });
+
+/** A registered entity holding personal data, with the tenant a subject's rows are scoped by. */
+export interface PersonalDataEntity {
+  readonly entity: EntityCore<unknown>;
+  /** The tenant column's property, or `null` for an untenanted entity. */
+  readonly tenantColumn: string | null;
+  readonly columns: readonly PersonalDataColumn[];
+}
+
+/** Every registered entity carrying at least one marked column, in the registry's name order. */
+export const personalDataEntities = (): readonly PersonalDataEntity[] =>
+  registeredEntities().flatMap((entry) => {
+    const core = entry.core;
+    if (core === undefined) return [];
+    const columns = personalDataOf(core);
+    return columns.length === 0
+      ? []
+      : [{ entity: core, tenantColumn: core.$tenantColumn, columns }];
+  });

package/src/pg-driver.ts

@@ -13,6 +13,7 @@ import {
   db,
   refuseAbandoned,
   type SqlFragment,
+  sql,
   withStatementAttribution,
 } from '@ultimat3/db';
 import { appendOnlyRepo } from './append-only';
@@ -21,8 +22,9 @@ import { writeChunks } from './bulk-write-send';
 import { entityNow } from './clock';
 import { coalesceFindById } from './coalesce';
 import { narrowRow } from './columns';
-import { cursorFor, seekFrom, valueAt } from './cursor';
+import { cursorFor, seekFrom } from './cursor';
 import type { Driver } from './database';
+import { noteDriverStore } from './driver-registry';
 import { type EntityCore, SOFT_DELETE_COLUMN } from './entity';
 import { notFound, repoClientPinned } from './errors';
 import { stampHistoryActor } from './history-actor';
@@ -37,7 +39,10 @@ import { postgresTransactor } from './pg-transactor';
 import { type ConflictTarget, deleteStatement, updateStatement } from './pg-write-sql';
 import { deletePlan, idPlan, readPlan, updatePlan } from './plan';
 import type { FindManyArgs, Repo, UpsertArgs } from './repo';
+import { rowKey } from './row-key';
 import { sealedRepo } from './sealed-repo';
+import { deletionStamp } from './search-index-version';
+import { writtenRowsSink } from './search-written-rows';
 import type { QueryPlan } from './tenancy';
 import { assertRowTenant } from './tenancy';
 import type { RowWrite } from './types';
@@ -104,14 +109,25 @@ export const postgresRepo = <Row>(
   const attributed = <T>(op: string, send: () => Promise<T>): Promise<T> =>
     withStatementAttribution(entity.$name, op, send);
 
-  const idOf = (row: Row): string =>
-    entity.$primaryKey.map((property) => String(valueAt(row, property))).join('');
+  /** The cursor's id — `row-key.ts`, the spelling the memory driver stores a row under. */
+  const idOf = (row: Row): string => rowKey(entity, row);
 
   /** Where a batch stops being wide — before `writeRows` or `upsertPlan` read a row. `narrowRow`
    *  says why the position matters, and `memoryRepo` narrows at the same one. */
   const narrowed = (batch: readonly RowWrite<Row>[]): readonly Row[] =>
     batch.map((row) => narrowRow<Row>(entity.$columns, row));
 
+  /**
+   * A soft delete's SET list: the stamp, plus a searchable entity's version — advanced past the
+   * stored one in the statement (`pg-write-sql.ts`), so a deletion is a newer version too.
+   */
+  const softDeletion = (): ReadonlyMap<string, unknown> =>
+    new Map(
+      Object.entries(
+        deletionStamp(entity as EntityCore<unknown>, SOFT_DELETE_COLUMN, entityNow()),
+      ).map(([property, value]) => [physicalName(entity, property), value]),
+    );
+
   const one = async (plan: QueryPlan, args: FindManyArgs): Promise<Row | null> => {
     const [found] = await client().query<PhysicalRow>(
       selectStatement(entity, plan, shapeOf(args), 1),
@@ -124,20 +140,54 @@ export const postgresRepo = <Row>(
    * delete hides the row without losing it; the column's presence is the switch, and `shapeOf({})`
    * keeps the `deleted_at is null` clause so a second call cannot move an existing stamp forward.
    *
-   * No `returning`: both callers send this through `execute()` and read a count, so the rows would
-   * be a whole tenant's table crossing the wire for nobody — the same cost `updateWhere` used to
-   * pay unconditionally, on the sibling path that looked like the one doing it right.
+   * `returning` only when the search mirror asked for the rows, and only up to the judged bound
+   * (`counted`): otherwise both callers read a count, and the rows would be a whole tenant's table
+   * crossing the wire for nobody.
    */
-  const removal = (plan: QueryPlan): SqlFragment =>
+  /** What a removal matches: the live rows of a soft delete, every row of a hard one. */
+  const removalShape = (): ReadShape => shapeOf({ includeDeleted: !entity.$softDelete });
+
+  const removal = (plan: QueryPlan, returning: boolean): SqlFragment =>
     entity.$softDelete
-      ? updateStatement(
-          entity,
-          plan,
-          new Map([[physicalName(entity, SOFT_DELETE_COLUMN), entityNow()]]),
-          shapeOf({}),
-          false,
-        )
-      : deleteStatement(entity, plan);
+      ? updateStatement(entity, plan, softDeletion(), shapeOf({}), returning)
+      : returning
+        ? sql`${deleteStatement(entity, plan)} returning *`
+        : deleteStatement(entity, plan);
+
+  /**
+   * A count, or — when the search mirror asked (`search-written-rows.ts`) — the same statement with
+   * `returning *`, its rows handed over and counted. The rows ride only for a searchable entity
+   * under an external index, and only when a count says they fit the judged bound: past
+   * `MAX_ASSERTED_ROWS` the write stays a count and the mirror logs the rows it could not name. A
+   * concurrent writer can drift the count, which moves the bound, never the write.
+   */
+  const counted = async (
+    op: string,
+    plan: QueryPlan,
+    shape: ReadShape,
+    statement: (returning: boolean) => SqlFragment,
+  ): Promise<number> => {
+    const sink = writtenRowsSink();
+    const fits =
+      sink !== undefined &&
+      Number(
+        (
+          await attributed(op, () =>
+            client().one<{ count: unknown }>(countStatement(entity, plan, shape)),
+          )
+        )?.count ?? 0,
+      ) <= MAX_ASSERTED_ROWS;
+    if (sink === undefined || !fits) {
+      return attributed(op, () => writing(() => client().execute(statement(false))));
+    }
+    return attributed(op, () =>
+      writing(async () => {
+        const rows = await client().query<PhysicalRow>(statement(true));
+        sink(rows.map((row) => decodeRow(entity, row)));
+        return rows.length;
+      }),
+    );
+  };
 
   /**
    * Every write goes out through here: the ONE place the request's preloaded rows are dropped (a
@@ -172,7 +222,9 @@ export const postgresRepo = <Row>(
     const statement = updateStatement(entity, plan, values, shape, true);
     return judged(async (send) => {
       const written = await attributed(op, () => writing(() => send.query<PhysicalRow>(statement)));
-      for (const row of written) entity.$assert(decodeRow(entity, row));
+      const decoded = written.map((row) => decodeRow(entity, row));
+      for (const row of decoded) entity.$assert(row);
+      writtenRowsSink()?.(decoded);
       return written.length;
     });
   };
@@ -183,13 +235,8 @@ export const postgresRepo = <Row>(
    */
   const removed = (op: string, plan: QueryPlan): Promise<number> =>
     entity.$softDelete && hasJsOnlyInvariant(entity.$invariants)
-      ? judgedRows(
-          op,
-          plan,
-          new Map([[physicalName(entity, SOFT_DELETE_COLUMN), entityNow()]]),
-          shapeOf({}),
-        )
-      : attributed(op, () => writing(() => client().execute(removal(plan))));
+      ? judgedRows(op, plan, softDeletion(), shapeOf({}))
+      : counted(op, plan, removalShape(), (returning) => removal(plan, returning));
 
   /**
    * The one insert path, for one row or ten thousand: the batch's own column list, split into as
@@ -372,8 +419,9 @@ export const postgresRepo = <Row>(
       // decoded each one, asserted nothing about any of them, and answered with a number.
       // `deleteWhere` was the tell: same predicate, same table, `execute()` and a count.
       if (!hasJsOnlyInvariant(entity.$invariants)) {
-        const statement = updateStatement(entity, plan, values, shape, false);
-        return attributed(op, () => writing(() => client().execute(statement)));
+        return counted(op, plan, shape, (returning) =>
+          updateStatement(entity, plan, values, shape, returning),
+        );
       }
       return judgedRows(op, plan, values, shape);
     },
@@ -399,8 +447,13 @@ export const postgresRepo = <Row>(
  * `database(entities, { driver: postgresDriver() })` — the one line that moves an app off the
  * in-memory default. Repos are stateless, so there is nothing to memoise.
  */
-export const postgresDriver = (config: PostgresDriverOptions = {}): Driver => ({
-  repo: <Row>(entity: EntityCore<Row>) => postgresRepo(entity, config),
-  transactor: () =>
-    postgresTransactor(config.client === undefined ? {} : { client: config.client }),
-});
+export const postgresDriver = (config: PostgresDriverOptions = {}): Driver => {
+  // Noted as it is built, so `x verify`'s drift step can tell a Postgres app from one whose store
+  // owns its own schema (`driver-registry.ts`). `testDriver()` builds one, so it is noted too.
+  noteDriverStore('sql', 'postgres');
+  return {
+    repo: <Row>(entity: EntityCore<Row>) => postgresRepo(entity, config),
+    transactor: () =>
+      postgresTransactor(config.client === undefined ? {} : { client: config.client }),
+  };
+};

package/src/pg-write-sql.ts

@@ -13,6 +13,7 @@ import { identifier, join, raw, type SqlFragment, sql } from '@ultimat3/db';
 import { columnName } from './column';
 import type { EntityCore } from './entity';
 import { conditions, type ReadShape } from './pg-sql';
+import { advancingVersionColumn } from './search-index-version';
 import type { QueryPlan } from './tenancy';
 
 /** `on conflict (…) do update set …`, or `do nothing` when there is nothing to overwrite. */
@@ -39,12 +40,41 @@ export interface InsertShape {
  */
 const DEFAULT_CELL = raw('default');
 
-const conflictSql = (conflict: ConflictTarget): SqlFragment => {
+/**
+ * A searchable entity's version column, physical — the one a write must ADVANCE past the stored
+ * value (`search-index-version.ts`), so two writes inside one millisecond are two versions.
+ */
+const advancingColumn = <Row>(entity: EntityCore<Row>): string | undefined => {
+  const property = advancingVersionColumn(entity as EntityCore<unknown>);
+  const meta = property === undefined ? undefined : entity.$columns[property]?.$meta;
+  return property === undefined || meta === undefined ? undefined : columnName(property, meta);
+};
+
+/**
+ * `VERSION_STEP_MS` as an interval — a closed literal like `default` above, and the same one
+ * millisecond (`search-version-monotonic.test.ts` pins the two together).
+ */
+export const VERSION_STEP_SQL = raw("interval '1 millisecond'");
+
+/** `greatest(next, stored + 1 ms)`: the version moves forward even when the clock has not. */
+const advanced = (column: string, next: SqlFragment, stored: SqlFragment): SqlFragment =>
+  sql`${identifier(column)} = greatest(${next}, ${stored} + ${VERSION_STEP_SQL})`;
+
+const conflictSql = <Row>(entity: EntityCore<Row>, conflict: ConflictTarget): SqlFragment => {
   const target = join(conflict.columns.map(identifier));
+  const version = advancingColumn(entity);
   return conflict.set.length === 0
     ? sql` on conflict (${target}) do nothing`
     : sql` on conflict (${target}) do update set ${join(
-        conflict.set.map((column) => sql`${identifier(column)} = excluded.${identifier(column)}`),
+        conflict.set.map((column) =>
+          column === version
+            ? advanced(
+                column,
+                sql`excluded.${identifier(column)}`,
+                sql`${identifier(entity.$table)}.${identifier(column)}`,
+              )
+            : sql`${identifier(column)} = excluded.${identifier(column)}`,
+        ),
       )}`;
 };
 
@@ -96,7 +126,7 @@ export const insertStatement = <Row>(
         ),
       )})`,
   );
-  const conflict = shape.conflict === undefined ? sql`` : conflictSql(shape.conflict);
+  const conflict = shape.conflict === undefined ? sql`` : conflictSql(entity, shape.conflict);
   return sql`insert into ${identifier(entity.$table)} (${join(
     shape.columns.map(identifier),
   )}) values ${join(tuples)}${conflict} returning *`;
@@ -117,8 +147,13 @@ export const updateStatement = <Row>(
   returning: boolean,
 ): SqlFragment => {
   const json = jsonColumns(entity);
+  const version = advancingColumn(entity);
   return sql`update ${identifier(entity.$table)} set ${join(
-    [...values].map(([column, value]) => sql`${identifier(column)} = ${cell(json, column, value)}`),
+    [...values].map(([column, value]) =>
+      column === version
+        ? advanced(column, cell(json, column, value), identifier(column))
+        : sql`${identifier(column)} = ${cell(json, column, value)}`,
+    ),
   )} where ${conditions(entity, plan, shape)}${returning ? sql` returning *` : sql``}`;
 };
 

package/src/plan.ts

@@ -12,6 +12,7 @@ import {
   patchEmpty,
   writeUnfiltered,
 } from './errors';
+import { screenPredicates } from './predicate-screen';
 import type { FindManyArgs, RepoOptions } from './repo';
 import type { Predicate, QueryPlan, SortKey } from './tenancy';
 import { scopedPlan } from './tenancy';
@@ -116,6 +117,7 @@ export const assertFinitePageSize = (entityName: string, rows: number): number =
 export const planFor = <Row>(entity: EntityCore<Row>, args: FindManyArgs): QueryPlan => {
   const orderBy = totalOrder(entity, args.orderBy ?? []);
   assertSeekable(entity, orderBy);
+  screenPredicates(entity, args.where ?? []);
   const scoped =
     args.orgId === undefined || entity.$tenantColumn === null
       ? []

package/src/predicate-screen.ts

@@ -0,0 +1,36 @@
+// Single responsibility: refuse, at plan time, a predicate no driver can answer — so the refusal
+// never depends on whether a table holds rows. Memory judged an operator only against a row, and
+// answered `[]` for an unknown one on an empty table where Postgres refused; a whole money value
+// compared as one column is the same shape: Postgres has no column to compare it with.
+
+import { columnFor } from './column';
+import type { EntityCore } from './entity';
+import { invariantViolated } from './errors';
+import { isOperator, operatorUnknown } from './operator';
+import type { Predicate } from './tenancy';
+
+/** A NULL test on a whole money property asks about its amount (`pg-sql.ts` `nullTestPath`). */
+const isNullTest = (predicate: Predicate): boolean =>
+  predicate.op === 'is-null' ||
+  predicate.op === 'is-not-null' ||
+  ((predicate.op === 'eq' || predicate.op === 'neq') && predicate.value === null);
+
+export const screenPredicates = <Row>(
+  entity: EntityCore<Row>,
+  where: readonly Predicate[],
+): void => {
+  for (const predicate of where) {
+    if (!isOperator(predicate.op)) {
+      throw operatorUnknown(entity.$name, predicate.column, predicate.op);
+    }
+    const column = columnFor(entity.$columns, predicate.column);
+    if (column?.$meta.kind === 'money' && !isNullTest(predicate)) {
+      const property = predicate.column;
+      throw invariantViolated(
+        entity.$name,
+        property,
+        `${property} is money: name ${property}.minor or ${property}.currency`,
+      );
+    }
+  }
+};

package/src/query.ts

@@ -18,6 +18,7 @@ import { type ReadAggregates, readAggregates } from './read-aggregates';
 import type { Repo, RepoOptions, UpsertArgs } from './repo';
 import { pickRow } from './sealed';
 import { SEARCH_PROPERTY } from './search';
+import { searchedRepo } from './search-repo';
 import { type TableLocks, tableLocks } from './table-locks';
 import type { Predicate, QueryPlan, SortDirection, SortKey } from './tenancy';
 import { stampedColumns, touch } from './touch';
@@ -74,6 +75,10 @@ export interface ReadBuilder<Row, S extends SealedKeySet = NoSealedKeys>
    * chain can serve: `ts_rank` is a computed value and the cursor carries columns, so the order
    * stays the one the caller declared. An entity with no searchable column is `X_SEARCH_UNDECLARED`
    * and the in-memory driver is `X_SEARCH_IN_MEMORY` — never a different answer from the two.
+   *
+   * The match is the INSTALLED `SearchIndex`'s (`searchIndex()`, `runtime.ts`'s `searchIndex`).
+   * The built-in Postgres one is this predicate in the statement, as above; any other index is
+   * asked for ids first (at most `MAX_PAGE_SIZE`), which the chain then reads (`search-repo.ts`).
    */
   search(term: string): ReadBuilder<Row, S>;
   orderBy(
@@ -379,10 +384,14 @@ export const tableFor = <Row, C extends ColumnMap>(
   related?: RelatedTables,
 ): Table<Row, C> => ({
   // One cast: `sealed-repo.ts` refuses at runtime, `SealedKeysOf<C>` is the same rule as a type.
-  ...(builder<Row, Row>(entity, repo, EMPTY, (row) => row, related) as ReadBuilder<
-    Row,
-    SealedKeysOf<C>
-  >),
+  // Reads go through `searchedRepo`: a `.search()` asks the installed `SearchIndex` (`search-repo.ts`).
+  ...(builder<Row, Row>(
+    entity,
+    searchedRepo(entity, repo),
+    EMPTY,
+    (row) => row,
+    related,
+  ) as ReadBuilder<Row, SealedKeysOf<C>>),
   insert: async (values, options) => repo.insert(entity.$parse(values), options),
   insertAll: async (rows, options) =>
     repo.insertAll(

package/src/registry.ts

@@ -12,6 +12,7 @@ import type {
 import type { EntityCore } from './entity';
 import { entityDuplicate } from './entity-error';
 import type { InvariantKind } from './invariants';
+import type { PartitionDescription } from './partition';
 import type { RecordProjection } from './record-projection';
 import type { ColumnDefault, OnDelete } from './types';
 
@@ -177,6 +178,8 @@ export interface EntityDescription {
   readonly retain?: RetainDescriptionLike;
   /** On a history entity: its parent's name. */
   readonly historyOf?: string;
+  /** `entity({ partitionBy })`, checked (`partition.ts`). Absent otherwise. `x db gen` reads it. */
+  readonly partition?: PartitionDescription;
 }
 
 export interface RegistryEntry {

package/src/row-key.ts

@@ -0,0 +1,19 @@
+// Single responsibility: a row's identity as one string — the key the memory driver stores a row
+// under and the id both drivers hand `cursorFor`. One definition, so the two cannot drift.
+
+import { keyOf } from './batch-read';
+import { kindOf, valueAt } from './cursor';
+import type { EntityCore } from './entity';
+
+/**
+ * Each primary-key value spelled as `keyOf` spells it (a `uuid` lower-cased, a number by value),
+ * so `findById(UPPER)` finds the row Postgres finds. A single key is that spelling alone; a
+ * composite one is a JSON array of them — joined with `''`, `('a', 'bc')` and `('ab', 'c')` were
+ * one key, and the memory driver stored the second row over the first.
+ */
+export const rowKey = <Row>(entity: EntityCore<Row>, row: unknown): string => {
+  const keys = entity.$primaryKey.map((property) =>
+    keyOf(kindOf(entity, property) ?? '', valueAt(row, property)),
+  );
+  return keys.length === 1 ? (keys[0] ?? '') : JSON.stringify(keys);
+};