@ultimat3/entity 1.2.0 → 2.0.0

package/src/invariants.ts

@@ -3,11 +3,12 @@
 // never disagree with the code — a bulk import, a psql session or a second service all hit the
 // same rule.
 //
-// `invariant()` is unbound on purpose: it is written next to the columns, and `entity()` binds
-// it to their physical names. That is what keeps a physical name from being typed twice.
+// `invariant()` names an already-built expression: `entity()` hands the whole `invariants:`
+// callback the typed column proxy once, so a rule is written against property keys (`c.likeCount`)
+// and `entity()` alone resolves them to physical names. A physical name is never typed twice.
 
-import { invariantViolated } from './errors';
-import type { Expr, InvariantColumns, Resolve, Row } from './expr';
+import { EntityError, invariantViolated } from './errors';
+import type { Expr, Resolve, Row } from './expr';
 
 /** `assert` is a rule only the app can run — a JS predicate with no SQL translation. */
 export type InvariantKind = 'check' | 'unique' | 'assert';
@@ -24,20 +25,25 @@ export interface Invariant<T> {
   readonly columns: readonly string[];
   /** Partial-constraint predicate, e.g. `deleted_at is null`. */
   readonly where?: string;
-  readonly holds: (row: T) => boolean;
+  /**
+   * A method, not a `readonly holds: (row: T) => boolean` property, and the difference is
+   * load-bearing: a function-typed property is checked contravariantly, which made `Invariant<Post>`
+   * unassignable to `Invariant<unknown>` and so made every real entity fail `EntitySet`. Every
+   * `database({ posts, … })` call then degraded to `Table<unknown>` and cascaded — 275 errors in
+   * the reference app from this one position. Method syntax is bivariant, which is what
+   * `EntityCore.$assert` beside it already relies on. Pinned by `database.variance.test.ts`.
+   */
+  holds(row: T): boolean;
 }
 
-/** What `invariant()` returns: a rule that does not yet know its physical column names. */
+/** What `invariant()` returns: a named rule that does not yet know its physical column names. */
 export interface InvariantDef {
   readonly name: string;
-  readonly build: (columns: InvariantColumns) => Expr;
+  readonly expr: Expr;
 }
 
-/** `invariant('post_like_count_non_negative', (c) => c.likeCount.atLeast(0))` */
-export const invariant = (
-  name: string,
-  build: (columns: InvariantColumns) => Expr,
-): InvariantDef => ({ name, build });
+/** `invariants: (c) => [invariant('post_like_count_non_negative', c.likeCount.atLeast(0))]` */
+export const invariant = (name: string, expr: Expr): InvariantDef => ({ name, expr });
 
 const asRow = (value: unknown): Row =>
   typeof value === 'object' && value !== null ? (value as Row) : {};
@@ -45,11 +51,10 @@ const asRow = (value: unknown): Row =>
 /** Called by `entity()`: resolves property paths to physical names and freezes the rule. */
 export const bindInvariant = <T>(
   def: InvariantDef,
-  columns: InvariantColumns,
   resolve: Resolve,
   partialWhere: string | undefined,
 ): Invariant<T> => {
-  const expr = def.build(columns);
+  const expr = def.expr;
   const sql = expr.toSql(resolve);
   const kind: InvariantKind = expr.kind === 'unique' ? 'unique' : sql === null ? 'assert' : 'check';
   return {
@@ -87,6 +92,43 @@ export const invariantsToSql = <T>(table: string, invariants: readonly Invariant
     .filter((statement): statement is string => statement !== null)
     .join('\n');
 
+/**
+ * Whether any rule here can only be judged in the app. This is what decides whether a FILTERED
+ * write has to read back the rows it wrote: a `check` or a `unique` is a constraint Postgres
+ * already enforced on the statement, so nothing has to come back to prove it, while an `assert`
+ * (`sql: null`) has no CHECK and can only be judged on the result. Read from `$invariants` and
+ * never from a flag, exactly as `uniqueTargets` (`bulk-write.ts`) reads the same list to classify
+ * a conflict target — one declaration, two questions.
+ */
+export const hasJsOnlyInvariant = <T>(invariants: readonly Invariant<T>[]): boolean =>
+  invariants.some((inv) => inv.kind === 'assert' && inv.sql === null);
+
+/**
+ * How many rows a filtered write may bring back to be judged. Only the case above needs any: with
+ * every rule expressible as a constraint the answer is a count, and `returning *` there is a whole
+ * table streamed into a process sized for one request — a tenant-wide
+ * `updateWhere({ orgId }, { marketingOptIn: false })` over twelve million rows is the shape of it.
+ * Past this the call is refused rather than answered, because the cheap form of the same sweep is
+ * already in the vocabulary: `inBatches(size)` reads one page per statement and judges one page at
+ * a time.
+ */
+export const MAX_ASSERTED_ROWS = 50_000;
+
+/**
+ * Not `invariantViolated`: its fix opens `x entity explain`, which describes a rule the author did
+ * not break — the rule is fine, the blast radius is not. What repairs this is one edit to the call.
+ */
+export const assertedRowsTooMany = (
+  entityName: string,
+  operation: string,
+  matched: number,
+): EntityError =>
+  new EntityError({
+    code: 'X_INVARIANT_VIOLATED',
+    cause: `${entityName}.${operation}() matches ${matched} rows and ${entityName} declares an invariant only the app can judge, so every one of them would be read back — past ${MAX_ASSERTED_ROWS} that is the whole table in memory`,
+    fix: `for await (const rows of ${entityName}.where(filter).inBatches(1000)) { … }   # one page per statement, judged one page at a time`,
+  });
+
 /** Runs on every write. Reports every violation at once so one round trip fixes all. */
 export const assertInvariants = <T>(
   entityName: string,

package/src/jit-preload.ts

@@ -0,0 +1,216 @@
+// Single responsibility: the sibling-aware preload. A point lookup whose id is a foreign key on a
+// page already read resolves that key for EVERY row of the page in one statement, and the rest of
+// the loop is served from memory. A sequential `for … of` loop awaits between iterations, so its
+// lookups never share a microtask and the coalescer cannot see them; this is what batches them.
+//
+// The trigger carries an id, not a row, so what a page leaves behind is an index of its foreign
+// key VALUES rather than a map keyed by row identity: an id is a thing that can be looked up in
+// it, and it holds values, so it pins no rows for the request's lifetime.
+//
+// The scope guard is a security boundary, not a tuning knob. A preloaded row is served only to a
+// lookup with the same scope key, the same client and no write since — anything else reads the
+// statement it always read.
+
+import type { Ctx } from '@ultimat3/core';
+import { tryUseContext } from '@ultimat3/core';
+import type { DbClient } from '@ultimat3/db';
+import { type Answer, keyOf, type PointRead, readByIds, statementChunks } from './batch-read';
+import type { EntityCore } from './entity';
+
+/** The rows one page's worth of foreign keys resolved to, under one scope. */
+interface Bucket {
+  /** The entity the rows belong to — what a write invalidates. */
+  readonly entity: string;
+  /** Where the rows were read from. A pinned client and the ambient pool are two places. */
+  readonly client: DbClient;
+  /** Writes to `entity` when the bucket was opened. A later write makes every row of it stale. */
+  readonly generation: number;
+  /** Never rejects: a failure is an `Answer`, so an id nobody asks for cannot go unhandled. */
+  readonly rows: Map<string, Promise<Answer>>;
+}
+
+interface Store {
+  /** `[targetEntity, targetProperty]` -> id key -> every id the page carried for that key. */
+  readonly siblings: Map<string, Map<string, readonly unknown[]>>;
+  /** Scope key -> the rows preloaded under it. */
+  readonly preloaded: Map<string, Bucket>;
+  /** Entity name -> writes this request has issued against it. */
+  readonly writes: Map<string, number>;
+}
+
+/**
+ * Per request, keyed by ctx identity, so a page's siblings and everything preloaded from them die
+ * with the request that read them — the shape the microtask coalescer's store has, one file over.
+ */
+const requests = new WeakMap<object, Store>();
+
+const storeFor = (ctx: Ctx): Store => {
+  const key: object = ctx;
+  const existing = requests.get(key);
+  if (existing !== undefined) return existing;
+  const created: Store = { siblings: new Map(), preloaded: new Map(), writes: new Map() };
+  requests.set(key, created);
+  return created;
+};
+
+/** Both ends of the edge: a key pointing at another column of the same entity is another edge. */
+const siblingKey = (targetEntity: string, targetProperty: string): string =>
+  JSON.stringify([targetEntity, targetProperty]);
+
+const writesTo = (store: Store, entity: string): number => store.writes.get(entity) ?? 0;
+
+/**
+ * What a page of rows leaves behind: for each foreign key it declares, the distinct values that
+ * page carried, filed under every one of them. A later `findById` for any of those values is a
+ * lookup this page can answer for all of them.
+ *
+ * Values only. Rows are not held, so a page read early in a long request costs its keys and not
+ * its rows, and a group of one still counts — a hundred posts by one author is one statement for
+ * the whole loop rather than a hundred.
+ */
+export const tagSiblings = <Row>(entity: EntityCore<Row>, rows: readonly Row[]): void => {
+  if (rows.length === 0) return;
+  const ctx = tryUseContext();
+  // No request, no store: a job or a script reads the statement it always read. Asked first —
+  // resolving the foreign keys is a pass over the columns, and nothing would read the answer.
+  if (ctx === undefined) return;
+  const references = entity.$references();
+  if (references.length === 0) return;
+  const store = storeFor(ctx);
+  for (const reference of references) {
+    // The declaring column's own kind: a foreign key mirrors the key it points at, and a value is
+    // filed here exactly as `findById` will spell it when it comes looking.
+    const kind = entity.$columns[reference.property]?.$meta.kind;
+    if (kind === undefined) continue;
+    const ids: unknown[] = [];
+    const keys = new Set<string>();
+    for (const row of rows) {
+      const value = (row as Record<string, unknown>)[reference.property];
+      // A nullable key that resolved to nothing is data, not a row to go looking for.
+      if (value === null || value === undefined) continue;
+      const key = keyOf(kind, value);
+      if (keys.has(key)) continue;
+      keys.add(key);
+      ids.push(value);
+    }
+    if (ids.length === 0) continue;
+    const at = siblingKey(reference.targetEntity, reference.targetProperty);
+    const index = store.siblings.get(at) ?? new Map<string, readonly unknown[]>();
+    for (const key of keys) index.set(key, ids);
+    store.siblings.set(at, index);
+  }
+};
+
+/**
+ * A write makes every preloaded row of that entity stale, so the bucket holding them is not read
+ * again. Called before the statement goes out: a row read back afterwards is the row the write
+ * left, and one read concurrently with it was concurrent either way.
+ */
+export const forgetPreloaded = (entityName: string): void => {
+  const ctx = tryUseContext();
+  if (ctx === undefined) return;
+  const store = requests.get(ctx);
+  if (store === undefined) return;
+  store.writes.set(entityName, writesTo(store, entityName) + 1);
+};
+
+/** The bucket this lookup may be served from, or `undefined` — a stale one is dropped, not read. */
+const usableBucket = <Row>(
+  store: Store,
+  read: PointRead<Row>,
+  scope: string,
+): Bucket | undefined => {
+  const open = store.preloaded.get(scope);
+  if (open === undefined) return undefined;
+  if (open.client === read.client && open.generation === writesTo(store, open.entity)) return open;
+  store.preloaded.delete(scope);
+  return undefined;
+};
+
+type Settlers = ReadonlyMap<string, (answer: Answer) => void>;
+
+/** One statement at a time: a page wide enough to split must not take the pool with it. */
+const fill = async <Row>(
+  read: PointRead<Row>,
+  ids: readonly unknown[],
+  settlers: Settlers,
+): Promise<void> => {
+  for (const chunk of statementChunks(ids)) {
+    let answers: ReadonlyMap<string, Answer>;
+    try {
+      answers = await readByIds(read, chunk);
+    } catch (error) {
+      // The statement failed, so it fails for everyone it was widened to cover — which is what
+      // the single statement each of them would have sent would have done.
+      for (const id of chunk) settlers.get(keyOf(read.key.kind, id))?.({ error });
+      continue;
+    }
+    for (const id of chunk) {
+      const at = keyOf(read.key.kind, id);
+      // An id the statement did not answer for is a row that is not there — `findById`'s null.
+      settlers.get(at)?.(answers.get(at) ?? { row: null });
+    }
+  }
+};
+
+/**
+ * The statement, and one promise per id it will answer for. Only the ids the bucket does not
+ * already hold are read: a second lookup into a page it already resolved is memory, not a wire.
+ */
+const preload = <Row>(read: PointRead<Row>, bucket: Bucket, ids: readonly unknown[]): void => {
+  const settlers = new Map<string, (answer: Answer) => void>();
+  const wanted: unknown[] = [];
+  for (const id of ids) {
+    const at = keyOf(read.key.kind, id);
+    if (bucket.rows.has(at)) continue;
+    // The executor runs synchronously, so `settle` is assigned before the promise is stored.
+    let settle!: (answer: Answer) => void;
+    bucket.rows.set(
+      at,
+      new Promise<Answer>((resolve) => {
+        settle = resolve;
+      }),
+    );
+    settlers.set(at, settle);
+    wanted.push(id);
+  }
+  // Nothing awaits this: an id nobody asks for still settles, and it settles with an `Answer`
+  // rather than a rejection, so a failed statement cannot go unhandled.
+  void fill(read, wanted, settlers);
+};
+
+const answered = <Row>(answer: Promise<Answer>): Promise<Row | null> =>
+  answer.then((settled) =>
+    'error' in settled ? Promise.reject(settled.error) : (settled.row as Row | null),
+  );
+
+/**
+ * The row this lookup is served from a page already read, or `undefined` when no page can answer
+ * it: nothing read in this request, an id that is no page's foreign key, a write since, or
+ * another client. Declining is always correct — the caller reads the statement it always read.
+ */
+export const preloadedFindById = <Row>(
+  ctx: Ctx,
+  read: PointRead<Row>,
+  scope: string,
+  id: unknown,
+): Promise<Row | null> | undefined => {
+  const store = requests.get(ctx);
+  if (store === undefined) return undefined;
+  const filedAt = keyOf(read.key.kind, id);
+  const bucket = usableBucket(store, read, scope);
+  const already = bucket?.rows.get(filedAt);
+  if (already !== undefined) return answered<Row>(already);
+  const ids = store.siblings.get(siblingKey(read.entity.$name, read.key.property))?.get(filedAt);
+  if (ids === undefined) return undefined;
+  const target = bucket ?? {
+    entity: read.entity.$name,
+    client: read.client,
+    generation: writesTo(store, read.entity.$name),
+    rows: new Map<string, Promise<Answer>>(),
+  };
+  store.preloaded.set(scope, target);
+  preload(read, target, ids);
+  const answer = target.rows.get(filedAt);
+  return answer === undefined ? undefined : answered<Row>(answer);
+};

package/src/n-plus-one.ts

@@ -0,0 +1,122 @@
+// Which fix a repeated statement has earned. The relations the schema already declared decide it:
+// a loop over a page reads what one `preload()` would have carried, so the map that names the
+// relation is the map that writes the fix line — never a name this file invents.
+//
+// Nothing here counts anything. A ledger elsewhere decides that a shape repeated; this turns that
+// verdict into the one error the surfaces render.
+
+import type { EntityError, PreloadCandidate } from './errors';
+import { nPlusOneQuery, nPlusOneWrite } from './errors';
+import type { RelationKind } from './relations';
+import { relationMap } from './relations';
+
+/**
+ * Statements of one shape, in one unit of work, before the loop is worth reporting. Five, because a
+ * page that reads the same shape four times is a page with four reads and one that reads it fifty
+ * times is a loop over rows — and the threshold has to sit far enough above the first number that a
+ * fixed-arity render never trips it.
+ *
+ * It lives here, with the codes and the fix, because it is the number that decides a *verdict*: the
+ * dev ledger (`x dev`) and the strict test fixture (`@ultimat3/testing`) are two detectors, and two
+ * numbers would mean a loop that fails a test and a loop that warns in dev are different loops.
+ * What a unit of work *is* stays each detector's — a request there, one test here.
+ */
+export const N_PLUS_ONE_THRESHOLD = 5;
+
+/** One statement shape, repeated inside one request — a ledger's verdict, as this layer reads it. */
+export interface StatementLoop {
+  /** Which of the two codes this is: decided from the statement, upstream, never re-derived here. */
+  readonly kind: 'read' | 'write';
+  /** What repeated: `members.findById` when a repository sent it, else the statement's own text. */
+  readonly subject: string;
+  /** Statements of this shape in the request. Reported, not judged — the threshold is the ledger's. */
+  readonly count: number;
+  /** The entity the repository call named. Absent for hand-written SQL, which names no chain. */
+  readonly entity?: string | undefined;
+  /** The repository operation — `findById`, `insert`. Absent for the same reason. */
+  readonly op?: string | undefined;
+}
+
+/**
+ * Which relation kind would have carried the rows this operation read one at a time.
+ *
+ * A point lookup per row is the `belongsTo` side — fifty `members.findById` are one page's authors,
+ * and `posts.preload('author')` is one statement for all of them. A filtered read per row is the
+ * `hasMany` side — fifty `comments.findMany({ postId })` are one page's comment lists, and
+ * `posts.preload('comments')` is the same one statement. Every other operation is left to the `in`
+ * form: a repeated `count` or `countBy` is answered by `countBy`, and a preload attaches rows to a
+ * page that a count never read.
+ */
+const PRELOADABLE_BY_OP: Readonly<Record<string, RelationKind>> = {
+  findById: 'belongsTo',
+  findMany: 'hasMany',
+};
+
+/**
+ * Every page that could have preloaded these rows, in the map's own sorted order.
+ *
+ * The edges are read by their `to` end, because that is the entity the loop repeated on: a lookup
+ * of `members` is fixed on whatever holds a `references()` to it, and this diagnostic only ever saw
+ * the statement — never the `for … of` above it — so it cannot know which of those pages was being
+ * iterated. Naming them all is what `preloadUnknownRelation` already does with relation names.
+ */
+export const preloadsFor = (
+  entityName: string,
+  op: string | undefined,
+): readonly PreloadCandidate[] => {
+  const kind = op === undefined ? undefined : PRELOADABLE_BY_OP[op];
+  if (kind === undefined) return [];
+  const candidates: PreloadCandidate[] = [];
+  for (const relations of Object.values(relationMap())) {
+    for (const relation of Object.values(relations)) {
+      if (relation.kind !== kind || relation.to !== entityName) continue;
+      candidates.push({ from: relation.from, relation: relation.name });
+    }
+  }
+  return candidates;
+};
+
+/**
+ * A schema whose relations cannot be named answers with the `in` form rather than with its own
+ * complaint: `relationMap()` throws `X_INVARIANT_VIOLATED` on two foreign keys it cannot tell
+ * apart, and a diagnostic that replaced the loop it was reporting with a schema error would hide
+ * the N+1 behind a fault the loop did not cause — in a dev process, as an uncaught throw.
+ */
+const preloadsOrNone = (
+  entityName: string,
+  op: string | undefined,
+): readonly PreloadCandidate[] => {
+  try {
+    return preloadsFor(entityName, op);
+  } catch {
+    return [];
+  }
+};
+
+/** The read half: the preload the schema names, else the `in` form of the statement it repeated. */
+const readLoop = (loop: StatementLoop): EntityError => {
+  if (loop.entity === undefined) return nPlusOneQuery(loop.subject, loop.count, { form: 'sql' });
+  // The destructure is what proves the fix has something to name — an empty candidate list is an
+  // empty `preload` fix, which is the one thing the error contract refuses outright.
+  const [first, ...rest] = preloadsOrNone(loop.entity, loop.op);
+  if (first === undefined) {
+    return nPlusOneQuery(loop.subject, loop.count, { form: 'in', entity: loop.entity });
+  }
+  return nPlusOneQuery(loop.subject, loop.count, { form: 'preload', candidates: [first, ...rest] });
+};
+
+/**
+ * One repeated shape, as the error every surface renders. One entry point and not two, because the
+ * caller already knows which of the two codes it holds and a second decision here is a second place
+ * for the read and the write halves to disagree about what a loop is.
+ */
+export const nPlusOne = (loop: StatementLoop): EntityError =>
+  loop.kind === 'read'
+    ? readLoop(loop)
+    : nPlusOneWrite(
+        loop.subject,
+        loop.count,
+        loop.entity === undefined
+          ? { form: 'sql' }
+          : { form: 'bulk', entity: loop.entity, op: loop.op },
+      );