latticesql 3.4.7 → 4.0.0

package/dist/index.d.ts

@@ -8,6 +8,15 @@ import { Server } from 'node:http';
 interface EntityFileManifestInfo {
     /** SHA-256 hex digest of the rendered content. */
     hash: string;
+    /**
+     * SHA-256 of the source entity row at render time (all columns, key-sorted).
+     * Reverse-sync compares this against the row's CURRENT version before applying
+     * a file edit: a mismatch means the DB row changed since this render, so the
+     * edit is rejected as a conflict instead of silently overwriting it. Optional
+     * for back-compat — manifests written before this field fall back to applying
+     * (the prior behavior) until the next render re-captures it.
+     */
+    rowVersion?: string;
 }
 interface EntityContextManifestEntry {
     directoryRoot: string;
@@ -15,14 +24,10 @@ interface EntityContextManifestEntry {
     declaredFiles: string[];
     protectedFiles: string[];
     /**
-     * Key = entity slug.
-     *
-     * **v2 format:** value is `Record<string, EntityFileManifestInfo>` (filename → info with hash).
-     * **v1 format (legacy):** value is `string[]` (bare filename list, no hashes).
-     *
-     * Use {@link normalizeEntityFiles} to convert v1 entries.
+     * Key = entity slug. Value = `Record<string, EntityFileManifestInfo>`
+     * (filename → info with content hash). This is the only shape we WRITE.
      */
-    entities: Record<string, Record<string, EntityFileManifestInfo> | string[]>;
+    entities: Record<string, Record<string, EntityFileManifestInfo>>;
 }
 /**
  * Monotonic cursor the rendered tree was produced FROM, recorded in the manifest
@@ -69,17 +74,16 @@ interface LatticeManifest {
      */
     cursor?: RenderCursor;
 }
-/** Type guard: is the entity files entry in v1 format (string[])? */
-declare function isV1EntityFiles(val: unknown): val is string[];
-/**
- * Convert a v1 entity files entry (string[]) to v2 format (Record with empty hashes).
- * Reverse-sync skips entries with empty hashes (no baseline to compare against).
- */
-declare function normalizeEntityFiles(val: Record<string, EntityFileManifestInfo> | string[]): Record<string, EntityFileManifestInfo>;
 /**
- * Get the filenames from an entity files entry (works for both v1 and v2).
+ * Get the filenames from an entity files entry.
+ *
+ * We only ever WRITE the v2 `Record<filename, info>` shape, but an OLD manifest
+ * already on disk may still carry a bare `string[]` entry (the pre-hash format).
+ * This accessor tolerates that legacy shape at the read boundary — a stale
+ * manifest is upgraded silently on the next render, never crashed over — and
+ * still returns the filenames so cleanup can detect orphaned files for it.
  */
-declare function entityFileNames(val: Record<string, EntityFileManifestInfo> | string[]): string[];
+declare function entityFileNames(val: Record<string, EntityFileManifestInfo> | readonly string[]): string[];
 declare function manifestPath(outputDir: string): string;
 declare function readManifest(outputDir: string): LatticeManifest | null;
 declare function writeManifest(outputDir: string, manifest: LatticeManifest): void;
@@ -624,10 +628,33 @@ interface WritebackStateStore {
 }
 /**
  * In-memory state store (default). State is lost on process exit.
+ *
+ * The seen-set dedupe is UNBOUNDED by default — every key ever marked for a
+ * file is retained for the process lifetime, exactly as before. For a
+ * long-running daemon tailing an append-only log this grows without limit, so
+ * an optional `maxSeenPerFile` cap is available: when set, only the most recent
+ * N keys per file are retained (oldest evicted first, by insertion order).
+ *
+ * The cap is a MEMORY-SAFETY guard for bounded-lifetime processes, NOT a
+ * durability substitute: an evicted key has no record, so if that same key
+ * reappears in the file later it would be re-processed (re-persisted). For
+ * long-running daemons over append-only logs prefer the durable
+ * {@link SQLiteStateStore} — its `_lattice_writeback_seen` table is durable and
+ * disk-bounded, so dedupe survives restarts without an in-memory ceiling.
  */
 declare class InMemoryStateStore implements WritebackStateStore {
     private readonly _offsets;
     private readonly _seen;
+    private readonly _maxSeenPerFile;
+    /**
+     * @param opts.maxSeenPerFile Cap the per-file dedupe set to this many keys
+     *   (oldest evicted first). Default `Infinity` — unbounded, preserving the
+     *   original behavior exactly. Set a finite cap only when you understand the
+     *   re-processing tradeoff documented on the class.
+     */
+    constructor(opts?: {
+        maxSeenPerFile?: number;
+    });
     getOffset(filePath: string): number;
     getSize(filePath: string): number;
     setOffset(filePath: string, offset: number, size: number): void;
@@ -1177,6 +1204,34 @@ interface WritebackDefinition {
         entries: unknown[];
         nextOffset: number;
     };
+    /**
+     * Opt into incremental file reads. Default (`undefined` / `false`): the
+     * pipeline reads the WHOLE file every tick with `readFileSync` and calls
+     * `parse(wholeFileContent, absoluteByteOffset)` — byte-for-byte the original
+     * behavior. No change for existing consumers.
+     *
+     * When `true`: each tick the pipeline reads ONLY the bytes at/after the
+     * stored byte offset (one `readSync` of `currentSize - offset` bytes) and
+     * passes that slice as `content` with `fromOffset = 0`. The parser's returned
+     * `nextOffset` is therefore RELATIVE to the slice; the pipeline adds the prior
+     * byte offset back before storing, so the persisted offset is always absolute.
+     * This avoids re-reading the whole file (and re-billing its egress) every tick
+     * on an append-only log.
+     *
+     * HARD PRECONDITION: the parser MUST operate purely on the byte-slice it
+     * receives — it may NOT rely on any bytes before the stored offset (no
+     * back-references into earlier content, no whole-file state). If your parser
+     * needs the full file (e.g. it re-parses a header on every tick), leave this
+     * off.
+     *
+     * Multi-byte safety: the slice is decoded with an incremental UTF-8 decoder
+     * (`StringDecoder`), so a multi-byte codepoint that straddles the slice's
+     * trailing edge is not split into a replacement char — its trailing bytes are
+     * simply not yet consumed and arrive on the next tick once the rest is written.
+     *
+     * @default undefined (false)
+     */
+    incrementalRead?: boolean;
     /** Persist a single parsed entry; called exactly once per unique dedupeKey */
     persist: (entry: unknown, filePath: string) => Promise<void>;
     /** Optional dedup key — if omitted, every entry is processed */
@@ -1491,6 +1546,22 @@ interface ReverseSyncError {
     /** Error description. */
     error: string;
 }
+/**
+ * A reverse-sync update that was NOT applied because the database row changed
+ * since the render that produced the file's baseline — applying the file edit
+ * would have silently overwritten that concurrent change. Reported (never
+ * applied) so a human can re-resolve; the DB row is left intact.
+ */
+interface ReverseSyncConflict {
+    /** Entity table the conflicting row belongs to. */
+    table: string;
+    /** Slug of the entity whose file changed. */
+    slug: string;
+    /** The changed file whose write was rejected. */
+    filename: string;
+    /** Why the write was rejected (e.g. the row changed since render). */
+    reason: string;
+}
 /**
  * Result of the reverse-sync phase in {@link Lattice.reconcile}.
  */
@@ -1503,6 +1574,12 @@ interface ReverseSyncResult {
     updatesApplied: number;
     /** Errors encountered (file-level — other files still processed). */
     errors: ReverseSyncError[];
+    /**
+     * Changed files whose write was REJECTED because the DB row changed since the
+     * render baseline (optimistic-concurrency conflict). Surfaced loudly, never
+     * applied — applying would have clobbered a concurrent DB/cloud edit.
+     */
+    conflicts: ReverseSyncConflict[];
 }
 /**
  * A missing row detected during reconcile: rendered files exist on disk
@@ -1639,6 +1716,32 @@ interface StorageAdapter {
      * natively.
      */
     addColumn(table: string, column: string, typeSpec: string): void;
+    /**
+     * Optional cheap, complete change-probe for the watch-loop render gate.
+     *
+     * Returns a token that is GUARANTEED to differ whenever ANY data change has
+     * been committed to the database since the previous call — by this connection
+     * OR any other connection/process. A caller may safely skip a render only when
+     * two consecutive tokens are strictly equal: equal tokens prove zero
+     * intervening commits, so nothing a render reads could have changed.
+     *
+     * The completeness guarantee is the whole point — a token that misses a class
+     * of writes would let the gate skip a render that was actually needed, leaving
+     * the rendered context stale. An adapter that cannot expose such a complete,
+     * cheap signal MUST leave this undefined; the gate then falls through to a
+     * full render every tick (the default, never-stale behavior).
+     *
+     * Must be O(1) — no table scan. It runs on every watch tick.
+     *
+     * - SQLite: `PRAGMA data_version` (catches other-connection commits) combined
+     *   with `total_changes()` (catches this connection's own row mutations) — the
+     *   pair is complete because between them they observe every committed row
+     *   change regardless of which connection made it. See the SQLite adapter.
+     * - Postgres: there is no equally-cheap GLOBAL complete counter, so the
+     *   Postgres adapter does NOT implement this — Postgres watch renders run every
+     *   tick, unchanged.
+     */
+    changeProbe?(): string | undefined;
     /** Async equivalent of run(). Returns when the statement has completed. */
     runAsync?(sql: string, params?: unknown[]): Promise<void>;
     /** Async equivalent of get(). */
@@ -1647,6 +1750,16 @@ interface StorageAdapter {
     allAsync?(sql: string, params?: unknown[]): Promise<Row[]>;
     /** Async equivalent of introspectColumns(). Used by the boot-path schema apply. */
     introspectColumnsAsync?(table: string): Promise<string[]>;
+    /**
+     * Introspect columns for MANY tables in one shot (one query on Postgres).
+     *
+     * Collapses the per-table `information_schema` round-trips the boot path
+     * would otherwise issue into a single whole-schema query. Implementations
+     * MUST throw on query failure rather than return a partial/empty map — a
+     * degraded read that silently yields an empty map would make the caller
+     * believe every table is missing.
+     */
+    introspectAllColumns?(tables: string[]): Promise<Map<string, Set<string>>>;
     /** Async equivalent of addColumn(). Used by the boot-path schema apply. */
     addColumnAsync?(table: string, column: string, typeSpec: string): Promise<void>;
     /**
@@ -1725,6 +1838,14 @@ interface TxClient {
     all(sql: string, params?: unknown[]): Promise<Row[]>;
 }
 
+/** Optional bounds for a list read. Omitting both is the default (no cap). */
+interface PageOptions {
+    /** Max rows to return. Must be a non-negative integer. */
+    limit?: number;
+    /** Rows to skip (only honored alongside `limit` — SQL OFFSET requires LIMIT). */
+    offset?: number;
+}
+
 /**
  * Options for {@link ReverseSyncEngine.process}.
  *
@@ -1920,6 +2041,18 @@ declare class ProgressThrottle {
     force(event: RenderProgress): void;
 }
 
+/**
+ * Thrown by Lattice.seed when onUnresolvedLink: 'throw' is set and one or more
+ * junction links could not be created because their target rows did not
+ * resolve. (Moved here from lattice.ts; re-exported from lattice.ts to
+ * preserve the public export name.)
+ */
+declare class SeedReconciliationError extends Error {
+    readonly table: string;
+    readonly unresolvedLinks: UnresolvedLink[];
+    constructor(table: string, unresolvedLinks: UnresolvedLink[]);
+}
+
 /**
  * Initialise Lattice from a YAML config file instead of an explicit path.
  *
@@ -1945,7 +2078,10 @@ type PkLookup = string | Record<string, unknown>;
 declare class Lattice {
     private readonly _adapter;
     private _changelogService?;
+    private _changelogWriterInstance?;
     private _reportBuilder?;
+    private _queryCoreInstance?;
+    private _seedEngine?;
     private readonly _schema;
     private readonly _sanitizer;
     private readonly _render;
@@ -1955,34 +2091,19 @@ declare class Lattice {
     private readonly _writeback;
     private _initialized;
     /**
-     * When set, insert/update/delete debounce-trigger a re-render into this dir.
-     * Configured by {@link enableAutoRender} / {@link Lattice.openWorkspace}.
-     * Undefined = inert: a bare `new Lattice(dbPath)` pays zero overhead and its
-     * behavior is unchanged.
-     */
-    private _autoRenderDir;
-    private _autoRenderTimer;
-    private _autoRenderPending;
-    private _autoRenderInFlight;
-    private _autoRenderDebounceMs;
-    /**
-     * Incremental auto-render scope, accumulated between debounced renders. A write
-     * or a remote (cloud) change records the AFFECTED table here, so the next
-     * auto-render re-renders only that entity (+ its cross-table dependents) instead
-     * of the whole tree. `_pendingRenderAll` forces a full render (the initial
-     * render, or a change with no known table). Captured + reset when a render
-     * starts, so changes during a render re-accumulate and re-trigger.
-     */
-    private _pendingRenderTables;
-    private _pendingRenderAll;
+     * The debounce + single-flight auto-render scheduler (see
+     * {@link AutoRenderScheduler}). Lazily constructed by the {@link _autoRender}
+     * getter and configured by {@link enableAutoRender} / {@link Lattice.openWorkspace}.
+     * Undefined = inert: a bare `new Lattice(dbPath)` never constructs the
+     * scheduler and pays zero overhead, so its behavior is unchanged.
+     */
+    private _autoRenderScheduler?;
     /** Cache of actual table columns (from PRAGMA), populated after init(). */
     private readonly _columnCache;
-    /** Derived encryption key (from options.encryptionKey via scrypt). */
-    private _encryptionKey?;
-    /** Map of table → set of column names that should be encrypted at rest. */
-    private readonly _encryptedTableColumns;
     /** Raw encryption key passphrase from constructor options. */
     private readonly _encryptionKeyRaw?;
+    /** Lazily-constructed column-encryption layer (see {@link _encryption}). */
+    private _encryptionLayer?;
     /** Changelog retention options. */
     private readonly _changelogOptions?;
     /** Set of table names that have changelog: true. */
@@ -2189,33 +2310,11 @@ declare class Lattice {
     /** Return the current task context string. */
     getTaskContext(): string;
     /**
-     * Throw-only validation of encryption-key configuration. Runs in the
-     * synchronous prefix of `init()` so `expect(() => db.init()).toThrow(...)`
-     * still observes the throw — moving this check into the async tail would
-     * convert the throw into a rejected Promise and break those tests.
-     * Column resolution happens later in {@link _finalizeEncryptionSetup} once
-     * the schema has been applied.
-     */
-    private _validateEncryptionConfig;
-    /**
-     * Resolve which columns to encrypt per table, using introspectColumns to
-     * see the post-migration schema. Runs in the async tail of init() after
-     * applySchema/applyMigrationsAsync.
-     */
-    private _finalizeEncryptionSetup;
-    /**
-     * Shared helper: derive the encryption key on first use, introspect the
-     * table's current columns, resolve which to encrypt, and record the set
-     * in `_encryptedTableColumns`. Called from both `_finalizeEncryptionSetup`
-     * (boot path) and `defineLate` (post-init table registration).
+     * Lazily-constructed column-encryption layer. Field-backed `??= new` getter
+     * mirrors `_report`/`_changelog`; the layer defers its scrypt key derivation
+     * to the first column registration, so merely touching this getter is cheap.
      */
-    private _registerEncryptedColumns;
-    /** Encrypt applicable columns in a row before writing. Returns a new row. */
-    private _encryptRow;
-    /** Decrypt applicable columns in a row after reading. Mutates in place. */
-    private _decryptRow;
-    /** Decrypt applicable columns in multiple rows. Mutates in place. */
-    private _decryptRows;
+    private get _encryption();
     /**
      * Defense-in-depth: validate a table (and any dynamic column) identifier
      * before it is interpolated into a SQL string. The library's threat model is
@@ -2329,7 +2428,7 @@ declare class Lattice {
      * Get all non-deleted rows from a table, ordered by the given column.
      * Works on any table, not just defined ones.
      */
-    getActive(table: string, orderBy?: string): Promise<Row[]>;
+    getActive(table: string, orderBy?: string, opts?: PageOptions): Promise<Row[]>;
     /**
      * Count non-deleted rows in a table.
      */
@@ -2353,6 +2452,8 @@ declare class Lattice {
      * and optionally soft-deletes records no longer in the data set.
      */
     seed(config: SeedConfig): Promise<SeedResult>;
+    /** Lazily-constructed seeding collaborator (see src/crud/seed-engine.ts). */
+    private get _seed();
     /** Infer FK column name from table name (e.g., 'rule' → 'rule_id'). */
     private _inferFk;
     /**
@@ -2362,6 +2463,12 @@ declare class Lattice {
     buildReport(config: ReportConfig): Promise<ReportResult>;
     /** Lazily-constructed report-generation collaborator (see src/report/builder.ts). */
     private get _report();
+    /** Lazily-constructed generic-read collaborator (see src/query/core.ts). The
+     *  decrypt deps are wired through `_encryption`, so the query/get decryption
+     *  asymmetry is preserved: only query/get invoke them. */
+    private get _queryCore();
+    /** Lazily-constructed auto-render scheduler (see src/render/auto-render.ts). */
+    private get _autoRender();
     /**
      * Update reward scores for a row. The total reward is recalculated as
      * the running average across all reward calls. Requires `rewardTracking`
@@ -2385,12 +2492,12 @@ declare class Lattice {
      * Render into `outputDir` through the shared single-flight guard, intended to
      * be called fire-and-forget (e.g. the GUI's instant-open background render).
      *
-     * The guard ({@link _renderGuarded}) holds {@link _autoRenderInFlight} for the
-     * render's duration, so a data mutation that lands while this render is in
-     * flight is deferred by {@link _runAutoRender} and coalesced — when this
-     * render settles, `finally` clears the flag and re-arms exactly one follow-up
-     * render via {@link _rearmAutoRenderIfPending}. Net invariant: at most one
-     * render to a given dir at a time.
+     * The guard ({@link AutoRenderScheduler.runGuarded}) holds the scheduler's
+     * in-flight flag for the render's duration, so a data mutation that lands while
+     * this render is in flight is deferred by the debounced auto-render path and
+     * coalesced — when this render settles, `finally` clears the flag and re-arms
+     * exactly one follow-up render. Net invariant: at most one render to a given
+     * dir at a time.
      *
      * Errors propagate to the caller (the GUI surfaces them, never silently swallowed); they are
      * not swallowed here.
@@ -2537,7 +2644,6 @@ declare class Lattice {
      * - `Record` → matches every PK column; all must be present in the object.
      */
     private _pkWhere;
-    private static readonly _PK_SEP;
     /**
      * The primary-key columns of `table` that PHYSICALLY exist, in declared
      * order. Empty when the table has no Lattice-addressable key — e.g. a table
@@ -2546,7 +2652,7 @@ declare class Lattice {
      * ACL is possible, so the row-perm SQL must not reference a pk column).
      */
     private _resolvedPkCols;
-    /** Canonical ACL / change-log `pk` string for a row. Matches {@link _pkSqlExpr}. */
+    /** Canonical ACL / change-log `pk` string for a row. Matches `db/pk.ts` `pkSqlExpr`. */
     private _serializeRowPk;
     /**
      * Canonical `pk` string for a {@link PkLookup} used by update/delete, so a
@@ -2554,19 +2660,6 @@ declare class Lattice {
      * {@link _serializeRowPk} keyed it at insert time.
      */
     private _serializePkLookup;
-    /**
-     * SQL expression reconstructing {@link _serializeRowPk} from a row aliased
-     * `t`. Returns null when the table is unkeyable (no pk columns present) — the
-     * caller must then avoid referencing a pk column at all. Dialect-aware tab
-     * separator: SQLite `char(9)`, Postgres `chr(9)` (both = U+0009), matching
-     * {@link _PK_SEP}.
-     */
-    private _pkSqlExpr;
-    /**
-     * Convert Filter objects into SQL clause strings and bound params.
-     * An `in` filter with an empty array is silently ignored (produces no clause).
-     */
-    private _buildFilters;
     /** Returns a rejected Promise if not initialized; null if ready. */
     private _fireWriteHooks;
     /**
@@ -2581,24 +2674,6 @@ declare class Lattice {
     }): this;
     /** Turn off automatic rendering and cancel any pending render. */
     disableAutoRender(): this;
-    private _scheduleAutoRender;
-    /** Arm the debounce timer if not already armed. Does NOT change the render
-     *  scope — used both by `_scheduleAutoRender` and the post-render re-arm so a
-     *  re-arm never escalates a pending incremental render to a full one. */
-    private _armAutoRenderTimer;
-    /**
-     * Shared single-flight render path used by {@link renderInBackground}.
-     *
-     * Holds {@link _autoRenderInFlight} for the render's duration so the
-     * mutation-driven {@link _runAutoRender} defers while this render runs (it
-     * sees the flag and marks itself pending instead of starting a second,
-     * overlapping render). On settle, `finally` clears the flag and re-arms a
-     * single coalesced follow-up render if any mutation arrived mid-flight.
-     * Errors propagate to the caller; the flag is always cleared.
-     */
-    private _renderGuarded;
-    private _runAutoRender;
-    private _rearmAutoRenderIfPending;
     /**
      * Update or remove the embedding for a row.
      * No-op if the table doesn't have `embeddings` configured.
@@ -2638,6 +2713,8 @@ declare class Lattice {
     /** Parse a raw changelog DB row into a ChangeEntry. */
     /** Lazily-constructed changelog read/replay collaborator (see src/changelog/service.ts). */
     private get _changelog();
+    /** Lazily-constructed changelog write/DDL collaborator (see src/changelog/writer.js). */
+    private get _changelogWriter();
     /**
      * Get change history for a specific row, newest first.
      */
@@ -2689,18 +2766,6 @@ declare class Lattice {
     private _invalidColumnError;
     private _assertNotInit;
 }
-/**
- * Thrown by {@link Lattice.seed} when `onUnresolvedLink: 'throw'` is set and
- * one or more junction links could not be created because their target rows
- * did not resolve. Carries the full list so the caller can report or
- * reconcile every missing target at once rather than discovering them one
- * silent drop at a time.
- */
-declare class SeedReconciliationError extends Error {
-    readonly table: string;
-    readonly unresolvedLinks: UnresolvedLink[];
-    constructor(table: string, unresolvedLinks: UnresolvedLink[]);
-}
 
 /**
  * Scalar types recognised in `lattice.config.yml` field definitions.
@@ -2728,7 +2793,7 @@ type LatticeFieldType = 'uuid' | 'text' | 'integer' | 'int' | 'real' | 'float' |
  * id:          { type: uuid,    primaryKey: true }
  * title:       { type: text,    required: true }
  * status:      { type: text,    default: open }
- * assignee_id: { type: uuid,    ref: user }
+ * assignee_id: { type: uuid }
  * score:       { type: integer, default: 0 }
  * ```
  */
@@ -2741,13 +2806,6 @@ interface LatticeFieldDef {
     required?: boolean;
     /** SQL DEFAULT value */
     default?: string | number | boolean;
-    /**
-     * Foreign-key reference to another entity (table name).
-     * Creates a `belongsTo` relation automatically.
-     * The relation name is derived from the field name — `_id` suffix is stripped
-     * (e.g. `assignee_id: { ref: user }` → relation name `assignee`).
-     */
-    ref?: string;
     /**
      * Per-column audience (Stage-0 scaffolding for the per-viewer enrichment
      * model). Names who may see this column's value in a cloud. Omitted ⇒
@@ -2757,6 +2815,15 @@ interface LatticeFieldDef {
      * cell-masking view from it; Stage-0 only records the metadata.
      */
     audience?: string;
+    /**
+     * DEPRECATED (3.x) per-field foreign-key shorthand: `ref: <targetTable>` declared
+     * a `belongsTo` whose relation name is the field name with a trailing `_id`
+     * stripped. Superseded by the explicit entity-level `relations:` block. 4.0 still
+     * PARSES it (converted to a `belongsTo` in-memory) so existing 3.0+ configs keep
+     * working, and the GUI silently rewrites it to `relations:` on open so configs
+     * migrate forward. A future major may drop this once configs have upgraded.
+     */
+    ref?: string;
 }
 /**
  * Inline render spec inside YAML — a flat object alternative to `TemplateRenderSpec`.
@@ -2779,8 +2846,14 @@ interface LatticeEntityRenderSpec {
  * ```yaml
  * ticket:
  *   fields:
- *     id:    { type: uuid, primaryKey: true }
- *     title: { type: text, required: true }
+ *     id:          { type: uuid, primaryKey: true }
+ *     title:       { type: text, required: true }
+ *     assignee_id: { type: uuid }
+ *   relations:
+ *     assignee:
+ *       type: belongsTo
+ *       table: user
+ *       foreignKey: assignee_id
  *   render: default-list
  *   outputFile: context/TICKETS.md
  * ```
@@ -2788,6 +2861,27 @@ interface LatticeEntityRenderSpec {
 interface LatticeEntityDef {
     /** Column definitions */
     fields: Record<string, LatticeFieldDef>;
+    /**
+     * Explicit `belongsTo` relations for this entity, keyed by relation name.
+     *
+     * Each entry declares a foreign key on THIS entity pointing at another
+     * table: `{ type: belongsTo, table, foreignKey, references? }`. The
+     * `foreignKey` names a plain field on this entity; `references` is the
+     * column on the related table (defaults to its primary key). The relation
+     * name (the map key) is whatever you choose — it is not derived from the
+     * field name.
+     *
+     * @example
+     * ```yaml
+     * relations:
+     *   assignee:
+     *     type: belongsTo
+     *     table: user
+     *     foreignKey: assignee_id
+     *     # references: id   # optional; defaults to the target's primary key
+     * ```
+     */
+    relations?: Record<string, BelongsToRelation>;
     /**
      * How to render rows into context text.
      * Accepts the same forms as `TableDefinition.render`:
@@ -3275,10 +3369,36 @@ declare class SQLiteAdapter implements StorageAdapter {
      * cannot add those via ALTER, and if the table exists it has its own PK.
      */
     addColumn(table: string, column: string, typeSpec: string): void;
+    /**
+     * Complete, O(1) change-probe for the watch-loop render gate (see
+     * `StorageAdapter.changeProbe`). Composes the two SQLite counters that, taken
+     * together, observe EVERY committed row change regardless of origin:
+     *
+     *   - `PRAGMA data_version`: a counter SQLite bumps whenever ANOTHER
+     *     connection (or process) commits a change to the database. It is
+     *     intentionally NOT bumped by commits on this same connection — so on its
+     *     own it would miss our own writes.
+     *   - `total_changes()` (`sqlite3_total_changes`): the cumulative count of
+     *     rows inserted/updated/deleted on THIS connection since it was opened,
+     *     INCLUDING rows changed indirectly by triggers and ON DELETE CASCADE. It
+     *     covers exactly the gap `data_version` leaves — our own connection's
+     *     writes.
+     *
+     * Neither counter moves on a no-op statement (an UPDATE matching zero rows,
+     * a bare SELECT), so an idle DB yields a stable token. The pair is therefore
+     * complete: the composite token changes if and only if at least one row was
+     * committed somewhere since the prior read. Both reads are single-value
+     * pragma/function lookups — no table scan, safe to call every tick.
+     *
+     * (DDL alone is not a watch-loop concern — the schema is fixed before watch
+     * starts — and any DATA a migration writes is counted by `total_changes()`.)
+     */
+    changeProbe(): string;
     runAsync(sql: string, params?: unknown[]): Promise<void>;
     getAsync(sql: string, params?: unknown[]): Promise<Row | undefined>;
     allAsync(sql: string, params?: unknown[]): Promise<Row[]>;
     introspectColumnsAsync(table: string): Promise<string[]>;
+    introspectAllColumns(tables: string[]): Promise<Map<string, Set<string>>>;
     addColumnAsync(table: string, column: string, typeSpec: string): Promise<void>;
     /**
      * Run `fn` inside a BEGIN/COMMIT block on the single SQLite connection.
@@ -3382,6 +3502,19 @@ declare class PostgresAdapter implements StorageAdapter {
      */
     prepareAsync(sql: string): PreparedStatementAsync;
     introspectColumnsAsync(table: string): Promise<string[]>;
+    /**
+     * Whole-schema column introspection in a single query — the batched
+     * equivalent of {@link introspectColumnsAsync}. The boot path uses this to
+     * collapse one `information_schema` round-trip per declared table into one,
+     * which on a high-RTT cloud is the difference between ~hundreds of serial
+     * round-trips and a single query.
+     *
+     * `tables` is accepted for interface symmetry but ignored: querying the
+     * whole `current_schema()` once is cheaper than constraining to a list, and
+     * the caller folds out tables it doesn't care about. Throws on query failure
+     * — a degraded read must never masquerade as "every table is missing".
+     */
+    introspectAllColumns(_tables: string[]): Promise<Map<string, Set<string>>>;
     addColumnAsync(table: string, column: string, typeSpec: string): Promise<void>;
     /**
      * Run `fn` against a single checked-out pool client wrapped in BEGIN/COMMIT.
@@ -3426,10 +3559,10 @@ declare class PostgresAdapter implements StorageAdapter {
  * file repository) that should not require every consumer to re-derive
  * the column shape from scratch.
  *
- * Columns are deliberately a *superset* of any earlier ad-hoc shapes
- * (e.g. older fixtures defined `files` with `path` + `kind` only). New
- * code should prefer the content-addressed columns (`sha256`, `blob_path`)
- * for files; legacy columns remain for backwards-compatibility.
+ * Columns are deliberately a *superset* of any earlier ad-hoc shapes. New
+ * code uses the content-addressed columns (`sha256`, `blob_path`) for owned
+ * bytes and the reference model (`ref_kind` / `ref_uri`) for files that live
+ * elsewhere.
  *
  * `secrets.value` is encrypted at rest. Registering native entities on a
  * Lattice without an `encryptionKey` configured will throw at init time
@@ -3969,7 +4102,6 @@ interface FilesRow {
     ref_uri?: string | null;
     ref_provider?: string | null;
     blob_path?: string | null;
-    path?: string | null;
     original_name?: string | null;
     mime?: string | null;
     size_bytes?: number | null;
@@ -4143,6 +4275,18 @@ interface MigrationProgress {
 interface MigrationResult {
     tablesCopied: string[];
     rowsCopied: number;
+    /**
+     * Count of copied `files` rows whose canonical bytes are OWNED-LOCAL
+     * (under `<source-root>/data/blobs/` or a machine-local absolute path)
+     * and were therefore NOT migrated — only the row (the pointer) was
+     * copied. After the source SQLite is archived those bytes are
+     * unreachable from the active cloud config, so the cloud `files` rows
+     * are dangling references. Present (and > 0) only when such rows exist;
+     * the caller surfaces it as an operator warning. Migrating the bytes
+     * themselves (S3 upload) is a deferred follow-up. Omitted (undefined)
+     * when every file is already cloud_ref or there are no files.
+     */
+    blobsNotMigrated?: number;
 }
 interface MigrationOptions {
     /** Rows copied per upsert batch. Default 500. */
@@ -4168,6 +4312,15 @@ interface MigrationOptions {
  * differ, encrypted values land on the target as plaintext-of-the-
  * source-ciphertext (i.e. unreadable). Callers are expected to thread
  * the same key through to both sides.
+ *
+ * After each table copy, re-counts the target with the same unfiltered
+ * (soft-delete-consistent) semantic as the source read and throws on
+ * mismatch — defensive insurance against a future write path that
+ * swallows partial failures (the throw lands before the caller archives
+ * the source, so a mismatch leaves the operator on the original local
+ * DB). Reports `blobsNotMigrated` counting copied `files` rows whose
+ * owned-local bytes were not migrated; the bytes themselves are NOT
+ * uploaded — that is a deferred follow-up.
  */
 declare function migrateLatticeData(source: Lattice, target: Lattice, options?: MigrationOptions): Promise<MigrationResult>;
 /**
@@ -4276,7 +4429,14 @@ declare function isPostgresUrl(url: string): boolean;
  * filters rows per individual login role (`session_user`). The group grants
  * *access*, never *visibility*.
  */
-declare const MEMBER_GROUP = "lattice_members";
+/**
+ * The role name a PRE-per-cloud-group cloud used for its member group. Postgres
+ * roles are cluster-global, so this single name was SHARED by every cloud on one
+ * Postgres cluster. Retained only so a legacy cloud can be recognized + migrated
+ * (see docs/MIGRATING-4.0.md) — never used to grant access on the live paths.
+ */
+declare const LEGACY_MEMBER_GROUP = "lattice_members";
+declare function memberGroupFor(db: Lattice): Promise<string>;
 /** Install the cloud RLS bootstrap (bookkeeping + helper functions). No-op on SQLite. */
 declare function installCloudRls(db: Lattice): Promise<void>;
 /**
@@ -4429,7 +4589,7 @@ declare function tableNeedsAudienceView(columnAudience: Record<string, string>):
  * Idempotent. `columns` is the table's full column list (stable order); `pkCols`
  * its primary key, so the row filter matches the RLS policy's pk serialization.
  */
-declare function audienceViewSql(table: string, columns: readonly string[], pkCols: readonly string[], columnAudience: Record<string, string>): string;
+declare function audienceViewSql(table: string, columns: readonly string[], pkCols: readonly string[], columnAudience: Record<string, string>, group: string): string;
 /**
  * Generate + install a table's cell-masking view (Postgres only; no-op on SQLite
  * and on a table with no audience columns). Versioned by a content hash of the
@@ -5090,4 +5250,4 @@ declare class FileSourceKeyStore implements SourceKeyStore {
     private encodeFile;
 }
 
-export { type AddWorkspaceOptions, type AdoptNativeOptions, type AdoptResult, type ApplyWriteResult, type AudienceRowCtx, type AuditEvent, type AutoUpdateResult, type BelongsToRelation, type BelongsToSource, type BlobMetadata, type BuiltinTemplateName, CLOUD_SETTING_SYSTEM_PROMPT, CLOUD_SETTING_WORKSPACE_LOGO, CLOUD_SETTING_WORKSPACE_LOGO_ETAG, CONFIG_SUBDIR, type CatalogEntity, type CatalogRecord, type ChangeEntry, type ChangelogOptions, type ClassifyMatch, type CleanupOptions, type CleanupResult, type CloudProbeResult, type CountOptions, type CrawlOptions, type CrawlResult, type CustomSource, DEFAULT_ENTRY_TYPES, DEFAULT_TYPE_ALIASES, type DiscoveredTable, type EmbeddingsConfig, type EnrichOptions, type EnrichResult, type EnrichedSource, type EnrichmentLookup, type EntityContextDefinition, type EntityContextManifestEntry, type EntityFileManifestInfo, type EntityFileSource, type EntityFileSpec, type EntityProfileField, type EntityProfileSection, type EntityProfileTemplate, type EntityRenderSpec, type EntityRenderTemplate, type EntitySectionPerRow, type EntitySectionsTemplate, type EntityTableColumn, type EntityTableTemplate, type ExtractedObject, FileSourceKeyStore, type FileSourceKeyStoreOptions, type FilesRow, type Filter, type FilterOp, FoldCache, type FtsConfig, type FtsGroup, type FtsHit, type FtsOptions, type FtsResult, type GuiServerHandle, type HasManyRelation, type HasManySource, InMemorySourceKeyStore, InMemoryStateStore, type InitOptions, LOCAL_DB_RELPATH, Lattice, type LatticeConfig, type LatticeConfigInput, type LatticeEntityDef, type LatticeEntityRenderSpec, type LatticeFieldDef, type LatticeFieldType, type LatticeManifest, type LatticeOptions, type LinkOptions, type LlmClient, type LlmMessage, MEMBER_GROUP, type ManyToManySource, type MarkdownTableColumn, type MigrateResult, type Migration, type MigrationOptions, type MigrationProgress, type MigrationResult, type MultiTableDefinition, NATIVE_ENTITY_DEFS, NATIVE_ENTITY_NAMES, NATIVE_REGISTRY_TABLE, type Observation, type OrderBySpec, type OrganizeOptions, type OrganizeResult, type OrganizedCreation, type OrganizedLink, type ParseError, type ParseResult, type ParsedConfig, type PdfOptions, type PdfSenderInput, type PkLookup, PostgresAdapter, type PostgresAdapterOptions, type PreparedStatement, type PrimaryKey, ProgressThrottle, type QueryOptions, READ_ONLY_HEADER, ROOT_DIRNAME, type ReadOnlyHeaderOptions, type ReconcileOptions, type ReconcileResult, type RefKind, type RefProvider, type ReferenceMetadata, ReferenceUnavailableError, type Relation, type RemoteBlobStore, type RenderHooks, type RenderOptions, type RenderProgress, type RenderProgressCallback, type RenderProgressKind, type RenderResult, type RenderSpec, type ReportConfig, type ReportResult, type ReportSection, type ReportSectionResult, type ResolveOptions, type ReverseSeedDetection, type ReverseSeedResult, type ReverseSeedTableResult, type ReverseSyncError, type ReverseSyncResult, type ReverseSyncUpdate, type RewardScores, type Row, type RowVisibilityDefault, type S3Config, type S3StoreConfig, S3UnavailableError, SQLiteAdapter, type SchemaEntity, type SearchOptions, type SearchResult, type SecurityOptions, type SeedConfig, type SeedLinkSpec, SeedReconciliationError, type SeedResult, type SelfSource, type SessionEntry, type SessionParseOptions, type SessionWriteEntry, type SessionWriteOp, type SessionWriteParseResult, type SourceHandle, type SourceKeyStore, type SourceMetadata, type SourceQueryOptions, SourceShreddedError, type StartGuiServerOptions, type StopFn, type StorageAdapter, type SyncResult, type TableDefinition, type TablePolicy, type TemplateRenderSpec, type TurnParams, type TurnResult, type UnresolvedLink, type UpsertByNaturalKeyOptions, type UserIdentity, type UserPreferences, type Viewer, type VisionOptions, type VisionSenderInput, WORKSPACES_SUBDIR, type WatchOptions, type WorkspacePaths, type WorkspaceRecord, type WorkspaceRegistry, type WriteHook, type WriteHookContext, type WritebackDefinition, type WritebackStateStore, type WritebackValidationResult, activeWorkspaceLabel, addWorkspace, adoptNativeEntities, analyticsEnabled, applyTokenBudget, applyWriteEntry, archiveLocalSqlite, assertSafeUrl, attachBlob, audiencePredicate, audienceViewSql, autoFtsColumns, autoUpdate, backfillOwnership, canManageRoles, classifyLinks, cloudRlsInstalled, configDir, contentHash, crawlUrl, createReadOnlyHeader, createS3Store, createSQLiteStateStore, decrypt, defaultWorkspaceYaml, deleteDbCredential, deleteToken, deriveCanonicalContexts, deriveKey, describeImage, describePdf, discoverCloudTables, enableAudienceView, enableChangelogRls, enableRlsForTable, encrypt, enrichKnowledge, ensureFtsIndex, ensureLatticeRoot, entityFileNames, estimateTokens, extractObjects, findLatticeRoot, fixSchemaConflicts, foldEntity, frontmatter, ftsTableName, fullTextSearch, generateEntryId, generateMemberPassword, generateWriteEntryId, getActiveWorkspace, getCloudSetting, getDbCredential, getOrCreateMasterKey, getTablePolicy, getWorkspace, hasFtsIndex, hashFile, importLegacyUserConfig, installCloudRls, installCloudSettings, isEncrypted, isNativeEntity, isPostgresUrl, isPrivateIp, isRowAudience, isV1EntityFiles, listDbCredentials, listNativeBindings, listTokens, listWorkspaces, loadColumnPolicy, manifestPath, markdownTable, memberRoleName, migrateLatticeData, normalizeEntityFiles, observationVisible, observationsFromChange, openTargetLatticeForMigration, openUnderSource, organizeSource, parseConfigFile, parseConfigString, parseMarkdownEntries, parseMatches, parseObjects, parseSessionMD, parseSessionWrites, probeCloud, providerForUrl, provisionMemberRole, readIdentity, readManifest, readPreferences, readRegistry, readToken, referenceLocalFile, referenceUrl, regenerateAudienceViewFromDb, registerNativeEntities, registryPath, resolveActiveS3Config, resolveLatticeRoot, resolveSource, resolveWorkspacePaths, revokeMemberRole, rootConfigDir, s3Key, saveDbCredential, saveDbCredentialForTeam, sealUnderSource, secureCloud, seedColumnPolicyFromYaml, setActiveWorkspace, setCloudSetting, setColumnAudience, setRowVisibility, setTableDefaultVisibility, setTableNeverShare, shredSource, slugify, startGuiServer, summarizeText, tableNeedsAudienceView, toSafeDirName, truncate, validateEntryId, workspaceBlobsDir, workspaceConfigPath, workspaceContextDir, workspaceDataDir, workspaceDbPath, workspaceDir, workspacesDir, writeIdentity, writeManifest, writePreferences, writeRegistry, writeToken };
+export { type AddWorkspaceOptions, type AdoptNativeOptions, type AdoptResult, type ApplyWriteResult, type AudienceRowCtx, type AuditEvent, type AutoUpdateResult, type BelongsToRelation, type BelongsToSource, type BlobMetadata, type BuiltinTemplateName, CLOUD_SETTING_SYSTEM_PROMPT, CLOUD_SETTING_WORKSPACE_LOGO, CLOUD_SETTING_WORKSPACE_LOGO_ETAG, CONFIG_SUBDIR, type CatalogEntity, type CatalogRecord, type ChangeEntry, type ChangelogOptions, type ClassifyMatch, type CleanupOptions, type CleanupResult, type CloudProbeResult, type CountOptions, type CrawlOptions, type CrawlResult, type CustomSource, DEFAULT_ENTRY_TYPES, DEFAULT_TYPE_ALIASES, type DiscoveredTable, type EmbeddingsConfig, type EnrichOptions, type EnrichResult, type EnrichedSource, type EnrichmentLookup, type EntityContextDefinition, type EntityContextManifestEntry, type EntityFileManifestInfo, type EntityFileSource, type EntityFileSpec, type EntityProfileField, type EntityProfileSection, type EntityProfileTemplate, type EntityRenderSpec, type EntityRenderTemplate, type EntitySectionPerRow, type EntitySectionsTemplate, type EntityTableColumn, type EntityTableTemplate, type ExtractedObject, FileSourceKeyStore, type FileSourceKeyStoreOptions, type FilesRow, type Filter, type FilterOp, FoldCache, type FtsConfig, type FtsGroup, type FtsHit, type FtsOptions, type FtsResult, type GuiServerHandle, type HasManyRelation, type HasManySource, InMemorySourceKeyStore, InMemoryStateStore, type InitOptions, LEGACY_MEMBER_GROUP, LOCAL_DB_RELPATH, Lattice, type LatticeConfig, type LatticeConfigInput, type LatticeEntityDef, type LatticeEntityRenderSpec, type LatticeFieldDef, type LatticeFieldType, type LatticeManifest, type LatticeOptions, type LinkOptions, type LlmClient, type LlmMessage, type ManyToManySource, type MarkdownTableColumn, type MigrateResult, type Migration, type MigrationOptions, type MigrationProgress, type MigrationResult, type MultiTableDefinition, NATIVE_ENTITY_DEFS, NATIVE_ENTITY_NAMES, NATIVE_REGISTRY_TABLE, type Observation, type OrderBySpec, type OrganizeOptions, type OrganizeResult, type OrganizedCreation, type OrganizedLink, type ParseError, type ParseResult, type ParsedConfig, type PdfOptions, type PdfSenderInput, type PkLookup, PostgresAdapter, type PostgresAdapterOptions, type PreparedStatement, type PrimaryKey, ProgressThrottle, type QueryOptions, READ_ONLY_HEADER, ROOT_DIRNAME, type ReadOnlyHeaderOptions, type ReconcileOptions, type ReconcileResult, type RefKind, type RefProvider, type ReferenceMetadata, ReferenceUnavailableError, type Relation, type RemoteBlobStore, type RenderHooks, type RenderOptions, type RenderProgress, type RenderProgressCallback, type RenderProgressKind, type RenderResult, type RenderSpec, type ReportConfig, type ReportResult, type ReportSection, type ReportSectionResult, type ResolveOptions, type ReverseSeedDetection, type ReverseSeedResult, type ReverseSeedTableResult, type ReverseSyncError, type ReverseSyncResult, type ReverseSyncUpdate, type RewardScores, type Row, type RowVisibilityDefault, type S3Config, type S3StoreConfig, S3UnavailableError, SQLiteAdapter, type SchemaEntity, type SearchOptions, type SearchResult, type SecurityOptions, type SeedConfig, type SeedLinkSpec, SeedReconciliationError, type SeedResult, type SelfSource, type SessionEntry, type SessionParseOptions, type SessionWriteEntry, type SessionWriteOp, type SessionWriteParseResult, type SourceHandle, type SourceKeyStore, type SourceMetadata, type SourceQueryOptions, SourceShreddedError, type StartGuiServerOptions, type StopFn, type StorageAdapter, type SyncResult, type TableDefinition, type TablePolicy, type TemplateRenderSpec, type TurnParams, type TurnResult, type UnresolvedLink, type UpsertByNaturalKeyOptions, type UserIdentity, type UserPreferences, type Viewer, type VisionOptions, type VisionSenderInput, WORKSPACES_SUBDIR, type WatchOptions, type WorkspacePaths, type WorkspaceRecord, type WorkspaceRegistry, type WriteHook, type WriteHookContext, type WritebackDefinition, type WritebackStateStore, type WritebackValidationResult, activeWorkspaceLabel, addWorkspace, adoptNativeEntities, analyticsEnabled, applyTokenBudget, applyWriteEntry, archiveLocalSqlite, assertSafeUrl, attachBlob, audiencePredicate, audienceViewSql, autoFtsColumns, autoUpdate, backfillOwnership, canManageRoles, classifyLinks, cloudRlsInstalled, configDir, contentHash, crawlUrl, createReadOnlyHeader, createS3Store, createSQLiteStateStore, decrypt, defaultWorkspaceYaml, deleteDbCredential, deleteToken, deriveCanonicalContexts, deriveKey, describeImage, describePdf, discoverCloudTables, enableAudienceView, enableChangelogRls, enableRlsForTable, encrypt, enrichKnowledge, ensureFtsIndex, ensureLatticeRoot, entityFileNames, estimateTokens, extractObjects, findLatticeRoot, fixSchemaConflicts, foldEntity, frontmatter, ftsTableName, fullTextSearch, generateEntryId, generateMemberPassword, generateWriteEntryId, getActiveWorkspace, getCloudSetting, getDbCredential, getOrCreateMasterKey, getTablePolicy, getWorkspace, hasFtsIndex, hashFile, importLegacyUserConfig, installCloudRls, installCloudSettings, isEncrypted, isNativeEntity, isPostgresUrl, isPrivateIp, isRowAudience, listDbCredentials, listNativeBindings, listTokens, listWorkspaces, loadColumnPolicy, manifestPath, markdownTable, memberGroupFor, memberRoleName, migrateLatticeData, observationVisible, observationsFromChange, openTargetLatticeForMigration, openUnderSource, organizeSource, parseConfigFile, parseConfigString, parseMarkdownEntries, parseMatches, parseObjects, parseSessionMD, parseSessionWrites, probeCloud, providerForUrl, provisionMemberRole, readIdentity, readManifest, readPreferences, readRegistry, readToken, referenceLocalFile, referenceUrl, regenerateAudienceViewFromDb, registerNativeEntities, registryPath, resolveActiveS3Config, resolveLatticeRoot, resolveSource, resolveWorkspacePaths, revokeMemberRole, rootConfigDir, s3Key, saveDbCredential, saveDbCredentialForTeam, sealUnderSource, secureCloud, seedColumnPolicyFromYaml, setActiveWorkspace, setCloudSetting, setColumnAudience, setRowVisibility, setTableDefaultVisibility, setTableNeverShare, shredSource, slugify, startGuiServer, summarizeText, tableNeedsAudienceView, toSafeDirName, truncate, validateEntryId, workspaceBlobsDir, workspaceConfigPath, workspaceContextDir, workspaceDataDir, workspaceDbPath, workspaceDir, workspacesDir, writeIdentity, writeManifest, writePreferences, writeRegistry, writeToken };