@jarenjs/db 0.43.1 → 0.46.4

package/dist/types/algebra.d.ts

@@ -9,8 +9,10 @@
  *
  * One plan shape covers this version: a guarded selection over ONE
  * collection with optional ordering, window, aggregate and a
- * whole-document projection. Constructs beyond it are residuals by
- * design (see ARCHITECTURE.md's deliberate-residual table).
+ * whole-document projection — or, instead of an ordering and a window,
+ * a k-nearest RANK the engine finishes over the rows the plan fetches.
+ * Constructs beyond it are residuals by design (see ARCHITECTURE.md's
+ * deliberate-residual table).
  */
 /** The plan format version, carried on every plan. */
 export declare const PLAN_VERSION = 2;
@@ -83,6 +85,18 @@ export type PlanOrderTerm = {
     desc: boolean;
     emptyGreatest: boolean;
 };
+export type PlanRank = {
+    column: string;
+    dims: number;
+    probe: {
+        lit: number[];
+    } | {
+        ext: string;
+    };
+    offset: number;
+    limit: number;
+    margin: number;
+};
 export type Plan = {
     planVersion: number;
     alg: 'select';
@@ -93,6 +107,7 @@ export type Plan = {
         offset: number;
         limit: number | null;
     } | null;
+    rank: PlanRank | null;
     aggregate: {
         fn: 'count' | 'sum' | 'avg' | 'min' | 'max';
         ref: PlanRef | null;
@@ -137,6 +152,20 @@ export type Plan = {
  *
  * @typedef {{ ref: PlanRef, desc: boolean, emptyGreatest: boolean }} PlanOrderTerm
  *
+ * @typedef {{ column: string, dims: number,
+ *   probe: { lit: number[] } | { ext: string },
+ *   offset: number, limit: number, margin: number }} PlanRank
+ *   The k-nearest stage: the packed vector column the ranking reads,
+ *   its declared width, the probe (a plan-time literal vector, or the
+ *   external that carries one at call time), the window the ENGINE
+ *   will apply, and the inclusive score margin of the candidate cut.
+ *   The column cuts — every row whose column score is within `margin`
+ *   of the `offset + limit`-th best is a candidate — and the engine
+ *   decides: the original document, its whole ordering and window
+ *   included, runs over the candidates' documents. A plan carrying a
+ *   rank carries no order and no window of its own: nothing in SQL
+ *   orders or limits the fetch.
+ *
  * @typedef {{
  *   planVersion: number,
  *   alg: 'select',
@@ -144,6 +173,7 @@ export type Plan = {
  *   filter: PlanPredicate | null,
  *   order: PlanOrderTerm[] | null,
  *   window: { offset: number, limit: number | null } | null,
+ *   rank: PlanRank | null,
  *   aggregate: { fn: 'count' | 'sum' | 'avg' | 'min' | 'max',
  *     ref: PlanRef | null } | null,
  *   project: 'document',

package/dist/types/ddl.d.ts

@@ -56,25 +56,35 @@ export declare function schemaTypeAt(schema: any, segments: import('./dialect.js
  * is false): there the columns are ordinary ones the store writes. The
  * two mappings produce different declared text on purpose — a database
  * built under one and opened under the other really does disagree, and
- * `verifyShape` says so rather than papering over it.
+ * `verifyShape` says so rather than papering over it. A
+ * `derive: 'vector'` column is the stored shape under BOTH mappings
+ * (`derivedMappingFor`), so for it the two agree.
  * @param {string} name - The collection name (also the table name)
  * @param {{ schema: any, keySegments: { name: string }[] | null,
  *   identity: string, indexes: { name: string, paths: string[],
  *   unique: boolean, derive?: string | null, precision?: number,
- *   docPath: string }[] }} collection - normalized
+ *   dims?: number, docPath: string }[] }} collection - normalized
  * @param {any} dialect
- * @param {{ derived?: 'virtual' | 'stored' }} [options] - the physical
- *   mapping for derived columns; `'virtual'` (a generated column over a
- *   registered function) unless the driver says it cannot index one
+ * @param {{ derived?: 'virtual' | 'stored', rtree?: boolean }} [options]
+ *   - `derived` is the physical mapping for derived columns:
+ *   `'virtual'` (a generated column over a registered function) unless
+ *   the driver says it cannot index one. `rtree` is whether the driver
+ *   carries the R\*Tree module; when it does not, a column set that
+ *   declared `physical: 'rtree'` falls back to the B-tree over its
+ *   columns and `explain().prefilters[].via` reports which shape ran
+ *   (MODEL-FORMAT §4) — a report, not a silent degradation
  * @returns {{
  *   table: string, keyColumn: string, docColumn: string,
  *   keyType: string,
  *   generated: { name: string, type: string, pathText: string,
  *     canonical: string }[],
  *   derived: { name: string, derive: string, precision?: number,
- *     component?: string, segments: any[] }[],
+ *     component?: string, dims?: number, segments: any[] }[],
  *   columnByCanonical: Map<string, string>,
  *   createSql: string[],
+ *   virtualTables: { stem: string, name: string, columns: string[],
+ *     edges: string[], createSql: string, fillSql: string,
+ *     triggers: { name: string, sql: string }[] }[],
  *   expected: { columns: { name: string, type: string,
  *     generated: boolean }[], indexes: { name: string, unique: boolean,
  *     columns: string[] }[] },
@@ -92,10 +102,12 @@ export declare function planCollection(name: string, collection: {
         unique: boolean;
         derive?: string | null;
         precision?: number;
+        dims?: number;
         docPath: string;
     }[];
 }, dialect: any, options?: {
     derived?: 'virtual' | 'stored';
+    rtree?: boolean;
 }): {
     table: string;
     keyColumn: string;
@@ -112,10 +124,23 @@ export declare function planCollection(name: string, collection: {
         derive: string;
         precision?: number;
         component?: string;
+        dims?: number;
         segments: any[];
     }[];
     columnByCanonical: Map<string, string>;
     createSql: string[];
+    virtualTables: {
+        stem: string;
+        name: string;
+        columns: string[];
+        edges: string[];
+        createSql: string;
+        fillSql: string;
+        triggers: {
+            name: string;
+            sql: string;
+        }[];
+    }[];
     expected: {
         columns: {
             name: string;

package/dist/types/derive.d.ts

@@ -2,20 +2,36 @@
  * @file Derived index columns: the one place a declared
  * `indexes[].derive` becomes a value. A spatial member is an array of
  * numbers or an object, and a generated column must be a scalar, so a
- * geohash cell or a bounding-box edge is what actually gets indexed.
+ * geohash cell or a bounding-box edge is what actually gets indexed;
+ * an embedding is an array of hundreds of numbers, and what gets
+ * stored is its packed, l2-normalized Float32 form.
  *
- * Every cell and every box comes from `@jarenjs/core/geo`; nothing
- * here computes spatial arithmetic of its own. The same functions
- * serve BOTH physical mappings — registered as deterministic SQL
- * functions inside a virtual generated column's expression where the
- * driver can index them, and called directly on the write path where
- * it cannot — so the two branches cannot drift into different answers.
+ * Every cell and every box comes from `@jarenjs/core/geo`, and every
+ * normalization and packing from `@jarenjs/core/vector`; nothing here
+ * computes arithmetic of its own. The spatial functions serve BOTH
+ * physical mappings — registered as deterministic SQL functions inside
+ * a virtual generated column's expression where the driver can index
+ * them, and called directly on the write path where it cannot — so
+ * the two branches cannot drift into different answers. The vector
+ * kind has ONE mapping, stored everywhere (`DERIVE_MAPPING`), and
+ * registers no function at all: a whole array re-derived per row as a
+ * host call is the cost the spatial work measured at 150×, a stored
+ * column is readable without any registration, and bun has no
+ * function API — one mapping is the only way every driver agrees.
  *
  * It is also this package's ONLY seam onto `@jarenjs/core/geo` (D1 —
- * one home for spatial arithmetic, grep-proven by test): the planner's
- * probe geometry — the box of a literal or bound region, the box of a
- * bounded-distance circle, a cell's neighbourhood — is computed by the
- * helpers below rather than by an import of its own.
+ * one home for spatial arithmetic, grep-proven by test) and onto
+ * `@jarenjs/core/vector` (the same rule, one home for vector
+ * arithmetic): the planner's probe geometry — the box of a literal or
+ * bound region, the box of a bounded-distance circle, a cell's
+ * neighbourhood — and the k-nearest plan's probe vector and column
+ * score are computed by the helpers below rather than by an import of
+ * their own.
+ *
+ * The switches over the kind are EXHAUSTIVE: a kind with no rule
+ * throws, here and in the dialect, so that adding a kind without
+ * teaching both is a failure at the first call rather than a bbox
+ * edge of the first two floats and `jaren_bbox_undefined(...)` SQL.
  *
  * Determinism is the contract, not a convenience: a value here is a
  * pure function of the document bytes. Nothing reads the clock, a
@@ -28,6 +44,33 @@
  */
 /** The closed set of derive kinds. */
 export declare const DERIVE_KINDS: Set<string>;
+/**
+ * The per-kind PHYSICAL MAPPING override. A spatial kind takes the
+ * mapping the driver's capability selects (`null` here); the vector
+ * kind is `'stored'` on every driver, for the three reasons in the
+ * file header, and never joins `registerDeriveFunctions`.
+ * @type {Readonly<Record<string, 'stored' | null>>}
+ */
+export declare const DERIVE_MAPPING: Readonly<Record<string, 'stored' | null>>;
+/**
+ * The physical mapping one derived column takes: the kind's override
+ * where it has one, the driver's mapping otherwise.
+ * @param {string} kind
+ * @param {'virtual' | 'stored'} driverMapping
+ * @returns {'virtual' | 'stored'}
+ */
+export declare function derivedMappingFor(kind: string, driverMapping: 'virtual' | 'stored'): 'virtual' | 'stored';
+/**
+ * The closed set of PHYSICAL realizations a `derive: 'bbox'` index may
+ * ask for. `'columns'` is the default and what an absent member means:
+ * four generated columns under one B-tree. `'rtree'` is the same four
+ * columns (they stay the box's one definition) beside a SQLite R\*Tree
+ * virtual table kept in sync by declared triggers, with no B-tree over
+ * them. The LOGICAL meaning of `derive: 'bbox'` is identical either
+ * way — same rows, same answers — which is the whole point of naming
+ * the shape separately from the derivation.
+ */
+export declare const PHYSICAL_KINDS: Set<string>;
 /** A bbox index's four columns, in the order they are declared —
  * `[west, south, east, north]`, the order the kernel's boxes carry. */
 export declare const BBOX_COMPONENTS: readonly string[];
@@ -42,6 +85,9 @@ export declare const BBOX_INDEX_ORDER: readonly string[];
 /** The declared geohash precision range (characters). */
 export declare const PRECISION_MIN = 1;
 export declare const PRECISION_MAX = 12;
+/** The declared vector width range (components). */
+export declare const DIMS_MIN = 1;
+export declare const DIMS_MAX = 8192;
 /**
  * The geohash cell of a value at a precision, or null when the value
  * carries no bounded position.
@@ -59,6 +105,20 @@ export declare function deriveGeohash(value: any, precision: number): string | n
  * @returns {number | null}
  */
 export declare function deriveBboxEdge(value: any, component: string): number | null;
+/**
+ * The value of a `derive: 'vector'` column for a document member — the
+ * ONE seam from this package onto `@jarenjs/core/vector`. The member
+ * round-trips through its stored form first, exactly as the spatial
+ * kinds do: a `Float32Array` in the document is held as an object and
+ * a `NaN` as `null`, and the column must be a function of what is
+ * held, so that a write, a migration backfill and a second open can
+ * never disagree about one row.
+ * @param {any} member - the value at the index path, or `undefined`
+ * @param {number} dims - the declared width
+ * @returns {Uint8Array | null} `4·dims` bytes, or `null` when the
+ *   member is not a vector of that width
+ */
+export declare function deriveVector(member: any, dims: number): Uint8Array | null;
 /**
  * A member as the database will hold it. A derived value must be a
  * function of the STORED document, not of the object handed to the
@@ -75,16 +135,21 @@ export declare function deriveBboxEdge(value: any, component: string): number |
  */
 export declare function storedMemberForm(member: any): any;
 /**
- * The value of one derived column for a document member.
- * @param {{ derive: string, precision?: number, component?: string }} column
- * @param {any} member - the value at the index path, or `undefined`
- * @returns {string | number | null}
+ * The value of one derived column for a document member, which the
+ * caller hands over in its STORED form (`storedMemberForm`): the write
+ * path round-trips it, and a migration backfill reads it from the row.
+ * Exhaustive over the kind — see the file header.
+ * @param {{ derive: string, precision?: number, component?: string,
+ *   dims?: number }} column
+ * @param {any} member - the stored value at the index path, or `undefined`
+ * @returns {string | number | Uint8Array | null}
  */
 export declare function derivedValue(column: {
     derive: string;
     precision?: number;
     component?: string;
-}, member: any): string | number | null;
+    dims?: number;
+}, member: any): string | number | Uint8Array | null;
 /**
  * Read the member one derived column is computed from, walking the
  * same typed segments the physical mapping was planned over.
@@ -159,3 +224,27 @@ export declare function derivedSlotValue(derived: {
     external: string;
     axis: string;
 }, value: any): number | null;
+/**
+ * The probe a k-nearest plan scores the column against: a literal or
+ * bound vector, l2-normalized once, so that its dot product with the
+ * column's normalized form IS the cosine of the raw vectors. `null`
+ * when the value is not a vector of exactly `dims` finite numbers —
+ * the binder's signal to divert the call to the residual, where the
+ * engine answers what it answers everywhere for such a probe (empty
+ * keys for another width, its own refusal for a non-array).
+ * @param {unknown} value
+ * @param {number} dims - the column's declared width
+ * @returns {Float32Array | null}
+ */
+export declare function probeVector(value: unknown, dims: number): Float32Array | null;
+/**
+ * One row's column score against a prepared probe: the packed column
+ * unpacked at the declared width and dotted with the probe. `null` for
+ * a row whose column holds no vector — SQL `NULL`, or bytes of another
+ * length — because such a row is unrankable and 0 is a real score.
+ * @param {unknown} bytes - the column value as the driver returns it
+ * @param {number} dims
+ * @param {Float32Array} probe - from {@link probeVector}
+ * @returns {number | null}
+ */
+export declare function columnScore(bytes: unknown, dims: number, probe: Float32Array): number | null;

package/dist/types/dialect.d.ts

@@ -36,6 +36,7 @@ export type JsonPathSegment = {
  *   capabilities: Record<string, any>,
  *   tableSuffix: string,
  *   docColumnType: string,
+ *   packedVectorType: string,
  *   quoteIdentifier: (s: string) => string,
  *   parameterRef: (i: number, name: string) => string,
  *   stringLiteral: (s: string) => string,
@@ -45,7 +46,7 @@ export type JsonPathSegment = {
  *   jsonPathText: (segments: JsonPathSegment[]) => string | null,
  *   jsonExtract: (columnSql: string, pathText: string) => string,
  *   derivedExpression?: (memberSql: string, column: { derive: string,
- *     precision?: number, component?: string }) => string,
+ *     precision?: number, component?: string, dims?: number }) => string,
  *   jsonSet: (exprSql: string, pathText: string, valueSql: string) => string,
  *   jsonRemove: (exprSql: string, pathText: string) => string,
  *   jsonAppend: (exprSql: string, arrayPathText: string, valueSql: string) => string,
@@ -60,6 +61,8 @@ export type JsonPathSegment = {
  *   strContains: (valueSql: string, patternSql: string) => string,
  *   orderNulls: (nullsFirst: boolean) => string,
  *   rowIdentity: () => string,
+ *   identityIn: (identitySql: string, paramSqls: string[]) => string,
+ *   rtree?: { module: string, columns: readonly string[] },
  *   explainQuery: (sql: string) => string,
  *   excludedRef: (columnSql: string) => string,
  *   tx: { begin: string, beginImmediate: string, commit: string,
@@ -83,6 +86,7 @@ export declare function createDialect(spec: {
     capabilities: Record<string, any>;
     tableSuffix: string;
     docColumnType: string;
+    packedVectorType: string;
     quoteIdentifier: (s: string) => string;
     parameterRef: (i: number, name: string) => string;
     stringLiteral: (s: string) => string;
@@ -95,6 +99,7 @@ export declare function createDialect(spec: {
         derive: string;
         precision?: number;
         component?: string;
+        dims?: number;
     }) => string;
     jsonSet: (exprSql: string, pathText: string, valueSql: string) => string;
     jsonRemove: (exprSql: string, pathText: string) => string;
@@ -110,6 +115,11 @@ export declare function createDialect(spec: {
     strContains: (valueSql: string, patternSql: string) => string;
     orderNulls: (nullsFirst: boolean) => string;
     rowIdentity: () => string;
+    identityIn: (identitySql: string, paramSqls: string[]) => string;
+    rtree?: {
+        module: string;
+        columns: readonly string[];
+    };
     explainQuery: (sql: string) => string;
     excludedRef: (columnSql: string) => string;
     tx: {

package/dist/types/index.d.ts

@@ -18,7 +18,8 @@ export { selectPlan, conjoin, assertNoSqlText, PLAN_VERSION } from './algebra.js
 export { typeOfPath, isNumericType } from './types.js';
 export { compileSetResidual, compileRowResidual, sequenceResult } from './residual.js';
 export { deterministicFragment, registerFragment } from './udf.js';
-export { DERIVE_KINDS, BBOX_COMPONENTS, BBOX_INDEX_ORDER, PRECISION_MIN, PRECISION_MAX, deriveGeohash, deriveBboxEdge, derivedValue, memberAt, storedMemberForm, registerDeriveFunctions, } from './derive.js';
+export { DERIVE_KINDS, DERIVE_MAPPING, PHYSICAL_KINDS, BBOX_COMPONENTS, BBOX_INDEX_ORDER, PRECISION_MIN, PRECISION_MAX, DIMS_MIN, DIMS_MAX, deriveGeohash, deriveBboxEdge, deriveVector, derivedValue, derivedMappingFor, memberAt, storedMemberForm, registerDeriveFunctions, probeVector, columnScore, } from './derive.js';
+export { KNN_MARGIN, IDENTITY_CHUNK, cutCandidates, identityBatches } from './knn.js';
 export { createQueryEngine, createQueryState, createEntityQueryEngine, createLoadEngine, INCLUDE_DEPTH_DEFAULT, } from './query.js';
 export { normalizeProfile, SAFE_PROFILE, translateProfilePredicate, applyMandatoryPredicate, applyRowBound, } from './profile.js';
 export { translatePatch } from './patch-sql.js';

package/dist/types/knn.d.ts

@@ -0,0 +1,69 @@
+/**
+ * @file The candidate cut of a k-nearest plan: the arithmetic between
+ * the scores a statement fetched and the row identities the engine
+ * will decide over. No vector arithmetic lives here — a score is
+ * `derive.js`'s, through `@jarenjs/core/vector` — and no SQL: the
+ * fetch of the winners is the dialect's. This is the one place the
+ * margin is applied and the one place candidate identities are
+ * batched for it.
+ */
+/**
+ * The inclusive score margin of the cut. The column's score is a dot
+ * product over binary32-normalized forms; the engine's key is the
+ * cosine of the raw doubles; measured, the two differ by at most
+ * ~1e-8. Any margin of at least twice that makes the engine's top
+ * `offset + limit` a SUBSET of the candidates: were a row the engine
+ * ranks inside the window cut, some candidate the engine ranks outside
+ * it would have to score higher by the column and lower by the engine,
+ * which two scores within half the margin of each other cannot do.
+ * This is a hundred times that bound — it admits, in practice, only
+ * true ties, and those the engine breaks by the document's own keys.
+ */
+export declare const KNN_MARGIN = 0.000001;
+/**
+ * The most identities one fetch statement binds: under the parameter
+ * cap of every SQLite build the store runs on. A guard, not a design —
+ * a k-nearest window is a handful of rows, and this only matters for a
+ * collection of many exact duplicates.
+ */
+export declare const IDENTITY_CHUNK = 512;
+/**
+ * The rows a k-nearest window can contain, from the scored fetch.
+ *
+ * `m = offset + limit` rows are needed. When at least `m` rows scored,
+ * the candidates are every scored row within `margin` of the m-th best
+ * score — ties at the boundary included by construction. When fewer
+ * did, the window reaches the unrankable tail (NULL columns, or a
+ * collection smaller than the window), which only the documents can
+ * order: every row is then a candidate, and the collection is no
+ * larger than the window.
+ * @param {{ identity: any, score: number | null }[]} rows - one per
+ *   fetched row; `score` is `null` where the column held no vector
+ * @param {number} m - `offset + limit`
+ * @param {number} margin
+ * @returns {{ identities: any[], scored: number, full: boolean }} the
+ *   candidate identities in ascending identity order — the
+ *   collection's own order, which a stable sort over the candidates
+ *   must see — with how many rows scored and whether every row was
+ *   taken
+ */
+export declare function cutCandidates(rows: {
+    identity: any;
+    score: number | null;
+}[], m: number, margin: number): {
+    identities: any[];
+    scored: number;
+    full: boolean;
+};
+/**
+ * The identities sliced into fetch batches: at most `IDENTITY_CHUNK`
+ * each, and each padded with `null` to the next power of two — a NULL
+ * in an IN list matches no row — so a handful of prepared statements
+ * serve every candidate count instead of one per count seen.
+ * @param {any[]} identities - in the order they should be fetched
+ * @returns {{ size: number, params: any[] }[]}
+ */
+export declare function identityBatches(identities: any[]): {
+    size: number;
+    params: any[];
+}[];

package/dist/types/migrate.d.ts

@@ -50,7 +50,7 @@ export declare function migrationChecksum(migration: any): string;
  * @param {any} fromModel
  * @param {any} toModel
  * @param {{ id?: string, dialect?: any,
- *   derived?: 'virtual' | 'stored' }} [options]
+ *   derived?: 'virtual' | 'stored', rtree?: boolean }} [options]
  * @returns {{ migration: any, report: {
  *   renamed: { from: string, to: string }[],
  *   added: string[], removed: string[],
@@ -61,6 +61,7 @@ export declare function planMigration(fromModel: any, toModel: any, options?: {
     id?: string;
     dialect?: any;
     derived?: 'virtual' | 'stored';
+    rtree?: boolean;
 }): {
     migration: any;
     report: {

package/dist/types/plan.d.ts

@@ -10,7 +10,7 @@
  *
  * The outcome of planning one document:
  *
- *   { plan, mode: 'native' | 'row' | 'set', reasons, rowReturn,
+ *   { plan, mode: 'native' | 'row' | 'set' | 'knn', reasons, rowReturn,
  *     prefilters }
  *
  * - `native` — everything translated; the plan alone answers.
@@ -21,6 +21,11 @@
  *   wrapper built anywhere else would have to guess it.
  * - `set`    — the pushed conjuncts narrow candidates; the WHOLE
  *   compiled document runs over the materialized candidates.
+ * - `knn`    — the pushed conjuncts narrow, the vector column CUTS the
+ *   candidates of a k-nearest window (`plan.rank`), and the whole
+ *   compiled document runs over the cut — a set residual whose
+ *   candidate set an ordering, not a predicate, chose (see "The
+ *   k-nearest promotion" below).
  *
  * `reasons` names every construct that forced work off the database,
  * with reason text drawn from the deliberate-residual table.
@@ -49,11 +54,12 @@ export declare function assertDecidedKind(node: any): void;
  * @returns {{
  *   analysis: any,
  *   plan: import('./algebra.js').Plan | null,
- *   mode: 'native' | 'row' | 'set',
+ *   mode: 'native' | 'row' | 'set' | 'knn',
  *   reasons: { construct: string, reason: string }[],
  *   rowReturn: any,
  *   udfs: string[],
- *   prefilters: { construct: string, columns: string[], exact: boolean }[],
+ *   prefilters: { construct: string, via: 'columns' | 'rtree',
+ *     columns: string[], exact: boolean }[],
  * }}
  */
 export declare function planQuery(document: any, shape: any, options?: {
@@ -64,7 +70,7 @@ export declare function planQuery(document: any, shape: any, options?: {
 }): {
     analysis: any;
     plan: import('./algebra.js').Plan | null;
-    mode: 'native' | 'row' | 'set';
+    mode: 'native' | 'row' | 'set' | 'knn';
     reasons: {
         construct: string;
         reason: string;
@@ -73,6 +79,7 @@ export declare function planQuery(document: any, shape: any, options?: {
     udfs: string[];
     prefilters: {
         construct: string;
+        via: 'columns' | 'rtree';
         columns: string[];
         exact: boolean;
     }[];

package/dist/types/query.d.ts

@@ -17,6 +17,17 @@
  * residual over the full collection instead of the native statement —
  * SQLite cannot bind a boolean, a `null` needs Jaren's semantics, and
  * a missing external must raise the ENGINE's error, not a driver's.
+ * Two externals never bind at all and divert by their own rule: a
+ * region reaching the statement through derived slots diverts when it
+ * has no box, and a k-nearest probe — which the plan scores in the
+ * engine, never in SQL — diverts when it is not a vector of the
+ * column's width, so the engine answers what it answers everywhere.
+ *
+ * The k-nearest mode (`plan.rank`) is a set residual whose candidates
+ * an ordering chose: the statement fetches (identity, column) under
+ * the pushed WHERE, the engine scores and cuts (`knn.js`), the
+ * winners' documents are fetched by identity through the dialect, and
+ * the whole document runs over them.
  */
 /**
  * The store-wide query state shared by every collection's engine: one

package/docs/LIVE-FORMAT.md

@@ -172,9 +172,39 @@ what the pushdown planner already means by it.
 | `orderBy` over extractable paths, optional `limit`, offset 0 | **maintained window**: a sorted structure; ties broken by the collection key, appended as the final sort term; an insert sorting beyond a full window is a no-op | the window rows and their sort keys |
 | whole-query `count` / `sum` / `avg` / `min` / `max` (the plan's aggregate), optional `where` | **running accumulator** plus a per-row contribution map — a delete can only be answered from retained contributions (§3: a `remove` carries no old value). `min`/`max` removal of the last extremum holder FALLS BACK to a recompute over the retained contributions; the accumulator alone cannot answer, and this fallback is the documented cost | one contribution per matching row |
 | single-level `groupBy` with aggregate returns, in the canonical form below | **per-group deltas**: the accumulator machinery, one instance per group; groups appear in first-appearance order, exactly the engine's order | per-group, per-row contributions |
+| `where` whose spatial predicate is **refined** (the plan pushed a bounding-box or cell-range pre-filter and left the exact `$within`, bounded `$distance` or over-long cell prefix to the residual — `explain().prefilters` with `exact: false`), no order, no aggregate; optional per-row `select` | **incremental rows** — the geofence: the initial fetch narrows through the derived index, and every touched row is re-evaluated by the engine's EXACT predicate, so a point emits `add` when it enters the region, `remove` when it leaves, and nothing while it moves within (a whole-document return sees a `replace` carrying the new position) | the result rows |
+| a refined spatial predicate over a collection with **no document key** (`key: null`, rowid identity) | **re-run on invalidation** — the per-row strategy tracks a row by its declared key, and a rowid is not one; the reason says `rows without a document key cannot be tracked` | the previous result, for diffing |
+| `orderBy` beside a refined spatial predicate — over `$distance` (not a path) or over a member (the set residual drops the planner's order terms) | **re-run on invalidation**, the ordering named as the reason | the previous result, for diffing |
+| a whole-query aggregate or a `groupBy` whose `where` is a refined spatial predicate | **re-run on invalidation** — the accumulator needs a fully translated selection and a refinement is not one; the reason says so | the previous result, for diffing |
+| a spatial predicate the planner **refused** (no `derive` index on the member, an untyped member, an unbounded probe) | **re-run on invalidation**, the refusal named — it never translated, so nothing narrows the fetch | the previous result, for diffing |
 | joins, multi-entity roots, graph loads, every entity query | **re-run on invalidation — declared, not attempted** in this version | the previous result, for diffing |
 | anything else: non-translatable predicates, `limit` without `orderBy`, `offset` > 0, windowed aggregates, `@jarenjs/linq`'s nested two-level `groupBy` emission, non-canonical group returns | **re-run on invalidation**, the reason named | the previous result, for diffing |
 
+The **physical mapping** of a `derive: 'bbox'` index (MODEL-FORMAT §2.1,
+`physical`) does not move a query between these rows. It can change one
+input to the choice — `$bbox-intersects` translates exactly over four
+generated columns and is refined over an R\*Tree, whose stored box is a
+32-bit-float superset — but both the exact row above and the refined one
+below it are the maintained per-row strategy, so the geofence behaves
+identically under either shape. That is checked, not assumed:
+`test/db/geofence.test.js` runs the same watch under both mappings and
+asserts the same strategy, the same emissions and the same rows.
+
+**What the geofence costs, stated rather than discovered.** A refined
+spatial predicate is maintained *per row*, not incrementally: there is
+no live spatial index, and none is planned. Every insert or update the
+collection sees runs the exact predicate — `$within` against the
+region, a geodesic `$distance` — once for that row, inside capture
+delivery, on the store's own connection. Over a large region (a ring of
+thousands of vertices) at a high write rate that is real work on every
+write, and a consumer with such a region either simplifies it for the
+fence (`$geo-simplify`) or accepts the cost knowingly. The region is
+bound at registration like every external (§8): the initial fetch binds
+it through the same derived parameter slots the planner uses for a
+one-off query, so an external region narrows exactly as a literal one
+does, and a region with no bounding box diverts to a full initial read
+and is still correct.
+
 Re-run is a first-class, documented outcome, not a failure. What is
 forbidden is *silently* re-running while the reader believes the query
 is incremental: `live.mode` reports `'incremental'` or `'rerun'`, the

package/docs/MIGRATION-FORMAT.md

@@ -84,6 +84,16 @@ A `jslt` transform on such a collection gets the same treatment for the
 same reason: it rewrites the documents the columns are computed from,
 so the planner follows it with a `derive` step that recomputes them.
 
+A `derive: 'vector'` column (MODEL-FORMAT §2.1) is stored under BOTH
+mappings, so the planner emits its backfill whatever `derived` says,
+and the column entry carries the width the value is packed to:
+
+```json
+{ "kind": "derive", "collection": "items",
+  "columns": [ { "name": "gx_embedding_v768", "derive": "vector", "dims": 768,
+                 "segments": [ { "name": "embedding" } ] } ] }
+```
+
 ## 3. Planning and the widening/narrowing rule
 
 `planMigration(fromModel, toModel, { dialect, id, derived })` produces