@jarenjs/db 0.34.0 → 0.43.1

package/src/query.js

@@ -31,6 +31,7 @@ import {
 import { emitPlan, emitEntityPlan, createEntityPredicateEmitters } from './emit.js';
 import { selectPlan } from './algebra.js';
 import { compileSetResidual, compileRowResidual, sequenceResult } from './residual.js';
+import { derivedSlotValue, probeBox } from './derive.js';
 import { deterministicFragment, registerFragment } from './udf.js';
 import {
   normalizeProfile, translateProfilePredicate,
@@ -63,6 +64,43 @@ function bindable(value) {
     || (typeof value === 'number' && Number.isFinite(value));
 }
 
+/**
+ * The value one parameter slot binds for a call. A DERIVED slot holds
+ * no value of its own: it names one edge of a bound external's
+ * bounding box, computed here because a GeoJSON object is not
+ * something any database binds.
+ * @param {import('./emit.js').ParamSlot} slot
+ * @param {any} externals
+ * @returns {any}
+ */
+function slotValue(slot, externals) {
+  if ('literal' in slot) return slot.literal;
+  if ('derived' in slot)
+    return derivedSlotValue(slot.derived, externals[slot.derived.external]);
+  return externals[slot.external];
+}
+
+/**
+ * How one external reaches the statement. An external reached ONLY
+ * through derived slots is bindable when its bound value HAS a box —
+ * the object itself never had to be bindable. One reached directly
+ * must be a string or a finite number, as before; and an external in
+ * the document that reaches no slot at all still forces the diversion,
+ * because the residual needs the engine's own semantics for it.
+ * @param {import('./emit.js').ParamSlot[]} slots
+ * @returns {Map<string, 'plain' | 'derived'>}
+ */
+function externalSlotKinds(slots) {
+  /** @type {Map<string, 'plain' | 'derived'>} */
+  const kinds = new Map();
+  for (const slot of slots) {
+    if ('external' in slot) kinds.set(slot.external, 'plain');
+    else if ('derived' in slot && !kinds.has(slot.derived.external))
+      kinds.set(slot.derived.external, 'derived');
+  }
+  return kinds;
+}
+
 /**
  * The query engine for one collection.
  * @param {{ connection: any, state: any, collection: any,
@@ -109,9 +147,9 @@ export function createQueryEngine(context) {
   };
 
   const udfHook = connection.capabilities.userFunctions
-    ? (fragment) => {
+    ? (/** @type {any} */ fragment, /** @type {string} */ binding) => {
       // Ring 3: admit the registry's pushable:'scalar' operators too
-      const qualified = deterministicFragment(fragment, operators);
+      const qualified = deterministicFragment(fragment, operators, binding);
       if (qualified === null) return null;
       // the store owns the final name: a fingerprint clash between two
       // distinct fragments is disambiguated at registration
@@ -165,6 +203,7 @@ export function createQueryEngine(context) {
         reasons: [{ construct: 'pushdown', reason: 'disabled by the harness switch' }],
         rowReturn: null,
         udfs: [],
+        prefilters: [],
       };
     }
 
@@ -213,6 +252,7 @@ export function createQueryEngine(context) {
       sql: emitted.sql,
       slots: emitted.slots,
       externalNames,
+      externalSlotKinds: externalSlotKinds(emitted.slots),
       dependencies: planned.analysis.dependencies,
       limits: planned.analysis.limits,
       residualLimits: limits,
@@ -305,11 +345,13 @@ export function createQueryEngine(context) {
 
   /** Bind slots against the call's externals. */
   const bindParams = (entry, externals) =>
-    entry.slots.map((slot) => ('literal' in slot ? slot.literal : externals[slot.external]));
+    entry.slots.map((slot) => slotValue(slot, externals));
 
   /** Must this call divert to the residual? */
   const mustDivert = (entry, externals) =>
-    entry.externalNames.some((name) => !bindable(externals[name]));
+    entry.externalNames.some((name) => (entry.externalSlotKinds.get(name) === 'derived'
+      ? probeBox(externals[name]) === null
+      : !bindable(externals[name])));
 
   const rowsToDocs = (rows) => rows.map((row) => JSON.parse(row.doc));
 
@@ -499,6 +541,10 @@ export function createQueryEngine(context) {
       if (pred === null) return;
       if (pred.p === 'and' || pred.p === 'or') pred.items.forEach(collectColumns);
       else if (pred.p === 'not') collectColumns(pred.item);
+      else if (pred.p === 'bboxOverlap')
+        for (const column of Object.values(pred.columns)) touchedColumns.add(column);
+      else if (pred.p === 'cellIn' || pred.p === 'cellPrefix')
+        touchedColumns.add(pred.column);
       else if ('ref' in pred && pred.ref?.column) touchedColumns.add(pred.ref.column);
     };
     collectColumns(entry.plan.filter);
@@ -510,11 +556,15 @@ export function createQueryEngine(context) {
       .filter((index) => index.columns.some((column) => touchedColumns.has(column)))
       .map((index) => index.name);
 
-    const params = entry.slots.map((slot) =>
-      ('external' in slot ? { external: slot.external } : { literal: slot.literal }));
-    const eqpParams = entry.slots.map((slot) => ('literal' in slot
-      ? slot.literal
-      : bindable(externals[slot.external]) ? externals[slot.external] : null));
+    const params = entry.slots.map((slot) => {
+      if ('external' in slot) return { external: slot.external };
+      if ('derived' in slot) return { derived: { ...slot.derived } };
+      return { literal: slot.literal };
+    });
+    const eqpParams = entry.slots.map((slot) => {
+      const value = slotValue(slot, externals);
+      return bindable(value) ? value : null;
+    });
 
     return chain(connection.prepare(dialect.explainQuery(entry.sql)), (statement) =>
       chain(statement.all(eqpParams), (rows) => ({
@@ -526,6 +576,8 @@ export function createQueryEngine(context) {
         sql: entry.sql,
         params,
         indexes,
+        prefilters: entry.planned.prefilters.map((prefilter) => ({ ...prefilter,
+          columns: [...prefilter.columns] })),
         residual: entry.planned.mode === 'native'
           ? null
           : { mode: entry.planned.mode, reasons: entry.planned.reasons },

package/src/residual.js

@@ -10,10 +10,17 @@
  *   materialized candidate array. Re-applying pushed conjuncts is
  *   idempotent, so SQL-side narrowing never changes the answer.
  * - `row` — only the projection stayed behind: each fetched row runs
- *   `{ $for: { it: '$[*]' }, $return: [ <ret> ] }` over the one-row
- *   array; the array wrapper packs the item sequence so an
- *   array-VALUED item stays unambiguous, and the per-row results
- *   concatenate in row order (streamable).
+ *   the planner's one-row document over the one-row array; the array
+ *   wrapper packs the item sequence so an array-VALUED item stays
+ *   unambiguous, and the per-row results concatenate in row order
+ *   (streamable).
+ *
+ * Neither mode builds a query document here. The collection binding is
+ * named by the caller's document — `it`, `user`, anything — so a wrapper
+ * synthesized in this file could only guess it, and a guess that
+ * disagreed with the projection's references would surface as an
+ * unbound-external error at request time rather than at compile time.
+ * The planner knows the name and hands both modes something complete.
  */
 
 import { compileJsonQuery } from '@jarenjs/json/query';
@@ -59,16 +66,17 @@ export function compileSetResidual(document, limits, operators) {
 
 /**
  * Compile the per-row projection for row-mode evaluation.
- * @param {any} returnExpression - The document's raw `$return` value
+ * @param {any} rowDocument - The planner's complete one-row document
+ *   (`{ $for: { <the document's own binding>: '$[*]' },
+ *   $return: [ <its $return> ] }`). It arrives whole because the binding
+ *   and the projection that references it must agree, and the planner is
+ *   the only place that knows the name.
  * @param {any} [limits]
  * @param {{ functions?: any, extensions?: any } | null} [operators]
  * @returns {(row: any, externals: any) => any[]} the row's items
  */
-export function compileRowResidual(returnExpression, limits, operators) {
-  const compiled = compileJsonQuery({
-    $for: { it: '$[*]' },
-    $return: [returnExpression],
-  }, residualOptions(limits, operators));
+export function compileRowResidual(rowDocument, limits, operators) {
+  const compiled = compileJsonQuery(rowDocument, residualOptions(limits, operators));
   return (row, externals) => {
     const packed = compiled([row], externals);
     // one binding → exactly one packed array of that row's items

package/src/store.js

@@ -36,6 +36,10 @@ import { createCaptureEngine, DEFAULT_RETENTION } from './capture.js';
 import { createLiveRegistry, classifyLiveQuery, LIVE_DEFAULTS } from './live.js';
 import { createJobEngine } from './jobs.js';
 import { collectEntityRoots } from './plan.js';
+import {
+  DERIVE_KINDS, PRECISION_MIN, PRECISION_MAX, derivedValue, memberAt,
+  storedMemberForm, registerDeriveFunctions,
+} from './derive.js';
 
 /** The model format version this store implements. */
 export const MODEL_VERSION = '0.1';
@@ -53,6 +57,70 @@ function modelError(code, reason, docPath) {
   return new DbCompileError(code, reason, docPath);
 }
 
+/**
+ * Normalize one index's `derive` declaration — the spatial storage
+ * vocabulary. `derive` says what is COMPUTED from the selected member,
+ * never how the member is selected, so the singular-path rule and its
+ * `JD0004` are untouched; these are the refusals a derived index adds.
+ *
+ * Every one of them is a mistake worth catching at open rather than at
+ * the first query that quietly returns nothing.
+ * @param {any} index - the declared index member
+ * @param {string[]} paths
+ * @param {string} docPath
+ * @returns {{ kind: string | null, precision: number | undefined }}
+ */
+function normalizeDerive(index, paths, docPath) {
+  const declared = index.derive;
+  if (declared === undefined) {
+    if (index.precision !== undefined) {
+      throw modelError('JD0004',
+        "precision belongs to a derived index — declare derive: 'geohash' beside it",
+        `${docPath}/precision`);
+    }
+    return { kind: null, precision: undefined };
+  }
+  if (typeof declared !== 'string' || !DERIVE_KINDS.has(declared)) {
+    throw modelError('JD0004',
+      `derive is a closed set ('geohash' or 'bbox'), got ${JSON.stringify(declared)} — `
+      + 'an open expression member would be a second query language inside the model',
+      `${docPath}/derive`);
+  }
+  if (paths.length !== 1) {
+    throw modelError('JD0004',
+      `a ${declared} index derives its columns from ONE member; a composite path declares several`,
+      `${docPath}/path`);
+  }
+  if (index.unique === true) {
+    throw modelError('JD0004',
+      `a ${declared} index is never unique: distinct positions share a cell (and a box edge) by construction`,
+      `${docPath}/unique`);
+  }
+  if (declared === 'bbox') {
+    if (index.precision !== undefined) {
+      throw modelError('JD0004',
+        'precision applies to a geohash index; a bbox index has no cell size',
+        `${docPath}/precision`);
+    }
+    return { kind: 'bbox', precision: undefined };
+  }
+  const precision = index.precision;
+  if (precision === undefined) {
+    throw modelError('JD0004',
+      `a geohash index must declare precision (${PRECISION_MIN}..${PRECISION_MAX} characters) — `
+      + 'there is no safe default: the right cell size depends on the query radius, '
+      + 'which the model cannot know',
+      `${docPath}/precision`);
+  }
+  if (typeof precision !== 'number' || !Number.isInteger(precision)
+    || precision < PRECISION_MIN || precision > PRECISION_MAX) {
+    throw modelError('JD0004',
+      `precision must be an integer ${PRECISION_MIN}..${PRECISION_MAX}, got ${JSON.stringify(precision)}`,
+      `${docPath}/precision`);
+  }
+  return { kind: 'geohash', precision };
+}
+
 /**
  * Normalize and check a model document. Every failure is `JD0005` with
  * a `docPath` into the model.
@@ -155,10 +223,13 @@ export function normalizeModel(model) {
           'an index path must be a JSONPath string (a composite index takes a non-empty array of them)',
           `${indexDocPath}/path`);
       }
+      const derive = normalizeDerive(index, paths, indexDocPath);
       indexes.push({
         name: index.name,
         paths,
         unique: index.unique === true,
+        derive: derive.kind,
+        precision: derive.precision,
         docPath: indexDocPath,
       });
     }
@@ -377,10 +448,39 @@ function ensureEntityShape(connection, entityPlans, entities, readOnly) {
  */
 function collectionCore(connection, collection, plan, validate, queryState, storeProfileRef) {
   const dialect = connection.dialect;
+  // the STORED branch (a driver that cannot index a registered
+  // function): the derived columns are ordinary ones, so every write
+  // carries their values. Empty everywhere else, and the statements are
+  // then byte-identical to what they were before derived indexes existed
+  const storedNames = new Set(plan.generated
+    .filter((column) => column.stored === true).map((column) => column.name));
+  const storedDerived = plan.derived.filter((column) => storedNames.has(column.name));
   const shape = {
     table: plan.table,
     keyColumn: plan.keyColumn,
     docColumn: plan.docColumn,
+    stored: storedDerived.length > 0
+      ? storedDerived.map((column) => column.name) : undefined,
+  };
+  /**
+   * The stored derived values for one document, in column order. The
+   * four columns of one bbox index share a segments array by identity,
+   * so the member is read and normalized once per index, not once per
+   * column.
+   * @param {any} doc
+   * @returns {any[]}
+   */
+  const derivedFor = (doc) => {
+    /** @type {Map<any, any>} */
+    const members = new Map();
+    return storedDerived.map((column) => {
+      let member = members.get(column.segments);
+      if (member === undefined) {
+        member = { value: storedMemberForm(memberAt(doc, column.segments)) };
+        members.set(column.segments, member);
+      }
+      return derivedValue(column, member.value);
+    });
   };
   /** @type {Map<string, any>} */
   const statements = new Map();
@@ -456,12 +556,12 @@ function collectionCore(connection, collection, plan, validate, queryState, stor
       if (key === null) {
         return chain(
           runWrite('insertAllocated', dialect.dml.insertAllocated(shape),
-            [JSON.stringify(doc)], undefined, true),
+            [JSON.stringify(doc), ...derivedFor(doc)], undefined, true),
           (row) => row.key);
       }
       return chain(
         runWrite('insert', dialect.dml.insert(shape),
-          [key, JSON.stringify(doc)], key, false),
+          [key, JSON.stringify(doc), ...derivedFor(doc)], key, false),
         () => key);
     },
     put(doc, explicitKey) {
@@ -470,12 +570,12 @@ function collectionCore(connection, collection, plan, validate, queryState, stor
       if (key === null) {
         return chain(
           runWrite('insertAllocated', dialect.dml.insertAllocated(shape),
-            [JSON.stringify(doc)], undefined, true),
+            [JSON.stringify(doc), ...derivedFor(doc)], undefined, true),
           (row) => row.key);
       }
       return chain(
         runWrite('upsert', dialect.dml.upsert(shape),
-          [key, JSON.stringify(doc)], key, false),
+          [key, JSON.stringify(doc), ...derivedFor(doc)], key, false),
         () => key);
     },
     patch(key, ops) {
@@ -495,7 +595,7 @@ function collectionCore(connection, collection, plan, validate, queryState, stor
           return chain(
             runWrite('patchFallback',
               dialect.dml.updateDoc(shape, dialect.jsonEncode(dialect.parameterRef(1, 'doc')), 2),
-              [JSON.stringify(next), key], key, false),
+              [JSON.stringify(next), ...derivedFor(next), key], key, false),
             () => next);
         }
         stats.patchTranslated++;
@@ -504,7 +604,7 @@ function collectionCore(connection, collection, plan, validate, queryState, stor
         const sql = dialect.dml.updateDoc(shape, expression, params.length + 1);
         return chain(prepared(`patch:${sql}`, sql), (statement) => {
           try {
-            statement.run([...params, key]);
+            statement.run([...params, ...derivedFor(next), key]);
           }
           catch (error) {
             throw wrapWriteError(error, plan, collection.name, collection.docPath, key);
@@ -747,10 +847,23 @@ export function openStore(model, options) {
       let topLevelTransaction = (fn) => withScope(opened.transaction, fn);
 
       const dialect = connection.dialect;
+      // The PHYSICAL MAPPING BRANCH for derived index columns. A driver
+      // that can index a registered deterministic function generates
+      // them; one that cannot has the store write them. It is a
+      // property of the driver that created the file, so a database
+      // built under one and opened under the other legitimately reports
+      // drift — that is a migration, not an open.
+      const derivedMapping = connection.capabilities.deterministicIndexableFunctions === true
+        ? 'virtual' : 'stored';
       /** @type {Map<string, any>} */
       const plans = new Map();
       for (const [name, collection] of collections)
-        plans.set(name, planCollection(name, collection, dialect));
+        plans.set(name, planCollection(name, collection, dialect, { derived: derivedMapping }));
+      // a table whose column expression calls a function this connection
+      // has not registered cannot even be SELECTed (probed), so the
+      // registration precedes every statement over it
+      const needsDeriveFunctions = derivedMapping === 'virtual'
+        && [...plans.values()].some((plan) => plan.derived.length > 0);
       /** @type {Map<string, any>} */
       const entityPlans = new Map();
       if (mapping !== null) {
@@ -815,6 +928,7 @@ export function openStore(model, options) {
           : Promise.reject(error);
       };
       const opening = () => chain(pragmas, () =>
+        chain(needsDeriveFunctions ? registerDeriveFunctions(connection) : null, () =>
         chain(ensureShape(connection, collections, plans, readOnly), () =>
         chain(ensureEntityShape(connection, entityPlans, entities, readOnly), () => {
           /** @type {Map<string, any>} */
@@ -1408,7 +1522,7 @@ export function openStore(model, options) {
           return chain(capture === null ? null : capture.ready,
             () => chain(jobsEngine === null ? null : jobsEngine.ready,
               () => Object.freeze(store)));
-        })));
+        }))));
 
       let opened_;
       try {

package/src/udf.js

@@ -55,16 +55,22 @@ const functionNameFor = (identity) => `jaren_p_${hashContent(identity)}`;
  *   [operators] - the store's registered operators (Ring 3); only its
  *   `pushable:'scalar'` subset is admitted. `null`/absent keeps the
  *   original engine-internal-only rule (no host function pushes).
+ * @param {string} [binding='it'] - the name the caller's document gave
+ *   the collection binding. The fragment references it, so the wrapper
+ *   below must bind it: under any other name every reference reads as an
+ *   external, the determinism check below rejects the fragment, and the
+ *   hatch silently never engages.
  * @returns {{ key: string, name: string,
  *   compile: () => (docText: string) => number } | null}
  */
-export function deterministicFragment(fragment, operators = null) {
+export function deterministicFragment(fragment, operators = null, binding = 'it') {
   const analyzeOpts = operators === null
     ? undefined
     : { functions: operators.functions, extensions: operators.extensions };
+  const wrap = (/** @type {any} */ body) => ({ $let: { [binding]: '$' }, $return: body });
   let dependencies;
   try {
-    dependencies = analyzeQuery({ $let: { it: '$' }, $return: fragment }, analyzeOpts).dependencies;
+    dependencies = analyzeQuery(wrap(fragment), analyzeOpts).dependencies;
   }
   catch {
     return null;
@@ -101,7 +107,10 @@ export function deterministicFragment(fragment, operators = null) {
     key,
     name: functionNameFor(key),
     compile: () => {
-      const compiled = compileJsonQuery({ $let: { it: '$' }, $return: fragment }, analyzeOpts);
+      // the SAME wrapper the analysis ran over: a compile that bound a
+      // different name than the analysis would judge one document and
+      // run another
+      const compiled = compileJsonQuery(wrap(fragment), analyzeOpts);
       return (docText) => (compiled.ebv(JSON.parse(docText)) ? 1 : 0);
     },
   };