@jarenjs/db 0.49.2 → 0.66.1

package/src/dialects/rtree-ddl.js

@@ -0,0 +1,129 @@
+//@ts-check
+/**
+ * @file The R\*Tree mapping's own DDL — the second physical
+ * realization of a `derive: 'bbox'` column set (MODEL-FORMAT §2.1,
+ * `physical: 'rtree'`): a virtual table beside the collection and the
+ * three row triggers that keep it in sync.
+ *
+ * It lives beside the dialects rather than in the shared statement
+ * builder because every line of it is one engine family's spelling —
+ * `CREATE VIRTUAL TABLE ... USING`, and a trigger body between `BEGIN`
+ * and `END`. A dialect whose `capabilities.virtualTables` is false
+ * composes none of it, and the planner maps a `physical: 'rtree'`
+ * column set back onto the B-tree over the four edge columns instead.
+ *
+ * Parameterized by the spelling spec it belongs to, so the two specs
+ * that DO carry an R\*Tree share one implementation and still produce
+ * their own quoting, their own module name and their own column order.
+ */
+
+/**
+ * The virtual-table DDL group for one spelling spec.
+ * @param {{ quoteIdentifier: (s: string) => string,
+ *   rowIdentity: () => string,
+ *   rtree: { module: string, columns: readonly string[] } }} spec
+ * @returns {Record<string, Function>}
+ */
+export function rtreeDdl(spec) {
+  const q = spec.quoteIdentifier;
+  return {
+  /**
+   * The second physical realization of a `derive: 'bbox'` column set
+   * (MODEL-FORMAT §2.1, `physical: 'rtree'`): an R\*Tree virtual
+   * table beside the collection, keyed by the collection's row id and
+   * carrying the four box edges as `(minx, maxx, miny, maxy)`.
+   *
+   * The coordinates are 32-bit floats rounded OUTWARD, so the stored
+   * box is a superset of the row's — no false negatives, which is
+   * what an implied conjunct needs, and the reason `$bbox-intersects`
+   * stops being exact under this mapping.
+   * @param {{ name: string }} shape
+   * @returns {string}
+   */
+  createVirtualTable({ name }) {
+    return `CREATE VIRTUAL TABLE ${q(name)} USING ${spec.rtree.module}(`
+      + `${spec.rtree.columns.map(q).join(', ')})`;
+  },
+  /**
+   * @param {string} name
+   * @returns {string}
+   */
+  dropVirtualTable(name) {
+    return `DROP TABLE ${q(name)}`;
+  },
+  /**
+   * @param {string} name
+   * @returns {string}
+   */
+  dropTrigger(name) {
+    return `DROP TRIGGER ${q(name)}`;
+  },
+  /**
+   * Fill an R\*Tree from the documents already stored — the migration
+   * step that turns a `columns` collection into an `rtree` one. The
+   * `IS NOT NULL` is §3.2's rule in SQL: a row with no bounded
+   * position is ABSENT from the index, not at `[0, 0]`.
+   * @param {{ table: string, virtualTable: string,
+   *   edges: { name: string }[] }} shape
+   * @returns {string}
+   */
+  fillVirtualTable({ table, virtualTable, edges }) {
+    const columns = spec.rtree.columns;
+    const sources = [spec.rowIdentity(), ...edges.map((edge) => q(edge.name))];
+    return `INSERT INTO ${q(virtualTable)} (${columns.map(q).join(', ')}) `
+      + `SELECT ${sources.join(', ')} FROM ${q(table)} `
+      + `WHERE ${q(edges[0].name)} IS NOT NULL`;
+  },
+  /**
+   * The three triggers that keep an R\*Tree in sync with its
+   * collection — insert, update, delete — as DECLARED objects of the
+   * collection table.
+   *
+   * Declared, and not a second write path in JavaScript: a trigger is
+   * inside the writing transaction by construction (SQLite cannot
+   * separate them), no write path can bypass it (`insert`,
+   * `insertAllocated`, `upsert`, a translated patch, the patch
+   * fallback, a delete and a migration backfill all fire it), and it
+   * belongs to the collection table, so the existing declared-text
+   * drift check sees it for free.
+   *
+   * The body reads the DERIVED COLUMNS through `NEW` rather than
+   * restating the box expression, so the columns stay the box's one
+   * definition — and that text works unchanged on the stored-column
+   * branch, where those columns are ordinary ones.
+   *
+   * The `IS NOT NULL` guard is load-bearing: an R\*Tree coerces a
+   * `NULL` coordinate to `0.0` without complaint, so without it every
+   * unbounded document would land on Null Island instead of being
+   * absent (MODEL-FORMAT §3.2).
+   * @param {{ table: string, virtualTable: string, prefix: string,
+   *   edges: { name: string }[] }} shape - `edges` are the four
+   *   derived columns in `(w, e, s, n)` order, which is the order the
+   *   virtual table's `(minx, maxx, miny, maxy)` carry
+   * @returns {{ name: string, sql: string }[]}
+   */
+  createSyncTriggers({ table, virtualTable, prefix, edges }) {
+    const rid = spec.rowIdentity();
+    const target = `${q(virtualTable)} (${spec.rtree.columns.map(q).join(', ')})`;
+    const guard = `${q(edges[0].name)} IS NOT NULL`;
+    const values = (row) => [`${row}.${rid}`,
+      ...edges.map((edge) => `${row}.${q(edge.name)}`)].join(', ');
+    return [
+      { name: `${prefix}_ai`,
+        sql: `CREATE TRIGGER ${q(`${prefix}_ai`)} AFTER INSERT ON ${q(table)} `
+          + `WHEN NEW.${guard} BEGIN `
+          + `INSERT INTO ${target} VALUES (${values('NEW')}); END` },
+      // one trigger, not two: the old id leaves and the new box
+      // arrives only when it exists, so a document that loses its
+      // geometry leaves the index rather than keeping a stale box
+      { name: `${prefix}_au`,
+        sql: `CREATE TRIGGER ${q(`${prefix}_au`)} AFTER UPDATE ON ${q(table)} BEGIN `
+          + `DELETE FROM ${q(virtualTable)} WHERE ${q(spec.rtree.columns[0])} = OLD.${rid}; `
+          + `INSERT INTO ${target} SELECT ${values('NEW')} WHERE NEW.${guard}; END` },
+      { name: `${prefix}_ad`,
+        sql: `CREATE TRIGGER ${q(`${prefix}_ad`)} AFTER DELETE ON ${q(table)} BEGIN `
+          + `DELETE FROM ${q(virtualTable)} WHERE ${q(spec.rtree.columns[0])} = OLD.${rid}; END` },
+    ];
+  },
+  };
+}

package/src/dialects/sqlite.js

@@ -9,6 +9,8 @@
  */
 
 import { createDialect } from '../dialect.js';
+import { rtreeDdl } from './rtree-ddl.js';
+import { readExpression } from './expression-read.js';
 
 /** @param {string} s */
 function quoteIdentifier(s) {
@@ -33,7 +35,8 @@ function jsonPathText(segments) {
   let text = '$';
   for (const segment of segments) {
     if ('index' in segment) {
-      text += `[${segment.index}]`;
+      // SQLite counts from the end as `[#-1]`; a bare `[-1]` is a bad path
+      text += segment.index < 0 ? `[#${segment.index}]` : `[${segment.index}]`;
       continue;
     }
     // eslint-disable-next-line no-control-regex
@@ -62,8 +65,9 @@ function typeFor(schemaType, hint) {
 }
 
 /**
- * A guarded PRAGMA argument: journal modes and similar keywords are a
- * closed word set, never interpolated user text.
+ * A guarded PRAGMA word: a pragma's name and a keyword value (journal
+ * modes and the like) are closed word sets, never interpolated user
+ * text.
  * @param {string} word
  * @returns {string}
  */
@@ -73,20 +77,209 @@ function pragmaWord(word) {
   return String(word);
 }
 
+/**
+ * A guarded PRAGMA value: an integer spelled whole, or a keyword from a
+ * closed set. Anything else is refused here, so no unvalidated value
+ * can reach the statement text.
+ * @param {number | string} value
+ * @returns {string}
+ */
+function pragmaValue(value) {
+  if (typeof value === 'number') {
+    if (!Number.isFinite(value)) throw new TypeError(`not a PRAGMA value: ${value}`);
+    return String(Math.trunc(value));
+  }
+  return pragmaWord(value);
+}
+
+/** SQLite's own per-row identity: the implicit `rowid` of every
+ * table this store creates, which is INSERTION order. */
+const rowIdentity = () => '"rowid"';
+
+/** The R*Tree module and the shape this store gives it: the row id and
+ * the four box edges in (minx, maxx, miny, maxy) order, which is the
+ * (w, e, s, n) a bbox index covers its columns in. The three shadow
+ * tables SQLite creates beside a virtual table are its own storage —
+ * deterministic from the name, and dropped with it. */
+const RTREE = Object.freeze({
+  module: 'rtree',
+  columns: Object.freeze(['id', 'minx', 'maxx', 'miny', 'maxy']),
+});
+
+/**
+ * The schema type a declared column type came from. `INTEGER` carries
+ * both `integer` and `boolean` in this mapping and `ANY` carries no
+ * type at all, so the inverse is partial by construction — which is
+ * what the introspector's loss report exists to say.
+ * @param {string} declaredType
+ * @returns {string | undefined}
+ */
+function schemaTypeOf(declaredType) {
+  switch (String(declaredType).toUpperCase()) {
+    case 'TEXT': return 'string';
+    case 'INTEGER': return 'integer';
+    case 'REAL': return 'number';
+    default: return undefined;
+  }
+}
+
+/**
+ * The member path a generated column's expression reads, recovered
+ * from this dialect's own spelling: `jsonb_extract("doc", '<path>')`
+ * over a path text this same module wrote. Anything else — a hand-made
+ * column, another tool's expression — answers `null`, and the caller
+ * reports the column rather than inventing a path for it.
+ * @param {string} expression
+ * @returns {import('../dialect.js').JsonPathSegment[] | null}
+ */
+function memberPathOf(expression) {
+  const match = /^\s*\(*\s*jsonb_extract\s*\(\s*"[^"]*"\s*,\s*'((?:[^']|'')*)'\s*\)\s*\)*\s*$/
+    .exec(String(expression));
+  if (match === null) return null;
+  return parsePathText(match[1].replace(/''/g, "'"));
+}
+
+/**
+ * `$."a"."b"[0]` back into typed segments — the inverse of
+ * {@link jsonPathText}, and only of that: a path shape this module
+ * cannot have written answers `null`.
+ * @param {string} text
+ * @returns {import('../dialect.js').JsonPathSegment[] | null}
+ */
+function parsePathText(text) {
+  if (!text.startsWith('$')) return null;
+  /** @type {import('../dialect.js').JsonPathSegment[]} */
+  const segments = [];
+  let i = 1;
+  while (i < text.length) {
+    if (text[i] === '.') {
+      if (text[i + 1] !== '"') return null;
+      const end = text.indexOf('"', i + 2);
+      if (end < 0) return null;
+      segments.push({ name: text.slice(i + 2, end) });
+      i = end + 1;
+      continue;
+    }
+    if (text[i] === '[') {
+      const end = text.indexOf(']', i + 1);
+      if (end < 0) return null;
+      const body = text.slice(i + 1, end);
+      // `[#-1]` is how this dialect writes a negative index
+      const index = body.startsWith('#') ? Number(body.slice(1)) : Number(body);
+      if (!Number.isInteger(index)) return null;
+      segments.push({ index });
+      i = end + 1;
+      continue;
+    }
+    return null;
+  }
+  return segments.length === 0 ? null : segments;
+}
+
+/**
+ * The declared index EXPRESSION a generated column computes, out of the
+ * SQL this dialect wrote for it. `byName` maps the engine's function
+ * name back to the model's — here that is the `jaren_x_` registration,
+ * which the store made from the model's own name.
+ * @param {string} expression
+ * @param {Record<string, string>} byName
+ * @returns {any | null}
+ */
+function expressionOf(expression, byName) {
+  return readExpression(expression, {
+    memberOf: (text) => {
+      const segments = memberPathOf(text);
+      return segments === null ? null : { member: pathOf(segments) };
+    },
+    nameOf: (name) => byName[name] ?? null,
+    stringOf: (text) => (/^'(?:[^']|'')*'$/.test(text)
+      ? text.slice(1, -1).replace(/''/g, "'") : null),
+  });
+}
+
+/** A recovered segment list, in the spelling a model's path takes. */
+function pathOf(segments) {
+  let text = '$';
+  for (const segment of segments) {
+    if ('index' in segment) { text += `[${segment.index}]`; continue; }
+    text += /^[A-Za-z_][A-Za-z0-9_]*$/.test(segment.name)
+      ? `.${segment.name}` : `[${JSON.stringify(segment.name)}]`;
+  }
+  return text;
+}
+
+/**
+ * The generated columns of one table, out of the CREATE text SQLite
+ * stored for it — the only place the expression lives here, since no
+ * pragma reports one.
+ * @param {any[]} rows - the `introspect.generated` answer
+ * @returns {{ name: string, expression: string }[]}
+ */
+function readGenerated(rows) {
+  const out = [];
+  for (const row of rows) {
+    const sql = String(row?.sql ?? '');
+    // `"<name>" <type> GENERATED ALWAYS AS (<expression>) VIRTUAL|STORED`
+    const pattern = /"((?:[^"]|"")*)"\s+\w+\s+GENERATED\s+ALWAYS\s+AS\s*\(/gi;
+    let match = pattern.exec(sql);
+    while (match !== null) {
+      // the expression runs to the parenthesis that closes the one the
+      // match ended on, so a nested call inside it is not the end
+      let depth = 1;
+      let i = pattern.lastIndex;
+      while (i < sql.length && depth > 0) {
+        if (sql[i] === "'") {
+          i = sql.indexOf("'", i + 1);
+          if (i < 0) break;
+        }
+        else if (sql[i] === '(') depth += 1;
+        else if (sql[i] === ')') depth -= 1;
+        i += 1;
+      }
+      if (depth !== 0) break;
+      out.push({ name: match[1].replace(/""/g, '"'), expression: sql.slice(pattern.lastIndex, i - 1) });
+      pattern.lastIndex = i;
+      match = pattern.exec(sql);
+    }
+  }
+  return out;
+}
+
 export const sqliteDialect = createDialect({
   name: 'sqlite',
   capabilities: {
     jsonb: true,
     generatedColumns: true,
+    indexableGeneratedColumns: true,
+    // `ANY` is a STRICT table's honest answer for a path the schema
+    // does not type, and it compares with a bound value of any type
+    untypedColumns: true,
     returning: true,
     upsert: true,
     savepoints: true,
+    // a SAVEPOINT outside a transaction starts one, which is why a
+    // top-level transaction here is one checkpoint rather than a block
+    savepointStartsTransaction: true,
+    immediateTransactions: true,
     alterTableFull: false,
+    virtualTables: true,
+    triggers: true,
+    pragmas: true,
+    declaredSqlText: true,
+    // foreign_keys defaults OFF and is set per connection, so this is
+    // the one engine in the suite that must verify it took
+    foreignKeysAlwaysOn: false,
+    // the implicit `rowid` every non-WITHOUT ROWID table carries
+    rowIdentity: true,
     // a GROUP BY / ORDER BY term may name a result alias, so a bucket
     // ladder is written once rather than three times
     groupByAlias: true,
   },
   tableSuffix: ' STRICT',
+  // a generated column over `jsonb_extract` is cheap to recompute and
+  // costs nothing on disk, so it is VIRTUAL; the value is materialised
+  // only in the index over it
+  generatedStorage: 'VIRTUAL',
   // RFC 3339 text → epoch milliseconds, in SQL: the migration planner
   // populates derived instant columns with it (rounded to the ms;
   // finer precision is the write contract's business, §10.3)
@@ -141,6 +334,16 @@ export const sqliteDialect = createDialect({
   jsonAgg: (exprSql) => `json_group_array(${exprSql})`,
   jsonTypeOf: (columnSql, pathText) =>
     `json_type(${columnSql}, ${stringLiteral(pathText)})`,
+  // SQLite keeps the two number types it stores apart, so a JSON number
+  // is one of two names here
+  numericTypeNames: ['integer', 'real'],
+  jsonObject: (pairsSql) => `json_object(${pairsSql})`,
+  // `json()` tags the value with SQLite's JSON subtype, which is what
+  // makes an enclosing `json_object` embed it rather than quote it
+  jsonEmbed: (columnSql) => `json(${columnSql})`,
+  // a dynamically typed engine binds an external as itself and compares
+  // it with whatever the member holds
+  externalEncoding: 'value',
   valueTypeOf: (paramSql) => `typeof(${paramSql})`,
   // the half-open range over the prefix, which an index on the value
   // can seek; `substr(value, 1, n) = p` and `value LIKE 'p%'` both read
@@ -171,19 +374,29 @@ export const sqliteDialect = createDialect({
   groupAggregate: (fn, valueSql) => (valueSql === null
     ? 'COUNT(*)'
     : `${{ sum: 'SUM', avg: 'AVG', min: 'MIN', max: 'MAX' }[fn]}(${valueSql})`),
-  rowIdentity: () => '"rowid"',
+  rowIdentity,
   // membership of the row identity in a bound list — the fetch of a
   // k-nearest plan's candidates. `IN` over the rowid is a primary-key
   // lookup per value; a NULL in the list matches no row, which is what
   // lets a caller pad a batch
   identityIn: (identitySql, paramSqls) => `${identitySql} IN (${paramSqls.join(', ')})`,
-  // the R*Tree module and the shape this store gives it: the row id
-  // and the four box edges in (minx, maxx, miny, maxy) order, which is
-  // the (w, e, s, n) a bbox index covers its columns in. The three
-  // shadow tables SQLite creates beside a virtual table are its own
-  // storage — deterministic from the name, and dropped with it.
-  rtree: { module: 'rtree', columns: ['id', 'minx', 'maxx', 'miny', 'maxy'] },
+  rtree: RTREE,
+  rtreeDdl: rtreeDdl({ quoteIdentifier, rowIdentity, rtree: RTREE }),
+  schemaTypeOf,
+  memberPathOf,
+  expressionOf,
+  readGenerated,
   explainQuery: (sql) => `EXPLAIN QUERY PLAN ${sql}`,
+  // SQLite's plan is PROSE, one `detail` column per row
+  explainLines: (rows) => rows.map((row) => String(row.detail)),
+  // `SCAN <table>` with no index behind it is the full read; a join
+  // statement's narrative names the ALIAS the emitter gave the table,
+  // which is why an alias shape counts as one too
+  isFullScan: (line, tables) =>
+    (/^SCAN t\d+\b/.test(line) || tables.some((table) => line.startsWith(`SCAN ${table}`)))
+    && !line.includes('USING INDEX'),
+  usesIndex: (line, index) =>
+    line.includes(`USING INDEX ${index}`) || line.includes(`USING COVERING INDEX ${index}`),
   excludedRef: (columnSql) => `excluded.${columnSql}`,
   tx: {
     begin: 'BEGIN',
@@ -195,13 +408,25 @@ export const sqliteDialect = createDialect({
     rollbackTo: (n) => `ROLLBACK TO SAVEPOINT ${quoteIdentifier(n)}`,
   },
   pragma: {
-    busyTimeout: (ms) => `PRAGMA busy_timeout = ${Math.trunc(ms)}`,
-    journalMode: (mode) => `PRAGMA journal_mode = ${pragmaWord(mode)}`,
+    // the one configuration spelling: the name comes from the store's
+    // closed pragma table and the value from its validators, and both
+    // are guarded again here
+    set: (name, value) => `PRAGMA ${pragmaWord(name)} = ${pragmaValue(value)}`,
     foreignKeys: (on) => `PRAGMA foreign_keys = ${on ? 'ON' : 'OFF'}`,
     foreignKeyCheck: () => 'PRAGMA foreign_key_check',
+    // the maintenance operations: a checkpoint mode is a closed word,
+    // an integrity-check limit a whole integer
+    walCheckpoint: (mode) => `PRAGMA wal_checkpoint(${pragmaWord(mode)})`,
+    integrityCheck: (limit) => (limit === undefined
+      ? 'PRAGMA integrity_check'
+      : `PRAGMA integrity_check(${pragmaValue(limit)})`),
+    optimize: () => 'PRAGMA optimize',
   },
   introspect: {
     version: () => 'SELECT sqlite_version() AS version',
+    // the read-back of one configuration pragma: `PRAGMA name` answers
+    // one row whose single column carries the value in effect
+    pragma: (name) => `PRAGMA ${pragmaWord(name)}`,
     compileOptions: () =>
       'SELECT compile_options AS name FROM pragma_compile_options',
     tableExists: () =>
@@ -229,6 +454,14 @@ export const sqliteDialect = createDialect({
       'SELECT type, name, sql FROM sqlite_schema '
       + `WHERE tbl_name = ${stringLiteral(table)} AND sql IS NOT NULL `
       + 'ORDER BY type, name',
+    // every table this store might own: the engine's own are excluded
+    // by name, and a VIEW is reported rather than derived
+    tables: () =>
+      "SELECT name, type FROM sqlite_schema WHERE type IN ('table', 'view') "
+      + "AND name NOT LIKE 'sqlite_%' ORDER BY type, name",
+    // the CREATE text is where a generated column's expression lives
+    generated: (table) =>
+      `SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = ${stringLiteral(table)}`,
     // the whole declared schema, for shape-equality comparison after a
     // rebuild: every object that carries SQL text, in a stable order
     schemaDump: () =>