@jarenjs/db 0.87.0 → 0.89.0

package/schemas/jaren-migration.draft-07.schema.json

@@ -29,7 +29,7 @@
       "type": "string"
     },
     "steps": {
-      "description": "Ordered steps; the order is the contract. A pure widening may carry NO steps \u2014 the migration then only moves the recorded shape.",
+      "description": "Ordered steps; the order is the contract. A pure widening may carry NO steps — the migration then only moves the recorded shape.",
       "type": "array",
       "items": {
         "$ref": "#/definitions/step"
@@ -80,6 +80,53 @@
             },
             "additionalProperties": false
           }
+        },
+        "target": {
+          "type": "object",
+          "properties": {
+            "objects": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "required": [
+                  "type",
+                  "name",
+                  "owner",
+                  "sql"
+                ],
+                "properties": {
+                  "type": {
+                    "enum": [
+                      "table",
+                      "view",
+                      "index",
+                      "trigger"
+                    ]
+                  },
+                  "name": {
+                    "type": "string"
+                  },
+                  "owner": {
+                    "type": "string"
+                  },
+                  "sql": {
+                    "type": "string"
+                  }
+                }
+              }
+            },
+            "tables": {
+              "type": "array",
+              "uniqueItems": true,
+              "items": {
+                "type": "string"
+              }
+            }
+          },
+          "required": [
+            "objects"
+          ],
+          "additionalProperties": false
         }
       },
       "additionalProperties": false
@@ -113,6 +160,9 @@
         },
         {
           "$ref": "#/definitions/rebuildStep"
+        },
+        {
+          "$ref": "#/definitions/tableStep"
         }
       ]
     },
@@ -158,6 +208,18 @@
         },
         "note": {
           "type": "string"
+        },
+        "model": {
+          "description": "The immutable model describing this step’s current physical layout.",
+          "type": "object",
+          "properties": {
+            "$model": {
+              "const": "0.1"
+            }
+          },
+          "required": [
+            "$model"
+          ]
         }
       },
       "required": [
@@ -189,6 +251,18 @@
         },
         "note": {
           "type": "string"
+        },
+        "model": {
+          "description": "The immutable model describing this step’s current physical layout.",
+          "type": "object",
+          "properties": {
+            "$model": {
+              "const": "0.1"
+            }
+          },
+          "required": [
+            "$model"
+          ]
         }
       },
       "required": [
@@ -199,7 +273,7 @@
       "additionalProperties": false
     },
     "deriveStep": {
-      "description": "Recompute named STORED derived index columns from the documents already in a collection. A spatial column is stored only under the physical mapping that cannot index a registered deterministic function (the generated mapping populates itself); a vector column is stored under both. The step is idempotent \u2014 a derived value is a pure function of the document \u2014 so a replay writes what the first run wrote.",
+      "description": "Recompute named STORED derived index columns from the documents already in a collection. A spatial column is stored only under the physical mapping that cannot index a registered deterministic function (the generated mapping populates itself); a vector column is stored under both. The step is idempotent — a derived value is a pure function of the document — so a replay writes what the first run wrote.",
       "type": "object",
       "properties": {
         "kind": {
@@ -273,7 +347,7 @@
       "additionalProperties": false
     },
     "sqlStep": {
-      "description": "A DATA step spelled directly as one SQL statement (MIGRATION-FORMAT \u00a79.4): the planner's fold of a dropped column into the document, the move of a document member into its new column, an R*Tree backfill. Executed exactly as a ddl step; distinct in kind so a reviewer reads intent and a dry run shows it as data.",
+      "description": "A DATA step spelled directly as one SQL statement (MIGRATION-FORMAT §9.4): the planner's fold of a dropped column into the document, the move of a document member into its new column, an R*Tree backfill. Executed exactly as a ddl step; distinct in kind so a reviewer reads intent and a dry run shows it as data.",
       "type": "object",
       "properties": {
         "kind": {
@@ -294,7 +368,7 @@
       "additionalProperties": false
     },
     "rebuildStep": {
-      "description": "The ALTER TABLE procedure SQLite lacks (MIGRATION-FORMAT \u00a710), self-contained: create the target shape under `<table>__rebuild`, copy the rows through the rendered column mapping, drop the old table, rename, recreate the indexes, then `PRAGMA foreign_key_check` inside the transaction.",
+      "description": "The ALTER TABLE procedure SQLite lacks (MIGRATION-FORMAT §10), self-contained: create the target shape under `<table>__rebuild`, copy the rows through the rendered column mapping, drop the old table, rename, recreate the indexes, then `PRAGMA foreign_key_check` inside the transaction.",
       "type": "object",
       "properties": {
         "kind": {
@@ -314,7 +388,7 @@
           }
         },
         "copy": {
-          "description": "The INSERT \u2026 SELECT that carries every row across the column mapping.",
+          "description": "The INSERT … SELECT that carries every row across the column mapping.",
           "type": "string",
           "minLength": 1
         },
@@ -338,6 +412,96 @@
         "indexes"
       ],
       "additionalProperties": false
+    },
+    "tableStep": {
+      "description": "A complete reviewed planTableMigration artifact, executed with its source, row, storage and identity guards.",
+      "type": "object",
+      "properties": {
+        "kind": {
+          "const": "table"
+        },
+        "plan": {
+          "type": "object",
+          "required": [
+            "version",
+            "id",
+            "table",
+            "checksum",
+            "source",
+            "after",
+            "rebuild",
+            "temporary",
+            "unchanged",
+            "statements",
+            "finish"
+          ],
+          "properties": {
+            "version": {
+              "const": 1
+            },
+            "id": {
+              "type": "string",
+              "minLength": 1
+            },
+            "table": {
+              "type": "string",
+              "minLength": 1
+            },
+            "checksum": {
+              "type": "string",
+              "minLength": 1
+            },
+            "source": {
+              "type": "array",
+              "items": {
+                "type": "object"
+              }
+            },
+            "after": {
+              "type": "array",
+              "items": {
+                "type": "object"
+              }
+            },
+            "rebuild": {
+              "type": "boolean"
+            },
+            "temporary": {
+              "type": "string",
+              "minLength": 1
+            },
+            "unchanged": {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "minLength": 1
+              }
+            },
+            "statements": {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "minLength": 1
+              }
+            },
+            "finish": {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "minLength": 1
+              }
+            }
+          }
+        },
+        "note": {
+          "type": "string"
+        }
+      },
+      "required": [
+        "kind",
+        "plan"
+      ],
+      "additionalProperties": false
     }
   }
 }

package/schemas/jaren-migration.schema.json

@@ -80,6 +80,53 @@
       },
       "additionalProperties": false
      }
+    },
+    "target": {
+     "type": "object",
+     "properties": {
+      "objects": {
+       "type": "array",
+       "items": {
+        "type": "object",
+        "required": [
+         "type",
+         "name",
+         "owner",
+         "sql"
+        ],
+        "properties": {
+         "type": {
+          "enum": [
+           "table",
+           "view",
+           "index",
+           "trigger"
+          ]
+         },
+         "name": {
+          "type": "string"
+         },
+         "owner": {
+          "type": "string"
+         },
+         "sql": {
+          "type": "string"
+         }
+        }
+       }
+      },
+      "tables": {
+       "type": "array",
+       "uniqueItems": true,
+       "items": {
+        "type": "string"
+       }
+      }
+     },
+     "required": [
+      "objects"
+     ],
+     "additionalProperties": false
     }
    },
    "additionalProperties": false
@@ -113,6 +160,9 @@
     },
     {
      "$ref": "#/$defs/rebuildStep"
+    },
+    {
+     "$ref": "#/$defs/tableStep"
     }
    ]
   },
@@ -158,6 +208,18 @@
     },
     "note": {
      "type": "string"
+    },
+    "model": {
+     "description": "The immutable model describing this step’s current physical layout.",
+     "type": "object",
+     "properties": {
+      "$model": {
+       "const": "0.1"
+      }
+     },
+     "required": [
+      "$model"
+     ]
     }
    },
    "required": [
@@ -189,6 +251,18 @@
     },
     "note": {
      "type": "string"
+    },
+    "model": {
+     "description": "The immutable model describing this step’s current physical layout.",
+     "type": "object",
+     "properties": {
+      "$model": {
+       "const": "0.1"
+      }
+     },
+     "required": [
+      "$model"
+     ]
     }
    },
    "required": [
@@ -338,6 +412,96 @@
     "indexes"
    ],
    "additionalProperties": false
+  },
+  "tableStep": {
+   "description": "A complete reviewed planTableMigration artifact, executed with its source, row, storage and identity guards.",
+   "type": "object",
+   "properties": {
+    "kind": {
+     "const": "table"
+    },
+    "plan": {
+     "type": "object",
+     "required": [
+      "version",
+      "id",
+      "table",
+      "checksum",
+      "source",
+      "after",
+      "rebuild",
+      "temporary",
+      "unchanged",
+      "statements",
+      "finish"
+     ],
+     "properties": {
+      "version": {
+       "const": 1
+      },
+      "id": {
+       "type": "string",
+       "minLength": 1
+      },
+      "table": {
+       "type": "string",
+       "minLength": 1
+      },
+      "checksum": {
+       "type": "string",
+       "minLength": 1
+      },
+      "source": {
+       "type": "array",
+       "items": {
+        "type": "object"
+       }
+      },
+      "after": {
+       "type": "array",
+       "items": {
+        "type": "object"
+       }
+      },
+      "rebuild": {
+       "type": "boolean"
+      },
+      "temporary": {
+       "type": "string",
+       "minLength": 1
+      },
+      "unchanged": {
+       "type": "array",
+       "items": {
+        "type": "string",
+        "minLength": 1
+       }
+      },
+      "statements": {
+       "type": "array",
+       "items": {
+        "type": "string",
+        "minLength": 1
+       }
+      },
+      "finish": {
+       "type": "array",
+       "items": {
+        "type": "string",
+        "minLength": 1
+       }
+      }
+     }
+    },
+    "note": {
+     "type": "string"
+    }
+   },
+   "required": [
+    "kind",
+    "plan"
+   ],
+   "additionalProperties": false
   }
  }
 }

package/src/ddl.js

@@ -21,6 +21,10 @@ import { chain } from './driver.js';
 import { BBOX_COMPONENTS, BBOX_INDEX_ORDER, derivedMappingFor } from './derive.js';
 import { expressionSql, expressionStem, expressionFunctions } from './expression.js';
 import { planTable } from './dialects/sqlite-schema.js';
+import { comparableDeclaredSql } from './schema-sql.js';
+export { normalizeDeclaredSql, comparableDeclaredSql } from './schema-sql.js';
+
+const MANAGED_SQL_OPTIONS = Object.freeze({ columnOrder: /** @type {const} */ ('ignore') });
 
 /** The fixed physical column names of the 0.1 mapping. */
 export const KEY_COLUMN = 'key';
@@ -620,88 +624,6 @@ export function planCollection(name, collection, dialect, options = undefined) {
   };
 }
 
-/**
- * Normalize a stored `CREATE` statement for comparison: collapse runs of
- * whitespace, drop whitespace around punctuation, and strip the
- * `IF NOT EXISTS` SQLite does not keep. What survives is every token that
- * carries meaning, so two statements compare equal exactly when they
- * declare the same physical object.
- * @param {string} sql
- * @returns {string}
- */
-export function normalizeDeclaredSql(sql) {
-  return String(sql)
-    .replace(/\s+/g, ' ')
-    .replace(/\s*([(),])\s*/g, '$1')
-    .replace(/\bIF NOT EXISTS\s+/i, '')
-    .trim();
-}
-
-/**
- * Split a comma-separated list at TOP-LEVEL commas only, so a
- * `CHECK(x IN (1,2))` or a multi-column constraint stays one item.
- * @param {string} body
- * @returns {string[]}
- */
-function splitTopLevel(body) {
-  /** @type {string[]} */
-  const parts = [];
-  let depth = 0;
-  let quote = '';
-  let start = 0;
-  for (let i = 0; i < body.length; i++) {
-    const c = body[i];
-    if (quote !== '') {
-      if (c === quote) quote = '';
-      continue;
-    }
-    if (c === '"' || c === "'") quote = c;
-    else if (c === '(') depth++;
-    else if (c === ')') depth--;
-    else if (c === ',' && depth === 0) {
-      parts.push(body.slice(start, i));
-      start = i + 1;
-    }
-  }
-  parts.push(body.slice(start));
-  return parts.map((part) => part.trim()).filter((part) => part !== '');
-}
-
-/**
- * A comparable form of one `CREATE` statement.
- *
- * For a TABLE the column definitions compare as a SET, because
- * `ALTER TABLE … ADD COLUMN` can only append — so a migrated table and a
- * freshly built one legitimately differ in column order, and this store
- * never reads a column positionally. Everything else is exact: each
- * column's full definition (type, `PRIMARY KEY`, `NOT NULL`, `DEFAULT`,
- * `CHECK`, `GENERATED … AS`, `REFERENCES … ON DELETE …`), the table
- * constraints, and the trailing table options (`STRICT`,
- * `WITHOUT ROWID`).
- *
- * For an INDEX the text compares whole, because an index IS its order —
- * `(a,b)` and `(b,a)` serve different lookups — as are its partial
- * predicate and each term's collation and direction.
- * @param {string} sql
- * @returns {string}
- */
-export function comparableDeclaredSql(sql) {
-  const normalized = normalizeDeclaredSql(sql);
-  const open = normalized.indexOf('(');
-  const close = normalized.lastIndexOf(')');
-  if (!/^CREATE\s+TABLE\b/i.test(normalized) || open < 0 || close < open)
-    return normalized;
-  const head = normalized.slice(0, open);
-  const options = normalized.slice(close + 1).trim();
-  const items = splitTopLevel(normalized.slice(open + 1, close));
-  // a column definition opens with the quoted column name; anything else
-  // (PRIMARY KEY(...), UNIQUE(...), CHECK(...), FOREIGN KEY(...)) is a
-  // table constraint, and those are unordered too
-  const columns = items.filter((item) => item.startsWith('"')).sort();
-  const constraints = items.filter((item) => !item.startsWith('"')).sort();
-  return `${head}(${[...columns, ...constraints].join(',')})${options}`;
-}
-
 /**
  * The declared-SQL half of verification: compare every schema object the
  * table owns against the statements the plan would have created.
@@ -738,7 +660,7 @@ function verifyDeclaredSql(connection, plan, disagree) {
   // comparison for free.
   const owned = new Set(plan.virtualTables?.map((virtual) => virtual.name) ?? []);
   for (const sql of plan.createSql) {
-    const comparable = comparableDeclaredSql(sql);
+    const comparable = comparableDeclaredSql(sql, MANAGED_SQL_OPTIONS);
     // the object's name is the first quoted identifier in the statement
     const name = /"((?:[^"]|"")*)"/.exec(comparable)?.[1]?.replace(/""/g, '"');
     if (name === undefined || owned.has(name)) continue;
@@ -756,8 +678,8 @@ function verifyDeclaredSql(connection, plan, disagree) {
           disagree(`the model declares the virtual table '${virtual.name}', `
             + 'which the database does not have');
         }
-        const have = comparableDeclaredSql(row.sql);
-        const wanted = comparableDeclaredSql(virtual.createSql);
+        const have = comparableDeclaredSql(row.sql, MANAGED_SQL_OPTIONS);
+        const wanted = comparableDeclaredSql(virtual.createSql, MANAGED_SQL_OPTIONS);
         if (have !== wanted) {
           disagree(`'${virtual.name}' is declared as\n  ${have}\nand the model declares\n  ${wanted}`);
         }
@@ -768,7 +690,7 @@ function verifyDeclaredSql(connection, plan, disagree) {
     (statement) => chain(statement.all([]), (rows) => {
       /** @type {Map<string, string>} */
       const actual = new Map();
-      for (const row of rows) actual.set(String(row.name), comparableDeclaredSql(row.sql));
+      for (const row of rows) actual.set(String(row.name), comparableDeclaredSql(row.sql, MANAGED_SQL_OPTIONS));
       for (const [name, wanted] of planned) {
         const have = actual.get(name);
         if (have === undefined)

package/src/dialects/sqlite.js

@@ -497,6 +497,7 @@ export const sqliteDialect = createDialect({
 
 /** SQLite schema inspection and identity preservation statements. */
 export const sqliteTableMigration = Object.freeze({
+    binaryCast: (sql) => `CAST(${sql} AS BLOB)`,
     schema: () => "SELECT type,name,tbl_name,sql FROM sqlite_schema WHERE sql IS NOT NULL AND substr(name,1,7) <> 'sqlite_' ORDER BY type,name",
     tableList: () => 'PRAGMA table_list',
     sequenceExists: () => "SELECT 1 AS present FROM sqlite_schema WHERE name='sqlite_sequence'",

package/src/document-steps.js

@@ -17,6 +17,7 @@
  */
 
 import { analyzeQuery, createQueryAccumulator } from '@jarenjs/json/query';
+import { setObjectMember } from '@jarenjs/core/object';
 import { DbCompileError, DbRuntimeError } from './errors.js';
 import { utf8Length } from './cursor.js';
 
@@ -31,13 +32,13 @@ export const DOCUMENT_STEP_KINDS = new Set(['jslt', 'query']);
 
 /**
  * The step kinds that act on a PHYSICAL database — rendered DDL, a
- * data statement spelled as SQL, the table-rebuild procedure, and the
+ * data statement spelled as SQL, guarded table plans, the table-rebuild procedure, and the
  * backfill that recomputes stored derived COLUMNS. A host without
  * tables cannot honour any of them, and silently skipping one would
  * leave a migration half-applied, so it refuses instead.
  * @type {ReadonlySet<string>}
  */
-export const PHYSICAL_STEP_KINDS = new Set(['ddl', 'sql', 'rebuild', 'derive']);
+export const PHYSICAL_STEP_KINDS = new Set(['ddl', 'sql', 'rebuild', 'derive', 'table']);
 
 /**
  * The refusal a failing step raises, spelled the one way — a reader
@@ -185,8 +186,12 @@ export function compileDocumentStep(step, index, context) {
         // a body that leaves it out keeps it, a body that rewrites it
         // is refused, as a collection's key is
         for (const key of keys) {
-          if (next[key] === undefined) next[key] = document[key];
-          else if (next[key] !== document[key]) {
+          const present = Object.hasOwn(document, key);
+          const previous = present ? document[key] : undefined;
+          if (!Object.hasOwn(next, key) || next[key] === undefined) {
+            if (present) setObjectMember(next, key, previous);
+          }
+          else if (next[key] !== previous) {
             fail(`the transform changed the key member '${key}' of row ${identity} — `
               + `key changes are not supported in ${MIGRATION_VERSION}`);
           }
@@ -271,7 +276,7 @@ export function compileDocumentStep(step, index, context) {
   };
 }
 
-const STEP_KINDS = new Set(['ddl', 'jslt', 'query', 'sql', 'rebuild', 'derive']);
+const STEP_KINDS = new Set([...DOCUMENT_STEP_KINDS, ...PHYSICAL_STEP_KINDS]);
 
 /**
  * Structural validation of one migration document, including the
@@ -304,6 +309,19 @@ export function checkMigrationDocument(migration) {
       throw new DbCompileError('JD0023',
         `migration '${migration.id}' step ${i} is a sql step without sql text`);
     }
+    if (step.kind === 'table' && (!step.plan || typeof step.plan !== 'object' || Array.isArray(step.plan) || step.plan.version !== 1
+      || ['id', 'table', 'checksum', 'temporary'].some((key) => typeof step.plan[key] !== 'string' || step.plan[key].length === 0)
+      || typeof step.plan.rebuild !== 'boolean'
+      || ['source', 'after'].some((key) => !Array.isArray(step.plan[key])
+        || step.plan[key].some((value) => !value || typeof value !== 'object' || Array.isArray(value)))
+      || ['unchanged', 'statements', 'finish'].some((key) => !Array.isArray(step.plan[key])
+        || step.plan[key].some((value) => typeof value !== 'string' || value.length === 0)))) {
+      throw new DbCompileError('JD0023', `migration '${migration.id}' step ${i} is a table step without its reviewed plan`);
+    }
+    if (['jslt', 'query'].includes(step.kind) && step.model !== undefined
+      && (!step.model || typeof step.model !== 'object' || step.model.$model !== '0.1')) {
+      throw new DbCompileError('JD0023', `migration '${migration.id}' step ${i} has an invalid current model`);
+    }
     if (step.kind === 'derive'
       && (typeof step.collection !== 'string' || !Array.isArray(step.columns)
         || step.columns.length === 0)) {

package/src/drivers/bun.js

@@ -20,8 +20,9 @@
 
 import { lazyOpen, openConnection } from '../driver.js';
 import { sqliteDialect } from '../dialects/sqlite.js';
-import { PRAGMA_NAMES } from '../pragmas.js';
+import { PRAGMAS, PRAGMA_NAMES } from '../pragmas.js';
 import { writeSqliteSnapshot } from './snapshot.js';
+import { nativeDatabaseIdentity } from './file-identity.js';
 export { snapshotDatabase } from './snapshot.js';
 
 /**
@@ -128,11 +129,19 @@ export function adaptBunDatabase(db, options) {
  * substitute module.
  * @param {any} mod - The `bun:sqlite` module (or a substitute)
  * @param {string} path
- * @param {{ readOnly?: boolean, queueTimeout?: number }} [options]
+ * @param {{ timeout?: number, readOnly?: boolean, queueTimeout?: number }} [options]
  * @returns {any}
  */
 export function fromBunModule(mod, path, options) {
+  const timeout = options?.timeout === undefined ? undefined
+    : PRAGMAS.busyTimeout.normalize(options.timeout, 'timeout');
   const db = options?.readOnly === true ? new mod.Database(path, { readonly: true }) : new mod.Database(path);
+  // Bun has no constructor timeout option. Configure newly owned handles
+  // before probing; adapting a caller-owned handle preserves its settings.
+  if (timeout !== undefined) {
+    try { db.run(sqliteDialect.pragma.set('busy_timeout', timeout)); }
+    catch (error) { db.close(); throw error; }
+  }
   const backup = {
     snapshot: true,
     copy: (target, copyOptions) => writeSqliteSnapshot(target, () => {
@@ -154,9 +163,10 @@ export function bunDriver() {
   return Object.freeze({
     name: 'bun-sqlite',
     dialect: sqliteDialect,
+    databaseIdentity: nativeDatabaseIdentity,
     /**
      * @param {string} path
-     * @param {{ readOnly?: boolean, queueTimeout?: number }} [options]
+     * @param {{ timeout?: number, readOnly?: boolean, queueTimeout?: number }} [options]
      * @returns {Promise<any>}
      */
     open: (path, options) => lazyOpen('bun:sqlite',