@jarenjs/db 0.49.2 → 0.66.1

package/src/cli.js

@@ -2,39 +2,77 @@
 //#region the jaren-db command
 // Migrations nobody drives by API stay undrifted by nobody: the CLI is
 // what puts `check` in CI and a reviewable migration document in the
-// repository. Five commands (MIGRATION-FORMAT §11): plan, status,
-// apply, check, shape.
+// repository. Six commands (MIGRATION-FORMAT §11): plan, snapshot,
+// status, apply, check, shape. A model or a migration is a JSON file or
+// a MODULE — the model pen's document, the migration pen's builder —
+// loaded twice, because a module that emits a different document on its
+// second load is one whose migration can never match its own history.
 
 import * as fs from 'fs';
 import * as path from 'path';
 import * as readline from 'readline';
 
+import { loadDocument as loadDocumentFile, isDocumentFile } from '@jarenjs/json/node';
+
 import {
-  planModelMigration, migrate, migrationStatus, shapeHash,
+  planModelMigration, migrate, migrationStatus, shapeHash, compareShapeToModel,
   sqliteDialect, normalizeModel, normalizeEntities, explainMapping,
-  planCollection, planEntity, planJoinTable, HISTORY_TABLE,
+  planCollection, planEntity, planJoinTable, HISTORY_TABLE, entityEmitModel,
+  migrateDocuments, streamDocuments, classifyAssertion,
 } from './index.js';
 import { nodeDriver } from './drivers/node.js';
+import {
+  readDocuments, openAtomicTarget, openStreamTarget, openNullTarget,
+  formatOf, DOCUMENT_FORMATS,
+} from './document-files.js';
 
 const USAGE = `jaren-db — model-driven SQLite migrations
 
 Usage:
-  jaren-db plan   --from <model> --to <model> [--store <db>] [--id <name>] [--out <file>]
-  jaren-db status --model <model> --store <db> --baseline <model> [--migrations <dir>]
-  jaren-db apply  --store <db> --baseline <model> --migrations <dir> [--model <m>] [--dry-run] [--yes]
-  jaren-db check  --model <model> --store <db> --baseline <model> [--migrations <dir>]
-  jaren-db shape  --model <model>
-
-plan    Diff two model FILES into a migration document (a database
-        stores shape hashes, not models — the from-model is the
-        previous model file). With --store, first verify the
-        from-model matches the database's recorded shape.
-status  Applied, pending, and drift (a hand-modified database).
-apply   Print every statement, then apply. Destructive steps (drop
-        table/column, rebuild) require --yes or an interactive
-        confirmation naming what is lost. --dry-run only prints.
-check   The CI command: exit 1 on pending migrations or drift.
-shape   Print the physical mapping a model produces.
+  jaren-db plan     --from <model> --to <model> [--store <db>] [--id <name>] [--out <file>]
+  jaren-db plan     --model <model> [--snapshot <file>] [--store <db>] [--id <name>] --out <file>
+  jaren-db snapshot --model <model> [--snapshot <file>] [--types <file>]
+  jaren-db status   --model <model> --store <db> [--migrations <dir>] [--snapshot <file>]
+  jaren-db apply    --store <db> --baseline <model> --migrations <dir> [--model <m>] [--dry-run] [--yes]
+  jaren-db check    --model <model> --store <db> [--migrations <dir>] [--snapshot <file>]
+  jaren-db shape    --model <model>
+  jaren-db documents --migrations <dir> --in <file|-> (--out <file|-> | --in-place --yes | --check)
+                     [--format json|jsonl] [--out-format json|jsonl] [--collection <name>] [--batch-size <n>]
+
+A <model> or a migration is a .json file, or a MODULE (.js, .mjs, .cjs —
+or .ts where Node strips types) whose default export, or its 'model' /
+'migration' export, is the document or a pen builder that emits one. A
+module is loaded twice and refused when its two emissions differ: no
+clock, no env, no randomness. --migrations reads .json files and
+modules, sorted by file name.
+
+plan      Diff two model FILES into a migration document (a database
+          stores shape hashes, not models — the from-model is the
+          previous model file), or diff the committed SNAPSHOT (default
+          model.snapshot.json beside the model) against the model: with
+          --out the migration is written and the snapshot advanced; a
+          model matching its snapshot plans nothing. With --store, first
+          verify the from-model matches the database's recorded shape.
+snapshot  Write the model's snapshot; with --types, emit's TypeScript
+          declaration for it (needs @jarenjs/emit beside @jarenjs/db).
+status    Applied, pending, drift (a hand-modified database) and — with
+          a snapshot — an unplanned model change.
+apply     Print every statement, then apply. Destructive steps (drop
+          table/column, rebuild) require --yes or an interactive
+          confirmation naming what is lost. --dry-run only prints.
+check     The CI command: exit 1 on an unplanned model change, pending
+          migrations or drift.
+shape     Print the physical mapping a model produces.
+documents Run a migration's DOCUMENT steps (jslt, query) over a file of
+          documents instead of a database — a JSON array or JSONL, a
+          path or stdio. A step that needs tables (ddl, sql, rebuild,
+          derive) is refused by name before the first document is read.
+          --out writes a new file, --in-place replaces the input (a
+          sibling temporary is renamed over it only once every document
+          has survived every step; any failure leaves the original byte
+          for byte), and --check transforms and validates everything
+          while writing nothing. Exit: 0 applicable and valid, 1 a
+          migration or source failure, 2 a misuse of this command line.
 `;
 
 function fail(message) {
@@ -42,10 +80,24 @@ function fail(message) {
   process.exit(1);
 }
 
+/**
+ * A command line that cannot be obeyed — a missing flag, two flags that
+ * contradict, a value that names nothing. Distinct from `fail`, because
+ * a caller scripting this command has to tell "you asked for the wrong
+ * thing" (2) from "what you asked for did not hold" (1).
+ */
+function misuse(message) {
+  console.error(`jaren-db: ${message}`);
+  process.exit(2);
+}
+
 function parseArgs(argv) {
   const options = {
     command: argv[2], from: null, to: null, model: null, store: null,
     baseline: null, migrations: null, id: null, out: null,
+    snapshot: null, types: null,
+    in: null, format: null, outFormat: null, collection: null,
+    batchSize: null, inPlace: false, check: false,
     dryRun: false, yes: false, help: false,
   };
   for (let i = 3; i < argv.length; i++) {
@@ -58,15 +110,28 @@ function parseArgs(argv) {
       case '--migrations': options.migrations = argv[++i]; break;
       case '--id': options.id = argv[++i]; break;
       case '--out': options.out = argv[++i]; break;
+      case '--snapshot': options.snapshot = argv[++i]; break;
+      case '--types': options.types = argv[++i]; break;
+      case '--in': options.in = argv[++i]; break;
+      case '--out-format': options.outFormat = argv[++i]; break;
+      case '--format': options.format = argv[++i]; break;
+      case '--collection': options.collection = argv[++i]; break;
+      case '--batch-size': options.batchSize = argv[++i]; break;
+      case '--in-place': options.inPlace = true; break;
+      case '--check': options.check = true; break;
       case '--dry-run': options.dryRun = true; break;
       case '--yes': options.yes = true; break;
       case '--help': case '-h': options.help = true; break;
-      default: fail(`unknown option: ${argv[i]}`);
+      // an unrecognised flag is a misuse of the command line, which is
+      // what `documents` promises exit 2 for; the five older commands
+      // keep the exit 1 their published contract has always used
+      default: (options.command === 'documents' ? misuse : fail)(`unknown option: ${argv[i]}`);
     }
   }
   return options;
 }
 
+/** A committed snapshot: JSON on disk, read as it is. */
 const readJson = (file, what) => {
   try {
     return JSON.parse(fs.readFileSync(file, 'utf8'));
@@ -76,13 +141,42 @@ const readJson = (file, what) => {
   }
 };
 
-/** The migrations directory, sorted — the full ordered chain. */
-const readMigrationsDir = (dir) => {
+/**
+ * A model or migration document: a `.json` file, or a module loaded
+ * TWICE — an emission that changes between loads is not pure (a clock,
+ * the environment, randomness), and a migration that hashes differently
+ * per load can never match its own history. The loader is the suite's
+ * one (`@jarenjs/json/node`, shared with `jaren-contract`); every
+ * refusal it names exits here under this CLI's prefix.
+ */
+async function loadDocument(file, what, exportName) {
+  try {
+    return await loadDocumentFile(file, { what, exportName, impure: 'no clock, no env, no randomness in a model or migration module' });
+  }
+  catch (error) {
+    return fail(error.message);
+  }
+}
+
+/** The migrations directory, sorted by file name — the full ordered chain. */
+const loadMigrationsDir = async (dir) => {
   if (dir === null) return [];
-  return fs.readdirSync(dir)
-    .filter((file) => file.endsWith('.json'))
-    .sort()
-    .map((file) => readJson(path.join(dir, file), 'migration'));
+  const files = fs.readdirSync(dir)
+    .filter((file) => isDocumentFile(file))
+    .sort();
+  const migrations = [];
+  for (const file of files) migrations.push(await loadDocument(path.join(dir, file), 'migration', 'migration'));
+  return migrations;
+};
+
+/** The committed snapshot beside a model, unless one is named. */
+const defaultSnapshotOf = (modelFile) => path.join(path.dirname(modelFile), 'model.snapshot.json');
+
+/** Write a file only when its text changed — two runs on one input change nothing. */
+const writeIfChanged = (file, text) => {
+  const same = fs.existsSync(file) && fs.readFileSync(file, 'utf8') === text;
+  if (!same) fs.writeFileSync(file, text);
+  return same;
 };
 
 /** What a migration will destroy, by note — the confirmation names it. */
@@ -105,17 +199,42 @@ const renderSteps = (migration) => {
 };
 
 async function commandPlan(options) {
-  if (options.from === null || options.to === null)
-    fail('plan needs --from and --to model files');
-  const fromModel = readJson(options.from, 'from-model');
-  const toModel = readJson(options.to, 'to-model');
+  let fromModel;
+  let toModel;
+  let snapshotFile = null;
+  if (options.model !== null) {
+    if (options.from !== null || options.to !== null)
+      fail('plan takes either --from and --to model files, or --model with its --snapshot — not both');
+    snapshotFile = options.snapshot ?? defaultSnapshotOf(options.model);
+    if (!fs.existsSync(snapshotFile)) {
+      fail(`no snapshot at '${snapshotFile}' — write one from the model the store was created with: `
+        + `jaren-db snapshot --model <baseline> --snapshot '${snapshotFile}'`);
+    }
+    fromModel = readJson(snapshotFile, 'snapshot');
+    toModel = await loadDocument(options.model, 'model', 'model');
+    if (shapeHash(fromModel) === shapeHash(toModel)) {
+      console.log(`no change — the model matches its snapshot (${snapshotFile}); nothing to plan`);
+      return;
+    }
+  }
+  else {
+    if (options.from === null || options.to === null)
+      fail('plan needs --from and --to model files, or --model with a committed --snapshot');
+    fromModel = await loadDocument(options.from, 'from-model', 'model');
+    toModel = await loadDocument(options.to, 'to-model', 'model');
+  }
   if (options.store !== null) {
-    const status = await migrationStatus(
-      { driver: nodeDriver(), path: options.store }, [], {
-        baseline: fromModel, model: fromModel,
-      }).catch((error) => fail(error.message));
-    if (status.drift !== null) {
-      fail(`the store does not match the from-model (${status.drift}) — `
+    // the from-model is compared with the database's SHAPE directly:
+    // asking the history with an empty chain refused every database
+    // that had applied a migration (JD0022), which is every database
+    // one plans a second migration for
+    const driver = nodeDriver();
+    const connection = await driver.open(options.store, {}).catch((error) => fail(error.message));
+    const drift = await Promise.resolve(compareShapeToModel(driver, connection, fromModel, undefined))
+      .catch((error) => fail(error.message));
+    await connection.close();
+    if (drift !== null) {
+      fail(`the store does not match the from-model (${drift}) — `
         + 'is this really the previous model?');
     }
   }
@@ -136,18 +255,45 @@ async function commandPlan(options) {
       + 'fill them in before applying');
   }
   if (report.destructive) console.error('NOTE: this migration is DESTRUCTIVE');
+  if (snapshotFile !== null) {
+    if (options.out === null) {
+      console.error(`NOTE: the snapshot was not advanced — plan with --out to write the migration and move ${snapshotFile}`);
+    }
+    else {
+      writeIfChanged(snapshotFile, JSON.stringify(toModel, null, 2) + '\n');
+      console.log(`advanced ${snapshotFile} (shape ${shapeHash(toModel)})`);
+    }
+  }
 }
 
 async function commandStatus(options, { asCheck }) {
-  if (options.store === null || options.baseline === null)
-    fail(`${asCheck ? 'check' : 'status'} needs --store and --baseline`);
-  const baseline = readJson(options.baseline, 'baseline model');
-  const model = options.model !== null ? readJson(options.model, 'model') : undefined;
-  const migrations = readMigrationsDir(options.migrations);
+  // the history and the model are what a status read needs; the
+  // baseline anchors an APPLY (the chain's first shape), not a read —
+  // `--baseline` is still accepted so an existing invocation keeps working
+  if (options.store === null)
+    fail(`${asCheck ? 'check' : 'status'} needs --store`);
+  // `check` without the model verified nothing and printed "in sync"
+  if (asCheck && options.model === null)
+    fail('check needs --model — drift is measured against the model the code carries');
+  const model = options.model !== null ? await loadDocument(options.model, 'model', 'model') : undefined;
+  const migrations = await loadMigrationsDir(options.migrations);
+  // the snapshot discipline, when it is in use: a model that moved
+  // without a plan is named as such, never as the database's drift
+  const snapshotFile = options.snapshot ?? (options.model !== null ? defaultSnapshotOf(options.model) : null);
+  let unplanned = null;
+  const snapshotInUse = snapshotFile !== null && (options.snapshot !== null || fs.existsSync(snapshotFile));
+  if (snapshotInUse) {
+    if (!fs.existsSync(snapshotFile)) fail(`no snapshot at '${snapshotFile}'`);
+    const recorded = shapeHash(readJson(snapshotFile, 'snapshot'));
+    const current = shapeHash(model);
+    if (recorded !== current) {
+      unplanned = `${snapshotFile} records shape ${recorded}, the model is ${current} — run jaren-db plan`;
+    }
+  }
   let status;
   try {
     status = await migrationStatus({ driver: nodeDriver(), path: options.store },
-      migrations, { baseline, model });
+      migrations, { model });
   }
   catch (error) {
     return fail(error.message);
@@ -157,7 +303,12 @@ async function commandStatus(options, { asCheck }) {
   if (model !== undefined && status.pending.length === 0) {
     console.log(`drift:    ${status.drift === null ? 'none — in sync' : status.drift}`);
   }
+  if (snapshotInUse) {
+    console.log(`model:    ${unplanned === null ? 'planned — matches its snapshot' : `UNPLANNED change — ${unplanned}`}`);
+  }
   if (asCheck) {
+    if (unplanned !== null)
+      fail(`unplanned model change: ${unplanned}`);
     if (status.pending.length > 0)
       fail(`${status.pending.length} pending migration(s) — run jaren-db apply`);
     if (status.drift !== null)
@@ -177,12 +328,12 @@ const confirm = (question) => new Promise((resolve) => {
 async function commandApply(options) {
   if (options.store === null || options.baseline === null || options.migrations === null)
     fail('apply needs --store, --baseline and --migrations');
-  const baseline = readJson(options.baseline, 'baseline model');
-  const model = options.model !== null ? readJson(options.model, 'model') : undefined;
-  const migrations = readMigrationsDir(options.migrations);
+  const baseline = await loadDocument(options.baseline, 'baseline model', 'model');
+  const model = options.model !== null ? await loadDocument(options.model, 'model', 'model') : undefined;
+  const migrations = await loadMigrationsDir(options.migrations);
   const target = { driver: nodeDriver(), path: options.store };
 
-  const status = await migrationStatus(target, migrations, { baseline })
+  const status = await migrationStatus(target, migrations)
     .catch((error) => fail(error.message));
   if (status.pending.length === 0) {
     console.log('nothing to apply — up to date');
@@ -206,7 +357,10 @@ async function commandApply(options) {
     const answer = await confirm('Apply anyway? [y/N] ');
     if (!answer) fail('aborted — nothing was applied');
   }
-  else if (!options.yes && process.stdin.isTTY) {
+  else if (!options.yes) {
+    // "default is dry-run + ask" (MIGRATION-FORMAT §11): where nobody
+    // can be asked, the statements above are the dry run and nothing runs
+    if (!process.stdin.isTTY) fail('apply needs --yes (no interactive terminal to ask) — nothing was applied');
     const answer = await confirm('Apply? [y/N] ');
     if (!answer) fail('aborted — nothing was applied');
   }
@@ -220,9 +374,37 @@ async function commandApply(options) {
   }
 }
 
-function commandShape(options) {
+async function commandSnapshot(options) {
+  if (options.model === null) fail('snapshot needs --model');
+  const model = await loadDocument(options.model, 'model', 'model');
+  const snapshotFile = options.snapshot ?? defaultSnapshotOf(options.model);
+  const same = writeIfChanged(snapshotFile, JSON.stringify(model, null, 2) + '\n');
+  console.log(`${same ? 'unchanged' : 'wrote'} ${snapshotFile} (shape ${shapeHash(model)})`);
+  if (options.types === null) return;
+  // emit is loaded lazily, and only here: db does not depend on it, so a
+  // host without it is told exactly what --types needs
+  let emit;
+  let typescript;
+  try {
+    emit = await import('@jarenjs/emit');
+    typescript = await import('@jarenjs/emit/typescript');
+  }
+  catch (error) {
+    if (error.code === 'ERR_MODULE_NOT_FOUND' || error.code === 'ERR_PACKAGE_PATH_NOT_EXPORTED') {
+      return fail(`--types needs @jarenjs/emit beside @jarenjs/db, and it does not resolve `
+        + `(${error.message}) — install it: npm install @jarenjs/emit`);
+    }
+    throw error;
+  }
+  const declaration = typescript.renderTypeScript(
+    entityEmitModel(model, { compile: emit.compileEmitModel, source: options.model }), {});
+  const sameTypes = writeIfChanged(options.types, declaration);
+  console.log(`${sameTypes ? 'unchanged' : 'wrote'} ${options.types}`);
+}
+
+async function commandShape(options) {
   if (options.model === null) fail('shape needs --model');
-  const model = readJson(options.model, 'model');
+  const model = await loadDocument(options.model, 'model', 'model');
   console.log(`shape hash: ${shapeHash(model)}`);
   for (const collection of normalizeModel(model).values()) {
     for (const sql of planCollection(collection.name, collection, sqliteDialect).createSql)
@@ -242,6 +424,109 @@ function commandShape(options) {
   console.log(`-- history rides in '${HISTORY_TABLE}'`);
 }
 
+/**
+ * Run a migration's document steps over a file of documents.
+ *
+ * The runner is chosen by what the migration asks for, and named in the
+ * report: a cross-document assertion needs every document at once, so
+ * its collection is read into memory; anything else streams, holding one
+ * batch. Reading a rewindable FILE for the first is not a compromise —
+ * it is the same source, read twice.
+ */
+async function commandDocuments(options) {
+  if (options.migrations === null) misuse('documents needs --migrations <dir>');
+  if (options.in === null) misuse('documents needs --in <file> (or - for standard input)');
+  const sinks = [options.out !== null, options.inPlace, options.check].filter(Boolean).length;
+  if (sinks === 0)
+    misuse('documents needs one of --out <file>, --in-place or --check');
+  if (sinks > 1)
+    misuse('documents takes exactly one of --out, --in-place and --check');
+  if (options.inPlace && options.in === '-')
+    misuse('--in-place needs a file to replace, not standard input');
+  if (options.inPlace && !options.yes)
+    misuse('--in-place rewrites the input file — pass --yes to confirm, or --out to write elsewhere');
+  const batchSize = options.batchSize === null ? 500 : Number(options.batchSize);
+  if (!Number.isInteger(batchSize) || batchSize < 1)
+    misuse(`--batch-size must be a positive integer, not '${options.batchSize}'`);
+
+  const fromStdin = options.in === '-';
+  const inFormat = options.format ?? (fromStdin ? 'jsonl' : formatOf(options.in));
+  if (!DOCUMENT_FORMATS.includes(inFormat))
+    misuse(`--format must be one of ${DOCUMENT_FORMATS.join(', ')}, not '${inFormat}'`);
+  const target = options.inPlace ? options.in : options.out;
+  const toStdout = target === '-';
+  const outFormat = options.outFormat
+    ?? (options.check || toStdout ? inFormat : formatOf(/** @type {string} */ (target)));
+  if (!DOCUMENT_FORMATS.includes(outFormat))
+    misuse(`--out-format must be one of ${DOCUMENT_FORMATS.join(', ')}, not '${outFormat}'`);
+  if (!fromStdin && !fs.existsSync(options.in)) fail(`no such file: '${options.in}'`);
+
+  const migrations = await loadMigrationsDir(options.migrations);
+  if (migrations.length === 0) fail(`no migration documents in '${options.migrations}'`);
+
+  // A document file holds ONE collection. The migrations say which:
+  // every document step must name it, or this chain cannot be applied to
+  // a file at all — running only the steps that match would leave the
+  // rest silently unapplied, which is the one outcome a migration runner
+  // may never produce.
+  const documentSteps = migrations.flatMap((migration) => migration.steps)
+    .filter((step) => step.kind === 'jslt' || step.kind === 'query');
+  const named = [...new Set(documentSteps.map((step) => step.collection))];
+  if (named.length > 1) {
+    fail(`a document file holds one collection, and these migrations touch ${named.length} `
+      + `(${named.join(', ')}) — run them against a store, or split the chain so each `
+      + 'migration touches the collection its file holds');
+  }
+  if (options.collection !== null && named.length === 1 && options.collection !== named[0]) {
+    misuse(`--collection names '${options.collection}', but these migrations touch `
+      + `'${named[0]}' — is this the right file for them?`);
+  }
+  // with no document step at all there is no collection to infer; the
+  // run still proceeds, because a physical step must be REFUSED by name
+  // rather than reported as a missing collection
+  const collection = named[0] ?? options.collection ?? 'documents';
+
+  // only a MATERIALIZING assertion needs the collection at once; a
+  // per-document predicate and an associative aggregate are both
+  // answered one batch at a time, so they stream
+  const materializes = documentSteps.some((step) => step.collection === collection
+    && step.kind === 'query' && classifyAssertion(step.assert).strategy === 'materialize');
+
+  const source = () => (fromStdin ? process.stdin : options.in);
+  let sink;
+  if (options.check) sink = openNullTarget();
+  else if (toStdout) sink = openStreamTarget(process.stdout, outFormat);
+  else sink = await openAtomicTarget(/** @type {string} */ (target), outFormat);
+
+  try {
+    let report;
+    if (materializes) {
+      const documents = [];
+      for await (const document of readDocuments(source(), inFormat)) documents.push(document);
+      const out = await migrateDocuments({ [collection]: documents }, migrations, { batchSize });
+      report = out.report;
+      for (const document of out.documents[collection]) await sink.write(document);
+    }
+    else {
+      report = await streamDocuments({ [collection]: readDocuments(source(), inFormat) },
+        migrations, { batchSize, write: (name, document) => sink.write(document) });
+    }
+    const written = await sink.commit();
+    const counts = report.counts[collection] ?? { read: 0, transformed: 0, asserted: 0 };
+    console.log(`${options.check ? 'checked' : 'migrated'} '${collection}': `
+      + `${counts.read} read, ${counts.transformed} transformed, ${counts.asserted} asserted `
+      + `(${report.strategy[collection]})`);
+    console.log(`applied: ${report.applied.join(', ')}`);
+    if (options.check) console.log('checked only — nothing was written');
+    else if (toStdout) console.log(`wrote ${written.documents} document(s) to standard output`);
+    else console.log(`wrote ${written.documents} document(s) to ${target} (${written.bytes} bytes)`);
+  }
+  catch (error) {
+    await sink.abort();
+    return fail(error.message);
+  }
+}
+
 async function main() {
   const options = parseArgs(process.argv);
   if (options.help || options.command === '--help' || options.command === '-h'
@@ -251,10 +536,12 @@ async function main() {
   }
   switch (options.command) {
     case 'plan': return commandPlan(options);
+    case 'snapshot': return commandSnapshot(options);
     case 'status': return commandStatus(options, { asCheck: false });
     case 'check': return commandStatus(options, { asCheck: true });
     case 'apply': return commandApply(options);
     case 'shape': return commandShape(options);
+    case 'documents': return commandDocuments(options);
     default: return fail(`unknown command '${options.command}' — try --help`);
   }
 }