mikser-io 9.46.0 → 9.47.0

package/docs/api-reference.md

@@ -646,6 +646,12 @@ happened, and losing them would be worse than not attributing them.
 rest into unattributed ones, which is what lets it offer undo for the first and
 not the second.
 
+The engine records the grouping and takes no position on what is done with it —
+it knows nothing about commits, branches or reverts. Versioning the paths
+together is one use; a snapshot, an audit trail, a draft-then-publish gate or a
+filesystem-level rollback all want the same fact. That is why change sets live
+in core and git does not.
+
 ## Search
 
 `queryEntities` sifts **meta**. `searchEntities` answers the other question —

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mikser-io",
-  "version": "9.46.0",
+  "version": "9.47.0",
   "description": "<p align=\"center\">   <img src=\"mikser-lockup-stacked.svg\" alt=\"mikser\" width=\"198\" /> </p>",
   "main": "index.js",
   "exports": {

package/src/changeset.js

@@ -8,8 +8,12 @@
 // second are indistinguishable, so removing one removes the other.
 //
 // A change set is the missing grain. The caller names it, the writes
-// accumulate under it, and a consumer — mikser-io-git today — can commit
-// exactly those paths and later remove exactly that contribution.
+// accumulate under it, and a consumer can act on exactly those paths.
+//
+// The engine records the grouping and takes no position on what is done with
+// it. Versioning the paths together is one use — mikser-io-git's — but a
+// snapshot, an audit trail, a draft-then-publish gate or a filesystem-level
+// rollback all want the same fact, and none of them is a commit.
 //
 // Deliberately NOT a transaction. Nothing is held back, nothing rolls back on
 // failure, and a half-finished set is a real set containing what actually
@@ -55,8 +59,11 @@ function store() {
     return runtime.changeSets
 }
 
-// Repo-relative, POSIX-separated: these end up in a git pathspec, and a
-// consumer should not have to redo that conversion or guess the root.
+// Relative to the working folder, POSIX-separated.
+//
+// The working folder is the root every consumer already reasons in, and
+// forward slashes are the separator entity ids use — so a path here matches
+// the vocabulary of the rest of the engine rather than the host's.
 function relativeToWorkingFolder(uri) {
     const root = runtime.options?.workingFolder
     if (!root || !uri) return null
@@ -82,9 +89,9 @@ export function recordChangeSetWrite({ changeSet, summary, principal, uri, opera
     undoOf ??= ambient?.undoOf
     if (!changeSet || !uri) return null
     const rel = relativeToWorkingFolder(uri)
-    // Outside the working folder there is nothing a repo-scoped consumer can
-    // do with the path, and silently keeping an absolute one would produce a
-    // pathspec that matches nothing.
+    // Outside the working folder there is nothing a consumer scoped to the
+    // project can do with the path, and silently keeping an absolute one would
+    // produce a selector that quietly matches nothing.
     if (!rel) return null
 
     const sets = store()
@@ -94,9 +101,9 @@ export function recordChangeSetWrite({ changeSet, summary, principal, uri, opera
             id: changeSet,
             summary: summary ?? null,
             principal: principal ?? null,
-            // Set when this change set exists to take another one back, so
-            // the undo is itself an ordinary, undoable change rather than a
-            // special history-rewriting operation.
+            // Set when this change set exists to take another one back, so an
+            // undo is itself an ordinary, undoable change rather than a
+            // privileged operation that rewrites the record.
             undoOf: undoOf ?? null,
             startedAt: Date.now(),
             paths: new Map(),
@@ -110,8 +117,8 @@ export function recordChangeSetWrite({ changeSet, summary, principal, uri, opera
     return set.id
 }
 
-// Every set with writes not yet consumed, oldest first — the order a consumer
-// should commit them in, so history reads the way the work happened.
+// Every set with writes not yet consumed, oldest first — the order the work
+// actually happened in, which is the order a consumer should record it in.
 export function pendingChangeSets() {
     return [...store().values()]
         .filter(set => set.paths.size)
@@ -129,10 +136,10 @@ export function pendingChangeSets() {
 
 // Drop sets a consumer has dealt with.
 //
-// Called after the paths are committed, not after they are written: a crash in
-// between loses the attribution but not the work, which then reaches the
-// consumer as an unclaimed write. That is the right way round — attribution is
-// a convenience, the bytes are not.
+// Called after a consumer has durably recorded the paths, not after they are
+// written: a crash in between loses the attribution but not the work, which
+// then reaches the consumer as an unclaimed write. That is the right way
+// round — attribution is a convenience, the bytes are not.
 export function clearChangeSets(ids = []) {
     const sets = store()
     for (const id of ids) sets.delete(id)

package/src/database/index.js

@@ -123,6 +123,57 @@ function tableNamesFrom(sqlScript) {
     return names
 }
 
+// What the given schema map says about each table it names: durable, or not.
+//
+// Takes the map rather than reading the module registry: createSqliteDatabase
+// receives its schemas as an option, and a caller that passes its own — every
+// test, and any embedder — has a registry the module never sees.
+function declaredTables(schemas) {
+    const durable = new Set()
+    const transient = new Set()
+    for (const value of schemas?.values?.() ?? []) {
+        const entry = schemaEntry(value)
+        for (const t of tableNamesFrom(entry.sql)) (entry.durable ? durable : transient).add(t)
+    }
+    return { durable, transient }
+}
+
+// The tables a wipe must keep.
+//
+// A declaration by THIS process wins, in both directions — marking a table
+// non-durable has to be able to take effect, or the flag is one-way forever
+// and a plugin can never undo it. The record only answers for tables this
+// process says nothing about, which is exactly the case it exists for: a
+// config that never loaded the plugin owning the data.
+function durableSet(schemas, handle) {
+    const { durable, transient } = declaredTables(schemas)
+    const keep = new Set(durable)
+    for (const t of recordedDurableTables(handle)) {
+        if (!transient.has(t)) keep.add(t)
+    }
+    return keep
+}
+
+// Table names any earlier open recorded as durable. Read defensively: a
+// database written before this was introduced simply has no row.
+function recordedDurableTables(handle) {
+    try {
+        const row = handle.prepare('SELECT value FROM mikser_meta WHERE key = ?').get('durable_tables')
+        const parsed = row?.value ? JSON.parse(row.value) : []
+        return Array.isArray(parsed) ? parsed.filter(t => typeof t === 'string') : []
+    } catch {
+        return []
+    }
+}
+
+// Write back what the next process should assume. Already reconciled by
+// durableSet, so a table this process demoted is genuinely dropped.
+function rememberDurableTables(handle, stmtStamp, keep) {
+    try {
+        stmtStamp.run('durable_tables', JSON.stringify([...keep].sort()))
+    } catch { /* a read-only run cannot record, and does not need to */ }
+}
+
 export function registerSchema(name, sqlScript, { durable = false } = {}) {
     if (typeof name !== 'string' || !name.length) {
         throw new Error('registerSchema: name must be a non-empty string')
@@ -218,6 +269,11 @@ export function useDatabase() {
 // onLoaded below.
 export function createSqliteDatabase({
     runtimeFolder, version, logger, config = {}, schemas, provisioners: provisionersArg,
+    // Clear the cache on this open even though nothing changed — what
+    // `--clear` asks for. Routed through the same wipe as a version change so
+    // it honours `durable`, rather than removing the file and taking the
+    // credentials with it.
+    forceWipe = false,
 }) {
     // Tests inject their own provisioners; the runtime path falls
     // through to the module-level `provisioners` array that plugins
@@ -300,7 +356,7 @@ export function createSqliteDatabase({
         const reportOnly = isReportOnlyRun()
 
         let upgradedFromVersion = null
-        if (reportOnly && ((recorded && recorded !== version) || configChanged)) {
+        if (reportOnly && !forceWipe && ((recorded && recorded !== version) || configChanged)) {
             logger?.warn(
                 'The cache is stale (%s changed since it was written) and this is a read-only run, '
                 + 'so it was NOT wiped — the answer below describes the last build, which may not '
@@ -314,7 +370,7 @@ export function createSqliteDatabase({
                 runtime.options.config,
             )
         }
-        if (!reportOnly && ((recorded && recorded !== version) || configChanged)) {
+        if (!reportOnly && (forceWipe || (recorded && recorded !== version) || configChanged)) {
             // Schema mismatch on upgrade or downgrade. Per ADR-0002 the
             // files on disk are the source of truth and this database
             // is a derived cache, so the right behavior is to wipe the
@@ -330,6 +386,8 @@ export function createSqliteDatabase({
                     'Database schema mismatch: stored=%s, current=%s. Wiping the cache and rebuilding from sources (files are the source of truth — no source data is affected).',
                     recorded, version,
                 )
+            } else if (forceWipe) {
+                logger?.info('Clearing the cache and rebuilding from sources (durable data is kept).')
             }
             // What must survive. A wipe exists because the cache is
             // DERIVED — ADR-0002, the files are the source of truth, so
@@ -343,11 +401,10 @@ export function createSqliteDatabase({
             // So a schema registered `durable` is kept and everything else
             // goes. mikser_meta stays too — its stamps are rewritten a few
             // lines down, and dropping it would only mean recreating it.
-            const durableTables = new Set(['mikser_meta'])
-            for (const value of schemas.values()) {
-                const { sql, durable } = schemaEntry(value)
-                if (durable) for (const t of tableNamesFrom(sql)) durableTables.add(t)
-            }
+            // What this process declares, plus what any earlier process
+            // recorded. The second half is what makes a wipe safe from a
+            // config that does not load the plugin owning the data.
+            const durableTables = new Set(['mikser_meta', ...durableSet(schemas, handle)])
 
             if (durableTables.size > 1 && dbPath !== ':memory:') {
                 // Drop table by table rather than unlinking, so the durable
@@ -391,6 +448,21 @@ export function createSqliteDatabase({
         stmtStamp.run('schema_version', version)
         if (currentConfig) stmtStamp.run('config_checksum', currentConfig)
 
+        // Remember which tables are durable, IN the database.
+        //
+        // Durability was otherwise a property of what happened to be loaded:
+        // the wipe asked the schema registry, so a run whose config does not
+        // include the plugin that owns a durable table saw no durable tables
+        // at all, took the unlink branch, and destroyed it. A dev config
+        // without the auth plugin, opening the same working folder, signed
+        // every connected agent out — and nothing in that run mentioned auth.
+        //
+        // Recorded here and unioned at wipe time, so the answer survives a
+        // process that has never heard of the plugin. Additive: a table stays
+        // on the list once recorded, because forgetting one costs data nothing
+        // can reproduce while keeping a stale name costs an empty table.
+        rememberDurableTables(handle, stmtStamp, durableSet(schemas, handle))
+
         // Build provisioning context. firstRun is true when the file
         // didn't exist before this open OR when the schema mismatch
         // wiped it (and the previous open's stamp is gone) — both shapes
@@ -499,6 +571,9 @@ onLoaded(async () => {
         logger,
         config,
         schemas,
+        // `--clear` asks for the cache to go. It is honoured here rather than
+        // by deleting the file, so the tables registered `durable` survive it.
+        forceWipe: Boolean(runtime.options.clear),
     })
 
     db.open()

package/src/engine.js

@@ -1,6 +1,6 @@
 import path from 'node:path'
 import { Command } from 'commander'
-import { rm, lstat, realpath, mkdir, unlink } from 'fs/promises'
+import { rm, lstat, realpath, mkdir, unlink, readdir } from 'fs/promises'
 import { existsSync } from 'fs'
 import _ from 'lodash'
 import Piscina from 'piscina'
@@ -186,11 +186,31 @@ export async function setup(options) {
         // and never appears in outputFolder.
         runtime.options.previewFolder = path.join(runtime.options.runtimeFolder, 'preview')
 
+        // The sqlite file and the WAL sidecars it leaves beside it.
+        const isDatabaseArtifact = (name) =>
+            name.endsWith('.sqlite') || name.endsWith('.sqlite-wal') || name.endsWith('.sqlite-shm')
+
         if (runtime.options.clear) {
             try {
                 runtime.engine.logger.info('Clearing folders')
                 await rm(runtime.options.outputFolder, { recursive: true })
-                await rm(runtime.options.runtimeFolder, { recursive: true })
+                // Everything in the runtime folder EXCEPT the database.
+                //
+                // Removing the folder wholesale took the database with it, and
+                // with it every table registered `durable` — an OAuth client
+                // registration, a refresh token, a form submission: data no
+                // file can reproduce, which is the whole reason that flag
+                // exists. `--clear` promising a rebuild and delivering a
+                // sign-out is the same bug the durable flag was added to fix,
+                // reached by a different route.
+                //
+                // The database is cleared too, but through its own wipe, which
+                // drops the derived tables and keeps the durable ones.
+                for (const entry of await readdir(runtime.options.runtimeFolder, { withFileTypes: true })
+                    .catch(() => [])) {
+                    if (isDatabaseArtifact(entry.name)) continue
+                    await rm(path.join(runtime.options.runtimeFolder, entry.name), { recursive: true, force: true })
+                }
             } catch (err) {
                 if (err.code != 'ENOENT')
                     throw err