mikser-io 9.46.1 → 9.48.0

package/docs/api-reference.md

@@ -601,8 +601,21 @@ clearChangeSets(['req-42'])
 | `withChangeSet({ changeSet, summary, principal }, fn)` | run `fn` with a set in effect |
 | `currentChangeSet()` | the set in effect, or null |
 | `recordChangeSetWrite({ changeSet, summary, principal, uri, operation, undoOf })` | attach one path to a set |
-| `pendingChangeSets()` | sets with unconsumed writes, oldest first |
-| `clearChangeSets(ids)` | drop what a consumer has committed |
+| `listChangeSets({ limit })` | the log, newest first |
+| `findChangeSet(id)` | resolve one id |
+| `pendingChangeSets()` | sets no consumer has recorded yet, oldest first |
+| `markChangeSetsRecorded(ids, recordedAs)` | mark recorded, and say what as |
+
+The log is **durable** and survives a restart: nothing else can reconstruct
+which writes belonged to one request. Not the files, which show the result and
+not the grouping — and not a consumer's own history, which may not exist yet,
+or at all. It keeps the most recent 200 sets.
+
+`recordedAs` is what a consumer recorded the set as — a commit sha. Its absence
+is meaningful: the set is real and listable, but there is nothing to revert
+from yet. `mikser_undo` reports that as `not-yet-committed`, which is a
+different answer from `unknown-change-set` and sends a reader somewhere
+different.
 
 ### Ambient, not threaded
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mikser-io",
-  "version": "9.46.1",
+  "version": "9.48.0",
   "description": "<p align=\"center\">   <img src=\"mikser-lockup-stacked.svg\" alt=\"mikser\" width=\"198\" /> </p>",
   "main": "index.js",
   "exports": {

package/src/changeset.js

@@ -27,6 +27,43 @@
 import path from 'node:path'
 import { AsyncLocalStorage } from 'node:async_hooks'
 import runtime from './runtime.js'
+import { registerSchema } from './database/index.js'
+
+// The log is DURABLE. It records which writes belonged to which request, and
+// nothing can reconstruct that: not the files, which show the result and not
+// the grouping, and not a consumer's own history, which may not exist yet or
+// at all. Losing it turns every id already handed to an agent into a dangling
+// handle.
+registerSchema('change_sets', `
+    CREATE TABLE IF NOT EXISTS mikser_change_sets (
+        id          TEXT PRIMARY KEY,
+        summary     TEXT,
+        principal   TEXT,
+        undo_of     TEXT,
+        created_at  INTEGER NOT NULL,
+        -- Set when a consumer has durably recorded the set somewhere of its
+        -- own — a commit, a snapshot. Until then the set is real and listable
+        -- but there is nothing to revert FROM, which is a different answer
+        -- from "no such change set".
+        recorded_at INTEGER,
+        recorded_as TEXT
+    );
+    CREATE INDEX IF NOT EXISTS idx_mikser_change_sets_created
+        ON mikser_change_sets (created_at DESC);
+
+    CREATE TABLE IF NOT EXISTS mikser_change_set_paths (
+        change_set TEXT NOT NULL,
+        path       TEXT NOT NULL,
+        operation  TEXT NOT NULL,
+        entity_id  TEXT,
+        PRIMARY KEY (change_set, path)
+    );
+`, { durable: true })
+
+// How many sets to keep. An undo log is only useful while the change is
+// recent enough to be worth taking back, and unbounded growth in a durable
+// table is a leak nothing cleans up.
+const KEEP_SETS = 200
 
 // The change set in effect for the current call.
 //
@@ -54,9 +91,87 @@ export function currentChangeSet() {
     return changeSetContext.getStore() ?? null
 }
 
-function store() {
-    runtime.changeSets ??= new Map()
-    return runtime.changeSets
+// The database when there is one, memory when there is not.
+//
+// A write can happen before the engine opens its database — a plugin acting at
+// load time, a unit test — and losing the attribution then would be worse than
+// keeping it somewhere weaker. Both back ends answer the same questions, so no
+// caller has to know which is in play.
+const memory = new Map()
+
+function db() {
+    // Read off the runtime rather than calling useDatabase(): this module is
+    // reached from utils.js, which loads before the database module can be
+    // imported without closing a cycle.
+    const handle = runtime.database?.handle
+    return handle?.prepare ? handle : null
+}
+
+function persist(handle, set, rel, operation, entityId) {
+    handle.prepare(`
+        INSERT INTO mikser_change_sets (id, summary, principal, undo_of, created_at)
+        VALUES (@id, @summary, @principal, @undoOf, @createdAt)
+        ON CONFLICT(id) DO UPDATE SET
+            summary   = COALESCE(mikser_change_sets.summary, excluded.summary),
+            principal = COALESCE(mikser_change_sets.principal, excluded.principal),
+            undo_of   = COALESCE(mikser_change_sets.undo_of, excluded.undo_of)
+    `).run({
+        id: set.id, summary: set.summary, principal: set.principal,
+        undoOf: set.undoOf, createdAt: set.startedAt,
+    })
+    handle.prepare(`
+        INSERT INTO mikser_change_set_paths (change_set, path, operation, entity_id)
+        VALUES (?, ?, ?, ?)
+        ON CONFLICT(change_set, path) DO UPDATE SET operation = excluded.operation
+    `).run(set.id, rel, operation, entityId ?? null)
+    prune(handle)
+}
+
+function prune(handle) {
+    handle.prepare(`
+        DELETE FROM mikser_change_set_paths WHERE change_set IN (
+            SELECT id FROM mikser_change_sets
+            ORDER BY created_at DESC LIMIT -1 OFFSET ?
+        )
+    `).run(KEEP_SETS)
+    handle.prepare(`
+        DELETE FROM mikser_change_sets WHERE id IN (
+            SELECT id FROM mikser_change_sets ORDER BY created_at DESC LIMIT -1 OFFSET ?
+        )
+    `).run(KEEP_SETS)
+}
+
+function rowsToSets(handle, rows) {
+    const stmt = handle.prepare(
+        'SELECT path, operation FROM mikser_change_set_paths WHERE change_set = ? ORDER BY path')
+    return rows.map(row => {
+        const paths = stmt.all(row.id)
+        return {
+            id: row.id,
+            summary: row.summary,
+            principal: row.principal,
+            undoOf: row.undo_of,
+            startedAt: row.created_at,
+            recordedAt: row.recorded_at ?? null,
+            recordedAs: row.recorded_as ?? null,
+            paths: paths.map(p => p.path),
+            deletions: paths.filter(p => p.operation === 'delete').map(p => p.path),
+        }
+    })
+}
+
+function memorySets(filter = () => true) {
+    return [...memory.values()].filter(set => set.paths.size).filter(filter).map(set => ({
+        id: set.id,
+        summary: set.summary,
+        principal: set.principal,
+        undoOf: set.undoOf,
+        startedAt: set.startedAt,
+        recordedAt: set.recordedAt ?? null,
+        recordedAs: set.recordedAs ?? null,
+        paths: [...set.paths.keys()],
+        deletions: [...set.paths.entries()].filter(([, op]) => op === 'delete').map(([p]) => p),
+    }))
 }
 
 // Relative to the working folder, POSIX-separated.
@@ -78,7 +193,9 @@ function relativeToWorkingFolder(uri) {
 // nothing downstream will ever know it as well — a reader choosing what to
 // undo needs "changed the hero text on the devices page", not a file count.
 // First one wins: later writes in the same set are the same request.
-export function recordChangeSetWrite({ changeSet, summary, principal, uri, operation = 'write', undoOf } = {}) {
+export function recordChangeSetWrite({
+    changeSet, summary, principal, uri, operation = 'write', undoOf, entityId,
+} = {}) {
     // An explicit id always wins; otherwise take whatever set is in effect.
     // A write with neither stays unclaimed, which is the correct outcome for
     // an API or human write that no request owns.
@@ -94,8 +211,7 @@ export function recordChangeSetWrite({ changeSet, summary, principal, uri, opera
     // produce a selector that quietly matches nothing.
     if (!rel) return null
 
-    const sets = store()
-    let set = sets.get(changeSet)
+    let set = memory.get(changeSet)
     if (!set) {
         set = {
             id: changeSet,
@@ -108,43 +224,98 @@ export function recordChangeSetWrite({ changeSet, summary, principal, uri, opera
             startedAt: Date.now(),
             paths: new Map(),
         }
-        sets.set(changeSet, set)
+        memory.set(changeSet, set)
     }
     if (!set.summary && summary) set.summary = summary
     if (!set.principal && principal) set.principal = principal
     if (!set.undoOf && undoOf) set.undoOf = undoOf
     set.paths.set(rel, operation)
+
+    const handle = db()
+    if (handle) {
+        try { persist(handle, set, rel, operation, entityId) } catch { /* memory still holds it */ }
+    }
     return set.id
 }
 
-// Every set with writes not yet consumed, oldest first — the order the work
-// actually happened in, which is the order a consumer should record it in.
+// Sets a consumer has not yet durably recorded, oldest first — the order the
+// work actually happened in, which is the order it should be recorded in.
 export function pendingChangeSets() {
-    return [...store().values()]
-        .filter(set => set.paths.size)
-        .sort((a, b) => a.startedAt - b.startedAt)
-        .map(set => ({
-            id: set.id,
-            summary: set.summary,
-            principal: set.principal,
-            undoOf: set.undoOf,
-            startedAt: set.startedAt,
-            paths: [...set.paths.keys()],
-            deletions: [...set.paths.entries()].filter(([, op]) => op === 'delete').map(([p]) => p),
-        }))
+    const handle = db()
+    if (handle) {
+        try {
+            const rows = handle.prepare(`
+                SELECT * FROM mikser_change_sets WHERE recorded_at IS NULL ORDER BY created_at ASC
+            `).all()
+            return rowsToSets(handle, rows).filter(set => set.paths.length)
+        } catch { /* fall through to memory */ }
+    }
+    return memorySets(set => !set.recordedAt).sort((a, b) => a.startedAt - b.startedAt)
 }
 
-// Drop sets a consumer has dealt with.
-//
-// Called after a consumer has durably recorded the paths, not after they are
-// written: a crash in between loses the attribution but not the work, which
-// then reaches the consumer as an unclaimed write. That is the right way
-// round — attribution is a convenience, the bytes are not.
-export function clearChangeSets(ids = []) {
-    const sets = store()
-    for (const id of ids) sets.delete(id)
+// The log an agent reads: every set, newest first, whether or not a consumer
+// has recorded it anywhere.
+export function listChangeSets({ limit = 20 } = {}) {
+    const handle = db()
+    if (handle) {
+        try {
+            const rows = handle.prepare(`
+                SELECT * FROM mikser_change_sets ORDER BY created_at DESC LIMIT ?
+            `).all(Math.max(1, Math.min(limit, 200)))
+            return rowsToSets(handle, rows)
+        } catch { /* fall through to memory */ }
+    }
+    return memorySets().sort((a, b) => b.startedAt - a.startedAt).slice(0, limit)
+}
+
+export function findChangeSet(id) {
+    if (!id) return null
+    const handle = db()
+    if (handle) {
+        try {
+            const row = handle.prepare('SELECT * FROM mikser_change_sets WHERE id = ?').get(id)
+            return row ? rowsToSets(handle, [row])[0] : null
+        } catch { /* fall through to memory */ }
+    }
+    return memorySets(set => set.id === id)[0] ?? null
+}
+
+// Mark sets a consumer has durably recorded, and say what it recorded them AS
+// — a commit sha, a snapshot id. That reference is what an undo reverts from,
+// and its absence is why "recorded but not yet committed" is a different
+// answer from "no such change set".
+export function markChangeSetsRecorded(ids = [], recordedAs = null) {
+    const at = Date.now()
+    for (const id of ids) {
+        const set = memory.get(id)
+        if (set) { set.recordedAt = at; set.recordedAs = recordedAs }
+    }
+    const handle = db()
+    if (!handle) return
+    try {
+        const stmt = handle.prepare(
+            'UPDATE mikser_change_sets SET recorded_at = ?, recorded_as = ? WHERE id = ?')
+        for (const id of ids) stmt.run(at, recordedAs, id)
+    } catch { /* memory still holds it */ }
+}
+
+// Kept as the name consumers already call. Marking recorded is what "done
+// with it" means now — the set stays in the log so it can still be undone.
+export function clearChangeSets(ids = [], recordedAs = null) {
+    markChangeSetsRecorded(ids, recordedAs)
 }
 
 export function forgetAllChangeSets() {
-    store().clear()
+    memory.clear()
+    const handle = db()
+    if (!handle) return
+    try {
+        handle.exec('DELETE FROM mikser_change_set_paths; DELETE FROM mikser_change_sets;')
+    } catch { /* nothing to clear */ }
 }
+
+// Published on the runtime so the write primitives in utils.js can record
+// without importing this module. utils.js loads early — before the database
+// module can be imported here without closing a cycle — and an import purely
+// to reach one function is what would close it.
+runtime.recordChangeSetWrite = recordChangeSetWrite

package/src/database/index.js

@@ -123,6 +123,57 @@ function tableNamesFrom(sqlScript) {
     return names
 }
 
+// What the given schema map says about each table it names: durable, or not.
+//
+// Takes the map rather than reading the module registry: createSqliteDatabase
+// receives its schemas as an option, and a caller that passes its own — every
+// test, and any embedder — has a registry the module never sees.
+function declaredTables(schemas) {
+    const durable = new Set()
+    const transient = new Set()
+    for (const value of schemas?.values?.() ?? []) {
+        const entry = schemaEntry(value)
+        for (const t of tableNamesFrom(entry.sql)) (entry.durable ? durable : transient).add(t)
+    }
+    return { durable, transient }
+}
+
+// The tables a wipe must keep.
+//
+// A declaration by THIS process wins, in both directions — marking a table
+// non-durable has to be able to take effect, or the flag is one-way forever
+// and a plugin can never undo it. The record only answers for tables this
+// process says nothing about, which is exactly the case it exists for: a
+// config that never loaded the plugin owning the data.
+function durableSet(schemas, handle) {
+    const { durable, transient } = declaredTables(schemas)
+    const keep = new Set(durable)
+    for (const t of recordedDurableTables(handle)) {
+        if (!transient.has(t)) keep.add(t)
+    }
+    return keep
+}
+
+// Table names any earlier open recorded as durable. Read defensively: a
+// database written before this was introduced simply has no row.
+function recordedDurableTables(handle) {
+    try {
+        const row = handle.prepare('SELECT value FROM mikser_meta WHERE key = ?').get('durable_tables')
+        const parsed = row?.value ? JSON.parse(row.value) : []
+        return Array.isArray(parsed) ? parsed.filter(t => typeof t === 'string') : []
+    } catch {
+        return []
+    }
+}
+
+// Write back what the next process should assume. Already reconciled by
+// durableSet, so a table this process demoted is genuinely dropped.
+function rememberDurableTables(handle, stmtStamp, keep) {
+    try {
+        stmtStamp.run('durable_tables', JSON.stringify([...keep].sort()))
+    } catch { /* a read-only run cannot record, and does not need to */ }
+}
+
 export function registerSchema(name, sqlScript, { durable = false } = {}) {
     if (typeof name !== 'string' || !name.length) {
         throw new Error('registerSchema: name must be a non-empty string')
@@ -218,6 +269,11 @@ export function useDatabase() {
 // onLoaded below.
 export function createSqliteDatabase({
     runtimeFolder, version, logger, config = {}, schemas, provisioners: provisionersArg,
+    // Clear the cache on this open even though nothing changed — what
+    // `--clear` asks for. Routed through the same wipe as a version change so
+    // it honours `durable`, rather than removing the file and taking the
+    // credentials with it.
+    forceWipe = false,
 }) {
     // Tests inject their own provisioners; the runtime path falls
     // through to the module-level `provisioners` array that plugins
@@ -300,7 +356,7 @@ export function createSqliteDatabase({
         const reportOnly = isReportOnlyRun()
 
         let upgradedFromVersion = null
-        if (reportOnly && ((recorded && recorded !== version) || configChanged)) {
+        if (reportOnly && !forceWipe && ((recorded && recorded !== version) || configChanged)) {
             logger?.warn(
                 'The cache is stale (%s changed since it was written) and this is a read-only run, '
                 + 'so it was NOT wiped — the answer below describes the last build, which may not '
@@ -314,7 +370,7 @@ export function createSqliteDatabase({
                 runtime.options.config,
             )
         }
-        if (!reportOnly && ((recorded && recorded !== version) || configChanged)) {
+        if (!reportOnly && (forceWipe || (recorded && recorded !== version) || configChanged)) {
             // Schema mismatch on upgrade or downgrade. Per ADR-0002 the
             // files on disk are the source of truth and this database
             // is a derived cache, so the right behavior is to wipe the
@@ -330,6 +386,8 @@ export function createSqliteDatabase({
                     'Database schema mismatch: stored=%s, current=%s. Wiping the cache and rebuilding from sources (files are the source of truth — no source data is affected).',
                     recorded, version,
                 )
+            } else if (forceWipe) {
+                logger?.info('Clearing the cache and rebuilding from sources (durable data is kept).')
             }
             // What must survive. A wipe exists because the cache is
             // DERIVED — ADR-0002, the files are the source of truth, so
@@ -343,11 +401,10 @@ export function createSqliteDatabase({
             // So a schema registered `durable` is kept and everything else
             // goes. mikser_meta stays too — its stamps are rewritten a few
             // lines down, and dropping it would only mean recreating it.
-            const durableTables = new Set(['mikser_meta'])
-            for (const value of schemas.values()) {
-                const { sql, durable } = schemaEntry(value)
-                if (durable) for (const t of tableNamesFrom(sql)) durableTables.add(t)
-            }
+            // What this process declares, plus what any earlier process
+            // recorded. The second half is what makes a wipe safe from a
+            // config that does not load the plugin owning the data.
+            const durableTables = new Set(['mikser_meta', ...durableSet(schemas, handle)])
 
             if (durableTables.size > 1 && dbPath !== ':memory:') {
                 // Drop table by table rather than unlinking, so the durable
@@ -391,6 +448,21 @@ export function createSqliteDatabase({
         stmtStamp.run('schema_version', version)
         if (currentConfig) stmtStamp.run('config_checksum', currentConfig)
 
+        // Remember which tables are durable, IN the database.
+        //
+        // Durability was otherwise a property of what happened to be loaded:
+        // the wipe asked the schema registry, so a run whose config does not
+        // include the plugin that owns a durable table saw no durable tables
+        // at all, took the unlink branch, and destroyed it. A dev config
+        // without the auth plugin, opening the same working folder, signed
+        // every connected agent out — and nothing in that run mentioned auth.
+        //
+        // Recorded here and unioned at wipe time, so the answer survives a
+        // process that has never heard of the plugin. Additive: a table stays
+        // on the list once recorded, because forgetting one costs data nothing
+        // can reproduce while keeping a stale name costs an empty table.
+        rememberDurableTables(handle, stmtStamp, durableSet(schemas, handle))
+
         // Build provisioning context. firstRun is true when the file
         // didn't exist before this open OR when the schema mismatch
         // wiped it (and the previous open's stamp is gone) — both shapes
@@ -499,6 +571,9 @@ onLoaded(async () => {
         logger,
         config,
         schemas,
+        // `--clear` asks for the cache to go. It is honoured here rather than
+        // by deleting the file, so the tables registered `durable` survive it.
+        forceWipe: Boolean(runtime.options.clear),
     })
 
     db.open()

package/src/engine.js

@@ -1,6 +1,6 @@
 import path from 'node:path'
 import { Command } from 'commander'
-import { rm, lstat, realpath, mkdir, unlink } from 'fs/promises'
+import { rm, lstat, realpath, mkdir, unlink, readdir } from 'fs/promises'
 import { existsSync } from 'fs'
 import _ from 'lodash'
 import Piscina from 'piscina'
@@ -186,11 +186,31 @@ export async function setup(options) {
         // and never appears in outputFolder.
         runtime.options.previewFolder = path.join(runtime.options.runtimeFolder, 'preview')
 
+        // The sqlite file and the WAL sidecars it leaves beside it.
+        const isDatabaseArtifact = (name) =>
+            name.endsWith('.sqlite') || name.endsWith('.sqlite-wal') || name.endsWith('.sqlite-shm')
+
         if (runtime.options.clear) {
             try {
                 runtime.engine.logger.info('Clearing folders')
                 await rm(runtime.options.outputFolder, { recursive: true })
-                await rm(runtime.options.runtimeFolder, { recursive: true })
+                // Everything in the runtime folder EXCEPT the database.
+                //
+                // Removing the folder wholesale took the database with it, and
+                // with it every table registered `durable` — an OAuth client
+                // registration, a refresh token, a form submission: data no
+                // file can reproduce, which is the whole reason that flag
+                // exists. `--clear` promising a rebuild and delivering a
+                // sign-out is the same bug the durable flag was added to fix,
+                // reached by a different route.
+                //
+                // The database is cleared too, but through its own wipe, which
+                // drops the derived tables and keeps the durable ones.
+                for (const entry of await readdir(runtime.options.runtimeFolder, { withFileTypes: true })
+                    .catch(() => [])) {
+                    if (isDatabaseArtifact(entry.name)) continue
+                    await rm(path.join(runtime.options.runtimeFolder, entry.name), { recursive: true, force: true })
+                }
             } catch (err) {
                 if (err.code != 'ENOENT')
                     throw err

package/src/utils.js

@@ -11,7 +11,6 @@ import yaml from 'yaml'
 import { contentType } from 'mime-types'
 import runtime from './runtime.js'
 import { trackedInfo, untrack, recordReads } from './track.js'
-import { recordChangeSetWrite } from './changeset.js'
 
 // Stable content fingerprint for entities — used by manifest snapshots,
 // engine mutation tracking, and the layouts dispatcher's hash-aware
@@ -794,7 +793,7 @@ export async function writeEntity(entity, patch = {}) {
     // The other file-writing primitive. A rename cascade rewrites every
     // referring file through here, which is the largest fan-out any single
     // request has and therefore the one most worth being able to take back.
-    recordChangeSetWrite({ uri: entity.uri })
+    runtime.recordChangeSetWrite?.({ uri: entity.uri })
 
     return entity.uri
 }
@@ -1114,14 +1113,14 @@ export function useCollection(runtime, name) {
             // of change sets still produces undoable work — the alternative is
             // every writer remembering, and the one that forgets is the one
             // whose edit cannot be taken back.
-            recordChangeSetWrite({ uri })
+            runtime.recordChangeSetWrite?.({ uri })
             return uri
         },
 
         async remove(relativePath) {
             const uri = resolveWithin(relativePath)
             await unlink(uri)
-            recordChangeSetWrite({ uri, operation: 'delete' })
+            runtime.recordChangeSetWrite?.({ uri, operation: 'delete' })
         },
     }
 }