mikser-io 9.48.0 → 9.49.1

package/docs/api-reference.md

@@ -605,6 +605,16 @@ clearChangeSets(['req-42'])
 | `findChangeSet(id)` | resolve one id |
 | `pendingChangeSets()` | sets no consumer has recorded yet, oldest first |
 | `markChangeSetsRecorded(ids, recordedAs)` | mark recorded, and say what as |
+| `closeChangeSet(id)` | the writer is finished with this set |
+
+`withChangeSet` takes `closeOnReturn` for the case where the call IS the whole
+request — true whenever the id was minted for it rather than supplied. That is
+exact, not a heuristic: an id nobody else can name cannot grow after the call
+that owns it returns, so a consumer can act on it at once instead of waiting to
+see whether more writes arrive. A caller-supplied id exists so several calls
+can join one set, so it stays open and closes on going quiet. Closing happens
+even when the request throws — work that landed before the failure is real, and
+a set left open forever holds it out of reach.
 
 The log is **durable** and survives a restart: nothing else can reconstruct
 which writes belonged to one request. Not the files, which show the result and

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mikser-io",
-  "version": "9.48.0",
+  "version": "9.49.1",
   "description": "<p align=\"center\">   <img src=\"mikser-lockup-stacked.svg\" alt=\"mikser\" width=\"198\" /> </p>",
   "main": "index.js",
   "exports": {

package/src/changeset.js

@@ -41,6 +41,15 @@ registerSchema('change_sets', `
         principal   TEXT,
         undo_of     TEXT,
         created_at  INTEGER NOT NULL,
+        -- Set when the writer said it was finished. A set that closed is
+        -- committable now; one still open is waiting to see whether more
+        -- writes join it.
+        closed_at   INTEGER,
+        -- When the set last grew. A set is one request, and a request is
+        -- finished when it stops writing — which is the only signal available,
+        -- since a caller grouping several tool calls under one id has not said
+        -- which call is the last.
+        updated_at  INTEGER,
         -- Set when a consumer has durably recorded the set somewhere of its
         -- own — a commit, a snapshot. Until then the set is real and listable
         -- but there is nothing to revert FROM, which is a different answer
@@ -77,14 +86,48 @@ const changeSetContext = new AsyncLocalStorage()
 
 // Run `fn` with a change set in effect. Writes inside it are attributed to
 // that set unless they name a different one explicitly.
+//
+// `closeOnReturn` says this call IS the whole request — which is true whenever
+// the id was minted for it rather than supplied by the caller. That is a
+// precise signal, not a heuristic: a set nobody else can name cannot grow
+// after the call that owns it returns, so it is committable immediately.
+//
+// A caller-supplied id is the opposite: it exists so several calls can join
+// one set, and nothing in this call knows whether another is coming. Those
+// close on going quiet instead.
 export function withChangeSet(set, fn) {
     if (!set?.changeSet) return fn()
-    return changeSetContext.run({
+    const context = {
         changeSet: set.changeSet,
         summary: set.summary ?? null,
         principal: set.principal ?? null,
         undoOf: set.undoOf ?? null,
-    }, fn)
+    }
+    if (!set.closeOnReturn) return changeSetContext.run(context, fn)
+    return changeSetContext.run(context, async () => {
+        try {
+            return await fn()
+        } finally {
+            // In `finally`: a request that failed part way still wrote what it
+            // wrote, and leaving that set open forever would hold real work
+            // out of the log's committable half.
+            closeChangeSet(set.changeSet)
+        }
+    })
+}
+
+// Mark a set finished. Idempotent, and silent for an id nothing recorded —
+// a request that wrote nothing has no set to close.
+export function closeChangeSet(id) {
+    if (!id) return
+    const at = Date.now()
+    const set = memory.get(id)
+    if (set) set.closedAt = at
+    const handle = db()
+    if (!handle) return
+    try {
+        handle.prepare('UPDATE mikser_change_sets SET closed_at = COALESCE(closed_at, ?) WHERE id = ?').run(at, id)
+    } catch { /* memory still holds it */ }
 }
 
 export function currentChangeSet() {
@@ -109,15 +152,16 @@ function db() {
 
 function persist(handle, set, rel, operation, entityId) {
     handle.prepare(`
-        INSERT INTO mikser_change_sets (id, summary, principal, undo_of, created_at)
-        VALUES (@id, @summary, @principal, @undoOf, @createdAt)
+        INSERT INTO mikser_change_sets (id, summary, principal, undo_of, created_at, updated_at)
+        VALUES (@id, @summary, @principal, @undoOf, @createdAt, @updatedAt)
         ON CONFLICT(id) DO UPDATE SET
-            summary   = COALESCE(mikser_change_sets.summary, excluded.summary),
-            principal = COALESCE(mikser_change_sets.principal, excluded.principal),
-            undo_of   = COALESCE(mikser_change_sets.undo_of, excluded.undo_of)
+            summary    = COALESCE(mikser_change_sets.summary, excluded.summary),
+            principal  = COALESCE(mikser_change_sets.principal, excluded.principal),
+            undo_of    = COALESCE(mikser_change_sets.undo_of, excluded.undo_of),
+            updated_at = excluded.updated_at
     `).run({
         id: set.id, summary: set.summary, principal: set.principal,
-        undoOf: set.undoOf, createdAt: set.startedAt,
+        undoOf: set.undoOf, createdAt: set.startedAt, updatedAt: set.updatedAt,
     })
     handle.prepare(`
         INSERT INTO mikser_change_set_paths (change_set, path, operation, entity_id)
@@ -141,6 +185,20 @@ function prune(handle) {
     `).run(KEEP_SETS)
 }
 
+// Said once per process, not once per write: a broken log is one condition,
+// and repeating it per write buries the builds that follow it.
+let failureReported = false
+function reportChangeSetFailure(err) {
+    if (failureReported) return
+    failureReported = true
+    const message = 'The change-set log could not be written (%s). Writes still land on disk, but they cannot be '
+        + 'listed or undone until this is fixed.'
+    try {
+        runtime.engine?.logger?.error(message, err.message)
+    } catch { /* no logger yet — the console is what is left */ }
+    if (!runtime.engine?.logger) console.error(message.replace('%s', err.message))
+}
+
 function rowsToSets(handle, rows) {
     const stmt = handle.prepare(
         'SELECT path, operation FROM mikser_change_set_paths WHERE change_set = ? ORDER BY path')
@@ -152,6 +210,8 @@ function rowsToSets(handle, rows) {
             principal: row.principal,
             undoOf: row.undo_of,
             startedAt: row.created_at,
+            updatedAt: row.updated_at ?? row.created_at,
+            closed: row.closed_at != null,
             recordedAt: row.recorded_at ?? null,
             recordedAs: row.recorded_as ?? null,
             paths: paths.map(p => p.path),
@@ -167,6 +227,8 @@ function memorySets(filter = () => true) {
         principal: set.principal,
         undoOf: set.undoOf,
         startedAt: set.startedAt,
+        updatedAt: set.updatedAt ?? set.startedAt,
+        closed: Boolean(set.closedAt),
         recordedAt: set.recordedAt ?? null,
         recordedAs: set.recordedAs ?? null,
         paths: [...set.paths.keys()],
@@ -222,6 +284,7 @@ export function recordChangeSetWrite({
             // privileged operation that rewrites the record.
             undoOf: undoOf ?? null,
             startedAt: Date.now(),
+            updatedAt: Date.now(),
             paths: new Map(),
         }
         memory.set(changeSet, set)
@@ -229,11 +292,21 @@ export function recordChangeSetWrite({
     if (!set.summary && summary) set.summary = summary
     if (!set.principal && principal) set.principal = principal
     if (!set.undoOf && undoOf) set.undoOf = undoOf
+    set.updatedAt = Date.now()
     set.paths.set(rel, operation)
 
     const handle = db()
     if (handle) {
-        try { persist(handle, set, rel, operation, entityId) } catch { /* memory still holds it */ }
+        try {
+            persist(handle, set, rel, operation, entityId)
+        } catch (err) {
+            // Loud. A swallowed failure here is invisible in exactly the way
+            // that matters: the write succeeds, an id comes back, and the log
+            // it points at silently never gains a row — which is how a stale
+            // column shape turned the whole feature into a no-op that looked
+            // like it was working.
+            reportChangeSetFailure(err)
+        }
     }
     return set.id
 }
@@ -248,7 +321,9 @@ export function pendingChangeSets() {
                 SELECT * FROM mikser_change_sets WHERE recorded_at IS NULL ORDER BY created_at ASC
             `).all()
             return rowsToSets(handle, rows).filter(set => set.paths.length)
-        } catch { /* fall through to memory */ }
+        } catch (err) {
+            reportChangeSetFailure(err)
+        }
     }
     return memorySets(set => !set.recordedAt).sort((a, b) => a.startedAt - b.startedAt)
 }
@@ -263,7 +338,9 @@ export function listChangeSets({ limit = 20 } = {}) {
                 SELECT * FROM mikser_change_sets ORDER BY created_at DESC LIMIT ?
             `).all(Math.max(1, Math.min(limit, 200)))
             return rowsToSets(handle, rows)
-        } catch { /* fall through to memory */ }
+        } catch (err) {
+            reportChangeSetFailure(err)
+        }
     }
     return memorySets().sort((a, b) => b.startedAt - a.startedAt).slice(0, limit)
 }

package/src/database/index.js

@@ -104,6 +104,78 @@ let db = null
 // duplicate detection. Same name twice = the later registration wins
 // (with a warning). Convention: `<owner>` matching the table prefix
 // (`catalog`, `manifest`, `vector`, etc.).
+// Column names a CREATE TABLE body declares, in order.
+//
+// Only the leading identifier of each top-level comma-separated clause, and
+// only when it is not a table constraint. Good enough for the schemas this
+// engine registers, and deliberately not a SQL parser.
+function columnsFrom(sqlScript, table) {
+    const re = new RegExp(
+        `CREATE\\s+TABLE\\s+(?:IF\\s+NOT\\s+EXISTS\\s+)?[\`"'\\[]?${table}[\`"'\\]]?\\s*\\(`, 'i')
+    const m = re.exec(sqlScript)
+    if (!m) return []
+    // Walk to the matching close paren so nested types and CHECK(...) do not
+    // end the body early.
+    let depth = 1
+    let i = m.index + m[0].length
+    const start = i
+    for (; i < sqlScript.length && depth > 0; i++) {
+        if (sqlScript[i] === '(') depth++
+        else if (sqlScript[i] === ')') depth--
+    }
+    const body = sqlScript.slice(start, i - 1)
+
+    const clauses = []
+    let current = ''
+    depth = 0
+    for (const ch of body) {
+        if (ch === '(') depth++
+        else if (ch === ')') depth--
+        if (ch === ',' && depth === 0) { clauses.push(current); current = '' } else current += ch
+    }
+    clauses.push(current)
+
+    const CONSTRAINTS = new Set(['primary', 'unique', 'foreign', 'check', 'constraint'])
+    return clauses
+        .map(clause => clause.replace(/--[^\n]*/g, '').trim())
+        .filter(Boolean)
+        .map(clause => clause.split(/\s+/)[0].replace(/["`\[\]]/g, ''))
+        .filter(name => name && !CONSTRAINTS.has(name.toLowerCase()))
+}
+
+// Add columns a DURABLE table has grown since it was created.
+//
+// `CREATE TABLE IF NOT EXISTS` does nothing to a table that already exists, so
+// re-applying a schema never adds a column. For a cache table that is
+// invisible: the version bump wipes and recreates it. A durable table is
+// exactly the one that SURVIVES the wipe, so it is the only kind that can go
+// stale — and it goes stale silently, with every insert naming the new column
+// failing against a table that still has the old shape.
+function migrateDurableColumns(handle, schemas, logger) {
+    for (const value of schemas?.values?.() ?? []) {
+        const { sql, durable } = schemaEntry(value)
+        if (!durable) continue
+        for (const table of tableNamesFrom(sql)) {
+            let existing
+            try {
+                existing = new Set(handle.prepare(`PRAGMA table_info("${table}")`).all().map(c => c.name))
+            } catch { continue }
+            if (!existing.size) continue
+            for (const column of columnsFrom(sql, table)) {
+                if (existing.has(column)) continue
+                // Only ever ADD. Dropping or retyping a column in a durable
+                // table would discard data the whole flag exists to keep.
+                try {
+                    handle.exec(`ALTER TABLE "${table}" ADD COLUMN "${column}"`)
+                    logger?.info('Durable table %s gained column %s', table, column)
+                } catch (err) {
+                    logger?.warn('Could not add column %s to %s: %s', column, table, err.message)
+                }
+            }
+        }
+    }
+}
+
 // Table names a schema script creates. Used to decide what a cache wipe
 // must leave alone — the registry knows schema NAMES, and the wipe works in
 // tables.
@@ -498,7 +570,8 @@ export function createSqliteDatabase({
         }
 
         // Apply each subsystem's registered schema script. Idempotent
-        // CREATE statements mean replay-safe across opens.
+        // CREATE statements mean replay-safe across opens — but only for
+        // tables they can recreate; see migrateDurableColumns below.
         for (const [name, value] of schemas) {
             const { sql: sqlScript } = schemaEntry(value)
             try {
@@ -511,6 +584,11 @@ export function createSqliteDatabase({
                 throw new Error(`Schema "${name}" failed to apply: ${err.message}`)
             }
         }
+
+        // A durable table survives the wipe, so re-applying its CREATE is not
+        // enough to give it a column the schema has grown. Reconciled here,
+        // additively.
+        migrateDurableColumns(handle, schemas, logger)
     }
 
     function close() {