mikser-io 9.2.1 → 9.3.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mikser-io",
-  "version": "9.2.1",
+  "version": "9.3.1",
   "description": "<p align=\"center\">   <img src=\"mikser-lockup-stacked.svg\" alt=\"mikser\" width=\"198\" /> </p>",
   "main": "index.js",
   "exports": {

package/src/catalog.js

@@ -319,6 +319,28 @@ onPersist(async () => {
 })
 
 onFinalize(async () => {
+    // Second drain, and the one that closes the cycle.
+    //
+    // onPersist runs BEFORE render, so the drain above only ever sees what
+    // the load and process phases journalled. Everything a render or a
+    // postprocess journals lands after it — and onFinalized's clearJournal()
+    // then throws those entries away unread. The manifest never had this
+    // problem because it drains at onFinalize; the catalog simply had a
+    // shorter view of the same journal.
+    //
+    // The visible symptom was pruning that silently did nothing: a render
+    // asking for `catalog: false` journals its DELETE once the render
+    // resolves, which is past persist, so the row stayed. gpoint's cms
+    // accumulated 1,134 scratch entities carrying 86 MB of render payload,
+    // took its public endpoint from 70ms to 8-15s, and blanked the site.
+    //
+    // Draining again here rather than moving the persist drain: the phases
+    // before render still need their mutations committed at persist (the
+    // render reads the catalog), and journal consumers are named and
+    // independent, so a second pass only ever picks up what the first could
+    // not have seen.
+    await applyJournalMutations()
+
     // Checkpoint the WAL so the main file size stays representative
     // and external tools (mikser --verify on a separate run, debug
     // scripts) see committed state. PASSIVE never blocks readers or

package/src/manifest.js

@@ -61,8 +61,6 @@ import { filterKey } from './track.js'
 import { findById } from './catalog.js'
 import { useDatabase, registerSchema } from './database/index.js'
 
-export { inputHashOf } from './utils.js'
-
 // Schema registration. Applied at db.open(). PRIMARY KEY (id,
 // destination) — leading id column means `WHERE id = ?` queries use the
 // PK index, no separate id index needed. Parent index is for pagination
@@ -542,11 +540,27 @@ onFinalize(async () => {
     const m = sharedManifest
 
     // 2a. Stage file unlinks for deleted entities + their children.
+    const deleted = new Set(deletedIds)
     const filesToUnlink = []
+    const staged = new Set()
+    const stageUnlink = (destination) => {
+        if (!destination || staged.has(destination)) return
+        staged.add(destination)
+        filesToUnlink.push({ destination, reason: 'Entity deleted' })
+    }
     for (const id of deletedIds) {
-        const rows = m._stmtSelectByIdOrParent.all(id, id)
-        for (const row of rows) {
-            filesToUnlink.push({ destination: row.destination, reason: 'Entity deleted' })
+        for (const row of m._stmtSelectByIdOrParent.all(id, id)) {
+            stageUnlink(row.destination)
+        }
+    }
+    // Recorded snapshots only describe PRIOR cycles. An entity rendered and
+    // deleted within the same cycle has no row to look its destination up in —
+    // on a first build there is no row at all — so the file it just wrote would
+    // survive both the entity and its snapshot. Take the destination from this
+    // cycle's render entries instead.
+    for (const { entity } of renderedEntries) {
+        if (deleted.has(entity.id) || (entity.parent && deleted.has(entity.parent))) {
+            stageUnlink(entity.destination)
         }
     }
 
@@ -581,8 +595,20 @@ onFinalize(async () => {
     }
 
     // 2e. Hash the rendered output files (async).
+    //
+    // An id can appear in BOTH drains within one cycle — rendered, then
+    // deleted. A render that opts out of the catalog journals its DELETE the
+    // moment it resolves, so the RENDER and the DELETE sit in the same
+    // journal. 3a below drops the snapshot and 3c would insert it straight
+    // back, undoing the delete inside its own transaction. The DELETE wins:
+    // the entity is gone, so nothing may go on describing it. Skipping here
+    // rather than in 3c also saves hashing an output that was just unlinked.
+    //
+    // `parent` is checked too, mirroring _stmtDeleteByIdOrParent — a deleted
+    // parent takes its paginated children's snapshots with it.
     const recordedSnapshots = []
     for (const { entity, deps } of renderedEntries) {
+        if (deleted.has(entity.id) || (entity.parent && deleted.has(entity.parent))) continue
         const outputHash = await hashOutputFile(entity.destination)
         recordedSnapshots.push(buildSnapshot(entity, deps, outputHash))
     }

package/src/plugins/api.js

@@ -394,6 +394,37 @@ export function api(options = {}) {
         //   api.endpoints.admin   { token: '...', operations: ['list','update','delete','render'] }
         for (const [name, ep] of Object.entries(endpoints)) {
             const router = express.Router()
+
+            // Nothing is served until the first build cycle has finished.
+            //
+            // The server binds at the end of the loaded phase — before
+            // process() has emitted an entity — so without this the endpoint
+            // spends the whole first build answering against an empty catalog.
+            // Renders fail outright (the layouts registry fills during
+            // process()), and reads are worse than that: a list returns
+            // whichever subset exists at that instant, which is a wrong answer
+            // wearing a 200. A consumer seeding its routes from that gets zero
+            // routes and renders blank pages.
+            //
+            // 503, not 4xx, and Retry-After: this is the one honest status
+            // here. It says the request was fine and the server was not, which
+            // is exactly the case, and it is the status every HTTP client
+            // already knows to retry. A 422 or a 500 invites the caller to
+            // treat a transient build as a permanent defect and give up.
+            //
+            // The window is normally sub-second and easy to miss. It stretches
+            // whenever the cache has to be rebuilt from scratch — most notably
+            // after an engine upgrade, where the schema stamp no longer matches
+            // and the catalog is wiped before the rebuild.
+            router.use((req, res, next) => {
+                if (runtime.ready) return next()
+                res.set('Retry-After', '1')
+                res.status(503).json({
+                    error: 'Mikser is still building — the catalog is not ready yet',
+                    ready: false,
+                })
+            })
+
             router.use(express.json({ limit: ep.bodyLimit ?? globalBodyLimit }))
 
             // Operations default to the safer shape when no token is set

package/src/runtime.js

@@ -12,6 +12,22 @@ const runtime = {
     journal: [],
     validators: [],
     started: false,
+    // Whether the FIRST build cycle has finished — i.e. whether the catalog
+    // reflects the sources yet. Not the same question as `started`, which is
+    // true from the moment the loaded phase ends, before process() has emitted
+    // a single entity.
+    //
+    // Transports need this. The server binds at the end of the loaded phase, by
+    // design, so that every plugin has registered its routes first — which also
+    // means requests are accepted while the catalog is still empty. A render
+    // arriving then cannot resolve its layout (the layouts registry is filled
+    // during process()), and a list returns whatever subset happens to exist,
+    // which is worse: it is wrong without being an error.
+    //
+    // Set once and never cleared. Later cycles rebuild against a catalog that
+    // is already populated, so they are serveable; flapping this on every watch
+    // rebuild would take the endpoint down for every keystroke.
+    ready: false,
     // Name of the lifecycle phase currently executing — null between
     // phases. Set inside start() / process() / render() etc. before
     // each callHooks(), cleared on completion. Read by mikser-io-mcp's
@@ -85,6 +101,7 @@ const runtime = {
 
         this.started = true
         await this.process()
+        this.ready = true
     },
 
     async process() {

package/src/subscriptions.js

@@ -35,6 +35,7 @@ import { onFinalize } from './lifecycle.js'
 import { useJournal } from './journal.js'
 import { OPERATION } from './constants.js'
 import { assertExpand, queryEntities } from './catalog.js'
+import sift from 'sift'
 
 // Per-module subscription registry. The dispatcher walks this Set
 // every onFinalize. A Set (not a Map) because subscriptions are keyed
@@ -50,10 +51,34 @@ const opNames = {
     [OPERATION.DELETE]: 'delete',
 }
 
+// A scope may be a predicate OR a sift object, and this is the ONE place that
+// difference is resolved.
+//
+// An endpoint declares its scope once and it then reaches several consumers:
+// queryEntities merges a sift object into the WHERE clause, while a dispatch
+// holding a single entity can only test it. Every consumer that reached for
+// `scope(entity)` therefore worked with the function form and threw
+// `TypeError: scope is not a function` with the object one — which took a
+// production render endpoint down and, because it needed a live subscription to
+// fire at all, reproduced in nothing smaller than production.
+//
+// Compiling here rather than at each call site is the point: a call site that
+// has to remember is a call site that will forget, and the next consumer added
+// inherits the fix instead of the bug.
+function toPredicate(scope) {
+    if (scope == null) return null
+    if (typeof scope === 'function') return scope
+    if (typeof scope === 'object') return sift(scope)
+    throw new Error('subscribe: scope must be a function or a sift filter object')
+}
+
 export function subscribe({ filter, scope, expand, onChange, signal } = {}) {
     if (typeof onChange !== 'function') {
         throw new Error('subscribe: onChange must be a function')
     }
+    // Normalised before it is stored, so both the journal-walk dispatch and the
+    // graph dispatch below see a predicate and neither has to care.
+    scope = toPredicate(scope)
 
     // Reject bad expand at registration. A misconfigured subscriber
     // can otherwise open a session that's expensive to re-dispatch on