mikser-io 8.3.1 → 8.3.6

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "mikser-io",
-	"version": "8.3.1",
+	"version": "8.3.6",
 	"description": "<p align=\"center\">   <img src=\"mikser-lockup-stacked.svg\" alt=\"mikser\" width=\"198\" /> </p>",
 	"main": "index.js",
 	"scripts": {

package/src/catalog.js

@@ -71,9 +71,38 @@ registerSchema('mikser_entities', `
     CREATE INDEX IF NOT EXISTS idx_mikser_entities_uri         ON mikser_entities(uri);
 `)
 
+// Per-process dedupe of the no-filter findEntities/iterateEntities
+// warning. Keyed by the rendering entity id (which is what gets
+// blamed in the recorded refClosure). Once per offending site is
+// enough — the warning is educational, not load-bearing.
+const _warnedNullFilter = new Set()
+
 function recordQuery(filter) {
     const ctx = queryContext.getStore()
     if (!ctx?.track) return
+    // Null/undefined filter records as `null` in the snapshot's
+    // refClosure, which manifest.shouldSkip and manifest.queryAffected
+    // treat as "any mutation could have affected this render."
+    // Architecturally correct, but it means an aggregate layout whose
+    // sidecar calls findEntities() with no args invalidates on every
+    // single CREATE/UPDATE/DELETE — including spurious ones (plugins
+    // re-emitting unchanged entities, etc.).
+    //
+    // Warn once per rendering entity so authors can narrow the filter.
+    // The fix is almost always to add the collection / type / format
+    // dimension the sidecar actually cares about; the JS-side .filter()
+    // chain that usually follows findEntities() is the signal that the
+    // filter belongs in the SQL.
+    if ((filter === undefined || filter === null) && ctx.entityId) {
+        if (!_warnedNullFilter.has(ctx.entityId)) {
+            _warnedNullFilter.add(ctx.entityId)
+            const logger = useLogger()
+            logger?.warn(
+                'findEntities()/iterateEntities() called with no filter from %s — recorded query dep invalidates on every mutation. For "all renderable entities" use findEntities({"meta.href": {$exists: true}}). For narrower scopes use any indexed column ({collection, type, format, "meta.layout", "meta.lang"}); pushing the filter into SQL keeps invalidation precise.',
+                ctx.entityId,
+            )
+        }
+    }
     ctx.track.query(normalizeFilter(filter))
 }
 

package/src/engine.js

@@ -410,7 +410,20 @@ export async function setup(options) {
                             sidecarQueries: context?.sidecarQueries,
                         })
                         entry.deps = edges
-                        runtime.refs?.replaceDynamic(entity.id, edges)
+                        // Pagination produces synthetic pageEntities
+                        // (index.2.html, index.3.html, ...) whose ids
+                        // are NOT in mikser_entities — they exist only
+                        // at render time. Roll their dynamic refs up to
+                        // entity.parent (set by layouts.onBeforeRender
+                        // for pages 2+) so the mikser_refs FK to
+                        // mikser_entities holds. The parent's own
+                        // render also writes to the same source_id;
+                        // INSERT OR IGNORE in stmtInsertEdge handles the
+                        // dedup across pages. Invalidation re-dispatches
+                        // the parent and the pagination expansion
+                        // produces the children from there, so granular
+                        // per-page refs aren't needed.
+                        runtime.refs?.replaceDynamic(entity.parent ?? entity.id, edges)
                         await runtime.complete(entry)
                         await updateEntry({ id, output: entry.output, deps: edges })
                     }

package/src/lifecycle.js

@@ -14,11 +14,23 @@ export async function createEntity(entity) {
     }
 }
 
-export async function deleteEntity({ id, collection, type }) {
+// deleteEntity accepts either a minimal { id, type, collection }
+// payload or a full entity. The full-entity form is preferred when
+// the caller has it on hand (source.sweepDeleted does — it just
+// findById'd the entity to compute the deletion), because the
+// manifest's query-affected dispatch (and any downstream sift-based
+// invalidation) can only match against the entity's actual fields.
+// A minimal {id,type,collection} still works for backwards
+// compatibility but query deps on fields not in that triple
+// (format, meta.*, name, etc.) won't fire.
+export async function deleteEntity(payload) {
     const logger = useLogger()
-    const entry = { operation: OPERATION.DELETE, entity: { id, type, collection } }
+    const entity = payload?.id
+        ? payload   // accept full entity payload as-is
+        : { id: payload?.id, type: payload?.type, collection: payload?.collection }
+    const entry = { operation: OPERATION.DELETE, entity }
     if (await runtime.validate(entry)) {
-        logger.debug('Delete %s entity: %s %s', collection, type, id)
+        logger.debug('Delete %s entity: %s %s', entity.collection, entity.type, entity.id)
         await addEntry(entry)
     }
 }

package/src/manifest.js

@@ -232,6 +232,15 @@ export function createManifest(db) {
           AND json_extract(value, '$.hash')   IS NOT NULL
         GROUP BY json_extract(value, '$.target')
     `)
+    // Snapshots whose refClosure contains at least one query dep. The
+    // LIKE pre-filter is cheap (JSON column scan with no parse) and
+    // narrows down to the small set of aggregate-layout renders.
+    // queryAffected then JSON.parses just those rows and sift-matches
+    // each query filter against each mutated entity.
+    const stmtSnapshotsWithQuery = db.prepare(`
+        SELECT id, refClosure FROM mikser_snapshots
+        WHERE refClosure LIKE '%"kind":"query"%'
+    `)
 
     const manifest = {
         // Look up a previously-recorded entry by entity (or by an
@@ -293,6 +302,45 @@ export function createManifest(db) {
             stmtUpsert.run(snapToRow(buildSnapshot(entity, deps)))
         },
 
+        // Return the set of entity ids whose recorded snapshots have a
+        // query dep that matches any of the cycle's mutated entities.
+        // The static-ref closure walk (refs.inverseClosureOf) only finds
+        // entities reachable via $-keyed edges in mikser_refs; aggregate
+        // layouts that depend on findEntities(...) results need this
+        // second-pass dispatch hint.
+        //
+        // Cheap when there are no aggregate layouts (LIKE pre-filter
+        // returns no rows). Cost scales with (snapshots-with-query) ×
+        // (mutated-entities), both typically small.
+        queryAffected(mutatedEntities) {
+            const affected = new Set()
+            if (!mutatedEntities?.size) return affected
+            for (const row of stmtSnapshotsWithQuery.iterate()) {
+                const refClosure = row.refClosure ? JSON.parse(row.refClosure) : []
+                let hit = false
+                for (const entry of refClosure) {
+                    if (entry.kind !== 'query') continue
+                    if (!entry.filter) {
+                        // Null filter = unserializable predicate captured
+                        // at render time. Conservative: invalidate on any
+                        // mutation.
+                        hit = true
+                        break
+                    }
+                    const matcher = sift(entry.filter)
+                    for (const mutated of mutatedEntities.values()) {
+                        if (matcher(mutated)) {
+                            hit = true
+                            break
+                        }
+                    }
+                    if (hit) break
+                }
+                if (hit) affected.add(row.id)
+            }
+            return affected
+        },
+
         // Drop all snapshots owned by entity id (direct outputs and any
         // paginated children whose `parent` is set to this id). Returns
         // the destinations that were removed so callers can unlink the

package/src/plugins/files.js

@@ -1,6 +1,8 @@
 import path from 'node:path'
 import { mkdir, symlink, unlink, lstat, realpath } from 'fs/promises'
 import { globby } from 'globby'
+import pMap from 'p-map'
+import { checksumsByCollection } from '../catalog.js'
 
 export default ({
     runtime,
@@ -125,25 +127,44 @@ export default ({
 
         const paths = await globby('**/*', { cwd: runtime.options.filesFolder })
         trackProgress('Files import', paths.length)
-        return Promise.all(paths.map(async relativePath => {
+        // Bulk-prefetch the catalog's existing (id → checksum) map for
+        // this collection once at scan start, so the gate below reads
+        // from it per-file instead of doing per-file SQL lookups. Same
+        // pattern source.js uses for documents/layouts via useSource.
+        // Without this gate the plugin re-emitted createEntity on every
+        // cycle for every file regardless of changes, inflating the
+        // journal with phantom mutations and triggering downstream
+        // re-dispatch of aggregate layouts whose recorded query deps
+        // matched the collection.
+        const priorChecksums = checksumsByCollection(collection)
+        await pMap(paths, async relativePath => {
             const { uri, source } = await ensureLink(relativePath)
             let name = relativePath
             if (runtime.config.files?.outputFolder) {
                 name = path.join(runtime.config.files.outputFolder, relativePath)
             }
+            const id = path.join(`/${collection}`, relativePath)
+            const newChecksum = await checksum(source)
+            updateProgress()
+            // Gate: if the catalog already has this entity with the same
+            // checksum, the file hasn't changed since the last cycle.
+            // Skip emitting a CREATE — the catalog row stays correct,
+            // the journal stays accurate (mutations = actual changes),
+            // and downstream aggregate-layout invalidation isn't fired
+            // spuriously.
+            if (priorChecksums.get(id) === newChecksum) return
             await createEntity({
-                id: path.join(`/${collection}`, relativePath),
+                id,
                 uri,
                 collection,
                 type,
                 format: path.extname(relativePath).substring(1).toLowerCase(),
                 name,
                 source,
-                checksum: await checksum(source),
-                link: await link(source)
+                checksum: newChecksum,
+                link: await link(source),
             })
-            updateProgress()
-        }))
+        }, { concurrency: 16 })
     })
 
     return {

package/src/plugins/layouts.js

@@ -565,12 +565,24 @@ export default ({
             // typical site has 0-10 of these, no full scan.
             const optOutEntities = await findEntities({ 'meta.cache': 0 })
 
-            if (seeds.length === 0 && optOutEntities.length === 0) return
+            // Query-dep affected snapshots: aggregate layouts that
+            // depend on findEntities(...) instead of static $-refs need
+            // a second-pass dispatch hint. manifest.queryAffected walks
+            // every snapshot whose refClosure contains a `query` entry
+            // and sift-matches the recorded filter against the cycle's
+            // mutated entities. Bounded by snapshots-with-query (small
+            // — index pages, sitemaps, RSS) × seeds.
+            const mutatedEntities = new Map(seeds.map(s => [s.id, s]))
+            const queryAffected = runtime.manifest?.queryAffected(mutatedEntities) ?? new Set()
+
+            if (seeds.length === 0 && optOutEntities.length === 0 && queryAffected.size === 0) return
 
             const closure = seeds.length ? runtime.refs.inverseClosureOf(seeds) : new Set()
-            // Combine closure ids + opt-out ids into one dispatch set.
+            // Combine closure ids + opt-out ids + query-affected ids
+            // into one dispatch set.
             const dispatchIds = new Set(closure)
             for (const e of optOutEntities) dispatchIds.add(e.id)
+            for (const id of queryAffected) dispatchIds.add(id)
 
             // Hydrate each id via findById. LRU cache absorbs
             // duplicates (refs BFS revisits, partial dispatches
@@ -700,7 +712,16 @@ export default ({
                             postprocessor: entity.layout.postprocessor,
                             tasks: entity.meta?.task || TASKS.INLINE
                         },
-                        context: { data, plugins }
+                        // sidecarQueries threads the sidecar load()'s
+                        // findEntities calls into manifest.collectEdges
+                        // as `{kind: 'query', filter}` refClosure entries.
+                        // Without it, aggregate layouts that don't
+                        // paginate (sitemap.xml, index pages, RSS feeds)
+                        // lose query-dep tracking and never invalidate
+                        // when matching entities are added/modified/
+                        // deleted. The paginated branch above already
+                        // does this.
+                        context: { data, plugins, sidecarQueries: sidecarTrack.queries }
                     })
                 }
             }

package/src/source.js

@@ -351,7 +351,12 @@ export function useSource(core, options) {
         }, { concurrency: SCAN_CONCURRENCY })
 
         scanStats.deleted = await sweepDeleted(collection, scanned, async (e) => {
-            await deleteEntity({ id: e.id, type, collection })
+            // Pass the full entity (e is hydrated from catalog by
+            // sweepDeleted) so the manifest's query-affected dispatch
+            // can sift-match aggregate-layout filters against the
+            // actual fields (format, meta.layout, etc.). Minimal
+            // {id,type,collection} would miss those filters.
+            await deleteEntity(e)
             logger.debug('%s removed (file gone): %s', collection, e.name)
         })