mikser-io 8.3.2 → 8.3.6

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "mikser-io",
-	"version": "8.3.2",
+	"version": "8.3.6",
 	"description": "<p align=\"center\">   <img src=\"mikser-lockup-stacked.svg\" alt=\"mikser\" width=\"198\" /> </p>",
 	"main": "index.js",
 	"scripts": {

package/src/catalog.js

@@ -71,9 +71,38 @@ registerSchema('mikser_entities', `
     CREATE INDEX IF NOT EXISTS idx_mikser_entities_uri         ON mikser_entities(uri);
 `)
 
+// Per-process dedupe of the no-filter findEntities/iterateEntities
+// warning. Keyed by the rendering entity id (which is what gets
+// blamed in the recorded refClosure). Once per offending site is
+// enough — the warning is educational, not load-bearing.
+const _warnedNullFilter = new Set()
+
 function recordQuery(filter) {
     const ctx = queryContext.getStore()
     if (!ctx?.track) return
+    // Null/undefined filter records as `null` in the snapshot's
+    // refClosure, which manifest.shouldSkip and manifest.queryAffected
+    // treat as "any mutation could have affected this render."
+    // Architecturally correct, but it means an aggregate layout whose
+    // sidecar calls findEntities() with no args invalidates on every
+    // single CREATE/UPDATE/DELETE — including spurious ones (plugins
+    // re-emitting unchanged entities, etc.).
+    //
+    // Warn once per rendering entity so authors can narrow the filter.
+    // The fix is almost always to add the collection / type / format
+    // dimension the sidecar actually cares about; the JS-side .filter()
+    // chain that usually follows findEntities() is the signal that the
+    // filter belongs in the SQL.
+    if ((filter === undefined || filter === null) && ctx.entityId) {
+        if (!_warnedNullFilter.has(ctx.entityId)) {
+            _warnedNullFilter.add(ctx.entityId)
+            const logger = useLogger()
+            logger?.warn(
+                'findEntities()/iterateEntities() called with no filter from %s — recorded query dep invalidates on every mutation. For "all renderable entities" use findEntities({"meta.href": {$exists: true}}). For narrower scopes use any indexed column ({collection, type, format, "meta.layout", "meta.lang"}); pushing the filter into SQL keeps invalidation precise.',
+                ctx.entityId,
+            )
+        }
+    }
     ctx.track.query(normalizeFilter(filter))
 }
 

package/src/engine.js

@@ -410,7 +410,20 @@ export async function setup(options) {
                             sidecarQueries: context?.sidecarQueries,
                         })
                         entry.deps = edges
-                        runtime.refs?.replaceDynamic(entity.id, edges)
+                        // Pagination produces synthetic pageEntities
+                        // (index.2.html, index.3.html, ...) whose ids
+                        // are NOT in mikser_entities — they exist only
+                        // at render time. Roll their dynamic refs up to
+                        // entity.parent (set by layouts.onBeforeRender
+                        // for pages 2+) so the mikser_refs FK to
+                        // mikser_entities holds. The parent's own
+                        // render also writes to the same source_id;
+                        // INSERT OR IGNORE in stmtInsertEdge handles the
+                        // dedup across pages. Invalidation re-dispatches
+                        // the parent and the pagination expansion
+                        // produces the children from there, so granular
+                        // per-page refs aren't needed.
+                        runtime.refs?.replaceDynamic(entity.parent ?? entity.id, edges)
                         await runtime.complete(entry)
                         await updateEntry({ id, output: entry.output, deps: edges })
                     }

package/src/plugins/files.js

@@ -1,6 +1,8 @@
 import path from 'node:path'
 import { mkdir, symlink, unlink, lstat, realpath } from 'fs/promises'
 import { globby } from 'globby'
+import pMap from 'p-map'
+import { checksumsByCollection } from '../catalog.js'
 
 export default ({
     runtime,
@@ -125,25 +127,44 @@ export default ({
 
         const paths = await globby('**/*', { cwd: runtime.options.filesFolder })
         trackProgress('Files import', paths.length)
-        return Promise.all(paths.map(async relativePath => {
+        // Bulk-prefetch the catalog's existing (id → checksum) map for
+        // this collection once at scan start, so the gate below reads
+        // from it per-file instead of doing per-file SQL lookups. Same
+        // pattern source.js uses for documents/layouts via useSource.
+        // Without this gate the plugin re-emitted createEntity on every
+        // cycle for every file regardless of changes, inflating the
+        // journal with phantom mutations and triggering downstream
+        // re-dispatch of aggregate layouts whose recorded query deps
+        // matched the collection.
+        const priorChecksums = checksumsByCollection(collection)
+        await pMap(paths, async relativePath => {
             const { uri, source } = await ensureLink(relativePath)
             let name = relativePath
             if (runtime.config.files?.outputFolder) {
                 name = path.join(runtime.config.files.outputFolder, relativePath)
             }
+            const id = path.join(`/${collection}`, relativePath)
+            const newChecksum = await checksum(source)
+            updateProgress()
+            // Gate: if the catalog already has this entity with the same
+            // checksum, the file hasn't changed since the last cycle.
+            // Skip emitting a CREATE — the catalog row stays correct,
+            // the journal stays accurate (mutations = actual changes),
+            // and downstream aggregate-layout invalidation isn't fired
+            // spuriously.
+            if (priorChecksums.get(id) === newChecksum) return
             await createEntity({
-                id: path.join(`/${collection}`, relativePath),
+                id,
                 uri,
                 collection,
                 type,
                 format: path.extname(relativePath).substring(1).toLowerCase(),
                 name,
                 source,
-                checksum: await checksum(source),
-                link: await link(source)
+                checksum: newChecksum,
+                link: await link(source),
             })
-            updateProgress()
-        }))
+        }, { concurrency: 16 })
     })
 
     return {