mikser-io 8.3.0 → 8.3.2

package/README.md

@@ -208,8 +208,8 @@ For a working starter — config with a real plugin set, sample `documents/`, ex
 The shape mikser fits cleanly:
 
 - **Marketing sites with editorial teams** — content authors work in files (via their editor, a Git client, or `mikser-io-decap`), engineers ship features without negotiating with a CMS schema, the site stays portable.
-- **Multilingual publishing platforms** — the `useHref()` / `useAlternates()` pattern in `sdk-vue` decouples logical references from per-locale URLs. One source tree, many language deployments.
-- **Content-heavy product catalogues** — `documents` + `mikser-io-schemas` + `data` plugin + a Vue frontend = typed product listings with live updates, semantic search via `vector`, and static-CDN-friendly JSON snapshots all at once.
+- **Multilingual publishing platforms** — the `useHref()` / `useAlternates()` pattern decouples logical references from per-locale URLs. One source tree, many language deployments.
+- **Content-heavy product catalogues** — `documents` + `mikser-io-schemas` + `data` plugin + a Frontend Framework = typed product listings with live updates, semantic search via `vector`, and static-CDN-friendly JSON snapshots all at once.
 - **AI-augmented media pipelines** — `assets` plugin presets call out to Replicate / OpenAI / local models to upscale images, transcribe audio, transcode video. The pipeline is JS code, so anything Node can do is in scope.
 - **Mixed-output publishing** — the same source document renders to HTML, PDF (via `post-pdf`), MJML email (via `post-mjml`), and JSON snapshots. One catalog, many output formats, all concurrent.
 - **Headless backends for static frontends** — pair the `api` plugin with `sdk-api` for SSE-driven live frontends; pair the `data` plugin output with any static host for pre-rendered consumption.

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "mikser-io",
-	"version": "8.3.0",
+	"version": "8.3.2",
 	"description": "<p align=\"center\">   <img src=\"mikser-lockup-stacked.svg\" alt=\"mikser\" width=\"198\" /> </p>",
 	"main": "index.js",
 	"scripts": {

package/src/engine.js

@@ -9,7 +9,7 @@ import { onInitialize, onInitialized, onRender, onCancel, onCancelled, onFinaliz
 import { useJournal, updateEntry } from './journal.js'
 import { globby } from 'globby'
 import { OPERATION, TASKS } from './constants.js'
-import { changeExtension, formatErrorContext, projectMeta } from './utils.js'
+import { changeExtension, formatErrorContext, projectMeta, lookupKeys } from './utils.js'
 import render from './render.js'
 import postprocess, { loadPlugin as loadPostPlugin } from './postprocess.js'
 import map from 'p-map'
@@ -221,8 +221,16 @@ export async function setup(options) {
         // cycle's CREATE/UPDATE/DELETE journal entries — RENDER entries
         // are this cycle's work, not the trigger.
         //
-        // - `mutatedRefs` is a Set of ids (and hrefs) — fast membership
-        //   check for layout/partial/$-ref edges.
+        // - `mutatedRefs` is a Map<key, Set<lang|null>>: the keys are
+        //   the ids / hrefs / id-minus-extension forms a refClosure
+        //   entry might target; the values are the set of languages
+        //   that touched that key this cycle (null for entities with
+        //   no meta.lang). The Map shape preserves the fast `.has(key)`
+        //   membership check the existing skip logic relied on AND
+        //   adds language information so multilingual sites don't
+        //   over-invalidate. When a French author entity changes,
+        //   English posts that reference the same /authors/<name>
+        //   href don't re-render — the lang sets disagree.
         // - `currentHashes` carries the current input hash for each
         //   mutated entity. Cold-start file discovery emits CREATE for
         //   every file even when content didn't change; without the
@@ -231,26 +239,40 @@ export async function setup(options) {
         // - `mutatedEntities` carries the entity payloads themselves so
         //   the query-match check can call `sift(filter)` against each
         //   mutation to decide whether a stored query dep is hit.
-        const mutatedRefs = new Set()
+        const mutatedRefs = new Map()
         const currentHashes = new Map()
         const mutatedEntities = new Map()
         for await (let { entity, operation } of useJournal('Manifest mutations', [OPERATION.CREATE, OPERATION.UPDATE, OPERATION.DELETE])) {
             if (!entity?.id) continue
-            mutatedRefs.add(entity.id)
-            if (entity.meta?.href) mutatedRefs.add(entity.meta.href)
             mutatedEntities.set(entity.id, entity)
+            const hash = operation === OPERATION.DELETE ? null : inputHashOf(entity)
+            const lang = entity.meta?.lang ?? null
+            // Expand the mutated entity into every form a refClosure
+            // entry might target — id, meta.href, AND id-minus-extension
+            // — via lookupKeys. Without the stripped form, a refClosure
+            // recorded against the natural author/blog-post pattern
+            // (`$author: /documents/authors/dick`) would never match
+            // the mutated `/documents/authors/dick.yml` here and
+            // manifest.shouldSkip would silently return true, pinning
+            // the post's output to bytes that reference stale author
+            // data. refs.inverseClosureOf and catalog.findEntity both
+            // use the same extension-tolerant resolution; the manifest
+            // layer has to match.
+            //
+            // Each key carries the language tag of the mutation so
+            // shouldSkip can constrain by language compatibility.
+            //
             // For DELETE we set `null` as the current hash so manifest.
             // shouldSkip can distinguish "target was deleted from the
             // catalog" from "target wasn't in this cycle's mutations
             // at all." Without this distinction, a consumer whose
             // refClosure points at a deleted partial/layout would
-            // silently skip re-rendering — leaving the disk output
-            // pinned to bytes that reference something no longer in
-            // the catalog.
-            currentHashes.set(
-                entity.id,
-                operation === OPERATION.DELETE ? null : inputHashOf(entity),
-            )
+            // silently skip re-rendering.
+            for (const key of lookupKeys(entity)) {
+                if (!mutatedRefs.has(key)) mutatedRefs.set(key, new Set())
+                mutatedRefs.get(key).add(lang)
+                currentHashes.set(key, hash)
+            }
         }
         let skipped = 0
 

package/src/lifecycle.js

@@ -14,11 +14,23 @@ export async function createEntity(entity) {
     }
 }
 
-export async function deleteEntity({ id, collection, type }) {
+// deleteEntity accepts either a minimal { id, type, collection }
+// payload or a full entity. The full-entity form is preferred when
+// the caller has it on hand (source.sweepDeleted does — it just
+// findById'd the entity to compute the deletion), because the
+// manifest's query-affected dispatch (and any downstream sift-based
+// invalidation) can only match against the entity's actual fields.
+// A minimal {id,type,collection} still works for backwards
+// compatibility but query deps on fields not in that triple
+// (format, meta.*, name, etc.) won't fire.
+export async function deleteEntity(payload) {
     const logger = useLogger()
-    const entry = { operation: OPERATION.DELETE, entity: { id, type, collection } }
+    const entity = payload?.id
+        ? payload   // accept full entity payload as-is
+        : { id: payload?.id, type: payload?.type, collection: payload?.collection }
+    const entry = { operation: OPERATION.DELETE, entity }
     if (await runtime.validate(entry)) {
-        logger.debug('Delete %s entity: %s %s', collection, type, id)
+        logger.debug('Delete %s entity: %s %s', entity.collection, entity.type, entity.id)
         await addEntry(entry)
     }
 }

package/src/manifest.js

@@ -232,6 +232,15 @@ export function createManifest(db) {
           AND json_extract(value, '$.hash')   IS NOT NULL
         GROUP BY json_extract(value, '$.target')
     `)
+    // Snapshots whose refClosure contains at least one query dep. The
+    // LIKE pre-filter is cheap (JSON column scan with no parse) and
+    // narrows down to the small set of aggregate-layout renders.
+    // queryAffected then JSON.parses just those rows and sift-matches
+    // each query filter against each mutated entity.
+    const stmtSnapshotsWithQuery = db.prepare(`
+        SELECT id, refClosure FROM mikser_snapshots
+        WHERE refClosure LIKE '%"kind":"query"%'
+    `)
 
     const manifest = {
         // Look up a previously-recorded entry by entity (or by an
@@ -253,6 +262,7 @@ export function createManifest(db) {
             if (!snapshot?.inputHash) return false
             if (inputHashOf(entity) !== snapshot.inputHash) return false
             if (!snapshot.refClosure?.length) return true
+            const sourceLang = entity?.meta?.lang ?? null
             for (const entry of snapshot.refClosure) {
                 if (entry.kind === 'query') {
                     if (!entry.filter) return false
@@ -264,6 +274,20 @@ export function createManifest(db) {
                     continue
                 }
                 if (!mutatedRefs?.has(entry.target)) continue
+                // Language scope: if the source has a meta.lang AND the
+                // mutation came from a different meta.lang, the ref
+                // doesn't actually depend on what changed. A `null` lang
+                // in the mutation set means a shared (un-localized)
+                // entity touched the same key — that does invalidate
+                // language-specific sources because the shared entity
+                // is the only variant. Sources without a meta.lang are
+                // language-agnostic and accept any mutation lang.
+                if (sourceLang) {
+                    const mutatedLangs = mutatedRefs.get(entry.target)
+                    if (mutatedLangs && !mutatedLangs.has(sourceLang) && !mutatedLangs.has(null)) {
+                        continue
+                    }
+                }
                 if (!entry.hash) return false
                 const currentHash = currentHashes?.get(entry.target)
                 if (currentHash === undefined) continue
@@ -278,6 +302,45 @@ export function createManifest(db) {
             stmtUpsert.run(snapToRow(buildSnapshot(entity, deps)))
         },
 
+        // Return the set of entity ids whose recorded snapshots have a
+        // query dep that matches any of the cycle's mutated entities.
+        // The static-ref closure walk (refs.inverseClosureOf) only finds
+        // entities reachable via $-keyed edges in mikser_refs; aggregate
+        // layouts that depend on findEntities(...) results need this
+        // second-pass dispatch hint.
+        //
+        // Cheap when there are no aggregate layouts (LIKE pre-filter
+        // returns no rows). Cost scales with (snapshots-with-query) ×
+        // (mutated-entities), both typically small.
+        queryAffected(mutatedEntities) {
+            const affected = new Set()
+            if (!mutatedEntities?.size) return affected
+            for (const row of stmtSnapshotsWithQuery.iterate()) {
+                const refClosure = row.refClosure ? JSON.parse(row.refClosure) : []
+                let hit = false
+                for (const entry of refClosure) {
+                    if (entry.kind !== 'query') continue
+                    if (!entry.filter) {
+                        // Null filter = unserializable predicate captured
+                        // at render time. Conservative: invalidate on any
+                        // mutation.
+                        hit = true
+                        break
+                    }
+                    const matcher = sift(entry.filter)
+                    for (const mutated of mutatedEntities.values()) {
+                        if (matcher(mutated)) {
+                            hit = true
+                            break
+                        }
+                    }
+                    if (hit) break
+                }
+                if (hit) affected.add(row.id)
+            }
+            return affected
+        },
+
         // Drop all snapshots owned by entity id (direct outputs and any
         // paginated children whose `parent` is set to this id). Returns
         // the destinations that were removed so callers can unlink the

package/src/plugins/layouts.js

@@ -565,12 +565,24 @@ export default ({
             // typical site has 0-10 of these, no full scan.
             const optOutEntities = await findEntities({ 'meta.cache': 0 })
 
-            if (seeds.length === 0 && optOutEntities.length === 0) return
+            // Query-dep affected snapshots: aggregate layouts that
+            // depend on findEntities(...) instead of static $-refs need
+            // a second-pass dispatch hint. manifest.queryAffected walks
+            // every snapshot whose refClosure contains a `query` entry
+            // and sift-matches the recorded filter against the cycle's
+            // mutated entities. Bounded by snapshots-with-query (small
+            // — index pages, sitemaps, RSS) × seeds.
+            const mutatedEntities = new Map(seeds.map(s => [s.id, s]))
+            const queryAffected = runtime.manifest?.queryAffected(mutatedEntities) ?? new Set()
+
+            if (seeds.length === 0 && optOutEntities.length === 0 && queryAffected.size === 0) return
 
             const closure = seeds.length ? runtime.refs.inverseClosureOf(seeds) : new Set()
-            // Combine closure ids + opt-out ids into one dispatch set.
+            // Combine closure ids + opt-out ids + query-affected ids
+            // into one dispatch set.
             const dispatchIds = new Set(closure)
             for (const e of optOutEntities) dispatchIds.add(e.id)
+            for (const id of queryAffected) dispatchIds.add(id)
 
             // Hydrate each id via findById. LRU cache absorbs
             // duplicates (refs BFS revisits, partial dispatches
@@ -700,7 +712,16 @@ export default ({
                             postprocessor: entity.layout.postprocessor,
                             tasks: entity.meta?.task || TASKS.INLINE
                         },
-                        context: { data, plugins }
+                        // sidecarQueries threads the sidecar load()'s
+                        // findEntities calls into manifest.collectEdges
+                        // as `{kind: 'query', filter}` refClosure entries.
+                        // Without it, aggregate layouts that don't
+                        // paginate (sitemap.xml, index pages, RSS feeds)
+                        // lose query-dep tracking and never invalidate
+                        // when matching entities are added/modified/
+                        // deleted. The paginated branch above already
+                        // does this.
+                        context: { data, plugins, sidecarQueries: sidecarTrack.queries }
                     })
                 }
             }

package/src/source.js

@@ -351,7 +351,12 @@ export function useSource(core, options) {
         }, { concurrency: SCAN_CONCURRENCY })
 
         scanStats.deleted = await sweepDeleted(collection, scanned, async (e) => {
-            await deleteEntity({ id: e.id, type, collection })
+            // Pass the full entity (e is hydrated from catalog by
+            // sweepDeleted) so the manifest's query-affected dispatch
+            // can sift-match aggregate-layout filters against the
+            // actual fields (format, meta.layout, etc.). Minimal
+            // {id,type,collection} would miss those filters.
+            await deleteEntity(e)
             logger.debug('%s removed (file gone): %s', collection, e.name)
         })