mikser-io 9.39.2 → 9.40.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mikser-io",
-  "version": "9.39.2",
+  "version": "9.40.0",
   "description": "<p align=\"center\">   <img src=\"mikser-lockup-stacked.svg\" alt=\"mikser\" width=\"198\" /> </p>",
   "main": "index.js",
   "exports": {

package/src/engine.js

@@ -541,7 +541,8 @@ export async function setup(options) {
                 // meta. Opt-out rather than opt-in: a contract that is only
                 // correct when someone remembered to enable it is a contract
                 // nobody can rely on. Set `metaReads: false` to switch it off.
-                const track = createTrack({ meta: runtime.options.metaReads !== false })
+                const track = createTrack({ meta: runtime.options.metaReads !== false,
+                                            consumed: runtime.options.metaReads !== false })
                 const renderOptions = {
                     entity: renderEntity,
                     options: {
@@ -639,10 +640,28 @@ export async function setup(options) {
                             ...(track?.metaReads ?? []),
                             ...(context?.sidecarMetaReads ?? []),
                         ]
+                        // Which keys of OTHER entities this render read, keyed
+                        // by the entity they belong to. Merged from the same two
+                        // places: the render's own track, and the sidecar's,
+                        // which runs earlier on the main thread.
+                        const consumedReads = new Map()
+                        for (const [cid, paths] of [
+                            ...(track?.consumedReads ?? []),
+                            ...(context?.sidecarConsumedReads ?? []),
+                        ]) {
+                            const into = consumedReads.get(cid) ?? new Set()
+                            for (const path of paths) into.add(path)
+                            consumedReads.set(cid, into)
+                        }
                         entry.output = {
                             success: true,
                             result,
                             ...(metaReads.length ? { metaReads: [...new Set(metaReads)].sort() } : {}),
+                            ...(consumedReads.size ? {
+                                consumedReads: [...consumedReads]
+                                    .map(([cid, paths]) => [cid, [...paths].sort()])
+                                    .sort(([a], [b]) => a.localeCompare(b)),
+                            } : {}),
                         }
                         // A render that produced nothing at all. Distinct from
                         // a render that THREW, which lands in `errors` and

package/src/manifest.js

@@ -86,6 +86,11 @@ export const SNAPSHOTS_SCHEMA = `
         -- JavaScript, and no parser for any template engine will ever find
         -- an optional chain like row.meta?.hero?.tags.
         metaReads   TEXT,
+        -- Keys of OTHER entities this render read, keyed by the entity they
+        -- belong to. The counterpart to metaReads, and the only record of what
+        -- a document that never renders is actually needed FOR: the engine knew
+        -- that a page queried it, never which of its fields mattered.
+        consumedReads TEXT,
         renderedAt  INTEGER,
         parent      TEXT,
         PRIMARY KEY (id, destination)
@@ -206,7 +211,7 @@ function buildRefClosure(entity, deps) {
     return closure
 }
 
-function buildSnapshot(entity, deps, outputHash, metaReads) {
+function buildSnapshot(entity, deps, outputHash, metaReads, consumedReads) {
     const snapshot = {
         id: entity.id,
         destination: entity.destination,
@@ -222,6 +227,7 @@ function buildSnapshot(entity, deps, outputHash, metaReads) {
     // Sorted and deduped: this is a SET of paths, and a stable order keeps a
     // snapshot comparable between runs.
     if (metaReads?.length) snapshot.metaReads = [...new Set(metaReads)].sort()
+    if (consumedReads?.length) snapshot.consumedReads = consumedReads
     if (entity.parent) snapshot.parent = entity.parent
     if (outputHash) snapshot.outputHash = outputHash
     return snapshot
@@ -239,6 +245,7 @@ function rowToSnap(row) {
         outputHash:  row.outputHash ?? undefined,
         refClosure:  row.refClosure ? JSON.parse(row.refClosure) : undefined,
         metaReads:   row.metaReads  ? JSON.parse(row.metaReads)  : undefined,
+        consumedReads: row.consumedReads ? JSON.parse(row.consumedReads) : undefined,
         renderedAt:  row.renderedAt ?? undefined,
         parent:      row.parent ?? undefined,
     }
@@ -253,6 +260,7 @@ function snapToRow(snap) {
         outputHash:  snap.outputHash  ?? null,
         refClosure:  snap.refClosure  ? JSON.stringify(snap.refClosure) : null,
         metaReads:   snap.metaReads   ? JSON.stringify(snap.metaReads)  : null,
+        consumedReads: snap.consumedReads ? JSON.stringify(snap.consumedReads) : null,
         renderedAt:  snap.renderedAt  ?? null,
         parent:      snap.parent      ?? null,
     }
@@ -376,7 +384,7 @@ export function createManifest(db) {
         SELECT id FROM mikser_snapshots WHERE destination = ?
     `)
     const stmtLookupByDestination = db.prepare(`
-        SELECT id, destination, inputHash, inputParts, outputHash, refClosure, metaReads, renderedAt, parent
+        SELECT id, destination, inputHash, inputParts, outputHash, refClosure, metaReads, consumedReads, renderedAt, parent
         FROM mikser_snapshots WHERE destination = ?
     `)
     const stmtDeleteByDestination = db.prepare(`
@@ -413,18 +421,18 @@ export function createManifest(db) {
     `)
 
     const stmtLookupById = db.prepare(`
-        SELECT id, destination, inputHash, inputParts, outputHash, refClosure, metaReads, renderedAt, parent
+        SELECT id, destination, inputHash, inputParts, outputHash, refClosure, metaReads, consumedReads, renderedAt, parent
         FROM mikser_snapshots WHERE id = ? ORDER BY destination
     `)
     const stmtLookup = db.prepare(`
-        SELECT id, destination, inputHash, inputParts, outputHash, refClosure, metaReads, renderedAt, parent
+        SELECT id, destination, inputHash, inputParts, outputHash, refClosure, metaReads, consumedReads, renderedAt, parent
         FROM mikser_snapshots WHERE id = ? AND destination = ?
     `)
     const stmtUpsert = db.prepare(`
         INSERT OR REPLACE INTO mikser_snapshots
-            (id, destination, inputHash, inputParts, outputHash, refClosure, metaReads, renderedAt, parent)
+            (id, destination, inputHash, inputParts, outputHash, refClosure, metaReads, consumedReads, renderedAt, parent)
         VALUES
-            (@id, @destination, @inputHash, @inputParts, @outputHash, @refClosure, @metaReads, @renderedAt, @parent)
+            (@id, @destination, @inputHash, @inputParts, @outputHash, @refClosure, @metaReads, @consumedReads, @renderedAt, @parent)
     `)
     const stmtDeleteByPK = db.prepare(`
         DELETE FROM mikser_snapshots WHERE id = ? AND destination = ?
@@ -440,7 +448,7 @@ export function createManifest(db) {
         SELECT id, destination FROM mikser_snapshots WHERE parent = ?
     `)
     const stmtSelectAll = db.prepare(`
-        SELECT id, destination, inputHash, inputParts, outputHash, refClosure, metaReads, renderedAt, parent
+        SELECT id, destination, inputHash, inputParts, outputHash, refClosure, metaReads, consumedReads, renderedAt, parent
         FROM mikser_snapshots
     `)
     const stmtCount = db.prepare(`SELECT COUNT(*) AS c FROM mikser_snapshots`)
@@ -859,8 +867,8 @@ export function createManifest(db) {
         },
 
         // Record a successful render. Single INSERT OR REPLACE.
-        record(entity, deps, metaReads) {
-            stmtUpsert.run(snapToRow(buildSnapshot(entity, deps, undefined, metaReads)))
+        record(entity, deps, metaReads, consumedReads) {
+            stmtUpsert.run(snapToRow(buildSnapshot(entity, deps, undefined, metaReads, consumedReads)))
         },
 
         // Return the set of entity ids whose recorded snapshots have a
@@ -1154,7 +1162,8 @@ onFinalize(async () => {
 
     for await (const { output, entity, deps } of useJournal('Output', [OPERATION.RENDER])) {
         if (output?.success && !output.skipped) {
-            renderedEntries.push({ entity, deps, metaReads: output.metaReads })
+            renderedEntries.push({ entity, deps, metaReads: output.metaReads,
+                                   consumedReads: output.consumedReads })
             if (entity.pages > 1) {
                 const parentId = entity.parent ?? entity.id
                 if (!newDestinationsByParent.has(parentId)) {
@@ -1287,10 +1296,10 @@ onFinalize(async () => {
     // `parent` is checked too, mirroring _stmtDeleteByIdOrParent — a deleted
     // parent takes its paginated children's snapshots with it.
     const recordedSnapshots = []
-    for (const { entity, deps, metaReads } of renderedEntries) {
+    for (const { entity, deps, metaReads, consumedReads } of renderedEntries) {
         if (deleted.has(entity.id) || (entity.parent && deleted.has(entity.parent))) continue
         const outputHash = await hashOutputFile(entity.destination)
-        recordedSnapshots.push(buildSnapshot(entity, deps, outputHash, metaReads))
+        recordedSnapshots.push(buildSnapshot(entity, deps, outputHash, metaReads, consumedReads))
     }
 
     // Apply all DB mutations atomically.

package/src/track.js

@@ -40,7 +40,7 @@ export function filterKey(filter) {
 // The returned object exposes `partials: Set<string>` and
 // `queries: Array<filter | null>` directly. Consumers iterate either
 // shape; both are owned by the track for the lifetime of the run.
-export function createTrack({ partial = true, query = true, lookup = true, meta = false } = {}) {
+export function createTrack({ partial = true, query = true, lookup = true, meta = false, consumed = false } = {}) {
     const track = {}
     if (lookup) {
         // Lookups a TEMPLATE made by name: runtime.href('/contacts'),
@@ -100,6 +100,28 @@ export function createTrack({ partial = true, query = true, lookup = true, meta
         track.metaReads = metaReads
         track.metaRead = (path) => { if (path) metaReads.add(path) }
     }
+    if (consumed) {
+        // Which fields of OTHER entities this render read.
+        //
+        // `metaReads` covers the entity being rendered. This covers the ones it
+        // pulled in — a page reads a navigation document through a query and
+        // then takes `items[].label` off it, and until now nothing recorded
+        // that. The consequence is that a document which never renders had no
+        // derivable contract at all: the engine knew THAT a page queried it,
+        // never which of its keys mattered.
+        //
+        // Keyed by the consumed entity's id, because the answer is per-entity:
+        // "what does anything read from navigation.yml" is the question, and it
+        // is asked from the other side.
+        const consumedReads = new Map()
+        track.consumedReads = consumedReads
+        track.consumedRead = (id, path) => {
+            if (!id || !path) return
+            let paths = consumedReads.get(id)
+            if (!paths) consumedReads.set(id, paths = new Set())
+            paths.add(path)
+        }
+    }
     if (query) {
         const queries = []
         const queryKeys = new Set()
@@ -261,6 +283,8 @@ export function serializeTrack(track) {
         queries:   track.queries   ? [...track.queries]   : undefined,
         metaReads: track.metaReads ? [...track.metaReads] : undefined,
         lookups:   track.lookups   ? [...track.lookups].map(([k, v]) => [k, [...v]]) : undefined,
+        consumedReads: track.consumedReads
+            ? [...track.consumedReads].map(([k, v]) => [k, [...v]]) : undefined,
     }
 }
 
@@ -273,5 +297,20 @@ export function mergeTrack(target, data) {
     for (const q of data.queries   ?? []) target.query?.(q)
     for (const m of data.metaReads ?? []) target.metaRead?.(m)
     for (const [name, ids] of data.lookups ?? []) target.lookup?.(name, ids)
+    for (const [id, paths] of data.consumedReads ?? []) {
+        for (const path of paths) target.consumedRead?.(id, path)
+    }
     return target
 }
+
+// A queried entity, wrapped so what a render reads off it is recorded against
+// THAT entity rather than the one being rendered.
+//
+// Only `meta` is wrapped: id, name and uri are how the caller identifies the
+// row, not content, and recording them would bury the keys that matter. Returns
+// the entity untouched when there is nothing to record into, so a call site can
+// wrap unconditionally.
+export function observeConsumed(entity, track) {
+    if (!track?.consumedRead || !entity?.id || !entity.meta || typeof entity.meta !== 'object') return entity
+    return { ...entity, meta: recordReads(entity.meta, '', (path) => track.consumedRead(entity.id, path)) }
+}