mikser-io 11.0.2 → 11.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mikser-io",
-  "version": "11.0.2",
+  "version": "11.1.0",
   "files": [
     "app.js",
     "index.js",

package/src/explain.js

@@ -9,7 +9,8 @@
 //
 // Follows --audit-output's shape: report and exit, no build phases run.
 import { existsSync } from 'node:fs'
-import { inputHashOf, inputPartsOf, diffInputParts, lookupKeys, checksum as fileChecksum } from './utils/index.js'
+import { inputHashOf, inputPartsOf, diffInputParts, lookupKeys, checksum as fileChecksum, isLocalUri, uriScheme } from './utils/index.js'
+import { recomputeSourceChecksum } from './source.js'
 import { outputMissing } from './invalidation.js'
 import { filterKey } from './track.js'
 import { findEntity, findEntities, findById } from './catalog.js'
@@ -149,19 +150,51 @@ export async function explain(reference) {
     // would say "skipped", which is true of the catalog and misleading about
     // the next build. So check the file too, and say which is being reported.
     //
-    // A caveat rather than a bug: some plugins compose a checksum from more
-    // than one file (layouts folds in its .js sidecar), so a difference does
-    // not always mean the entity's own source moved. Both values are reported
-    // and the wording avoids claiming more than is known.
+    // Compared like with like.
+    //
+    // Some collections store a COMPOSED checksum rather than a file hash —
+    // layouts stores md5("<template>:<sidecar>:<shared>") so a sidecar edit
+    // invalidates the layout. Comparing that against a fresh md5 of the
+    // template is comparing two recipes: it never matches, and this reported
+    // `differs: true` for every layout on every site, permanently, for files
+    // nobody had touched.
+    //
+    // So the collection is asked how to recompute its own value. Only where
+    // nothing is registered — every ordinary source — is the file hash the
+    // right comparison, and there it still is.
     let source = null
-    if (entity.uri) {
+    // A provider-backed entity has no local file, and asking the filesystem
+    // about `https://...` returns ENOENT — which this used to report as
+    // "file is gone", and the verdict then told the reader a build would
+    // DELETE the entity. It would do no such thing: mikser-io-csv pulls rows
+    // over http and the entity is perfectly healthy.
+    //
+    // Whether the remote moved is a real question, but it is not one a local
+    // checksum can answer, and answering it here would put a network fetch
+    // inside a read-only explain. So the comparison is declined, out loud.
+    if (entity.uri && !isLocalUri(entity.uri)) {
+        source = {
+            uri: entity.uri,
+            scheme: uriScheme(entity.uri),
+            catalogChecksum: entity.checksum ?? null,
+            remote: true,
+            // No `differs`. Absent would read as false to anything checking
+            // truthiness, so the reason it is absent is stated instead.
+            notCompared: 'the source is fetched by a provider, so there is no local file to compare against',
+        }
+    } else if (entity.uri) {
         try {
             const onDisk = await fileChecksum(entity.uri)
+            const composed = await recomputeSourceChecksum(entity)
+            const comparable = composed ?? onDisk
             source = {
                 uri: entity.uri,
                 catalogChecksum: entity.checksum ?? null,
                 fileChecksum: onDisk,
-                differs: entity.checksum != null && entity.checksum !== onDisk,
+                // Reported only when it is not simply the file hash, so its
+                // presence means "this collection composes" rather than noise.
+                ...(composed ? { comparableChecksum: composed } : {}),
+                differs: entity.checksum != null && entity.checksum !== comparable,
             }
         } catch (err) {
             source = { uri: entity.uri, error: err.code === 'ENOENT' ? 'file is gone' : err.message }
@@ -293,7 +326,10 @@ export async function explain(reference) {
         })),
         // What a plain build would do next, stated plainly.
         verdict: contestedVerdict(competingFor(snapshots, entity.id))
-            ?? (source?.error === 'file is gone'
+            ?? (source?.remote
+            ? `source is fetched over ${source.scheme} — a build asks its provider, and no local `
+              + 'file was compared here'
+            : source?.error === 'file is gone'
             ? 'source file is gone — a build would DELETE this entity and unlink its output'
             : source?.differs
                 ? 'source differs from the catalog — a build would re-import it first, then re-render. '

package/src/manifest/cycle.js

@@ -12,6 +12,7 @@ import { useJournal } from '../journal.js'
 import { onFinalize, onLoaded } from '../lifecycle.js'
 import { unlink } from 'fs/promises'
 import { buildSnapshot, hashOutputFile, snapToRow } from './snapshot.js'
+import { renderedByDependency } from '../report.js'
 
 // The manifest instance and its database, owned here because this is the only
 // place either is assigned: onLoaded builds them, onFinalize commits through
@@ -249,8 +250,26 @@ onFinalize(async () => {
         // still is knowable at the moment it happens, for the cost of one
         // lookup. That is a rendering change nobody asked for: an upgraded
         // renderer, a changed helper, a dependency that shifted under the
-        // build. Every entity input is in inputHash by construction, so if
-        // that did not move, the cause was not the content.
+        // build.
+        //
+        // "Every entity input is in inputHash by construction" is what this
+        // used to say, and it is false. inputHash covers the entity's OWN
+        // meta and checksum. An entity assembled from a query — a CSS bundle
+        // globbing styles/**/*.css, a page listing its collection — has every
+        // real input outside it, reaching the render through refClosure. Its
+        // inputHash is CONSTANT by construction, so editing any part tripped
+        // this check on every build, forever, telling the reader the cause was
+        // an upgraded renderer when it was the file they had just saved.
+        //
+        // The refClosure cannot settle it either: a query edge records the
+        // filter and how many matched, not a hash of what they contained, so
+        // it is identical before and after the edit.
+        //
+        // What does settle it is WHY the render happened. `ref-changed` and
+        // `query-matched` mean something this entity consumes moved — that is
+        // an input change the hashes cannot see, and not drift. Everything
+        // else with an unchanged inputHash still is: --force sweeps, retries,
+        // a renderer that stopped being a function of its inputs.
         //
         // Recorded here rather than checked later because later is too late —
         // the render rewrites its own snapshot, so by the time anything asks,
@@ -261,6 +280,7 @@ onFinalize(async () => {
         // build the unchanged ones are skipped and never reach here, so this
         // reports on what moved; under --force everything re-renders with
         // unchanged inputs, which makes it a full sweep.
+        const dependencyDriven = renderedByDependency()
         for (const snap of recordedSnapshots) {
             const prior = m._stmtLookup.get(snap.id, snap.destination)
             if (prior
@@ -268,7 +288,8 @@ onFinalize(async () => {
                 && prior.inputHash === snap.inputHash
                 && prior.outputHash
                 && snap.outputHash
-                && prior.outputHash !== snap.outputHash) {
+                && prior.outputHash !== snap.outputHash
+                && !dependencyDriven.has(snap.id)) {
                 drifted.push({ id: snap.id, destination: snap.destination })
             }
             m._stmtUpsert.run(snapToRow(snap))

package/src/report.js

@@ -128,7 +128,7 @@ export function resetReport() {
     // not in later ones, which is what actually happened.
     runtime.state.timings = {}
     runtime.state.pluginTimings = {}
-    runtime.state.activity = { rendered: 0, changed: 0 }
+    runtime.state.activity = { rendered: 0, changed: 0, dependencyDriven: new Set() }
 }
 
 // Published on the runtime so runtime.js can start a fresh cycle for a
@@ -281,7 +281,19 @@ export function reportGated(count = 1) {
 // `matched`, `dependency` each mean something specific, and a single
 // polymorphic key would push the type switch onto every consumer.
 export function reportRendered(entity, reason, decision = {}) {
-    activity().rendered++
+    const seen = activity()
+    seen.rendered++
+    // Recorded BEFORE the reportWanted gate, because the drift check depends
+    // on it and a check that only works under --report is not a check.
+    //
+    // These two reasons mean the entity re-rendered because something it
+    // consumes moved — a $-ref, a partial, an entity matching a recorded
+    // query. Its own inputHash did not move and cannot have: for a bundle
+    // assembled from a query, every real input is OUTSIDE inputHash by
+    // construction. See the drift check in manifest/cycle.js.
+    if (reason === 'ref-changed' || reason === 'query-matched') {
+        if (entity?.id) seen.dependencyDriven.add(entity.id)
+    }
     if (!reportWanted()) return
     store().rendered.push({
         id: entity?.id,
@@ -442,9 +454,17 @@ export function renderErrorCount() {
 // "nothing moved" on a build that had just rendered.
 //
 // Two integers cost nothing and are always true.
+// The entities that re-rendered this cycle because a DEPENDENCY moved, rather
+// than because their own inputs did. Read by the drift check, which cannot
+// tell the difference from hashes alone.
+export function renderedByDependency() {
+    return activity().dependencyDriven
+}
+
 function activity() {
     runtime.state ??= {}
-    runtime.state.activity ??= { rendered: 0, changed: 0 }
+    runtime.state.activity ??= { rendered: 0, changed: 0, dependencyDriven: new Set() }
+    runtime.state.activity.dependencyDriven ??= new Set()
     return runtime.state.activity
 }
 

package/src/source.js

@@ -42,6 +42,7 @@ import { mkdir, readFile } from 'node:fs/promises'
 import { globby } from 'globby'
 import pMap from 'p-map'
 import runtime from './runtime.js'
+import { useLogger } from './engine/index.js'
 import { ACTION } from './constants.js'
 import { checksum as fileChecksum, checksumOf, junkIgnore } from './utils/index.js'
 import { reportGated, reportChanged } from './report.js'
@@ -82,6 +83,53 @@ const SCAN_CONCURRENCY = 16
 // disagree, and the losing combination (empty content + a checksum correct
 // for the finished file) is permanent, because every later sync then
 // short-circuits on "unchanged".
+// How to recompute a collection's CATALOG checksum, for the collections that
+// do not store a plain file hash.
+//
+// `gateChecksum` accepts `bytes`, so a plugin can store a checksum composed
+// from several files — layouts folds in its .js sidecar and the shared digest,
+// as md5("<template>:<sidecar>:<shared>"). That is deliberate and correct.
+//
+// What was not correct is anything comparing that stored value against a fresh
+// md5 of the file: two different recipes, so they never match, and
+// mikser_explain reported `differs: true` for EVERY layout on every site —
+// telling the reader that someone had edited a file outside the build when
+// nobody had. A permanent false alarm is worse than no alarm, because it
+// trains the reader to skip the real one.
+//
+// A collection with nothing registered keeps the plain file hash, which is
+// right for every ordinary source. A registered recompute may ALSO return null
+// for an individual entity, meaning "this one is not composed" — layouts needs
+// that for its sidecars, which live in the same collection and are gated on
+// their own bytes.
+const sourceChecksums = new Map()
+
+export function registerSourceChecksum(collection, recompute) {
+    if (!collection || typeof recompute !== 'function') {
+        throw new Error('registerSourceChecksum(collection, recompute) requires both')
+    }
+    sourceChecksums.set(collection, recompute)
+    return () => { sourceChecksums.delete(collection) }
+}
+
+// The checksum the CATALOG would hold for this entity as it stands on disk —
+// composed the way its own collection composes it. Returns null when the
+// collection composes nothing, which tells the caller to use the file hash.
+export async function recomputeSourceChecksum(entity) {
+    const recompute = sourceChecksums.get(entity?.collection)
+    if (!recompute) return null
+    try {
+        return await recompute(entity)
+    } catch (err) {
+        // "Cannot tell" rather than a crash — explain must still answer. But
+        // it says so: a silently swallowed recompute would send the reader
+        // back to comparing raw hashes with no hint that anything went wrong.
+        useLogger?.()?.debug('Source checksum for %s could not be recomputed: %s',
+            entity?.collection, err.message)
+        return null
+    }
+}
+
 export async function gateChecksum(file, id, { reload = false, priorChecksums, bytes } = {}) {
     const compute = () => (bytes !== undefined ? checksumOf(bytes) : fileChecksum(file))
     // What overrides the checksum is not this function's to decide — see

package/src/utils/entity.js

@@ -149,6 +149,25 @@ const providerModuleCache = new Map()
 // built-in filesystem read.
 const URI_SCHEME_RE = /^([a-z][a-z0-9+\-.]*):\/\//i
 
+// The scheme a uri names, lowercased, or null when it names none.
+//
+// `null` and `file` both mean the local filesystem — that is the split
+// readEntityContent below dispatches on, and anything else asking "is there a
+// file here to read" has to make the same distinction. Exported so it is made
+// once: mikser_explain used to run a filesystem checksum over `https://...`,
+// get ENOENT, and report "source file is gone — a build would DELETE this
+// entity", about a healthy entity a provider had fetched.
+export function uriScheme(uri) {
+    if (typeof uri !== 'string') return null
+    return URI_SCHEME_RE.exec(uri)?.[1]?.toLowerCase() ?? null
+}
+
+// Does this uri point at something on the local filesystem?
+export function isLocalUri(uri) {
+    const scheme = uriScheme(uri)
+    return scheme === null || scheme === 'file'
+}
+
 // Resolve `<scheme>` to the package `mikser-io-provider-<scheme>` and
 // import it. Cached per scheme; same package convention as renderers
 // (`mikser-io-render-<name>`) and postprocessors (`mikser-io-post-<name>`).

package/src/utils/index.js

@@ -8,7 +8,7 @@
 // This is the surface. Everything imported from './utils/index.js' is
 // re-exported here, so the split costs no caller anything.
 
-export { changeExtension, getFormatInfo, isTextEntity, looksTextual, matchEntity, mimeForEntity, projectMeta, readEntityContent, useCollection } from './entity.js'
+export { changeExtension, getFormatInfo, isLocalUri, isTextEntity, looksTextual, matchEntity, mimeForEntity, projectMeta, readEntityContent, uriScheme, useCollection } from './entity.js'
 export { AbortError, formatErrorContext, formatLogArgs } from './errors.js'
 export { ExpandError, expandEntity } from './expand.js'
 export { checksum, checksumOf, diffInputParts, inputHashOf, inputPartsOf, normalize } from './hash.js'