mikser-io 11.0.2 → 11.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mikser-io",
-  "version": "11.0.2",
+  "version": "11.2.0",
   "files": [
     "app.js",
     "index.js",

package/src/changeset.js

@@ -84,6 +84,27 @@ registerMigrations('change-sets', [
             })
         },
     },
+    {
+        // What the write actually put there.
+        //
+        // Without it, a set that produced no commit is ambiguous in a way that
+        // matters: the file on disk being identical to the last commit means
+        // EITHER the write matched what was already committed, or something
+        // replaced it before the commit pass ran. The first is a no-op; the
+        // second is lost work. From disk alone they are the same picture, and
+        // a real one was misread as the harmless case on a live site.
+        //
+        // Recorded at write time, so a consumer can ask afterwards whether
+        // what it wrote is still there. Nullable: a writer that does not know
+        // its own bytes records nothing, and a consumer must then say it
+        // cannot tell rather than assume either way.
+        name: '002-write-checksums',
+        up: async (knex) => {
+            await knex.schema.alterTable('mikser_change_set_paths', (table) => {
+                table.string('checksum')
+            })
+        },
+    },
 ])
 
 // How many sets to keep. An undo log is only useful while the change is
@@ -197,7 +218,7 @@ async function settled() {
     try { await queue } catch { /* reported at the point of failure */ }
 }
 
-async function persist(knex, set, rel, operation, entityId) {
+async function persist(knex, set, rel, operation, entityId, checksum) {
     // COALESCE rather than a plain merge: the first write's summary is the
     // request's own description of itself, and a later write in the same set
     // must not overwrite it with its own. `excluded` is the pseudo-table on
@@ -220,9 +241,17 @@ async function persist(knex, set, rel, operation, entityId) {
         })
 
     await knex('mikser_change_set_paths')
-        .insert({ change_set: set.id, path: rel, operation, entity_id: entityId ?? null })
+        .insert({ change_set: set.id, path: rel, operation, entity_id: entityId ?? null,
+                  checksum: checksum ?? null })
         .onConflict(['change_set', 'path'])
-        .merge({ operation: knex.raw('excluded.operation') })
+        // The LATEST write to a path is the one that says what should be there
+        // now, so its checksum replaces the earlier one — but only when it has
+        // one, or a writer that knows its bytes is overwritten by one that
+        // does not and the path silently becomes unverifiable.
+        .merge({
+            operation: knex.raw('excluded.operation'),
+            checksum: knex.raw('COALESCE(excluded.checksum, mikser_change_set_paths.checksum)'),
+        })
 
     await prune(knex)
 }
@@ -267,7 +296,7 @@ async function rowsToSets(knex, rows) {
     const paths = await knex('mikser_change_set_paths')
         .whereIn('change_set', rows.map(row => row.id))
         .orderBy('path')
-        .select('change_set', 'path', 'operation')
+        .select('change_set', 'path', 'operation', 'checksum')
 
     const bySet = new Map()
     for (const row of paths) {
@@ -292,6 +321,10 @@ async function rowsToSets(knex, rows) {
             outcome: row.outcome ?? null,
             paths: mine.map(p => p.path),
             deletions: mine.filter(p => p.operation === 'delete').map(p => p.path),
+            // path -> the checksum of what was written there, for the paths
+            // whose writer knew. Absent means unverifiable, not unchanged.
+            checksums: Object.fromEntries(
+                mine.filter(p => p.checksum).map(p => [p.path, p.checksum])),
         }
     })
 }
@@ -312,6 +345,9 @@ function memorySets(filter = () => true) {
         outcome: set.outcome ?? null,
         paths: [...set.paths.keys()],
         deletions: [...set.paths.entries()].filter(([, op]) => op === 'delete').map(([p]) => p),
+        // Same shape as the durable path, so a consumer reads one contract
+        // whether or not a database is configured.
+        checksums: Object.fromEntries(set.checksums ?? []),
     }))
 }
 
@@ -336,6 +372,11 @@ function relativeToWorkingFolder(uri) {
 // First one wins: later writes in the same set are the same request.
 export function recordChangeSetWrite({
     changeSet, summary, principal, uri, operation = 'write', undoOf, entityId,
+    // The checksum of the bytes just written, when the caller knows them.
+    // Optional on purpose: a writer that does not have the content in hand
+    // records nothing rather than a wrong value, and a consumer reading the
+    // log must treat its absence as "cannot tell".
+    checksum,
 } = {}) {
     // An explicit id always wins; otherwise take whatever set is in effect.
     // A write with neither stays unclaimed, which is the correct outcome for
@@ -373,6 +414,7 @@ export function recordChangeSetWrite({
     if (!set.undoOf && undoOf) set.undoOf = undoOf
     set.updatedAt = Date.now()
     set.paths.set(rel, operation)
+    if (checksum) (set.checksums ??= new Map()).set(rel, checksum)
 
     if (db()) {
         // Snapshotted, because the queued write runs later and the set keeps
@@ -381,7 +423,7 @@ export function recordChangeSetWrite({
         const snapshot = { ...set }
         enqueue(async () => {
             const knex = db()
-            if (knex) await persist(knex, snapshot, rel, operation, entityId)
+            if (knex) await persist(knex, snapshot, rel, operation, entityId, checksum)
         })
     }
     return set.id

package/src/explain.js

@@ -9,7 +9,8 @@
 //
 // Follows --audit-output's shape: report and exit, no build phases run.
 import { existsSync } from 'node:fs'
-import { inputHashOf, inputPartsOf, diffInputParts, lookupKeys, checksum as fileChecksum } from './utils/index.js'
+import { inputHashOf, inputPartsOf, diffInputParts, lookupKeys, checksum as fileChecksum, isLocalUri, uriScheme } from './utils/index.js'
+import { recomputeSourceChecksum } from './source.js'
 import { outputMissing } from './invalidation.js'
 import { filterKey } from './track.js'
 import { findEntity, findEntities, findById } from './catalog.js'
@@ -149,19 +150,51 @@ export async function explain(reference) {
     // would say "skipped", which is true of the catalog and misleading about
     // the next build. So check the file too, and say which is being reported.
     //
-    // A caveat rather than a bug: some plugins compose a checksum from more
-    // than one file (layouts folds in its .js sidecar), so a difference does
-    // not always mean the entity's own source moved. Both values are reported
-    // and the wording avoids claiming more than is known.
+    // Compared like with like.
+    //
+    // Some collections store a COMPOSED checksum rather than a file hash —
+    // layouts stores md5("<template>:<sidecar>:<shared>") so a sidecar edit
+    // invalidates the layout. Comparing that against a fresh md5 of the
+    // template is comparing two recipes: it never matches, and this reported
+    // `differs: true` for every layout on every site, permanently, for files
+    // nobody had touched.
+    //
+    // So the collection is asked how to recompute its own value. Only where
+    // nothing is registered — every ordinary source — is the file hash the
+    // right comparison, and there it still is.
     let source = null
-    if (entity.uri) {
+    // A provider-backed entity has no local file, and asking the filesystem
+    // about `https://...` returns ENOENT — which this used to report as
+    // "file is gone", and the verdict then told the reader a build would
+    // DELETE the entity. It would do no such thing: mikser-io-csv pulls rows
+    // over http and the entity is perfectly healthy.
+    //
+    // Whether the remote moved is a real question, but it is not one a local
+    // checksum can answer, and answering it here would put a network fetch
+    // inside a read-only explain. So the comparison is declined, out loud.
+    if (entity.uri && !isLocalUri(entity.uri)) {
+        source = {
+            uri: entity.uri,
+            scheme: uriScheme(entity.uri),
+            catalogChecksum: entity.checksum ?? null,
+            remote: true,
+            // No `differs`. Absent would read as false to anything checking
+            // truthiness, so the reason it is absent is stated instead.
+            notCompared: 'the source is fetched by a provider, so there is no local file to compare against',
+        }
+    } else if (entity.uri) {
         try {
             const onDisk = await fileChecksum(entity.uri)
+            const composed = await recomputeSourceChecksum(entity)
+            const comparable = composed ?? onDisk
             source = {
                 uri: entity.uri,
                 catalogChecksum: entity.checksum ?? null,
                 fileChecksum: onDisk,
-                differs: entity.checksum != null && entity.checksum !== onDisk,
+                // Reported only when it is not simply the file hash, so its
+                // presence means "this collection composes" rather than noise.
+                ...(composed ? { comparableChecksum: composed } : {}),
+                differs: entity.checksum != null && entity.checksum !== comparable,
             }
         } catch (err) {
             source = { uri: entity.uri, error: err.code === 'ENOENT' ? 'file is gone' : err.message }
@@ -293,7 +326,10 @@ export async function explain(reference) {
         })),
         // What a plain build would do next, stated plainly.
         verdict: contestedVerdict(competingFor(snapshots, entity.id))
-            ?? (source?.error === 'file is gone'
+            ?? (source?.remote
+            ? `source is fetched over ${source.scheme} — a build asks its provider, and no local `
+              + 'file was compared here'
+            : source?.error === 'file is gone'
             ? 'source file is gone — a build would DELETE this entity and unlink its output'
             : source?.differs
                 ? 'source differs from the catalog — a build would re-import it first, then re-render. '

package/src/manifest/cycle.js

@@ -12,6 +12,7 @@ import { useJournal } from '../journal.js'
 import { onFinalize, onLoaded } from '../lifecycle.js'
 import { unlink } from 'fs/promises'
 import { buildSnapshot, hashOutputFile, snapToRow } from './snapshot.js'
+import { renderedByDependency } from '../report.js'
 
 // The manifest instance and its database, owned here because this is the only
 // place either is assigned: onLoaded builds them, onFinalize commits through
@@ -249,8 +250,26 @@ onFinalize(async () => {
         // still is knowable at the moment it happens, for the cost of one
         // lookup. That is a rendering change nobody asked for: an upgraded
         // renderer, a changed helper, a dependency that shifted under the
-        // build. Every entity input is in inputHash by construction, so if
-        // that did not move, the cause was not the content.
+        // build.
+        //
+        // "Every entity input is in inputHash by construction" is what this
+        // used to say, and it is false. inputHash covers the entity's OWN
+        // meta and checksum. An entity assembled from a query — a CSS bundle
+        // globbing styles/**/*.css, a page listing its collection — has every
+        // real input outside it, reaching the render through refClosure. Its
+        // inputHash is CONSTANT by construction, so editing any part tripped
+        // this check on every build, forever, telling the reader the cause was
+        // an upgraded renderer when it was the file they had just saved.
+        //
+        // The refClosure cannot settle it either: a query edge records the
+        // filter and how many matched, not a hash of what they contained, so
+        // it is identical before and after the edit.
+        //
+        // What does settle it is WHY the render happened. `ref-changed` and
+        // `query-matched` mean something this entity consumes moved — that is
+        // an input change the hashes cannot see, and not drift. Everything
+        // else with an unchanged inputHash still is: --force sweeps, retries,
+        // a renderer that stopped being a function of its inputs.
         //
         // Recorded here rather than checked later because later is too late —
         // the render rewrites its own snapshot, so by the time anything asks,
@@ -261,6 +280,7 @@ onFinalize(async () => {
         // build the unchanged ones are skipped and never reach here, so this
         // reports on what moved; under --force everything re-renders with
         // unchanged inputs, which makes it a full sweep.
+        const dependencyDriven = renderedByDependency()
         for (const snap of recordedSnapshots) {
             const prior = m._stmtLookup.get(snap.id, snap.destination)
             if (prior
@@ -268,7 +288,8 @@ onFinalize(async () => {
                 && prior.inputHash === snap.inputHash
                 && prior.outputHash
                 && snap.outputHash
-                && prior.outputHash !== snap.outputHash) {
+                && prior.outputHash !== snap.outputHash
+                && !dependencyDriven.has(snap.id)) {
                 drifted.push({ id: snap.id, destination: snap.destination })
             }
             m._stmtUpsert.run(snapToRow(snap))

package/src/report.js

@@ -128,7 +128,7 @@ export function resetReport() {
     // not in later ones, which is what actually happened.
     runtime.state.timings = {}
     runtime.state.pluginTimings = {}
-    runtime.state.activity = { rendered: 0, changed: 0 }
+    runtime.state.activity = { rendered: 0, changed: 0, dependencyDriven: new Set() }
 }
 
 // Published on the runtime so runtime.js can start a fresh cycle for a
@@ -281,7 +281,19 @@ export function reportGated(count = 1) {
 // `matched`, `dependency` each mean something specific, and a single
 // polymorphic key would push the type switch onto every consumer.
 export function reportRendered(entity, reason, decision = {}) {
-    activity().rendered++
+    const seen = activity()
+    seen.rendered++
+    // Recorded BEFORE the reportWanted gate, because the drift check depends
+    // on it and a check that only works under --report is not a check.
+    //
+    // These two reasons mean the entity re-rendered because something it
+    // consumes moved — a $-ref, a partial, an entity matching a recorded
+    // query. Its own inputHash did not move and cannot have: for a bundle
+    // assembled from a query, every real input is OUTSIDE inputHash by
+    // construction. See the drift check in manifest/cycle.js.
+    if (reason === 'ref-changed' || reason === 'query-matched') {
+        if (entity?.id) seen.dependencyDriven.add(entity.id)
+    }
     if (!reportWanted()) return
     store().rendered.push({
         id: entity?.id,
@@ -442,9 +454,17 @@ export function renderErrorCount() {
 // "nothing moved" on a build that had just rendered.
 //
 // Two integers cost nothing and are always true.
+// The entities that re-rendered this cycle because a DEPENDENCY moved, rather
+// than because their own inputs did. Read by the drift check, which cannot
+// tell the difference from hashes alone.
+export function renderedByDependency() {
+    return activity().dependencyDriven
+}
+
 function activity() {
     runtime.state ??= {}
-    runtime.state.activity ??= { rendered: 0, changed: 0 }
+    runtime.state.activity ??= { rendered: 0, changed: 0, dependencyDriven: new Set() }
+    runtime.state.activity.dependencyDriven ??= new Set()
     return runtime.state.activity
 }
 

package/src/source.js

@@ -42,6 +42,7 @@ import { mkdir, readFile } from 'node:fs/promises'
 import { globby } from 'globby'
 import pMap from 'p-map'
 import runtime from './runtime.js'
+import { useLogger } from './engine/index.js'
 import { ACTION } from './constants.js'
 import { checksum as fileChecksum, checksumOf, junkIgnore } from './utils/index.js'
 import { reportGated, reportChanged } from './report.js'
@@ -82,6 +83,53 @@ const SCAN_CONCURRENCY = 16
 // disagree, and the losing combination (empty content + a checksum correct
 // for the finished file) is permanent, because every later sync then
 // short-circuits on "unchanged".
+// How to recompute a collection's CATALOG checksum, for the collections that
+// do not store a plain file hash.
+//
+// `gateChecksum` accepts `bytes`, so a plugin can store a checksum composed
+// from several files — layouts folds in its .js sidecar and the shared digest,
+// as md5("<template>:<sidecar>:<shared>"). That is deliberate and correct.
+//
+// What was not correct is anything comparing that stored value against a fresh
+// md5 of the file: two different recipes, so they never match, and
+// mikser_explain reported `differs: true` for EVERY layout on every site —
+// telling the reader that someone had edited a file outside the build when
+// nobody had. A permanent false alarm is worse than no alarm, because it
+// trains the reader to skip the real one.
+//
+// A collection with nothing registered keeps the plain file hash, which is
+// right for every ordinary source. A registered recompute may ALSO return null
+// for an individual entity, meaning "this one is not composed" — layouts needs
+// that for its sidecars, which live in the same collection and are gated on
+// their own bytes.
+const sourceChecksums = new Map()
+
+export function registerSourceChecksum(collection, recompute) {
+    if (!collection || typeof recompute !== 'function') {
+        throw new Error('registerSourceChecksum(collection, recompute) requires both')
+    }
+    sourceChecksums.set(collection, recompute)
+    return () => { sourceChecksums.delete(collection) }
+}
+
+// The checksum the CATALOG would hold for this entity as it stands on disk —
+// composed the way its own collection composes it. Returns null when the
+// collection composes nothing, which tells the caller to use the file hash.
+export async function recomputeSourceChecksum(entity) {
+    const recompute = sourceChecksums.get(entity?.collection)
+    if (!recompute) return null
+    try {
+        return await recompute(entity)
+    } catch (err) {
+        // "Cannot tell" rather than a crash — explain must still answer. But
+        // it says so: a silently swallowed recompute would send the reader
+        // back to comparing raw hashes with no hint that anything went wrong.
+        useLogger?.()?.debug('Source checksum for %s could not be recomputed: %s',
+            entity?.collection, err.message)
+        return null
+    }
+}
+
 export async function gateChecksum(file, id, { reload = false, priorChecksums, bytes } = {}) {
     const compute = () => (bytes !== undefined ? checksumOf(bytes) : fileChecksum(file))
     // What overrides the checksum is not this function's to decide — see

package/src/utils/entity.js

@@ -3,6 +3,7 @@ import path from 'path'
 import runtime from '../runtime.js'
 import yaml from 'yaml'
 
+import { checksumOf } from './hash.js'
 import { isRefKey } from './refs.js'
 import { contentType } from 'mime-types'
 import { minimatch } from 'minimatch'
@@ -149,6 +150,25 @@ const providerModuleCache = new Map()
 // built-in filesystem read.
 const URI_SCHEME_RE = /^([a-z][a-z0-9+\-.]*):\/\//i
 
+// The scheme a uri names, lowercased, or null when it names none.
+//
+// `null` and `file` both mean the local filesystem — that is the split
+// readEntityContent below dispatches on, and anything else asking "is there a
+// file here to read" has to make the same distinction. Exported so it is made
+// once: mikser_explain used to run a filesystem checksum over `https://...`,
+// get ENOENT, and report "source file is gone — a build would DELETE this
+// entity", about a healthy entity a provider had fetched.
+export function uriScheme(uri) {
+    if (typeof uri !== 'string') return null
+    return URI_SCHEME_RE.exec(uri)?.[1]?.toLowerCase() ?? null
+}
+
+// Does this uri point at something on the local filesystem?
+export function isLocalUri(uri) {
+    const scheme = uriScheme(uri)
+    return scheme === null || scheme === 'file'
+}
+
 // Resolve `<scheme>` to the package `mikser-io-provider-<scheme>` and
 // import it. Cached per scheme; same package convention as renderers
 // (`mikser-io-render-<name>`) and postprocessors (`mikser-io-post-<name>`).
@@ -430,7 +450,11 @@ export function useCollection(runtime, name) {
             // of change sets still produces undoable work — the alternative is
             // every writer remembering, and the one that forgets is the one
             // whose edit cannot be taken back.
-            runtime.recordChangeSetWrite?.({ uri })
+            // The checksum goes with it, from the bytes just written. Same
+            // reason the recording is here: a consumer asking later whether
+            // its write is still on disk needs to know what it wrote, and the
+            // only place that reliably knows is the write itself.
+            runtime.recordChangeSetWrite?.({ uri, checksum: checksumOf(content) })
             return uri
         },
 

package/src/utils/index.js

@@ -8,7 +8,7 @@
 // This is the surface. Everything imported from './utils/index.js' is
 // re-exported here, so the split costs no caller anything.
 
-export { changeExtension, getFormatInfo, isTextEntity, looksTextual, matchEntity, mimeForEntity, projectMeta, readEntityContent, useCollection } from './entity.js'
+export { changeExtension, getFormatInfo, isLocalUri, isTextEntity, looksTextual, matchEntity, mimeForEntity, projectMeta, readEntityContent, uriScheme, useCollection } from './entity.js'
 export { AbortError, formatErrorContext, formatLogArgs } from './errors.js'
 export { ExpandError, expandEntity } from './expand.js'
 export { checksum, checksumOf, diffInputParts, inputHashOf, inputPartsOf, normalize } from './hash.js'

package/src/write.js

@@ -22,7 +22,7 @@ import { readdir, readFile } from 'node:fs/promises'
 
 import runtime from './runtime.js'
 import { readEntity, findEntities } from './catalog.js'
-import { useCollection, checksum, readEntityContent, lookupKeys, validateSource } from './utils/index.js'
+import { useCollection, checksum, checksumOf, readEntityContent, lookupKeys, validateSource } from './utils/index.js'
 import { nextCycleId, whenCycleCompletes } from './report.js'
 import { recordChangeSetWrite, currentChangeSet } from './changeset.js'
 
@@ -275,7 +275,10 @@ export async function writeEntitySource({
     // AFTER the write, so a set only ever claims paths that actually moved.
     // Claiming on intent would make a failed write undoable, and undoing a
     // write that never happened is a way to delete someone else's file.
-    if (changeSet) recordChangeSetWrite({ changeSet, summary, principal, uri })
+    // The checksum of what was just written, so a consumer can tell later
+    // whether what it wrote is still there. See the change-set log's
+    // 002-write-checksums migration.
+    if (changeSet) recordChangeSetWrite({ changeSet, summary, principal, uri, checksum: checksumOf(content) })
 
     const result = {
         ok: true, collection, relativePath,