mikser-io 10.0.1 → 10.0.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mikser-io",
-  "version": "10.0.1",
+  "version": "10.0.3",
   "files": [
     "app.js",
     "index.js",
@@ -12,7 +12,7 @@
     "mikser-mark.svg",
     "mikser-lockup-stacked.svg"
   ],
-  "description": "A mixer for content: entities in, configurable render pipelines, outputs of any kind. Static sites are the canonical recipe, not the definition \u2014 the same engine renders PDFs, emails and whatever a renderer plugin produces. Files are the source of truth, every lifecycle phase is observable, and the build graph is queryable by an agent.",
+  "description": "A mixer for content: entities in, configurable render pipelines, outputs of any kind. Static sites are the canonical recipe, not the definition — the same engine renders PDFs, emails and whatever a renderer plugin produces. Files are the source of truth, every lifecycle phase is observable, and the build graph is queryable by an agent.",
   "main": "index.js",
   "exports": {
     ".": "./index.js",

package/src/explain.js

@@ -8,7 +8,9 @@
 // needs knowledge a user of the tool should not need.
 //
 // Follows --audit-output's shape: report and exit, no build phases run.
+import { existsSync } from 'node:fs'
 import { inputHashOf, inputPartsOf, diffInputParts, lookupKeys, checksum as fileChecksum } from './utils.js'
+import { resolveOutputPath } from './manifest.js'
 import { filterKey } from './track.js'
 import { findEntity, findEntities, findById } from './catalog.js'
 import runtime from './runtime.js'
@@ -134,6 +136,13 @@ export async function explain(reference) {
     const failedSnapshots = snapshots
         .map(snap => runtime.manifest?.failureAt?.(entity.id, snap.destination))
         .filter(Boolean)
+    // Same position in the verdict for the same reason: a destination whose
+    // file is gone re-renders whatever the hashes say. Ranked below a failed
+    // attempt — if the last render threw, that is the more useful sentence,
+    // and it explains the absence too.
+    const missingDestinations = snapshots
+        .filter(snap => snap.destination && !existsSync(resolveOutputPath(snap.destination)))
+        .map(snap => snap.destination)
 
     // The catalog is as of the LAST BUILD. If the file has been edited since,
     // nothing here knows it yet — the hashes would all agree and the verdict
@@ -226,6 +235,18 @@ export async function explain(reference) {
                 : snap.inputParts
                     ? diffInputParts(snap.inputParts, currentParts)
                     : 'unknown',
+            // The file this render wrote, gone from disk.
+            //
+            // Same shape as `failed` above and added for the same reason: the
+            // recorded state is entirely consistent — the input hash matches,
+            // the snapshot is intact — so without this the report reads
+            // `[current]` and `would be SKIPPED` for an entity a build now
+            // re-renders. --explain contradicting the build is worse than
+            // --explain being incomplete, because it is the tool someone
+            // reaches for when the build has already surprised them.
+            missing: snap.destination
+                ? !existsSync(resolveOutputPath(snap.destination))
+                : false,
             outputHash: snap.outputHash ?? null,
             parent: snap.parent ?? null,
             // Which keys of its OWN meta the render read, and which keys it
@@ -286,6 +307,10 @@ export async function explain(reference) {
                   + `(${failedSnapshots[0].error})`
             : snapshots.some(s => s.inputHash !== currentHash)
                 ? renderVerdict(snapshots, currentHash, currentParts)
+            : missingDestinations.length
+                ? `would re-render — the output is gone from disk (${missingDestinations.join(', ')}). `
+                  + 'The inputs are unchanged, so nothing about the entity says this; the file being '
+                  + 'absent is the whole reason.'
                 : 'would be SKIPPED — input hash unchanged. A dependency in refClosure changing is the only other thing that would re-render it.'),
         lookupKeys: lookupKeys(entity),
         // Other entities rendering to the same path as this one.
@@ -360,6 +385,7 @@ export function formatExplain(report) {
     for (const r of report.renders) {
         row('rendered', `${r.renderedAt ?? 'unknown'}   → ${r.destination}`
             + (r.failed ? '   [STALE: last render attempt failed]'
+                : r.missing ? '   [MISSING: the file is not on disk]'
                 : r.stale ? '   [STALE: input hash moved since]'
                 : '   [current]'))
         if (r.failed) {

package/src/manifest.js

@@ -372,6 +372,11 @@ export async function sourcesOf(destination) {
 export function createManifest(db) {
     if (!db) throw new Error('createManifest: db is required')
 
+    // Per-instance, not module-level: tests build their own manifest via
+    // createManifest(db), and a shared memo would leak one test's output
+    // folder into the next.
+    let cachedMissingOutputs = null
+
     const stmtCollisions = db.prepare(`
         SELECT destination, count(*) AS n, group_concat(id) AS ids
         FROM mikser_snapshots
@@ -657,6 +662,7 @@ export function createManifest(db) {
         //
         //   unchanged        nothing this render depends on moved
         //   never-rendered   no snapshot: first build, or it never rendered
+        //   output-missing   the file the last render wrote is gone from disk
         //   inputs-changed   the entity's own hash moved
         //   ref-changed      a $-ref or partial it depends on moved
         //   query-matched    an entity matching a recorded query mutated
@@ -715,6 +721,32 @@ export function createManifest(db) {
                     changed: describeInputChange(entity, snapshot),
                 }
             }
+            // The inputs say nothing moved. That is only a reason to skip if
+            // the thing the last render PRODUCED is still there.
+            //
+            // Every branch above this one reasons about inputs, and inputs are
+            // not where `rm -rf out` shows up: the documents are unchanged,
+            // because what was deleted is the output. So the build skips every
+            // entity, renders nothing, prints no `Rendered:` line and exits 0
+            // with an empty output folder — correct by its own reasoning and
+            // wrong about the only question the caller asked. A missing output
+            // is a reason to re-render, not a reason to report unchanged.
+            //
+            // Placed after the input comparison and before the refClosure walk
+            // so one check covers both `unchanged` exits, and placed here
+            // rather than in the render loop because this is the function that
+            // owns the word: anything asking shouldSkip() gets the same answer.
+            //
+            // Checked unconditionally rather than behind a flag. It is one stat
+            // per entity that reached this point — 2.2ms for 1842 entities,
+            // the same whether the files are there or not — against a failure
+            // mode whose whole character is that nobody thinks to look for it.
+            // auditOutput() resolves destinations exactly this way, and it has
+            // to stay that way: the two disagreeing is the bug, since `--audit-output`
+            // finding what the build just called clean is what sent someone here.
+            if (snapshot.destination && !existsSync(resolveOutputPath(snapshot.destination))) {
+                return { skip: false, reason: 'output-missing', destination: snapshot.destination }
+            }
             if (!snapshot.refClosure?.length) return { skip: true, reason: 'unchanged' }
             const sourceLang = entity?.meta?.lang ?? null
             for (const entry of snapshot.refClosure) {
@@ -988,12 +1020,25 @@ export function createManifest(db) {
         // for the same id (matches the prior first-seen-wins semantic;
         // entity inputHashes were always added before deps in the
         // earlier loop).
+        // An entity whose output is gone is omitted entirely. The consumer
+        // uses a recorded hash to decide "this needs no render", and that
+        // conclusion only follows while the file the hash describes is still
+        // there. Omitting is the whole fix at this gate: a seed with no
+        // recorded hash is always dispatched.
+        //
+        // Done here rather than in the dispatcher because this map exists for
+        // that one decision, and a second consumer would otherwise have to
+        // rediscover the same caveat. missingOutputIds() is memoized for the
+        // cycle, so this shares the walk the source gate already paid for.
         recordedHashes() {
+            const missingOutputs = this.missingOutputIds()
             const map = new Map()
             for (const row of stmtEntityInputHashes.iterate()) {
+                if (missingOutputs.has(row.id)) continue
                 map.set(row.id, row.inputHash)
             }
             for (const row of stmtDepHashes.iterate()) {
+                if (missingOutputs.has(row.target)) continue
                 if (!map.has(row.target)) map.set(row.target, row.hash)
             }
             return map
@@ -1061,6 +1106,56 @@ export function createManifest(db) {
             return edges
         },
 
+        // The entity ids whose last render wrote a file that is no longer
+        // on disk. A dispatch hint, the same shape as queryAffected: the
+        // source checksum gate consults it so those entities are re-emitted
+        // instead of being short-circuited as unchanged.
+        //
+        // This exists because the gate that stops `rm -rf out` from being
+        // noticed is the FIRST one, not the last. An unchanged file never
+        // gets a CREATE, so it never reaches the journal, so the render loop
+        // never sees it and manifest.skipDecision — which does check for a
+        // missing output — is never asked. Fixing only the render gate makes
+        // the build correct for entities that get that far and changes
+        // nothing for the case in the report, where none of them do.
+        //
+        // One stat per recorded snapshot, once per cycle, hoisted out of the
+        // per-file path exactly like checksumsByCollection. The syscalls are
+        // ~2.2ms per 1842 paths and cost the same whether the files are there
+        // or not. At the build level it does not register: warm rebuilds of
+        // the 10k perf corpus ran 3.36s median with this against 3.56s
+        // without, i.e. nominally faster, which is only to say the difference
+        // is well inside the run-to-run spread (3.0-4.1 vs 3.5-4.7 over five
+        // runs each). Re-measure with `npm run test:perf` before trusting a
+        // claim that it got slower.
+        //
+        // Resolves destinations through resolveOutputPath, which is what
+        // auditOutput uses. The two must agree — `--audit-output` reporting
+        // files that the build just called unchanged is the symptom, not a
+        // separate diagnostic.
+        // Memoized for the cycle. Two callers ask — the source checksum gate
+        // once per scan, recordedHashes() once per layouts dispatch — and both
+        // ask before anything has rendered, so one walk answers both. Cleared
+        // at the end of onFinalize, which is where this cycle's renders are
+        // recorded and the answer stops being true.
+        missingOutputIds() {
+            if (cachedMissingOutputs) return cachedMissingOutputs
+            const missing = new Set()
+            for (const row of stmtSelectAll.iterate()) {
+                const snap = rowToSnap(row)
+                if (!snap.destination) continue
+                if (!existsSync(resolveOutputPath(snap.destination))) missing.add(snap.id)
+            }
+            cachedMissingOutputs = missing
+            return missing
+        },
+
+        // Drops the memo. Called at the end of onFinalize; exported on the
+        // surface so a test can force a re-read without a full cycle.
+        forgetMissingOutputs() {
+            cachedMissingOutputs = null
+        },
+
         // Walk the output folder against recorded snapshots, returning
         // a diff describing missing / mismatched / orphaned /
         // unverifiable. Backs `mikser --audit-output`. Pure: no mutations.
@@ -1442,4 +1537,11 @@ onFinalize(async () => {
             + 'a render rewrites its own snapshot, so afterwards the new bytes agree with themselves.',
             drifted.length, drifted.length > SHOWN ? `, ${SHOWN} shown` : '')
     }
+
+    // This cycle just wrote files and recorded snapshots for them, so the
+    // memoized missing-output set describes a state that no longer exists.
+    // Dropped here rather than at the start of the next cycle because in
+    // watch mode there may not be one for hours, and a stale set held that
+    // long would keep re-dispatching entities it saw as missing.
+    m.forgetMissingOutputs()
 })

package/src/plugins/assets.js

@@ -348,6 +348,17 @@ export function assets(options = {}) {
     }
 
     async function isPresetRendered(entity) {
+        // The marker says a render HAPPENED. It does not say the file it
+        // produced is still there, and the two come apart the moment anyone
+        // deletes a derivative — which is the obvious thing to do when you
+        // want one rebuilt. The marker survives, this returned true, and the
+        // image never came back on any number of ordinary builds; only
+        // --force or a config change brought it round, and --audit-output was
+        // the only thing that ever said a file was missing.
+        //
+        // Checked first and cheaply: one stat, and if the derivative is gone
+        // no amount of marker archaeology changes the answer.
+        if (!existsSync(entity.destination)) return false
         let result = false
         let revisions = []
         const assetChecksum = `${entity.destination}.${entity.preset.checksum}.md5`

package/src/plugins/files.js

@@ -4,7 +4,7 @@ import { mkdir, symlink, unlink, lstat, realpath } from 'fs/promises'
 import { globby } from 'globby'
 import pMap from 'p-map'
 import { checksumsByCollection } from '../catalog.js'
-import { sweepDeleted } from '../source.js'
+import { gateChecksum, sweepDeleted } from '../source.js'
 
 export function files(options = {}) {
     return ({
@@ -187,23 +187,32 @@ export function files(options = {}) {
 
             const paths = await globby('**/*', { cwd: runtime.options.filesFolder })
             trackProgress('Files import', paths.length)
-            // Bulk-prefetch the catalog's existing (id → checksum) map for
-            // this collection once at scan start, so the gate below reads
-            // from it per-file instead of doing per-file SQL lookups. Same
-            // pattern source.js uses for documents/layouts via useSource.
-            // Without this gate the plugin re-emitted createEntity on every
-            // cycle for every file regardless of changes, inflating the
-            // journal with phantom mutations and triggering downstream
-            // re-dispatch of aggregate layouts whose recorded query deps
-            // matched the collection.
-            // --force (and a wiped catalog) must defeat this gate, the
-            // same way source.js's gateChecksum lets them defeat its own.
-            // This is a second, independent gate: if it ignores them, no
-            // amount of forcing re-derives a file's name / meta.url /
-            // meta.presets, and a catalog holding bad `files` rows has no
-            // repair path short of deleting them.
-            const forced = runtime.options.force || runtime.catalog?.cacheInvalidated
+            // The SHARED gate, not a copy of it.
+            //
+            // Without a gate the plugin re-emitted createEntity on every cycle
+            // for every file regardless of changes, inflating the journal with
+            // phantom mutations and triggering downstream re-dispatch of
+            // aggregate layouts whose recorded query deps matched the
+            // collection. So it grew one — and then it was a SECOND,
+            // independent implementation of the decision source.js already
+            // owns, which is how it silently missed the missing-output bypass
+            // that was added to the original: an image derivative deleted from
+            // the assets folder was never regenerated, because the file it
+            // derives from was gated here as unchanged and never reached the
+            // assets plugin. Green build, missing image, and only
+            // --audit-output said otherwise.
+            //
+            // Every reason to defeat the gate — --force, a wiped catalog, a
+            // recorded output that is gone — now lives in exactly one place,
+            // so the next one cannot be added to that place and missed here.
+            //
+            // priorChecksums is still bulk-prefetched per scan so the gate
+            // reads a map instead of doing per-file SQL, and missingOutputs
+            // alongside it for the same reason; the manifest memoizes the
+            // latter for the cycle, so this shares the walk with useSource's
+            // own scans rather than paying for a second one.
             const priorChecksums = checksumsByCollection(collection)
+            const missingOutputs = runtime.manifest?.missingOutputIds() ?? new Set()
             const scanned = new Set()
             await pMap(paths, async relativePath => {
                 const { source } = await ensureLink(relativePath)
@@ -217,15 +226,13 @@ export function files(options = {}) {
                 // to re-emit; the sweep asks what still EXISTS, and a file
                 // skipped for being unchanged very much exists.
                 scanned.add(id)
-                const newChecksum = await checksum(source)
+                // Returns the checksum to stamp on the entity, or null when
+                // the catalog already has this file unchanged and there is
+                // nothing to emit. Progress ticks either way — a gated file
+                // was still looked at.
+                const newChecksum = await gateChecksum(source, id, { priorChecksums, missingOutputs })
                 updateProgress()
-                // Gate: if the catalog already has this entity with the same
-                // checksum, the file hasn't changed since the last cycle.
-                // Skip emitting a CREATE — the catalog row stays correct,
-                // the journal stays accurate (mutations = actual changes),
-                // and downstream aggregate-layout invalidation isn't fired
-                // spuriously.
-                if (!forced && priorChecksums.get(id) === newChecksum) return
+                if (newChecksum === null) return
                 await createEntity({
                     id,
                     uri,

package/src/source.js

@@ -81,11 +81,19 @@ const SCAN_CONCURRENCY = 16
 // disagree, and the losing combination (empty content + a checksum correct
 // for the finished file) is permanent, because every later sync then
 // short-circuits on "unchanged".
-export async function gateChecksum(file, id, { reload = false, priorChecksums, bytes } = {}) {
+// `missingOutputs`, when given, is the set of entity ids whose last render
+// wrote a file that is no longer on disk (manifest.missingOutputIds()). The
+// gate answers "has this INPUT changed", and deleting the output folder does
+// not change any input — so without this, `rm -rf out` followed by a build
+// short-circuits every file here, emits nothing, renders nothing, and reports
+// success over an empty folder. An entity whose output is gone is not
+// unchanged in any sense the caller means.
+export async function gateChecksum(file, id, { reload = false, priorChecksums, bytes, missingOutputs } = {}) {
     const compute = () => (bytes !== undefined ? checksumOf(bytes) : fileChecksum(file))
     const canGate = !reload
         && !runtime.options.force
         && !runtime.catalog?.cacheInvalidated
+        && !missingOutputs?.has(id)
     if (canGate) {
         const priorChecksum = priorChecksums
             ? priorChecksums.get(id)
@@ -422,6 +430,12 @@ export function useSource(core, options) {
         // per-file instead of doing per-file SQL lookups. ~14× faster
         // at 14k entities (column projection vs full-entity JSON.parse).
         const priorChecksums = checksumsByCollection(collection)
+        // Prefetched once per scan, beside priorChecksums and for the same
+        // reason: the gate needs an O(1) answer per file, and this is one
+        // pass over the snapshots rather than a stat inside the per-file
+        // path. Entities listed here defeat the gate — their input is
+        // unchanged but the file they produced is gone.
+        const missingOutputs = runtime.manifest?.missingOutputIds() ?? new Set()
         // Parallel register — file read + checksum is I/O-bound. Set
         // additions, scanStats increments, and journal addEntry calls
         // are all single-threaded-JS-atomic so no locking required;
@@ -429,7 +443,7 @@ export function useSource(core, options) {
         // and INSERTs are ~10μs vs ~ms-per-file-read, so the journal
         // is never the bottleneck.
         await pMap(files, async (file) => {
-            await registerFile(file, { logger, scanned, stats: scanStats, priorChecksums })
+            await registerFile(file, { logger, scanned, stats: scanStats, priorChecksums, missingOutputs })
             if (phase === 'import') updateProgress()
         }, { concurrency: SCAN_CONCURRENCY })
 
@@ -450,7 +464,7 @@ export function useSource(core, options) {
         logger.info(scanSummary({ cap, loaded: files.length, ...scanStats }))
     })
 
-    async function registerFile(file, { logger, action = ACTION.CREATE, scanned, priorChecksums } = {}) {
+    async function registerFile(file, { logger, action = ACTION.CREATE, scanned, priorChecksums, missingOutputs } = {}) {
         // stats — when called from the scanHook the outer scan provides
         // a Map to accumulate counts. For onSync (chokidar single-file
         // events) the counter is absent and per-file tally is not
@@ -483,7 +497,7 @@ export function useSource(core, options) {
             }
         }
 
-        const chksum = await gateChecksum(file, id, { reload, priorChecksums, bytes })
+        const chksum = await gateChecksum(file, id, { reload, priorChecksums, bytes, missingOutputs })
         if (chksum === null) {
             if (stats) stats.skipped++
             // Never becomes a render task, so it would otherwise be invisible