mikser-io 10.0.0 → 10.0.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mikser-io",
-  "version": "10.0.0",
+  "version": "10.0.2",
   "files": [
     "app.js",
     "index.js",
@@ -96,5 +96,9 @@
   "bugs": {
     "url": "https://github.com/almero-digital-marketing/mikser-io/issues"
   },
-  "homepage": "https://github.com/almero-digital-marketing/mikser-io#readme"
+  "homepage": "https://github.com/almero-digital-marketing/mikser-io#readme",
+  "allowScripts": {
+    "better-sqlite3": true,
+    "sharp": true
+  }
 }

package/src/explain.js

@@ -8,7 +8,9 @@
 // needs knowledge a user of the tool should not need.
 //
 // Follows --audit-output's shape: report and exit, no build phases run.
+import { existsSync } from 'node:fs'
 import { inputHashOf, inputPartsOf, diffInputParts, lookupKeys, checksum as fileChecksum } from './utils.js'
+import { resolveOutputPath } from './manifest.js'
 import { filterKey } from './track.js'
 import { findEntity, findEntities, findById } from './catalog.js'
 import runtime from './runtime.js'
@@ -134,6 +136,13 @@ export async function explain(reference) {
     const failedSnapshots = snapshots
         .map(snap => runtime.manifest?.failureAt?.(entity.id, snap.destination))
         .filter(Boolean)
+    // Same position in the verdict for the same reason: a destination whose
+    // file is gone re-renders whatever the hashes say. Ranked below a failed
+    // attempt — if the last render threw, that is the more useful sentence,
+    // and it explains the absence too.
+    const missingDestinations = snapshots
+        .filter(snap => snap.destination && !existsSync(resolveOutputPath(snap.destination)))
+        .map(snap => snap.destination)
 
     // The catalog is as of the LAST BUILD. If the file has been edited since,
     // nothing here knows it yet — the hashes would all agree and the verdict
@@ -226,6 +235,18 @@ export async function explain(reference) {
                 : snap.inputParts
                     ? diffInputParts(snap.inputParts, currentParts)
                     : 'unknown',
+            // The file this render wrote, gone from disk.
+            //
+            // Same shape as `failed` above and added for the same reason: the
+            // recorded state is entirely consistent — the input hash matches,
+            // the snapshot is intact — so without this the report reads
+            // `[current]` and `would be SKIPPED` for an entity a build now
+            // re-renders. --explain contradicting the build is worse than
+            // --explain being incomplete, because it is the tool someone
+            // reaches for when the build has already surprised them.
+            missing: snap.destination
+                ? !existsSync(resolveOutputPath(snap.destination))
+                : false,
             outputHash: snap.outputHash ?? null,
             parent: snap.parent ?? null,
             // Which keys of its OWN meta the render read, and which keys it
@@ -286,6 +307,10 @@ export async function explain(reference) {
                   + `(${failedSnapshots[0].error})`
             : snapshots.some(s => s.inputHash !== currentHash)
                 ? renderVerdict(snapshots, currentHash, currentParts)
+            : missingDestinations.length
+                ? `would re-render — the output is gone from disk (${missingDestinations.join(', ')}). `
+                  + 'The inputs are unchanged, so nothing about the entity says this; the file being '
+                  + 'absent is the whole reason.'
                 : 'would be SKIPPED — input hash unchanged. A dependency in refClosure changing is the only other thing that would re-render it.'),
         lookupKeys: lookupKeys(entity),
         // Other entities rendering to the same path as this one.
@@ -360,6 +385,7 @@ export function formatExplain(report) {
     for (const r of report.renders) {
         row('rendered', `${r.renderedAt ?? 'unknown'}   → ${r.destination}`
             + (r.failed ? '   [STALE: last render attempt failed]'
+                : r.missing ? '   [MISSING: the file is not on disk]'
                 : r.stale ? '   [STALE: input hash moved since]'
                 : '   [current]'))
         if (r.failed) {

package/src/manifest.js

@@ -372,6 +372,11 @@ export async function sourcesOf(destination) {
 export function createManifest(db) {
     if (!db) throw new Error('createManifest: db is required')
 
+    // Per-instance, not module-level: tests build their own manifest via
+    // createManifest(db), and a shared memo would leak one test's output
+    // folder into the next.
+    let cachedMissingOutputs = null
+
     const stmtCollisions = db.prepare(`
         SELECT destination, count(*) AS n, group_concat(id) AS ids
         FROM mikser_snapshots
@@ -657,6 +662,7 @@ export function createManifest(db) {
         //
         //   unchanged        nothing this render depends on moved
         //   never-rendered   no snapshot: first build, or it never rendered
+        //   output-missing   the file the last render wrote is gone from disk
         //   inputs-changed   the entity's own hash moved
         //   ref-changed      a $-ref or partial it depends on moved
         //   query-matched    an entity matching a recorded query mutated
@@ -715,6 +721,32 @@ export function createManifest(db) {
                     changed: describeInputChange(entity, snapshot),
                 }
             }
+            // The inputs say nothing moved. That is only a reason to skip if
+            // the thing the last render PRODUCED is still there.
+            //
+            // Every branch above this one reasons about inputs, and inputs are
+            // not where `rm -rf out` shows up: the documents are unchanged,
+            // because what was deleted is the output. So the build skips every
+            // entity, renders nothing, prints no `Rendered:` line and exits 0
+            // with an empty output folder — correct by its own reasoning and
+            // wrong about the only question the caller asked. A missing output
+            // is a reason to re-render, not a reason to report unchanged.
+            //
+            // Placed after the input comparison and before the refClosure walk
+            // so one check covers both `unchanged` exits, and placed here
+            // rather than in the render loop because this is the function that
+            // owns the word: anything asking shouldSkip() gets the same answer.
+            //
+            // Checked unconditionally rather than behind a flag. It is one stat
+            // per entity that reached this point — 2.2ms for 1842 entities,
+            // the same whether the files are there or not — against a failure
+            // mode whose whole character is that nobody thinks to look for it.
+            // auditOutput() resolves destinations exactly this way, and it has
+            // to stay that way: the two disagreeing is the bug, since `--audit-output`
+            // finding what the build just called clean is what sent someone here.
+            if (snapshot.destination && !existsSync(resolveOutputPath(snapshot.destination))) {
+                return { skip: false, reason: 'output-missing', destination: snapshot.destination }
+            }
             if (!snapshot.refClosure?.length) return { skip: true, reason: 'unchanged' }
             const sourceLang = entity?.meta?.lang ?? null
             for (const entry of snapshot.refClosure) {
@@ -988,12 +1020,25 @@ export function createManifest(db) {
         // for the same id (matches the prior first-seen-wins semantic;
         // entity inputHashes were always added before deps in the
         // earlier loop).
+        // An entity whose output is gone is omitted entirely. The consumer
+        // uses a recorded hash to decide "this needs no render", and that
+        // conclusion only follows while the file the hash describes is still
+        // there. Omitting is the whole fix at this gate: a seed with no
+        // recorded hash is always dispatched.
+        //
+        // Done here rather than in the dispatcher because this map exists for
+        // that one decision, and a second consumer would otherwise have to
+        // rediscover the same caveat. missingOutputIds() is memoized for the
+        // cycle, so this shares the walk the source gate already paid for.
         recordedHashes() {
+            const missingOutputs = this.missingOutputIds()
             const map = new Map()
             for (const row of stmtEntityInputHashes.iterate()) {
+                if (missingOutputs.has(row.id)) continue
                 map.set(row.id, row.inputHash)
             }
             for (const row of stmtDepHashes.iterate()) {
+                if (missingOutputs.has(row.target)) continue
                 if (!map.has(row.target)) map.set(row.target, row.hash)
             }
             return map
@@ -1061,6 +1106,53 @@ export function createManifest(db) {
             return edges
         },
 
+        // The entity ids whose last render wrote a file that is no longer
+        // on disk. A dispatch hint, the same shape as queryAffected: the
+        // source checksum gate consults it so those entities are re-emitted
+        // instead of being short-circuited as unchanged.
+        //
+        // This exists because the gate that stops `rm -rf out` from being
+        // noticed is the FIRST one, not the last. An unchanged file never
+        // gets a CREATE, so it never reaches the journal, so the render loop
+        // never sees it and manifest.skipDecision — which does check for a
+        // missing output — is never asked. Fixing only the render gate makes
+        // the build correct for entities that get that far and changes
+        // nothing for the case in the report, where none of them do.
+        //
+        // One stat per recorded snapshot, once per cycle, hoisted out of the
+        // per-file path exactly like checksumsByCollection. The syscalls are
+        // ~2.2ms per 1842 paths and cost the same whether the files are there
+        // or not; at the build level a warm 2000-document rebuild measured
+        // 1.05s against 1.08s without it, which is inside the run-to-run
+        // spread — the walk disappears into the cycle it is part of.
+        //
+        // Resolves destinations through resolveOutputPath, which is what
+        // auditOutput uses. The two must agree — `--audit-output` reporting
+        // files that the build just called unchanged is the symptom, not a
+        // separate diagnostic.
+        // Memoized for the cycle. Two callers ask — the source checksum gate
+        // once per scan, recordedHashes() once per layouts dispatch — and both
+        // ask before anything has rendered, so one walk answers both. Cleared
+        // at the end of onFinalize, which is where this cycle's renders are
+        // recorded and the answer stops being true.
+        missingOutputIds() {
+            if (cachedMissingOutputs) return cachedMissingOutputs
+            const missing = new Set()
+            for (const row of stmtSelectAll.iterate()) {
+                const snap = rowToSnap(row)
+                if (!snap.destination) continue
+                if (!existsSync(resolveOutputPath(snap.destination))) missing.add(snap.id)
+            }
+            cachedMissingOutputs = missing
+            return missing
+        },
+
+        // Drops the memo. Called at the end of onFinalize; exported on the
+        // surface so a test can force a re-read without a full cycle.
+        forgetMissingOutputs() {
+            cachedMissingOutputs = null
+        },
+
         // Walk the output folder against recorded snapshots, returning
         // a diff describing missing / mismatched / orphaned /
         // unverifiable. Backs `mikser --audit-output`. Pure: no mutations.
@@ -1442,4 +1534,11 @@ onFinalize(async () => {
             + 'a render rewrites its own snapshot, so afterwards the new bytes agree with themselves.',
             drifted.length, drifted.length > SHOWN ? `, ${SHOWN} shown` : '')
     }
+
+    // This cycle just wrote files and recorded snapshots for them, so the
+    // memoized missing-output set describes a state that no longer exists.
+    // Dropped here rather than at the start of the next cycle because in
+    // watch mode there may not be one for hours, and a stale set held that
+    // long would keep re-dispatching entities it saw as missing.
+    m.forgetMissingOutputs()
 })

package/src/source.js

@@ -81,11 +81,19 @@ const SCAN_CONCURRENCY = 16
 // disagree, and the losing combination (empty content + a checksum correct
 // for the finished file) is permanent, because every later sync then
 // short-circuits on "unchanged".
-export async function gateChecksum(file, id, { reload = false, priorChecksums, bytes } = {}) {
+// `missingOutputs`, when given, is the set of entity ids whose last render
+// wrote a file that is no longer on disk (manifest.missingOutputIds()). The
+// gate answers "has this INPUT changed", and deleting the output folder does
+// not change any input — so without this, `rm -rf out` followed by a build
+// short-circuits every file here, emits nothing, renders nothing, and reports
+// success over an empty folder. An entity whose output is gone is not
+// unchanged in any sense the caller means.
+export async function gateChecksum(file, id, { reload = false, priorChecksums, bytes, missingOutputs } = {}) {
     const compute = () => (bytes !== undefined ? checksumOf(bytes) : fileChecksum(file))
     const canGate = !reload
         && !runtime.options.force
         && !runtime.catalog?.cacheInvalidated
+        && !missingOutputs?.has(id)
     if (canGate) {
         const priorChecksum = priorChecksums
             ? priorChecksums.get(id)
@@ -422,6 +430,12 @@ export function useSource(core, options) {
         // per-file instead of doing per-file SQL lookups. ~14× faster
         // at 14k entities (column projection vs full-entity JSON.parse).
         const priorChecksums = checksumsByCollection(collection)
+        // Prefetched once per scan, beside priorChecksums and for the same
+        // reason: the gate needs an O(1) answer per file, and this is one
+        // pass over the snapshots rather than a stat inside the per-file
+        // path. Entities listed here defeat the gate — their input is
+        // unchanged but the file they produced is gone.
+        const missingOutputs = runtime.manifest?.missingOutputIds() ?? new Set()
         // Parallel register — file read + checksum is I/O-bound. Set
         // additions, scanStats increments, and journal addEntry calls
         // are all single-threaded-JS-atomic so no locking required;
@@ -429,7 +443,7 @@ export function useSource(core, options) {
         // and INSERTs are ~10μs vs ~ms-per-file-read, so the journal
         // is never the bottleneck.
         await pMap(files, async (file) => {
-            await registerFile(file, { logger, scanned, stats: scanStats, priorChecksums })
+            await registerFile(file, { logger, scanned, stats: scanStats, priorChecksums, missingOutputs })
             if (phase === 'import') updateProgress()
         }, { concurrency: SCAN_CONCURRENCY })
 
@@ -450,7 +464,7 @@ export function useSource(core, options) {
         logger.info(scanSummary({ cap, loaded: files.length, ...scanStats }))
     })
 
-    async function registerFile(file, { logger, action = ACTION.CREATE, scanned, priorChecksums } = {}) {
+    async function registerFile(file, { logger, action = ACTION.CREATE, scanned, priorChecksums, missingOutputs } = {}) {
         // stats — when called from the scanHook the outer scan provides
         // a Map to accumulate counts. For onSync (chokidar single-file
         // events) the counter is absent and per-file tally is not
@@ -483,7 +497,7 @@ export function useSource(core, options) {
             }
         }
 
-        const chksum = await gateChecksum(file, id, { reload, priorChecksums, bytes })
+        const chksum = await gateChecksum(file, id, { reload, priorChecksums, bytes, missingOutputs })
         if (chksum === null) {
             if (stats) stats.skipped++
             // Never becomes a render task, so it would otherwise be invisible