mikser-io 10.0.4 → 10.1.0

package/index.js

@@ -1,6 +1,7 @@
 export { default as runtime } from './src/runtime.js'
 export * as constants from './src/constants.js'
 export * from './src/utils.js'
+export * from './src/invalidation.js'
 export * from './src/auth.js'
 export * from './src/roles.js'
 export * from './src/inventory.js'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mikser-io",
-  "version": "10.0.4",
+  "version": "10.1.0",
   "files": [
     "app.js",
     "index.js",

package/src/explain.js

@@ -10,7 +10,7 @@
 // Follows --audit-output's shape: report and exit, no build phases run.
 import { existsSync } from 'node:fs'
 import { inputHashOf, inputPartsOf, diffInputParts, lookupKeys, checksum as fileChecksum } from './utils.js'
-import { resolveOutputPath } from './manifest.js'
+import { outputMissing } from './invalidation.js'
 import { filterKey } from './track.js'
 import { findEntity, findEntities, findById } from './catalog.js'
 import runtime from './runtime.js'
@@ -141,7 +141,7 @@ export async function explain(reference) {
     // attempt — if the last render threw, that is the more useful sentence,
     // and it explains the absence too.
     const missingDestinations = snapshots
-        .filter(snap => snap.destination && !existsSync(resolveOutputPath(snap.destination)))
+        .filter(snap => outputMissing(snap.destination))
         .map(snap => snap.destination)
 
     // The catalog is as of the LAST BUILD. If the file has been edited since,
@@ -244,9 +244,7 @@ export async function explain(reference) {
             // re-renders. --explain contradicting the build is worse than
             // --explain being incomplete, because it is the tool someone
             // reaches for when the build has already surprised them.
-            missing: snap.destination
-                ? !existsSync(resolveOutputPath(snap.destination))
-                : false,
+            missing: outputMissing(snap.destination),
             outputHash: snap.outputHash ?? null,
             parent: snap.parent ?? null,
             // Which keys of its OWN meta the render read, and which keys it

package/src/invalidation.js

@@ -0,0 +1,158 @@
+// Should this work be redone?
+//
+// One question, asked at four layers with different evidence in hand:
+//
+//   source.js      a file's checksum against the catalog's
+//   manifest       a recorded hash, for the layouts dispatcher's seeding
+//   manifest       a snapshot plus its refClosure, at the render gate
+//   assets         a `.md5` marker at the current preset revision
+//
+// The evidence differs and belongs to each layer. What does NOT differ is
+// what OVERRIDES the evidence — and that is what this module owns, because
+// it was written four times and drifted the moment a fifth override was
+// added to one of them.
+//
+// The drift, concretely: `rm -rf out` changes no input, so every layer
+// answered "unchanged" and a build rendered nothing, printed no `Rendered:`
+// line and exited 0 over an empty output folder. Adding an output-existence
+// check to the render gate fixed nothing, because the source gate had
+// already dropped the entity; adding it there fixed the site but not image
+// derivatives, because files.js held a private copy of that gate; and fixing
+// that still left the assets marker, which had never asked about the file at
+// all. Five gates, one question, four implementations of the override rules.
+//
+// So the rule is: a layer may own what it KNOWS. It may not own what
+// overrides it. Anything that makes a gate not apply — --force, a wiped
+// cache, a reload event, an output that is gone — is declared here once, and
+// a new one added here reaches every gate that ever asks.
+//
+// Deliberately NOT moved here: the dependency-graph walk in
+// manifest.skipDecision. That reasons over snapshots, refClosures and sift
+// filters the manifest owns, and hauling it out would trade a real coupling
+// for a fake one. It asks this module what overrides its answer, which is the
+// part that was duplicated.
+//
+// A leaf module: runtime and node builtins only. It reads `runtime.manifest`
+// lazily rather than importing it, the way every other consumer of engine
+// state does, so the manifest can import this without a cycle.
+
+import path from 'node:path'
+import { existsSync } from 'node:fs'
+import runtime from './runtime.js'
+
+// The vocabulary, in one place because it is asserted against.
+//
+// `--json` carries these strings out to callers and the scenario suite
+// matches on them, so they are API. Defined here rather than in the gate
+// that happens to emit each one, so the set can be read as a set.
+export const REASON = Object.freeze({
+    // Overrides — this module's business.
+    FORCE: 'force',
+    CACHE_INVALIDATED: 'cache-invalidated',
+    RELOAD: 'reload',
+    OUTPUT_MISSING: 'output-missing',
+    // Evidence — each layer's business, named here so the vocabulary is
+    // legible as a whole.
+    UNCHANGED: 'unchanged',
+    NEVER_RENDERED: 'never-rendered',
+    INPUTS_CHANGED: 'inputs-changed',
+    REF_CHANGED: 'ref-changed',
+    QUERY_MATCHED: 'query-matched',
+    CACHE_DISABLED: 'cache-disabled',
+    RETRY_FAILED: 'retry-failed',
+})
+
+// Where an output lives, given what a snapshot recorded.
+//
+// Destinations come in two shapes and both are legitimate: output-relative
+// for a rendered page (`/index.html`), absolute for a derivative the assets
+// plugin places itself. Resolving relative-first and falling back to the
+// literal path covers both without the caller having to know which it holds.
+//
+// Returns the joined path even when nothing exists there, so a caller can
+// report WHICH path it looked for — "missing" without a path is a fact
+// nobody can act on.
+export function resolveOutputPath(destination, outputFolder = runtime.options?.outputFolder) {
+    if (!destination) return undefined
+    const joined = path.join(outputFolder ?? '', destination)
+    if (existsSync(joined)) return joined
+    if (path.isAbsolute(destination) && existsSync(destination)) return destination
+    return joined
+}
+
+// Is the file this destination names gone?
+export function outputMissing(destination) {
+    if (!destination) return false
+    return !existsSync(resolveOutputPath(destination))
+}
+
+// Memoized for the cycle.
+//
+// Every gate asks, and they all ask before anything has rendered, so one walk
+// answers all of them. Dropped at the end of onFinalize, where this cycle's
+// renders are recorded and the answer stops being true — not at the start of
+// the next cycle, because in watch mode there may not be one for hours and a
+// stale set held that long would re-dispatch entities it saw as missing.
+let cachedMissingOutputs = null
+
+// The entity ids whose last render wrote a file that is no longer on disk.
+//
+// One stat per recorded snapshot. The syscalls are ~2.2ms per 1842 paths and
+// cost the same whether the files are there or not; at the build level it does
+// not register — warm rebuilds of the 10k perf corpus ran 3.36s median with
+// this against 3.56s without, i.e. nominally faster, which is only to say the
+// difference is inside the run-to-run spread. Re-measure with
+// `npm run test:perf` before trusting a claim that it got slower.
+export function missingOutputIds() {
+    if (cachedMissingOutputs) return cachedMissingOutputs
+    const missing = new Set()
+    for (const snapshot of runtime.manifest?.all?.() ?? []) {
+        if (!snapshot.destination) continue
+        if (outputMissing(snapshot.destination)) missing.add(snapshot.id)
+    }
+    cachedMissingOutputs = missing
+    return missing
+}
+
+export function forgetMissingOutputs() {
+    cachedMissingOutputs = null
+}
+
+// What overrides a gate's own evidence, or null when nothing does.
+//
+// Callers pass the evidence they hold: `reload` if they are a watch event,
+// `id` if they are gating an entity whose outputs are recorded. Passing
+// neither asks only the universal question, which is what the render gate
+// wants — it checks the output separately and later, so that an entity whose
+// inputs ALSO moved reports `inputs-changed` rather than losing that detail
+// to a broader answer.
+export function bypassReason({ reload = false, id } = {}) {
+    if (reload) return REASON.RELOAD
+    if (runtime.options?.force) return REASON.FORCE
+    if (runtime.catalog?.cacheInvalidated) return REASON.CACHE_INVALIDATED
+    if (id !== undefined && missingOutputIds().has(id)) return REASON.OUTPUT_MISSING
+    return null
+}
+
+// Is every entity being presented for evaluation this cycle?
+//
+// The same fact as bypassReason, asked for a different purpose: a "declared X
+// matched nothing" warning is only worth printing when everything was looked
+// at. On an incremental cycle a pattern legitimately matches nothing in a run
+// of two, and a warning that always fires is one people filter out — taking
+// the real instance with it.
+//
+// `firstRun` counts here and not in bypassReason because there is nothing to
+// bypass on a first run: no prior checksum, no snapshot, so every gate opens
+// on its own evidence.
+//
+// Takes the runtime rather than reading the singleton, so a plugin passes the
+// one it was injected with — the singleton in a real build, the harness's
+// stand-in under test.
+export function isFullCycle(engineRuntime = runtime) {
+    return !!(
+        engineRuntime?.options?.force
+        || engineRuntime?.options?.firstRun
+        || engineRuntime?.catalog?.cacheInvalidated
+    )
+}

package/src/manifest.js

@@ -60,6 +60,9 @@ import { extractRefs, inputHashOf, inputPartsOf, diffInputParts, lookupKeys } fr
 import { filterKey } from './track.js'
 import { findById, findEntities } from './catalog.js'
 import { useDatabase, registerSchema } from './database/index.js'
+import {
+    REASON, bypassReason, resolveOutputPath, outputMissing, missingOutputIds, forgetMissingOutputs,
+} from './invalidation.js'
 
 // Schema registration. Applied at db.open(). PRIMARY KEY (id,
 // destination) — leading id column means `WHERE id = ?` queries use the
@@ -284,13 +287,6 @@ function snapToRow(snap) {
 // becomes two symptoms — one loud (every asset reported missing, at its
 // real and present path) and one silent (no outputHash recorded, leaving
 // most snapshots presence-checked only).
-export function resolveOutputPath(destination, outputFolder = runtime.options?.outputFolder) {
-    if (!destination) return undefined
-    const joined = path.join(outputFolder ?? '', destination)
-    if (existsSync(joined)) return joined
-    if (path.isAbsolute(destination) && existsSync(destination)) return destination
-    return joined
-}
 
 async function hashOutputFile(destination) {
     const filePath = resolveOutputPath(destination)
@@ -372,11 +368,6 @@ export async function sourcesOf(destination) {
 export function createManifest(db) {
     if (!db) throw new Error('createManifest: db is required')
 
-    // Per-instance, not module-level: tests build their own manifest via
-    // createManifest(db), and a shared memo would leak one test's output
-    // folder into the next.
-    let cachedMissingOutputs = null
-
     const stmtCollisions = db.prepare(`
         SELECT destination, count(*) AS n, group_concat(id) AS ids
         FROM mikser_snapshots
@@ -682,8 +673,12 @@ export function createManifest(db) {
             // That is the situation --force exists for — the invalidation
             // graph being under suspicion — including where the preset
             // no-match warning tells the operator to use it.
-            if (runtime.options?.force) return { skip: false, reason: 'force' }
-            if (entity?.meta?.cache === false) return { skip: false, reason: 'cache-disabled' }
+            // --force and a wiped cache both mean "ignore what you think you
+            // know", and both are declared in invalidation.js so a third one
+            // added there reaches this gate without being remembered.
+            const override = bypassReason()
+            if (override) return { skip: false, reason: override }
+            if (entity?.meta?.cache === false) return { skip: false, reason: REASON.CACHE_DISABLED }
             // A render whose last attempt threw must be retried, and checked
             // before anything else: every other branch reasons about hashes,
             // and the hashes are consistent — the entity did not change, the
@@ -737,15 +732,13 @@ export function createManifest(db) {
             // rather than in the render loop because this is the function that
             // owns the word: anything asking shouldSkip() gets the same answer.
             //
-            // Checked unconditionally rather than behind a flag. It is one stat
-            // per entity that reached this point — 2.2ms for 1842 entities,
-            // the same whether the files are there or not — against a failure
-            // mode whose whole character is that nobody thinks to look for it.
-            // auditOutput() resolves destinations exactly this way, and it has
-            // to stay that way: the two disagreeing is the bug, since `--audit-output`
-            // finding what the build just called clean is what sent someone here.
-            if (snapshot.destination && !existsSync(resolveOutputPath(snapshot.destination))) {
-                return { skip: false, reason: 'output-missing', destination: snapshot.destination }
+            // Asked of invalidation.js rather than answered here, so that
+            // auditOutput, the source gate and the assets marker resolve a
+            // destination the same way. The two disagreeing is the bug —
+            // `--audit-output` finding what the build just called clean is
+            // what sends people to this function in the first place.
+            if (outputMissing(snapshot.destination)) {
+                return { skip: false, reason: REASON.OUTPUT_MISSING, destination: snapshot.destination }
             }
             if (!snapshot.refClosure?.length) return { skip: true, reason: 'unchanged' }
             const sourceLang = entity?.meta?.lang ?? null
@@ -1031,7 +1024,7 @@ export function createManifest(db) {
         // rediscover the same caveat. missingOutputIds() is memoized for the
         // cycle, so this shares the walk the source gate already paid for.
         recordedHashes() {
-            const missingOutputs = this.missingOutputIds()
+            const missingOutputs = missingOutputIds()
             const map = new Map()
             for (const row of stmtEntityInputHashes.iterate()) {
                 if (missingOutputs.has(row.id)) continue
@@ -1106,56 +1099,6 @@ export function createManifest(db) {
             return edges
         },
 
-        // The entity ids whose last render wrote a file that is no longer
-        // on disk. A dispatch hint, the same shape as queryAffected: the
-        // source checksum gate consults it so those entities are re-emitted
-        // instead of being short-circuited as unchanged.
-        //
-        // This exists because the gate that stops `rm -rf out` from being
-        // noticed is the FIRST one, not the last. An unchanged file never
-        // gets a CREATE, so it never reaches the journal, so the render loop
-        // never sees it and manifest.skipDecision — which does check for a
-        // missing output — is never asked. Fixing only the render gate makes
-        // the build correct for entities that get that far and changes
-        // nothing for the case in the report, where none of them do.
-        //
-        // One stat per recorded snapshot, once per cycle, hoisted out of the
-        // per-file path exactly like checksumsByCollection. The syscalls are
-        // ~2.2ms per 1842 paths and cost the same whether the files are there
-        // or not. At the build level it does not register: warm rebuilds of
-        // the 10k perf corpus ran 3.36s median with this against 3.56s
-        // without, i.e. nominally faster, which is only to say the difference
-        // is well inside the run-to-run spread (3.0-4.1 vs 3.5-4.7 over five
-        // runs each). Re-measure with `npm run test:perf` before trusting a
-        // claim that it got slower.
-        //
-        // Resolves destinations through resolveOutputPath, which is what
-        // auditOutput uses. The two must agree — `--audit-output` reporting
-        // files that the build just called unchanged is the symptom, not a
-        // separate diagnostic.
-        // Memoized for the cycle. Two callers ask — the source checksum gate
-        // once per scan, recordedHashes() once per layouts dispatch — and both
-        // ask before anything has rendered, so one walk answers both. Cleared
-        // at the end of onFinalize, which is where this cycle's renders are
-        // recorded and the answer stops being true.
-        missingOutputIds() {
-            if (cachedMissingOutputs) return cachedMissingOutputs
-            const missing = new Set()
-            for (const row of stmtSelectAll.iterate()) {
-                const snap = rowToSnap(row)
-                if (!snap.destination) continue
-                if (!existsSync(resolveOutputPath(snap.destination))) missing.add(snap.id)
-            }
-            cachedMissingOutputs = missing
-            return missing
-        },
-
-        // Drops the memo. Called at the end of onFinalize; exported on the
-        // surface so a test can force a re-read without a full cycle.
-        forgetMissingOutputs() {
-            cachedMissingOutputs = null
-        },
-
         // Walk the output folder against recorded snapshots, returning
         // a diff describing missing / mismatched / orphaned /
         // unverifiable. Backs `mikser --audit-output`. Pure: no mutations.
@@ -1543,5 +1486,5 @@ onFinalize(async () => {
     // Dropped here rather than at the start of the next cycle because in
     // watch mode there may not be one for hours, and a stale set held that
     // long would keep re-dispatching entities it saw as missing.
-    m.forgetMissingOutputs()
+    forgetMissingOutputs()
 })

package/src/plugins/assets.js

@@ -6,9 +6,10 @@ import { mkdir, writeFile, unlink, rm, readFile, symlink, } from 'fs/promises'
 import { existsSync } from 'node:fs'
 import { createRequire } from 'node:module'
 import { globby } from 'globby'
+import { isFullCycle, outputMissing } from '../invalidation.js'
 import _ from 'lodash'
 import map from 'p-map'
-import { isFullCycle } from '../utils.js'
+
 
 // Normalize a `options.presets[name]` value to a consistent
 // { matches, options } shape so callers don't have to inspect which form
@@ -357,8 +358,10 @@ export function assets(options = {}) {
         // the only thing that ever said a file was missing.
         //
         // Checked first and cheaply: one stat, and if the derivative is gone
-        // no amount of marker archaeology changes the answer.
-        if (!existsSync(entity.destination)) return false
+        // no amount of marker archaeology changes the answer. Asked of
+        // invalidation.js so a derivative and a rendered page agree on what
+        // "the output is gone" means.
+        if (outputMissing(entity.destination)) return false
         let result = false
         let revisions = []
         const assetChecksum = `${entity.destination}.${entity.preset.checksum}.md5`

package/src/plugins/files.js

@@ -207,12 +207,9 @@ export function files(options = {}) {
             // so the next one cannot be added to that place and missed here.
             //
             // priorChecksums is still bulk-prefetched per scan so the gate
-            // reads a map instead of doing per-file SQL, and missingOutputs
-            // alongside it for the same reason; the manifest memoizes the
-            // latter for the cycle, so this shares the walk with useSource's
-            // own scans rather than paying for a second one.
+            // reads a map instead of doing per-file SQL. What overrides the
+            // checksum, the gate asks invalidation.js for itself.
             const priorChecksums = checksumsByCollection(collection)
-            const missingOutputs = runtime.manifest?.missingOutputIds() ?? new Set()
             const scanned = new Set()
             await pMap(paths, async relativePath => {
                 const { source } = await ensureLink(relativePath)
@@ -230,7 +227,7 @@ export function files(options = {}) {
                 // the catalog already has this file unchanged and there is
                 // nothing to emit. Progress ticks either way — a gated file
                 // was still looked at.
-                const newChecksum = await gateChecksum(source, id, { priorChecksums, missingOutputs })
+                const newChecksum = await gateChecksum(source, id, { priorChecksums })
                 updateProgress()
                 if (newChecksum === null) return
                 await createEntity({

package/src/source.js

@@ -46,6 +46,7 @@ import { ACTION } from './constants.js'
 import { checksum as fileChecksum, checksumOf, junkIgnore } from './utils.js'
 import { reportGated, reportChanged } from './report.js'
 import { findById, findEntities, checksumsByCollection } from './catalog.js'
+import { bypassReason } from './invalidation.js'
 import { useDatabase } from './database/index.js'
 
 // Per-source-scan registerFile concurrency. The work is dominated by
@@ -81,20 +82,12 @@ const SCAN_CONCURRENCY = 16
 // disagree, and the losing combination (empty content + a checksum correct
 // for the finished file) is permanent, because every later sync then
 // short-circuits on "unchanged".
-// `missingOutputs`, when given, is the set of entity ids whose last render
-// wrote a file that is no longer on disk (manifest.missingOutputIds()). The
-// gate answers "has this INPUT changed", and deleting the output folder does
-// not change any input — so without this, `rm -rf out` followed by a build
-// short-circuits every file here, emits nothing, renders nothing, and reports
-// success over an empty folder. An entity whose output is gone is not
-// unchanged in any sense the caller means.
-export async function gateChecksum(file, id, { reload = false, priorChecksums, bytes, missingOutputs } = {}) {
+export async function gateChecksum(file, id, { reload = false, priorChecksums, bytes } = {}) {
     const compute = () => (bytes !== undefined ? checksumOf(bytes) : fileChecksum(file))
-    const canGate = !reload
-        && !runtime.options.force
-        && !runtime.catalog?.cacheInvalidated
-        && !missingOutputs?.has(id)
-    if (canGate) {
+    // What overrides the checksum is not this function's to decide — see
+    // invalidation.js. It used to be, and the copy in files.js then missed
+    // the output-existence clause when that was added here.
+    if (!bypassReason({ reload, id })) {
         const priorChecksum = priorChecksums
             ? priorChecksums.get(id)
             : findById(id)?.checksum
@@ -430,12 +423,6 @@ export function useSource(core, options) {
         // per-file instead of doing per-file SQL lookups. ~14× faster
         // at 14k entities (column projection vs full-entity JSON.parse).
         const priorChecksums = checksumsByCollection(collection)
-        // Prefetched once per scan, beside priorChecksums and for the same
-        // reason: the gate needs an O(1) answer per file, and this is one
-        // pass over the snapshots rather than a stat inside the per-file
-        // path. Entities listed here defeat the gate — their input is
-        // unchanged but the file they produced is gone.
-        const missingOutputs = runtime.manifest?.missingOutputIds() ?? new Set()
         // Parallel register — file read + checksum is I/O-bound. Set
         // additions, scanStats increments, and journal addEntry calls
         // are all single-threaded-JS-atomic so no locking required;
@@ -443,7 +430,7 @@ export function useSource(core, options) {
         // and INSERTs are ~10μs vs ~ms-per-file-read, so the journal
         // is never the bottleneck.
         await pMap(files, async (file) => {
-            await registerFile(file, { logger, scanned, stats: scanStats, priorChecksums, missingOutputs })
+            await registerFile(file, { logger, scanned, stats: scanStats, priorChecksums })
             if (phase === 'import') updateProgress()
         }, { concurrency: SCAN_CONCURRENCY })
 
@@ -464,7 +451,7 @@ export function useSource(core, options) {
         logger.info(scanSummary({ cap, loaded: files.length, ...scanStats }))
     })
 
-    async function registerFile(file, { logger, action = ACTION.CREATE, scanned, priorChecksums, missingOutputs } = {}) {
+    async function registerFile(file, { logger, action = ACTION.CREATE, scanned, priorChecksums } = {}) {
         // stats — when called from the scanHook the outer scan provides
         // a Map to accumulate counts. For onSync (chokidar single-file
         // events) the counter is absent and per-file tally is not
@@ -497,7 +484,7 @@ export function useSource(core, options) {
             }
         }
 
-        const chksum = await gateChecksum(file, id, { reload, priorChecksums, bytes, missingOutputs })
+        const chksum = await gateChecksum(file, id, { reload, priorChecksums, bytes })
         if (chksum === null) {
             if (stats) stats.skipped++
             // Never becomes a render task, so it would otherwise be invisible

package/src/utils.js

@@ -1187,29 +1187,6 @@ export async function writeOutput(file, bytes) {
     return true
 }
 
-// Did this cycle evaluate the whole corpus, or only what changed?
-//
-// The distinction is what makes a "declared X matched nothing" warning
-// worth printing. On an incremental cycle only changed entities are
-// re-evaluated, so a pattern or preset legitimately matches nothing in a
-// run of two — printing it there means the warning fires on every healthy
-// build, and a warning that always fires is one people filter out, taking
-// the real instance with it.
-//
-// True for: --force, a first-run or wiped database, and a cache
-// invalidation. Those are exactly the cases where the import gate is
-// bypassed and every entity is presented for evaluation.
-// Takes the runtime rather than reading the singleton, so a plugin passes
-// the one it was injected with — which is the singleton in a real build and
-// the harness's stand-in under test.
-export function isFullCycle(rt = runtime) {
-    return !!(
-        rt?.options?.force
-        || rt?.options?.firstRun
-        || rt?.catalog?.cacheInvalidated
-    )
-}
-
 // ── operating-system and file-manager litter ────────────────────────────
 //
 // Exposing a source folder over a network filesystem (mikser-io-drive) or