mikser-io 10.0.4 → 10.2.0

package/index.js

@@ -1,6 +1,7 @@
 export { default as runtime } from './src/runtime.js'
 export * as constants from './src/constants.js'
 export * from './src/utils.js'
+export * from './src/invalidation.js'
 export * from './src/auth.js'
 export * from './src/roles.js'
 export * from './src/inventory.js'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mikser-io",
-  "version": "10.0.4",
+  "version": "10.2.0",
   "files": [
     "app.js",
     "index.js",

package/src/database/index.js

@@ -48,7 +48,7 @@ import Database from 'better-sqlite3'
 import runtime from '../runtime.js'
 import { isReportOnlyRun } from '../tools.js'
 import { reportWipe } from '../report.js'
-import { onLoaded } from '../lifecycle.js'
+import { onLoaded, onFinalized } from '../lifecycle.js'
 import packageInfo from '../../package.json' with { type: 'json' }
 
 // Local logger resolver — same one-liner engine.js exports as
@@ -242,6 +242,10 @@ export function createSqliteDatabase({
     // `--clear` asks for. Removes the file; nothing durable is in it.
     forceWipe = false,
 }) {
+    // Set at open, written at the first successful finalize. See the note
+    // where it is assigned.
+    let pendingStamp = null
+
     // Tests inject their own provisioners; the runtime path falls
     // through to the module-level `provisioners` array that plugins
     // populate via onProvision() at module-eval.
@@ -391,9 +395,24 @@ export function createSqliteDatabase({
             handle = new Database(dbPath)
             setupConnection()
         }
-        const stmtStamp = handle.prepare('INSERT OR REPLACE INTO mikser_meta (key, value) VALUES (?, ?)')
-        stmtStamp.run('schema_version', version)
-        if (currentConfig) stmtStamp.run('config_checksum', currentConfig)
+        // Stamped when a cycle FINISHES, not when the database opens.
+        //
+        // The stamp is the only record of "this cache was rebuilt for this
+        // version", and writing it at open made it a record of "this cache
+        // was opened", which is a different and much weaker claim. A wipe
+        // followed by an interrupted rebuild left the version current and the
+        // cache half-built: the import had finished, so the catalog matched
+        // disk, but nothing had rendered, so there were no snapshots and the
+        // output folder still held the previous build. Every later start read
+        // a matching version, wiped nothing, correctly reported "N unchanged"
+        // and rendered nothing — a site frozen on the last good output, green
+        // on every signal, recoverable only by --force.
+        //
+        // Deferred, the same sequence self-heals: the interrupted run leaves
+        // no stamp, so the next start sees a mismatch and rebuilds properly.
+        // config_checksum goes with it for the same reason — a config change
+        // that half-applied must not look applied.
+        pendingStamp = { version, config: currentConfig }
 
         // Build provisioning context. firstRun is true when the file
         // didn't exist before this open OR when the schema mismatch
@@ -444,6 +463,43 @@ export function createSqliteDatabase({
             }
         }
 
+        // Did the last rebuild finish?
+        //
+        // Now that the stamp means "a cycle completed", its ABSENCE beside a
+        // populated catalog is a fact worth acting on: entities were imported
+        // and nothing ever finalized. The catalog matches disk, so every gate
+        // that reasons about inputs correctly says "unchanged"; the manifest
+        // is empty, so nothing has been rendered; and the output folder still
+        // holds whatever the last complete build wrote, because a cache wipe
+        // unlinks the database and nothing else.
+        //
+        // No layer can see this on its own. The source gate's evidence is
+        // sound, the dispatcher seeds from a journal that was discarded with
+        // the interrupted run, and the render gate — which would answer
+        // `never-rendered` — is never asked, because nothing dispatches to it.
+        // So it is declared as an override, and invalidation.js hands it to
+        // every gate at once.
+        //
+        // An empty catalog here is an ordinary first run or a completed wipe:
+        // nothing to reconcile, and emptiness already opens every gate.
+        if (!handle.prepare('SELECT value FROM mikser_meta WHERE key = ?').get('schema_version')) {
+            const table = handle.prepare(
+                "SELECT name FROM sqlite_master WHERE type='table' AND name='mikser_entities'").get()
+            const entities = table
+                ? handle.prepare('SELECT count(*) AS count FROM mikser_entities').get()?.count ?? 0
+                : 0
+            if (entities > 0) {
+                logger?.warn(
+                    'The last rebuild did not finish — %d entities were imported and no cycle completed, '
+                    + 'so the catalog describes your sources while nothing has been rendered from them and '
+                    + 'the output folder still holds the previous build. Rebuilding everything on this run. '
+                    + '(A cache wipe followed by a restart mid-cycle does this; without the check the site '
+                    + 'stays on the old output and every build reports "unchanged".)',
+                    entities,
+                )
+                runtime.options.cacheRebuildInterrupted = true
+            }
+        }
     }
 
     function close() {
@@ -467,6 +523,17 @@ export function createSqliteDatabase({
         path: dbPath,
         open,
         close,
+        // Called once a cycle has finalized — see pendingStamp. Idempotent:
+        // the second cycle of a watch run has nothing left to write.
+        commitStamp() {
+            if (!pendingStamp || !handle) return false
+            const stamp = handle.prepare('INSERT OR REPLACE INTO mikser_meta (key, value) VALUES (?, ?)')
+            stamp.run('schema_version', pendingStamp.version)
+            if (pendingStamp.config) stamp.run('config_checksum', pendingStamp.config)
+            pendingStamp = null
+            runtime.options.cacheRebuildInterrupted = false
+            return true
+        },
         get isOpen() { return handle !== null },
         // Provisioning context from the most recent open. Plugins'
         // onLoaded handlers can read this without subscribing to
@@ -492,6 +559,20 @@ export function createSqliteDatabase({
     }
 }
 
+// The cache is stamped for this version only once a cycle has run to the
+// end. Registered at module level rather than per-open so a watch server
+// that re-opens nothing still stamps its first completed cycle, and so the
+// hook cannot accumulate across opens.
+//
+// onFinalized, not onFinalize: the manifest writes this cycle's snapshots in
+// onFinalize, and the stamp claims that work happened. Claiming it one hook
+// early would reintroduce the failure in miniature.
+onFinalized(async () => {
+    if (db?.commitStamp?.()) {
+        useLogger()?.debug('Cache stamped — a full cycle completed for this schema version')
+    }
+})
+
 onLoaded(async () => {
     if (db?.isOpen) return  // multi-cycle watch mode — keep the open connection
 

package/src/engine.js

@@ -1575,10 +1575,34 @@ The full version, with what each code means: docs/diagnostics.md`)
     // undefined here. The logger has no such problem — it writes during the
     // run, by which time options exist.
     const quietStdout = ['--json', '--tool', '--tools'].some(flag => process.argv.includes(flag))
+    // Through the logger when nobody is watching, so it gets a timestamp.
+    //
+    // This line marks a process start, which in a supervisor's log is the
+    // most useful thing on the page — it is how a restart is found at all.
+    // Written straight to the stream it was undated and, worse, carried its
+    // own hardcoded escapes into a file that no terminal would ever render.
+    //
+    // The decorated form stays for a terminal, where it is a banner rather
+    // than a record.
     if (runtime.options?.json || quietStdout) {
+        // Written straight to stderr, NOT through the logger: the logger picks
+        // its sink per-write from runtime.options.json, which commander has
+        // not parsed yet — the same reason quietStdout reads argv above. A
+        // banner routed through it here lands on stdout and turns the
+        // document into a parse error, which is the failure this branch was
+        // added to prevent in the first place.
         process.stderr.write(`mikser. ${packageInfo.version}\n`)
-    } else {
+    } else if (process.stdout.isTTY && !process.env.NO_COLOR) {
         console.info('\x1b[1mmikser\x1b[22;5;38;2;255;63;0m.\x1b[0m %s\n', packageInfo.version)
+    } else {
+        // Redirected: through the logger, so the line that marks a process
+        // start carries a timestamp. In a supervisor's log that is the most
+        // useful line on the page — it is how a restart is found at all — and
+        // written straight to the stream it was undated and carried its own
+        // escapes into a file no terminal will render.
+        const logger = useLogger()
+        if (logger) logger.info('mikser. %s', packageInfo.version)
+        else process.stdout.write(`mikser. ${packageInfo.version}\n`)
     }
     return runtime
 }

package/src/explain.js

@@ -10,7 +10,7 @@
 // Follows --audit-output's shape: report and exit, no build phases run.
 import { existsSync } from 'node:fs'
 import { inputHashOf, inputPartsOf, diffInputParts, lookupKeys, checksum as fileChecksum } from './utils.js'
-import { resolveOutputPath } from './manifest.js'
+import { outputMissing } from './invalidation.js'
 import { filterKey } from './track.js'
 import { findEntity, findEntities, findById } from './catalog.js'
 import runtime from './runtime.js'
@@ -141,7 +141,7 @@ export async function explain(reference) {
     // attempt — if the last render threw, that is the more useful sentence,
     // and it explains the absence too.
     const missingDestinations = snapshots
-        .filter(snap => snap.destination && !existsSync(resolveOutputPath(snap.destination)))
+        .filter(snap => outputMissing(snap.destination))
         .map(snap => snap.destination)
 
     // The catalog is as of the LAST BUILD. If the file has been edited since,
@@ -244,9 +244,7 @@ export async function explain(reference) {
             // re-renders. --explain contradicting the build is worse than
             // --explain being incomplete, because it is the tool someone
             // reaches for when the build has already surprised them.
-            missing: snap.destination
-                ? !existsSync(resolveOutputPath(snap.destination))
-                : false,
+            missing: outputMissing(snap.destination),
             outputHash: snap.outputHash ?? null,
             parent: snap.parent ?? null,
             // Which keys of its OWN meta the render read, and which keys it

package/src/invalidation.js

@@ -0,0 +1,165 @@
+// Should this work be redone?
+//
+// One question, asked at four layers with different evidence in hand:
+//
+//   source.js      a file's checksum against the catalog's
+//   manifest       a recorded hash, for the layouts dispatcher's seeding
+//   manifest       a snapshot plus its refClosure, at the render gate
+//   assets         a `.md5` marker at the current preset revision
+//
+// The evidence differs and belongs to each layer. What does NOT differ is
+// what OVERRIDES the evidence — and that is what this module owns, because
+// it was written four times and drifted the moment a fifth override was
+// added to one of them.
+//
+// The drift, concretely: `rm -rf out` changes no input, so every layer
+// answered "unchanged" and a build rendered nothing, printed no `Rendered:`
+// line and exited 0 over an empty output folder. Adding an output-existence
+// check to the render gate fixed nothing, because the source gate had
+// already dropped the entity; adding it there fixed the site but not image
+// derivatives, because files.js held a private copy of that gate; and fixing
+// that still left the assets marker, which had never asked about the file at
+// all. Five gates, one question, four implementations of the override rules.
+//
+// So the rule is: a layer may own what it KNOWS. It may not own what
+// overrides it. Anything that makes a gate not apply — --force, a wiped
+// cache, a reload event, an output that is gone — is declared here once, and
+// a new one added here reaches every gate that ever asks.
+//
+// Deliberately NOT moved here: the dependency-graph walk in
+// manifest.skipDecision. That reasons over snapshots, refClosures and sift
+// filters the manifest owns, and hauling it out would trade a real coupling
+// for a fake one. It asks this module what overrides its answer, which is the
+// part that was duplicated.
+//
+// A leaf module: runtime and node builtins only. It reads `runtime.manifest`
+// lazily rather than importing it, the way every other consumer of engine
+// state does, so the manifest can import this without a cycle.
+
+import path from 'node:path'
+import { existsSync } from 'node:fs'
+import runtime from './runtime.js'
+
+// The vocabulary, in one place because it is asserted against.
+//
+// `--json` carries these strings out to callers and the scenario suite
+// matches on them, so they are API. Defined here rather than in the gate
+// that happens to emit each one, so the set can be read as a set.
+export const REASON = Object.freeze({
+    // Overrides — this module's business.
+    FORCE: 'force',
+    CACHE_INVALIDATED: 'cache-invalidated',
+    RELOAD: 'reload',
+    REBUILD_INTERRUPTED: 'rebuild-interrupted',
+    OUTPUT_MISSING: 'output-missing',
+    // Evidence — each layer's business, named here so the vocabulary is
+    // legible as a whole.
+    UNCHANGED: 'unchanged',
+    NEVER_RENDERED: 'never-rendered',
+    INPUTS_CHANGED: 'inputs-changed',
+    REF_CHANGED: 'ref-changed',
+    QUERY_MATCHED: 'query-matched',
+    CACHE_DISABLED: 'cache-disabled',
+    RETRY_FAILED: 'retry-failed',
+})
+
+// Where an output lives, given what a snapshot recorded.
+//
+// Destinations come in two shapes and both are legitimate: output-relative
+// for a rendered page (`/index.html`), absolute for a derivative the assets
+// plugin places itself. Resolving relative-first and falling back to the
+// literal path covers both without the caller having to know which it holds.
+//
+// Returns the joined path even when nothing exists there, so a caller can
+// report WHICH path it looked for — "missing" without a path is a fact
+// nobody can act on.
+export function resolveOutputPath(destination, outputFolder = runtime.options?.outputFolder) {
+    if (!destination) return undefined
+    const joined = path.join(outputFolder ?? '', destination)
+    if (existsSync(joined)) return joined
+    if (path.isAbsolute(destination) && existsSync(destination)) return destination
+    return joined
+}
+
+// Is the file this destination names gone?
+export function outputMissing(destination) {
+    if (!destination) return false
+    return !existsSync(resolveOutputPath(destination))
+}
+
+// Memoized for the cycle.
+//
+// Every gate asks, and they all ask before anything has rendered, so one walk
+// answers all of them. Dropped at the end of onFinalize, where this cycle's
+// renders are recorded and the answer stops being true — not at the start of
+// the next cycle, because in watch mode there may not be one for hours and a
+// stale set held that long would re-dispatch entities it saw as missing.
+let cachedMissingOutputs = null
+
+// The entity ids whose last render wrote a file that is no longer on disk.
+//
+// One stat per recorded snapshot. The syscalls are ~2.2ms per 1842 paths and
+// cost the same whether the files are there or not; at the build level it does
+// not register — warm rebuilds of the 10k perf corpus ran 3.36s median with
+// this against 3.56s without, i.e. nominally faster, which is only to say the
+// difference is inside the run-to-run spread. Re-measure with
+// `npm run test:perf` before trusting a claim that it got slower.
+export function missingOutputIds() {
+    if (cachedMissingOutputs) return cachedMissingOutputs
+    const missing = new Set()
+    for (const snapshot of runtime.manifest?.all?.() ?? []) {
+        if (!snapshot.destination) continue
+        if (outputMissing(snapshot.destination)) missing.add(snapshot.id)
+    }
+    cachedMissingOutputs = missing
+    return missing
+}
+
+export function forgetMissingOutputs() {
+    cachedMissingOutputs = null
+}
+
+// What overrides a gate's own evidence, or null when nothing does.
+//
+// Callers pass the evidence they hold: `reload` if they are a watch event,
+// `id` if they are gating an entity whose outputs are recorded. Passing
+// neither asks only the universal question, which is what the render gate
+// wants — it checks the output separately and later, so that an entity whose
+// inputs ALSO moved reports `inputs-changed` rather than losing that detail
+// to a broader answer.
+export function bypassReason({ reload = false, id } = {}) {
+    if (reload) return REASON.RELOAD
+    if (runtime.options?.force) return REASON.FORCE
+    if (runtime.catalog?.cacheInvalidated) return REASON.CACHE_INVALIDATED
+    // The previous rebuild imported entities and never finalized, so the
+    // catalog describes sources that nothing has rendered. Set by
+    // database/index.js at open, cleared when a cycle stamps the cache.
+    // Declared here so it reaches every gate — which is the whole reason
+    // this module exists, and this is the first override added since.
+    if (runtime.options?.cacheRebuildInterrupted) return REASON.REBUILD_INTERRUPTED
+    if (id !== undefined && missingOutputIds().has(id)) return REASON.OUTPUT_MISSING
+    return null
+}
+
+// Is every entity being presented for evaluation this cycle?
+//
+// The same fact as bypassReason, asked for a different purpose: a "declared X
+// matched nothing" warning is only worth printing when everything was looked
+// at. On an incremental cycle a pattern legitimately matches nothing in a run
+// of two, and a warning that always fires is one people filter out — taking
+// the real instance with it.
+//
+// `firstRun` counts here and not in bypassReason because there is nothing to
+// bypass on a first run: no prior checksum, no snapshot, so every gate opens
+// on its own evidence.
+//
+// Takes the runtime rather than reading the singleton, so a plugin passes the
+// one it was injected with — the singleton in a real build, the harness's
+// stand-in under test.
+export function isFullCycle(engineRuntime = runtime) {
+    return !!(
+        engineRuntime?.options?.force
+        || engineRuntime?.options?.firstRun
+        || engineRuntime?.catalog?.cacheInvalidated
+    )
+}

package/src/logger.js

@@ -154,16 +154,44 @@ function createTerminalStream() {
 // trace} resolves to, defaulting to 'info'.
 export function createMikserLogger(level = 'info') {
     const terminalStream = createTerminalStream()
+
+    // A terminal is watched as it happens. A file is read afterwards.
+    //
+    // The minimal format below is right for the first and wrong for the
+    // second, and until now it was used for both — so a supervisor's log was
+    // a wall of undated lines wearing ANSI escapes. Reconstructing an
+    // incident from one meant ordering events by file mtimes and git commit
+    // dates because the build's own log could not say when anything happened,
+    // and every excerpt had to be piped through sed to be readable.
+    //
+    // Decided from the stream these lines actually land on, which is stderr
+    // under --json / --tool (stdout carries the document there). The logger is
+    // rebuilt at onLoad, by which point those options are parsed, so the
+    // second construction gets it right even if the first cannot.
+    //
+    // Same signal the progress bar already uses — a gauge is pointless in a
+    // file for the same reason a timestamp is pointless on a terminal.
+    const target = (runtime.options?.json || runtime.options?.tool || runtime.options?.tools)
+        ? process.stderr : process.stdout
+    const attended = Boolean(target.isTTY)
+
     const prettyStream = pretty({
         destination:   terminalStream,
-        colorize:      true,
-        // apt-like minimal format: hide timestamp / pid / hostname, and
-        // suppress the level prefix entirely via a customPrettifier
-        // that returns an empty string. The icon prepended by
-        // messageFormat (🟡 / 🔴 / 🟢 / …) is what signals level. The
-        // raw pino record still carries `level`, so third-party
-        // transports get full structured data.
-        ignore:        'pid,hostname,time',
+        // NO_COLOR is honoured on a terminal too; nobody wants escapes in a
+        // file regardless.
+        colorize:      attended && !process.env.NO_COLOR,
+        // `SYS:standard` carries the date, milliseconds AND the UTC offset.
+        // The offset is not decoration: the incident that prompted this
+        // needed a container's clock lined up against commit dates in
+        // another zone, and a bare wall-clock time cannot answer that.
+        ...(attended ? {} : { translateTime: 'SYS:standard' }),
+        // apt-like minimal format: hide pid / hostname, and suppress the
+        // level prefix entirely via a customPrettifier that returns an empty
+        // string. The icon prepended by messageFormat (🟡 / 🔴 / 🟢 / …) is
+        // what signals level. The raw pino record still carries `level`, so
+        // third-party transports get full structured data. `time` is dropped
+        // only when someone is watching.
+        ignore:        attended ? 'pid,hostname,time' : 'pid,hostname',
         // The terminal gets the SENTENCE; the structured fields go to the
         // report and to transports.
         //

package/src/manifest.js

@@ -60,6 +60,9 @@ import { extractRefs, inputHashOf, inputPartsOf, diffInputParts, lookupKeys } fr
 import { filterKey } from './track.js'
 import { findById, findEntities } from './catalog.js'
 import { useDatabase, registerSchema } from './database/index.js'
+import {
+    REASON, bypassReason, resolveOutputPath, outputMissing, missingOutputIds, forgetMissingOutputs,
+} from './invalidation.js'
 
 // Schema registration. Applied at db.open(). PRIMARY KEY (id,
 // destination) — leading id column means `WHERE id = ?` queries use the
@@ -284,13 +287,6 @@ function snapToRow(snap) {
 // becomes two symptoms — one loud (every asset reported missing, at its
 // real and present path) and one silent (no outputHash recorded, leaving
 // most snapshots presence-checked only).
-export function resolveOutputPath(destination, outputFolder = runtime.options?.outputFolder) {
-    if (!destination) return undefined
-    const joined = path.join(outputFolder ?? '', destination)
-    if (existsSync(joined)) return joined
-    if (path.isAbsolute(destination) && existsSync(destination)) return destination
-    return joined
-}
 
 async function hashOutputFile(destination) {
     const filePath = resolveOutputPath(destination)
@@ -372,11 +368,6 @@ export async function sourcesOf(destination) {
 export function createManifest(db) {
     if (!db) throw new Error('createManifest: db is required')
 
-    // Per-instance, not module-level: tests build their own manifest via
-    // createManifest(db), and a shared memo would leak one test's output
-    // folder into the next.
-    let cachedMissingOutputs = null
-
     const stmtCollisions = db.prepare(`
         SELECT destination, count(*) AS n, group_concat(id) AS ids
         FROM mikser_snapshots
@@ -682,8 +673,12 @@ export function createManifest(db) {
             // That is the situation --force exists for — the invalidation
             // graph being under suspicion — including where the preset
             // no-match warning tells the operator to use it.
-            if (runtime.options?.force) return { skip: false, reason: 'force' }
-            if (entity?.meta?.cache === false) return { skip: false, reason: 'cache-disabled' }
+            // --force and a wiped cache both mean "ignore what you think you
+            // know", and both are declared in invalidation.js so a third one
+            // added there reaches this gate without being remembered.
+            const override = bypassReason()
+            if (override) return { skip: false, reason: override }
+            if (entity?.meta?.cache === false) return { skip: false, reason: REASON.CACHE_DISABLED }
             // A render whose last attempt threw must be retried, and checked
             // before anything else: every other branch reasons about hashes,
             // and the hashes are consistent — the entity did not change, the
@@ -737,15 +732,13 @@ export function createManifest(db) {
             // rather than in the render loop because this is the function that
             // owns the word: anything asking shouldSkip() gets the same answer.
             //
-            // Checked unconditionally rather than behind a flag. It is one stat
-            // per entity that reached this point — 2.2ms for 1842 entities,
-            // the same whether the files are there or not — against a failure
-            // mode whose whole character is that nobody thinks to look for it.
-            // auditOutput() resolves destinations exactly this way, and it has
-            // to stay that way: the two disagreeing is the bug, since `--audit-output`
-            // finding what the build just called clean is what sent someone here.
-            if (snapshot.destination && !existsSync(resolveOutputPath(snapshot.destination))) {
-                return { skip: false, reason: 'output-missing', destination: snapshot.destination }
+            // Asked of invalidation.js rather than answered here, so that
+            // auditOutput, the source gate and the assets marker resolve a
+            // destination the same way. The two disagreeing is the bug —
+            // `--audit-output` finding what the build just called clean is
+            // what sends people to this function in the first place.
+            if (outputMissing(snapshot.destination)) {
+                return { skip: false, reason: REASON.OUTPUT_MISSING, destination: snapshot.destination }
             }
             if (!snapshot.refClosure?.length) return { skip: true, reason: 'unchanged' }
             const sourceLang = entity?.meta?.lang ?? null
@@ -1031,7 +1024,7 @@ export function createManifest(db) {
         // rediscover the same caveat. missingOutputIds() is memoized for the
         // cycle, so this shares the walk the source gate already paid for.
         recordedHashes() {
-            const missingOutputs = this.missingOutputIds()
+            const missingOutputs = missingOutputIds()
             const map = new Map()
             for (const row of stmtEntityInputHashes.iterate()) {
                 if (missingOutputs.has(row.id)) continue
@@ -1106,56 +1099,6 @@ export function createManifest(db) {
             return edges
         },
 
-        // The entity ids whose last render wrote a file that is no longer
-        // on disk. A dispatch hint, the same shape as queryAffected: the
-        // source checksum gate consults it so those entities are re-emitted
-        // instead of being short-circuited as unchanged.
-        //
-        // This exists because the gate that stops `rm -rf out` from being
-        // noticed is the FIRST one, not the last. An unchanged file never
-        // gets a CREATE, so it never reaches the journal, so the render loop
-        // never sees it and manifest.skipDecision — which does check for a
-        // missing output — is never asked. Fixing only the render gate makes
-        // the build correct for entities that get that far and changes
-        // nothing for the case in the report, where none of them do.
-        //
-        // One stat per recorded snapshot, once per cycle, hoisted out of the
-        // per-file path exactly like checksumsByCollection. The syscalls are
-        // ~2.2ms per 1842 paths and cost the same whether the files are there
-        // or not. At the build level it does not register: warm rebuilds of
-        // the 10k perf corpus ran 3.36s median with this against 3.56s
-        // without, i.e. nominally faster, which is only to say the difference
-        // is well inside the run-to-run spread (3.0-4.1 vs 3.5-4.7 over five
-        // runs each). Re-measure with `npm run test:perf` before trusting a
-        // claim that it got slower.
-        //
-        // Resolves destinations through resolveOutputPath, which is what
-        // auditOutput uses. The two must agree — `--audit-output` reporting
-        // files that the build just called unchanged is the symptom, not a
-        // separate diagnostic.
-        // Memoized for the cycle. Two callers ask — the source checksum gate
-        // once per scan, recordedHashes() once per layouts dispatch — and both
-        // ask before anything has rendered, so one walk answers both. Cleared
-        // at the end of onFinalize, which is where this cycle's renders are
-        // recorded and the answer stops being true.
-        missingOutputIds() {
-            if (cachedMissingOutputs) return cachedMissingOutputs
-            const missing = new Set()
-            for (const row of stmtSelectAll.iterate()) {
-                const snap = rowToSnap(row)
-                if (!snap.destination) continue
-                if (!existsSync(resolveOutputPath(snap.destination))) missing.add(snap.id)
-            }
-            cachedMissingOutputs = missing
-            return missing
-        },
-
-        // Drops the memo. Called at the end of onFinalize; exported on the
-        // surface so a test can force a re-read without a full cycle.
-        forgetMissingOutputs() {
-            cachedMissingOutputs = null
-        },
-
         // Walk the output folder against recorded snapshots, returning
         // a diff describing missing / mismatched / orphaned /
         // unverifiable. Backs `mikser --audit-output`. Pure: no mutations.
@@ -1543,5 +1486,5 @@ onFinalize(async () => {
     // Dropped here rather than at the start of the next cycle because in
     // watch mode there may not be one for hours, and a stale set held that
     // long would keep re-dispatching entities it saw as missing.
-    m.forgetMissingOutputs()
+    forgetMissingOutputs()
 })

package/src/plugins/assets.js

@@ -6,9 +6,10 @@ import { mkdir, writeFile, unlink, rm, readFile, symlink, } from 'fs/promises'
 import { existsSync } from 'node:fs'
 import { createRequire } from 'node:module'
 import { globby } from 'globby'
+import { isFullCycle, outputMissing } from '../invalidation.js'
 import _ from 'lodash'
 import map from 'p-map'
-import { isFullCycle } from '../utils.js'
+
 
 // Normalize a `options.presets[name]` value to a consistent
 // { matches, options } shape so callers don't have to inspect which form
@@ -357,8 +358,10 @@ export function assets(options = {}) {
         // the only thing that ever said a file was missing.
         //
         // Checked first and cheaply: one stat, and if the derivative is gone
-        // no amount of marker archaeology changes the answer.
-        if (!existsSync(entity.destination)) return false
+        // no amount of marker archaeology changes the answer. Asked of
+        // invalidation.js so a derivative and a rendered page agree on what
+        // "the output is gone" means.
+        if (outputMissing(entity.destination)) return false
         let result = false
         let revisions = []
         const assetChecksum = `${entity.destination}.${entity.preset.checksum}.md5`

package/src/plugins/files.js

@@ -207,12 +207,9 @@ export function files(options = {}) {
             // so the next one cannot be added to that place and missed here.
             //
             // priorChecksums is still bulk-prefetched per scan so the gate
-            // reads a map instead of doing per-file SQL, and missingOutputs
-            // alongside it for the same reason; the manifest memoizes the
-            // latter for the cycle, so this shares the walk with useSource's
-            // own scans rather than paying for a second one.
+            // reads a map instead of doing per-file SQL. What overrides the
+            // checksum, the gate asks invalidation.js for itself.
             const priorChecksums = checksumsByCollection(collection)
-            const missingOutputs = runtime.manifest?.missingOutputIds() ?? new Set()
             const scanned = new Set()
             await pMap(paths, async relativePath => {
                 const { source } = await ensureLink(relativePath)
@@ -230,7 +227,7 @@ export function files(options = {}) {
                 // the catalog already has this file unchanged and there is
                 // nothing to emit. Progress ticks either way — a gated file
                 // was still looked at.
-                const newChecksum = await gateChecksum(source, id, { priorChecksums, missingOutputs })
+                const newChecksum = await gateChecksum(source, id, { priorChecksums })
                 updateProgress()
                 if (newChecksum === null) return
                 await createEntity({

package/src/source.js

@@ -46,6 +46,7 @@ import { ACTION } from './constants.js'
 import { checksum as fileChecksum, checksumOf, junkIgnore } from './utils.js'
 import { reportGated, reportChanged } from './report.js'
 import { findById, findEntities, checksumsByCollection } from './catalog.js'
+import { bypassReason } from './invalidation.js'
 import { useDatabase } from './database/index.js'
 
 // Per-source-scan registerFile concurrency. The work is dominated by
@@ -81,20 +82,12 @@ const SCAN_CONCURRENCY = 16
 // disagree, and the losing combination (empty content + a checksum correct
 // for the finished file) is permanent, because every later sync then
 // short-circuits on "unchanged".
-// `missingOutputs`, when given, is the set of entity ids whose last render
-// wrote a file that is no longer on disk (manifest.missingOutputIds()). The
-// gate answers "has this INPUT changed", and deleting the output folder does
-// not change any input — so without this, `rm -rf out` followed by a build
-// short-circuits every file here, emits nothing, renders nothing, and reports
-// success over an empty folder. An entity whose output is gone is not
-// unchanged in any sense the caller means.
-export async function gateChecksum(file, id, { reload = false, priorChecksums, bytes, missingOutputs } = {}) {
+export async function gateChecksum(file, id, { reload = false, priorChecksums, bytes } = {}) {
     const compute = () => (bytes !== undefined ? checksumOf(bytes) : fileChecksum(file))
-    const canGate = !reload
-        && !runtime.options.force
-        && !runtime.catalog?.cacheInvalidated
-        && !missingOutputs?.has(id)
-    if (canGate) {
+    // What overrides the checksum is not this function's to decide — see
+    // invalidation.js. It used to be, and the copy in files.js then missed
+    // the output-existence clause when that was added here.
+    if (!bypassReason({ reload, id })) {
         const priorChecksum = priorChecksums
             ? priorChecksums.get(id)
             : findById(id)?.checksum
@@ -430,12 +423,6 @@ export function useSource(core, options) {
         // per-file instead of doing per-file SQL lookups. ~14× faster
         // at 14k entities (column projection vs full-entity JSON.parse).
         const priorChecksums = checksumsByCollection(collection)
-        // Prefetched once per scan, beside priorChecksums and for the same
-        // reason: the gate needs an O(1) answer per file, and this is one
-        // pass over the snapshots rather than a stat inside the per-file
-        // path. Entities listed here defeat the gate — their input is
-        // unchanged but the file they produced is gone.
-        const missingOutputs = runtime.manifest?.missingOutputIds() ?? new Set()
         // Parallel register — file read + checksum is I/O-bound. Set
         // additions, scanStats increments, and journal addEntry calls
         // are all single-threaded-JS-atomic so no locking required;
@@ -443,7 +430,7 @@ export function useSource(core, options) {
         // and INSERTs are ~10μs vs ~ms-per-file-read, so the journal
         // is never the bottleneck.
         await pMap(files, async (file) => {
-            await registerFile(file, { logger, scanned, stats: scanStats, priorChecksums, missingOutputs })
+            await registerFile(file, { logger, scanned, stats: scanStats, priorChecksums })
             if (phase === 'import') updateProgress()
         }, { concurrency: SCAN_CONCURRENCY })
 
@@ -464,7 +451,7 @@ export function useSource(core, options) {
         logger.info(scanSummary({ cap, loaded: files.length, ...scanStats }))
     })
 
-    async function registerFile(file, { logger, action = ACTION.CREATE, scanned, priorChecksums, missingOutputs } = {}) {
+    async function registerFile(file, { logger, action = ACTION.CREATE, scanned, priorChecksums } = {}) {
         // stats — when called from the scanHook the outer scan provides
         // a Map to accumulate counts. For onSync (chokidar single-file
         // events) the counter is absent and per-file tally is not
@@ -497,7 +484,7 @@ export function useSource(core, options) {
             }
         }
 
-        const chksum = await gateChecksum(file, id, { reload, priorChecksums, bytes, missingOutputs })
+        const chksum = await gateChecksum(file, id, { reload, priorChecksums, bytes })
         if (chksum === null) {
             if (stats) stats.skipped++
             // Never becomes a render task, so it would otherwise be invisible

package/src/utils.js

@@ -1187,29 +1187,6 @@ export async function writeOutput(file, bytes) {
     return true
 }
 
-// Did this cycle evaluate the whole corpus, or only what changed?
-//
-// The distinction is what makes a "declared X matched nothing" warning
-// worth printing. On an incremental cycle only changed entities are
-// re-evaluated, so a pattern or preset legitimately matches nothing in a
-// run of two — printing it there means the warning fires on every healthy
-// build, and a warning that always fires is one people filter out, taking
-// the real instance with it.
-//
-// True for: --force, a first-run or wiped database, and a cache
-// invalidation. Those are exactly the cases where the import gate is
-// bypassed and every entity is presented for evaluation.
-// Takes the runtime rather than reading the singleton, so a plugin passes
-// the one it was injected with — which is the singleton in a real build and
-// the harness's stand-in under test.
-export function isFullCycle(rt = runtime) {
-    return !!(
-        rt?.options?.force
-        || rt?.options?.firstRun
-        || rt?.catalog?.cacheInvalidated
-    )
-}
-
 // ── operating-system and file-manager litter ────────────────────────────
 //
 // Exposing a source folder over a network filesystem (mikser-io-drive) or