mikser-io 10.1.0 → 10.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mikser-io",
-  "version": "10.1.0",
+  "version": "10.3.0",
   "files": [
     "app.js",
     "index.js",

package/src/database/index.js

@@ -44,11 +44,12 @@
 
 import path from 'node:path'
 import { mkdirSync, unlinkSync, existsSync, readFileSync, writeFileSync } from 'node:fs'
+import { createHash } from 'node:crypto'
 import Database from 'better-sqlite3'
 import runtime from '../runtime.js'
 import { isReportOnlyRun } from '../tools.js'
 import { reportWipe } from '../report.js'
-import { onLoaded } from '../lifecycle.js'
+import { onLoaded, onFinalized } from '../lifecycle.js'
 import packageInfo from '../../package.json' with { type: 'json' }
 
 // Local logger resolver — same one-liner engine.js exports as
@@ -232,6 +233,43 @@ export function useDatabase() {
     return db
 }
 
+// What the cache's SHAPE is, as a value that changes only when the cache
+// stops being reusable.
+//
+// It used to be the package version, so every release wiped every
+// deployment's cache and rebuilt from cold — a patch that touched a README
+// discarded 14k entities and re-rendered a site. The version was standing in
+// for "something might have changed", which is true of every release and
+// therefore says nothing.
+//
+// Two things actually invalidate a cache, and both are in here:
+//
+//   the SQL   — a column, index or table that moved. Hashed from the
+//               registered scripts themselves, so it cannot drift from what
+//               is applied: the fingerprint and the schema come from one
+//               string.
+//   the SHAPE — what the rows MEAN, when the SQL is untouched. A change to
+//               how inputHash is computed, to the edge kinds in a
+//               refClosure, to what a destination is relative to. No parser
+//               can see these, so they are a number someone bumps.
+//
+// BUMP DERIVED_SHAPE when a release changes the meaning of anything already
+// stored. Getting that wrong is a silent stale cache, which is the failure
+// this whole subsystem keeps producing — so when in doubt, bump. A needless
+// wipe costs one cold rebuild; a missed one costs a site serving wrong
+// output with every signal green.
+const DERIVED_SHAPE = 1
+
+// `<shape>:<sql-hash>` rather than a bare hash: the stored value is read by
+// a human when a wipe happens, and the two halves say WHICH moved.
+function cacheFingerprint(registered) {
+    const sql = [...registered.entries()]
+        .map(([name, value]) => `${name}\n${schemaEntry(value).sql}`)
+        .sort()
+        .join('\n')
+    return `${DERIVED_SHAPE}:${createHash('sha1').update(sql).digest('hex').slice(0, 12)}`
+}
+
 // Build a sqlite-backed database handle. Exported so tests can exercise
 // the lifecycle and transaction semantics in isolation (without driving
 // the full onLoaded chain). The runtime path uses this internally from
@@ -242,6 +280,10 @@ export function createSqliteDatabase({
     // `--clear` asks for. Removes the file; nothing durable is in it.
     forceWipe = false,
 }) {
+    // Set at open, written at the first successful finalize. See the note
+    // where it is assigned.
+    let pendingStamp = null
+
     // Tests inject their own provisioners; the runtime path falls
     // through to the module-level `provisioners` array that plugins
     // populate via onProvision() at module-eval.
@@ -292,6 +334,11 @@ export function createSqliteDatabase({
 
         const stmtMeta = handle.prepare('SELECT value FROM mikser_meta WHERE key = ?')
         const recorded = stmtMeta.get('schema_version')?.value
+        // The identity of the cache's shape, not of the release that wrote
+        // it. `version` is still carried — into `built_by_version` at stamp
+        // time — because "which mikser built this" is worth knowing when
+        // reading a cache; it is simply not a reason to throw one away.
+        const fingerprint = cacheFingerprint(schemas)
 
         // A config change invalidates the cache for the same reason a version
         // change does: the derived state was computed under different rules.
@@ -323,13 +370,13 @@ export function createSqliteDatabase({
         const reportOnly = isReportOnlyRun()
 
         let upgradedFromVersion = null
-        if (reportOnly && !forceWipe && ((recorded && recorded !== version) || configChanged)) {
+        if (reportOnly && !forceWipe && ((recorded && recorded !== fingerprint) || configChanged)) {
             logger?.warn(
                 'The cache is stale (%s changed since it was written) and this is a read-only run, '
                 + 'so it was NOT wiped — the answer below describes the last build, which may not '
                 + 'match your sources. Run a build to refresh it.',
                 configChanged ? 'config' : 'schema version')
-        } else if (configChanged && !(recorded && recorded !== version)) {
+        } else if (configChanged && !(recorded && recorded !== fingerprint)) {
             logger?.warn(
                 'Config changed since the last run. Wiping the cache and rebuilding from sources '
                 + '(files are the source of truth — no source data is affected). The stamp covers %s and '
@@ -337,7 +384,7 @@ export function createSqliteDatabase({
                 runtime.options.config,
             )
         }
-        if (!reportOnly && (forceWipe || (recorded && recorded !== version) || configChanged)) {
+        if (!reportOnly && (forceWipe || (recorded && recorded !== fingerprint) || configChanged)) {
             // Schema mismatch on upgrade or downgrade. Per ADR-0002 the
             // files on disk are the source of truth and this database
             // is a derived cache, so the right behavior is to wipe the
@@ -348,10 +395,27 @@ export function createSqliteDatabase({
             // expect a cold-start rebuild on this run. No data loss
             // beyond the cache itself; everything in mikser.sqlite is
             // recoverable from the working folder.
-            if (recorded && recorded !== version) {
+            if (recorded && recorded !== fingerprint) {
+                // Names WHICH half moved, because the two mean different
+                // things to whoever is reading. A shape bump is a deliberate
+                // decision someone made in this release; a SQL change is a
+                // table that moved. "stored=10.0.1, current=10.1.0" said
+                // neither, and could not — it only ever meant "a release
+                // happened".
+                const [storedShape, storedSql] = String(recorded).split(':')
+                const [shape, sql] = fingerprint.split(':')
+                const cause = storedSql === undefined
+                    ? 'the cache predates schema fingerprints'
+                    : storedShape !== shape && storedSql !== sql
+                    ? 'the schema tables AND the meaning of what is stored both changed'
+                    : storedShape !== shape
+                    ? 'this release changed the meaning of what is already stored'
+                    : 'the schema tables changed'
                 logger?.warn(
-                    'Database schema mismatch: stored=%s, current=%s. Wiping the cache and rebuilding from sources (files are the source of truth — no source data is affected).',
-                    recorded, version,
+                    'Cache shape changed — %s (stored=%s, current=%s, this is mikser %s). Wiping the cache '
+                    + 'and rebuilding from sources (files are the source of truth — no source data is '
+                    + 'affected). A release that changes neither reuses the cache.',
+                    cause, recorded, fingerprint, version,
                 )
             } else if (forceWipe) {
                 logger?.info('Clearing the cache and rebuilding from sources.')
@@ -361,8 +425,8 @@ export function createSqliteDatabase({
             // the same output whether the version moved, the config moved, or
             // someone passed --clear.
             reportWipe(
-                recorded && recorded !== version ? 'version' : forceWipe ? 'clear' : 'config',
-                recorded && recorded !== version ? { from: recorded, to: version } : {},
+                recorded && recorded !== fingerprint ? 'version' : forceWipe ? 'clear' : 'config',
+                recorded && recorded !== fingerprint ? { from: recorded, to: fingerprint } : {},
             )
 
             // Unlink, rather than dropping table by table.
@@ -391,9 +455,24 @@ export function createSqliteDatabase({
             handle = new Database(dbPath)
             setupConnection()
         }
-        const stmtStamp = handle.prepare('INSERT OR REPLACE INTO mikser_meta (key, value) VALUES (?, ?)')
-        stmtStamp.run('schema_version', version)
-        if (currentConfig) stmtStamp.run('config_checksum', currentConfig)
+        // Stamped when a cycle FINISHES, not when the database opens.
+        //
+        // The stamp is the only record of "this cache was rebuilt for this
+        // version", and writing it at open made it a record of "this cache
+        // was opened", which is a different and much weaker claim. A wipe
+        // followed by an interrupted rebuild left the version current and the
+        // cache half-built: the import had finished, so the catalog matched
+        // disk, but nothing had rendered, so there were no snapshots and the
+        // output folder still held the previous build. Every later start read
+        // a matching version, wiped nothing, correctly reported "N unchanged"
+        // and rendered nothing — a site frozen on the last good output, green
+        // on every signal, recoverable only by --force.
+        //
+        // Deferred, the same sequence self-heals: the interrupted run leaves
+        // no stamp, so the next start sees a mismatch and rebuilds properly.
+        // config_checksum goes with it for the same reason — a config change
+        // that half-applied must not look applied.
+        pendingStamp = { fingerprint, version, config: currentConfig }
 
         // Build provisioning context. firstRun is true when the file
         // didn't exist before this open OR when the schema mismatch
@@ -444,6 +523,43 @@ export function createSqliteDatabase({
             }
         }
 
+        // Did the last rebuild finish?
+        //
+        // Now that the stamp means "a cycle completed", its ABSENCE beside a
+        // populated catalog is a fact worth acting on: entities were imported
+        // and nothing ever finalized. The catalog matches disk, so every gate
+        // that reasons about inputs correctly says "unchanged"; the manifest
+        // is empty, so nothing has been rendered; and the output folder still
+        // holds whatever the last complete build wrote, because a cache wipe
+        // unlinks the database and nothing else.
+        //
+        // No layer can see this on its own. The source gate's evidence is
+        // sound, the dispatcher seeds from a journal that was discarded with
+        // the interrupted run, and the render gate — which would answer
+        // `never-rendered` — is never asked, because nothing dispatches to it.
+        // So it is declared as an override, and invalidation.js hands it to
+        // every gate at once.
+        //
+        // An empty catalog here is an ordinary first run or a completed wipe:
+        // nothing to reconcile, and emptiness already opens every gate.
+        if (!handle.prepare('SELECT value FROM mikser_meta WHERE key = ?').get('schema_version')) {
+            const table = handle.prepare(
+                "SELECT name FROM sqlite_master WHERE type='table' AND name='mikser_entities'").get()
+            const entities = table
+                ? handle.prepare('SELECT count(*) AS count FROM mikser_entities').get()?.count ?? 0
+                : 0
+            if (entities > 0) {
+                logger?.warn(
+                    'The last rebuild did not finish — %d entities were imported and no cycle completed, '
+                    + 'so the catalog describes your sources while nothing has been rendered from them and '
+                    + 'the output folder still holds the previous build. Rebuilding everything on this run. '
+                    + '(A cache wipe followed by a restart mid-cycle does this; without the check the site '
+                    + 'stays on the old output and every build reports "unchanged".)',
+                    entities,
+                )
+                runtime.options.cacheRebuildInterrupted = true
+            }
+        }
     }
 
     function close() {
@@ -467,6 +583,21 @@ export function createSqliteDatabase({
         path: dbPath,
         open,
         close,
+        // Called once a cycle has finalized — see pendingStamp. Idempotent:
+        // the second cycle of a watch run has nothing left to write.
+        commitStamp() {
+            if (!pendingStamp || !handle) return false
+            const stamp = handle.prepare('INSERT OR REPLACE INTO mikser_meta (key, value) VALUES (?, ?)')
+            stamp.run('schema_version', pendingStamp.fingerprint)
+            // Which release wrote this cache. Recorded because it is the
+            // first thing a person wants when reading one, and deliberately
+            // NOT compared against anything — that was the bug.
+            stamp.run('built_by_version', pendingStamp.version)
+            if (pendingStamp.config) stamp.run('config_checksum', pendingStamp.config)
+            pendingStamp = null
+            runtime.options.cacheRebuildInterrupted = false
+            return true
+        },
         get isOpen() { return handle !== null },
         // Provisioning context from the most recent open. Plugins'
         // onLoaded handlers can read this without subscribing to
@@ -492,6 +623,20 @@ export function createSqliteDatabase({
     }
 }
 
+// The cache is stamped for this version only once a cycle has run to the
+// end. Registered at module level rather than per-open so a watch server
+// that re-opens nothing still stamps its first completed cycle, and so the
+// hook cannot accumulate across opens.
+//
+// onFinalized, not onFinalize: the manifest writes this cycle's snapshots in
+// onFinalize, and the stamp claims that work happened. Claiming it one hook
+// early would reintroduce the failure in miniature.
+onFinalized(async () => {
+    if (db?.commitStamp?.()) {
+        useLogger()?.debug('Cache stamped — a full cycle completed for this schema version')
+    }
+})
+
 onLoaded(async () => {
     if (db?.isOpen) return  // multi-cycle watch mode — keep the open connection
 

package/src/engine.js

@@ -1575,10 +1575,34 @@ The full version, with what each code means: docs/diagnostics.md`)
     // undefined here. The logger has no such problem — it writes during the
     // run, by which time options exist.
     const quietStdout = ['--json', '--tool', '--tools'].some(flag => process.argv.includes(flag))
+    // Through the logger when nobody is watching, so it gets a timestamp.
+    //
+    // This line marks a process start, which in a supervisor's log is the
+    // most useful thing on the page — it is how a restart is found at all.
+    // Written straight to the stream it was undated and, worse, carried its
+    // own hardcoded escapes into a file that no terminal would ever render.
+    //
+    // The decorated form stays for a terminal, where it is a banner rather
+    // than a record.
     if (runtime.options?.json || quietStdout) {
+        // Written straight to stderr, NOT through the logger: the logger picks
+        // its sink per-write from runtime.options.json, which commander has
+        // not parsed yet — the same reason quietStdout reads argv above. A
+        // banner routed through it here lands on stdout and turns the
+        // document into a parse error, which is the failure this branch was
+        // added to prevent in the first place.
         process.stderr.write(`mikser. ${packageInfo.version}\n`)
-    } else {
+    } else if (process.stdout.isTTY && !process.env.NO_COLOR) {
         console.info('\x1b[1mmikser\x1b[22;5;38;2;255;63;0m.\x1b[0m %s\n', packageInfo.version)
+    } else {
+        // Redirected: through the logger, so the line that marks a process
+        // start carries a timestamp. In a supervisor's log that is the most
+        // useful line on the page — it is how a restart is found at all — and
+        // written straight to the stream it was undated and carried its own
+        // escapes into a file no terminal will render.
+        const logger = useLogger()
+        if (logger) logger.info('mikser. %s', packageInfo.version)
+        else process.stdout.write(`mikser. ${packageInfo.version}\n`)
     }
     return runtime
 }

package/src/invalidation.js

@@ -50,6 +50,7 @@ export const REASON = Object.freeze({
     FORCE: 'force',
     CACHE_INVALIDATED: 'cache-invalidated',
     RELOAD: 'reload',
+    REBUILD_INTERRUPTED: 'rebuild-interrupted',
     OUTPUT_MISSING: 'output-missing',
     // Evidence — each layer's business, named here so the vocabulary is
     // legible as a whole.
@@ -130,6 +131,12 @@ export function bypassReason({ reload = false, id } = {}) {
     if (reload) return REASON.RELOAD
     if (runtime.options?.force) return REASON.FORCE
     if (runtime.catalog?.cacheInvalidated) return REASON.CACHE_INVALIDATED
+    // The previous rebuild imported entities and never finalized, so the
+    // catalog describes sources that nothing has rendered. Set by
+    // database/index.js at open, cleared when a cycle stamps the cache.
+    // Declared here so it reaches every gate — which is the whole reason
+    // this module exists, and this is the first override added since.
+    if (runtime.options?.cacheRebuildInterrupted) return REASON.REBUILD_INTERRUPTED
     if (id !== undefined && missingOutputIds().has(id)) return REASON.OUTPUT_MISSING
     return null
 }

package/src/logger.js

@@ -154,16 +154,44 @@ function createTerminalStream() {
 // trace} resolves to, defaulting to 'info'.
 export function createMikserLogger(level = 'info') {
     const terminalStream = createTerminalStream()
+
+    // A terminal is watched as it happens. A file is read afterwards.
+    //
+    // The minimal format below is right for the first and wrong for the
+    // second, and until now it was used for both — so a supervisor's log was
+    // a wall of undated lines wearing ANSI escapes. Reconstructing an
+    // incident from one meant ordering events by file mtimes and git commit
+    // dates because the build's own log could not say when anything happened,
+    // and every excerpt had to be piped through sed to be readable.
+    //
+    // Decided from the stream these lines actually land on, which is stderr
+    // under --json / --tool (stdout carries the document there). The logger is
+    // rebuilt at onLoad, by which point those options are parsed, so the
+    // second construction gets it right even if the first cannot.
+    //
+    // Same signal the progress bar already uses — a gauge is pointless in a
+    // file for the same reason a timestamp is pointless on a terminal.
+    const target = (runtime.options?.json || runtime.options?.tool || runtime.options?.tools)
+        ? process.stderr : process.stdout
+    const attended = Boolean(target.isTTY)
+
     const prettyStream = pretty({
         destination:   terminalStream,
-        colorize:      true,
-        // apt-like minimal format: hide timestamp / pid / hostname, and
-        // suppress the level prefix entirely via a customPrettifier
-        // that returns an empty string. The icon prepended by
-        // messageFormat (🟡 / 🔴 / 🟢 / …) is what signals level. The
-        // raw pino record still carries `level`, so third-party
-        // transports get full structured data.
-        ignore:        'pid,hostname,time',
+        // NO_COLOR is honoured on a terminal too; nobody wants escapes in a
+        // file regardless.
+        colorize:      attended && !process.env.NO_COLOR,
+        // `SYS:standard` carries the date, milliseconds AND the UTC offset.
+        // The offset is not decoration: the incident that prompted this
+        // needed a container's clock lined up against commit dates in
+        // another zone, and a bare wall-clock time cannot answer that.
+        ...(attended ? {} : { translateTime: 'SYS:standard' }),
+        // apt-like minimal format: hide pid / hostname, and suppress the
+        // level prefix entirely via a customPrettifier that returns an empty
+        // string. The icon prepended by messageFormat (🟡 / 🔴 / 🟢 / …) is
+        // what signals level. The raw pino record still carries `level`, so
+        // third-party transports get full structured data. `time` is dropped
+        // only when someone is watching.
+        ignore:        attended ? 'pid,hostname,time' : 'pid,hostname',
         // The terminal gets the SENTENCE; the structured fields go to the
         // report and to transports.
         //