mikser-io 9.7.0 → 9.12.0

package/docs/configuration.md

@@ -59,6 +59,7 @@ These options are part of `runtime.options` and apply to the engine itself.
 | `threads` | — | number | `4` | Worker thread count for the Piscina pools (`renderWorkers`, `postprocessWorkers`). Both pools are lazy (`minThreads: 0` + `idleTimeout: 30_000`) so INLINE-only workloads spin up zero workers. |
 | `server` | `-s, --server [port]` | number\|boolean | — | When set, the engine creates a shared Express app on `runtime.options.app` and listens on the given port (default `3001`) after all plugins have mounted their routes. Plugins like `api` attach to it instead of starting their own server. The `outputFolder` is also served as a static catch-all route at `/` (plugin routes match first; anything that doesn't match falls through to the rendered output). Requires `express` to be installed. |
 | `junk` | — | array\|false | built-in list | OS and file-manager litter, filtered out of both the scan and the watcher. The dot-prefixed files (`.DS_Store`, `._*`) were already invisible — globby defaults to `dot: false` and the watcher ignores leading dots — but the Windows ones are **not** dotfiles: `Thumbs.db` and `desktop.ini` were measurably scanned *and* watched, and became entities. The list is deliberately conservative (OS/file-manager artifacts and application lock files only, no `*.tmp`, `*.bak` or editor backups), because a filter that silently drops content is worse than the litter it prevents. `false` disables it; an array replaces it. See `isJunkPath` / `JUNK_IGNORE` in `src/utils.js`. Plugins that write metadata next to content add their own patterns with `registerJunk({ ignore, match })` — the engine provides the mechanism and the plugin the knowledge of what its files are called (`mikser-io-webdav` registers `*.nephelemeta`). Plugin registrations survive an array override, since narrowing the OS list is not a request to start importing a library's sidecars. |
+| — (plugin) | — | object | — | `sources({ styles: { folder: 'styles', extensions: ['css'] } })` registers build inputs as catalog entities, one collection per key. A sidecar can then read them with `findEntities()`, whose queries land in the render's `refClosure` — so editing, adding or removing a part re-renders the bundle and nothing else. Reading the same files with `fs` instead works for one build and silently breaks watch, because the engine has no dependency on a file it never saw. Nothing is linked into `outputFolder`: these are inputs, not output. Named `sources` rather than `inputs` because `entity.inputs` already means something adjacent — bytes an output depends on without being entities at all. |
 | `cors` / `no-cors` | `--cors` / `--no-cors` | boolean | — | Toggle CORS on the engine's shared Express app. See `src/server.js` for the extensible header arrays plugins push onto. |
 | `server.requestTimeout` | — | number | node default (`300000`) | Milliseconds a single request may take, set on the underlying `http.Server`. Node's 5-minute default is effectively an **upload size limit expressed in seconds** — a large file over a slow link is indistinguishable from a stalled request, so it is cut off and the caller sees a truncated write rather than a readable error. Only reachable from the `http.Server`, which the engine owns, so a plugin that mounts an upload surface (`mikser-io-webdav`, `forms` with large attachments) cannot raise it for itself. `0` disables the cap: reasonable on a trusted-network build server, bad facing the internet, where it removes the only bound on how long a client can hold a connection open doing nothing. `headersTimeout` is clamped to stay at or below it. The server is also exposed as `runtime.options.httpServer`. |
 | `url` | `-u, --url <url>` | string | — | Public URL where this mikser is reachable (e.g. `https://blog.me.com`). Validated, trailing slash stripped, stamped on `runtime.options.url`. Read by webhook-capable plugins for push-vs-poll gating (`url.startsWith('https://')`); used by anything that surfaces absolute URLs externally — MCP preview URLs returned to agents, forms share links, email tracking pixels. Plugins that just need internal URLs keep using `runtime.options.port`. |

package/index.js

@@ -24,6 +24,7 @@ export * from './src/routes.js'
 // for the v9 plugin shape.
 export { api }           from './src/plugins/api.js'
 export { assets }        from './src/plugins/assets.js'
+export { sources }       from './src/plugins/sources.js'
 export { commands }      from './src/plugins/commands.js'
 export { data }          from './src/plugins/data.js'
 export { documents }     from './src/plugins/documents.js'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mikser-io",
-  "version": "9.7.0",
+  "version": "9.12.0",
   "description": "<p align=\"center\">   <img src=\"mikser-lockup-stacked.svg\" alt=\"mikser\" width=\"198\" /> </p>",
   "main": "index.js",
   "exports": {

package/src/config.js

@@ -1,12 +1,34 @@
 import runtime from './runtime.js'
 import { useLogger } from './engine.js'
 import { onLoad } from './lifecycle.js'
+import { checksum } from './utils.js'
 import path from 'node:path'
 
 onLoad(async () => {
     const logger = useLogger()
     const configFile = path.resolve(runtime.options.config)
     logger.info('Config: %s', configFile)
+
+    // Stamp the config so a change to it can invalidate the derived cache.
+    //
+    // Without this, editing mikser.config.js invalidated NOTHING: flipping
+    // an option that changes every page's destination reported "36 unchanged"
+    // and left the previous output in place. The config was genuinely read —
+    // --force applied it immediately — it simply took part in no
+    // invalidation, so the only symptom was output that did not match the
+    // config, with nothing saying so.
+    //
+    // The file's bytes only. A config that imports other modules will not
+    // notice a change in those, which is a real limit worth knowing rather
+    // than a reason to hash the whole module graph.
+    try {
+        runtime.options.configChecksum = await checksum(configFile)
+    } catch {
+        // No config file is a legitimate state (defaults all the way down);
+        // absent stamp means "nothing to compare", not "changed".
+        runtime.options.configChecksum = null
+    }
+
     try {
         const config = await import(configFile)
         if (typeof config.default == 'function') {

package/src/database/index.js

@@ -247,10 +247,38 @@ export function createSqliteDatabase({
         handle = new Database(dbPath)
         setupConnection()
 
-        const recorded = handle.prepare('SELECT value FROM mikser_meta WHERE key = ?')
-            .get('schema_version')?.value
+        const stmtMeta = handle.prepare('SELECT value FROM mikser_meta WHERE key = ?')
+        const recorded = stmtMeta.get('schema_version')?.value
+
+        // A config change invalidates the cache for the same reason a version
+        // change does: the derived state was computed under different rules.
+        //
+        // Before this, editing mikser.config.js invalidated nothing — flipping
+        // an option that changes every page's destination reported "36
+        // unchanged" and left the previous output in place. The config was
+        // read; it simply took part in no invalidation, so the only symptom
+        // was output that did not match the config and nothing saying so.
+        //
+        // Treated exactly like a version mismatch rather than something
+        // narrower: a config edit can change how sources are PARSED (a mapper
+        // transform, documents() options) as well as how they are rendered,
+        // so invalidating only the render manifest would still leave stale
+        // entities. Per ADR-0002 the files are the source of truth, so
+        // rebuilding is always safe — just slower.
+        const recordedConfig = stmtMeta.get('config_checksum')?.value
+        const currentConfig = runtime.options.configChecksum ?? null
+        const configChanged = Boolean(recordedConfig && currentConfig && recordedConfig !== currentConfig)
+
         let upgradedFromVersion = null
-        if (recorded && recorded !== version) {
+        if (configChanged && !(recorded && recorded !== version)) {
+            logger?.warn(
+                'Config changed since the last run. Wiping the cache and rebuilding from sources ' +
+                '(files are the source of truth — no source data is affected). Note this tracks the ' +
+                'bytes of %s only: a change in a module it imports is not seen.',
+                runtime.options.config,
+            )
+        }
+        if ((recorded && recorded !== version) || configChanged) {
             // Schema mismatch on upgrade or downgrade. Per ADR-0002 the
             // files on disk are the source of truth and this database
             // is a derived cache, so the right behavior is to wipe the
@@ -261,10 +289,12 @@ export function createSqliteDatabase({
             // expect a cold-start rebuild on this run. No data loss
             // beyond the cache itself; everything in mikser.sqlite is
             // recoverable from the working folder.
-            logger?.warn(
-                'Database schema mismatch: stored=%s, current=%s. Wiping the cache and rebuilding from sources (files are the source of truth — no source data is affected).',
-                recorded, version,
-            )
+            if (recorded && recorded !== version) {
+                logger?.warn(
+                    'Database schema mismatch: stored=%s, current=%s. Wiping the cache and rebuilding from sources (files are the source of truth — no source data is affected).',
+                    recorded, version,
+                )
+            }
             handle.close()
             handle = null
 
@@ -277,12 +307,16 @@ export function createSqliteDatabase({
                 }
             }
 
-            upgradedFromVersion = recorded
+            // Non-null marks the provisioning context as firstRun/upgraded,
+            // which is the right shape for a config change too: what the
+            // provisioners see is an empty-state database either way.
+            upgradedFromVersion = recorded ?? 'config'
             handle = new Database(dbPath)
             setupConnection()
         }
-        handle.prepare('INSERT OR REPLACE INTO mikser_meta (key, value) VALUES (?, ?)')
-            .run('schema_version', version)
+        const stmtStamp = handle.prepare('INSERT OR REPLACE INTO mikser_meta (key, value) VALUES (?, ?)')
+        stmtStamp.run('schema_version', version)
+        if (currentConfig) stmtStamp.run('config_checksum', currentConfig)
 
         // Build provisioning context. firstRun is true when the file
         // didn't exist before this open OR when the schema mismatch

package/src/plugins/assets.js

@@ -86,19 +86,53 @@ export function assets(options = {}) {
     const type = 'preset'
     const checksumMap = new Set()
 
+    // A preset that matches nothing builds green and says nothing, which is
+    // how a mistyped pattern ships. `files({ outputFolder })` prefixes `name`
+    // and `meta.url` but NOT `id`, and `match` runs against `id` — so
+    // '/files/media/devices/**' looks right and matches zero. Tallied here,
+    // reported once per process at onFinalize.
+    const matchTally = { evaluated: 0, matched: new Set(), reported: false }
+
     async function getEntityPresets(entity) {
         const entityPresets = []
+        matchTally.evaluated++
         for (let preset in (options.presets || {})) {
             const { matches } = normalizePresetConfig(options.presets[preset])
             for (let match of matches) {
                 if (matchEntity(entity, match)) {
                     entityPresets.push(preset)
+                    matchTally.matched.add(preset)
                 }
             }
         }
         return entityPresets
     }
 
+    // Report presets that matched none of the entities this run evaluated.
+    //
+    // Deliberately phrased as what was OBSERVED rather than "this preset is
+    // broken": an incremental cycle only re-evaluates changed entities, so a
+    // preset can legitimately match nothing in a run of three. The count and
+    // the --force hint let the reader tell the two apart, which a bare
+    // "matched nothing" would not. Once per process, so watch mode is quiet.
+    function reportUnmatchedPresets(logger) {
+        if (matchTally.reported) return
+        const configured = Object.keys(options.presets || {})
+        if (!configured.length || !matchTally.evaluated) return
+        matchTally.reported = true
+
+        for (const preset of configured) {
+            if (matchTally.matched.has(preset)) continue
+            const { matches } = normalizePresetConfig(options.presets[preset])
+            logger.warn(
+                'Assets preset %j matched none of the %d entities evaluated (patterns: %s). ' +
+                'Patterns run against entity.id, which files({ outputFolder }) does NOT prefix — ' +
+                'the prefix appears on name and meta.url only. On an incremental run unchanged ' +
+                'entities are not re-evaluated; use --force to check the whole catalog.',
+                preset, matchTally.evaluated, matches.join(', '))
+        }
+    }
+
     // Resolve a preset name to an importable module location. Local
     // files in presetsFolder win; names with no local file fall back to
     // an npm package named `mikser-io-preset-<name>`, resolved from the
@@ -437,6 +471,8 @@ export function assets(options = {}) {
         const logger = useLogger()
         const { presets } = runtime.state.assets
 
+        reportUnmatchedPresets(logger)
+
         let revisions = await globby('**/*.md5', { cwd: runtime.options.assetsFolder })
         for (let revision of revisions) {
             const [preset] = revision.split(path.sep)

package/src/plugins/sources.js

@@ -0,0 +1,69 @@
+// Register build inputs — styles/, js/ — as catalog entities.
+//
+// The gap this fills: a site that bundles assets needs a layout sidecar to
+// read those files. Reading them with `fs` works for a one-shot build and
+// SILENTLY breaks watch — the engine has no dependency on a file it never
+// saw, so editing styles/sections/hero.css rebuilds nothing and the page is
+// stale with nothing saying so. The first build is correct, which is what
+// makes it expensive: the mistake only surfaces later as "why didn't my edit
+// take effect", and the obvious workaround is the broken one.
+//
+// Registered as entities instead, a sidecar reads them with findEntities(),
+// whose queries are recorded into the render's refClosure — so touching one
+// part re-renders the bundle and nothing else, by the same mechanism that
+// re-renders a page when a document it references changes.
+//
+// Deliberately NOT the `files` plugin: these are inputs, not output. Nothing
+// is linked into out/, no meta.url is stamped, and they take no part in
+// ADR-0011 served-path resolution. A stylesheet part is not a thing the site
+// serves; it is a thing the site is built from.
+//
+// Named `sources` rather than `inputs` because `entity.inputs` already means
+// something adjacent but different — bytes an entity's output depends on
+// without them being entities at all (see inputHashOf). Two spellings of
+// "input" meaning two things would be worse than a slightly generic name.
+import path from 'node:path'
+import { useSource } from '../source.js'
+
+export function sources(options = {}) {
+    const collections = Object.entries(options)
+    return (core) => {
+        const { useLogger } = core
+        if (!collections.length) {
+            // No collections is a legitimate config (a flag turned them all
+            // off); nothing to register, and nothing to complain about.
+            return
+        }
+        for (const [collection, config] of collections) {
+            const {
+                folder = collection,
+                extensions = ['*'],
+                ignore = [],
+                // Content is the point — a sidecar bundling CSS needs the
+                // bytes, not just the path. Overridable for a collection
+                // that only needs to be *known* (an image manifest, say).
+                content = true,
+                load,
+                // Code-shaped, so the extension stays in `id` (two parts of
+                // the same name in different languages must not collide) and
+                // `name` keeps the folder-relative path without it.
+                stripExtensionFromId = false,
+            } = config ?? {}
+
+            useSource(core, {
+                collection,
+                type: 'source',
+                folder,
+                extensions,
+                ignore,
+                content,
+                stripExtensionFromId,
+                load: load ?? (async () => ({})),
+                progress: `${collection.replace(/^./, c => c.toUpperCase())} import`,
+            })
+        }
+        useLogger?.()?.debug('Sources registered: %s', collections.map(([c]) => c).join(', '))
+    }
+}
+
+export default sources

package/src/source.js

@@ -43,7 +43,7 @@ import { globby } from 'globby'
 import pMap from 'p-map'
 import runtime from './runtime.js'
 import { ACTION } from './constants.js'
-import { checksum as fileChecksum, junkIgnore } from './utils.js'
+import { checksum as fileChecksum, checksumOf, junkIgnore } from './utils.js'
 import { findById, findEntities, checksumsByCollection } from './catalog.js'
 import { useDatabase } from './database/index.js'
 
@@ -74,7 +74,14 @@ const SCAN_CONCURRENCY = 16
 // no SQL) instead of per-file findById. The single-file chokidar
 // event handler (no scan context) doesn't pass it and falls back to
 // findById, which is fine for low-frequency one-off mutations.
-export async function gateChecksum(file, id, { reload = false, priorChecksums } = {}) {
+// `bytes`, when given, is content the caller has ALREADY read: the checksum
+// is derived from those exact bytes rather than from a second, independent
+// read. That is the whole point — two reads of a file being written can
+// disagree, and the losing combination (empty content + a checksum correct
+// for the finished file) is permanent, because every later sync then
+// short-circuits on "unchanged".
+export async function gateChecksum(file, id, { reload = false, priorChecksums, bytes } = {}) {
+    const compute = () => (bytes !== undefined ? checksumOf(bytes) : fileChecksum(file))
     const canGate = !reload
         && !runtime.options.force
         && !runtime.catalog?.cacheInvalidated
@@ -83,12 +90,12 @@ export async function gateChecksum(file, id, { reload = false, priorChecksums }
             ? priorChecksums.get(id)
             : findById(id)?.checksum
         if (priorChecksum) {
-            const current = await fileChecksum(file)
+            const current = await compute()
             if (priorChecksum === current) return null
             return current
         }
     }
-    return await fileChecksum(file)
+    return await compute()
 }
 
 // Delete sweep. After a scan, find every catalog entity in `collection`
@@ -263,9 +270,17 @@ export function useSource(core, options) {
     const prefix = idPrefix ?? `/${collection}`
     const cap = collection.replace(/^./, c => c.toUpperCase())
     const progressLabel = progress ?? `${cap} import`
+    // A SINGLE extension must not go through brace syntax: `**/*.{css}`
+    // matches NOTHING in minimatch/globby — a one-element brace is not
+    // expanded — so a source declaring one extension silently imported zero
+    // files and reported "Styles loaded: 0" as though the folder were empty.
+    // Same shape as the other silent-declaration failures: green build,
+    // nothing there.
     const pattern = extensions.includes('*')
         ? '**/*'
-        : `**/*.{${extensions.join(',')}}`
+        : extensions.length === 1
+            ? `**/*.${extensions[0]}`
+            : `**/*.{${extensions.join(',')}}`
 
     // Hot-reload — chokidar dispatches into onSync(collection) when
     // a file inside the folder changes.
@@ -413,7 +428,26 @@ export function useSource(core, options) {
             : `${prefix}/${relativePath.replace(/\\/g, '/')}`
         scanned?.add(id)
 
-        const chksum = await gateChecksum(file, id, { reload, priorChecksums })
+        // When this source stores content, read the file ONCE and let both
+        // the checksum and the stored body come from the same bytes. The
+        // previous shape hashed the file in gateChecksum and then read it
+        // again below — two reads, and a torn write between them persisted
+        // an empty body next to a valid checksum, permanently.
+        //
+        // No extra cost: the gate already read the whole file to hash it.
+        // Only for `content` sources; large media goes through files.js,
+        // which stores no body and must not be slurped into memory.
+        let bytes
+        if (content) {
+            try {
+                bytes = await readFile(file)
+            } catch (err) {
+                logger.warn('%s read failed for %s: %s', collection, name, err.message)
+                return
+            }
+        }
+
+        const chksum = await gateChecksum(file, id, { reload, priorChecksums, bytes })
         if (chksum === null) {
             if (stats) stats.skipped++
             return
@@ -430,12 +464,9 @@ export function useSource(core, options) {
             checksum: chksum,
         }
         if (content) {
-            try {
-                base.content = await readFile(file, 'utf8')
-            } catch (err) {
-                logger.warn('%s read failed for %s: %s', collection, name, err.message)
-                return
-            }
+            // Decoded from the bytes the checksum was taken over — not
+            // re-read. See the gate above.
+            base.content = bytes.toString('utf8')
         }
         try {
             const extra = await load({ file, name, relativePath, entity: base })

package/src/utils.js

@@ -1,9 +1,8 @@
 import crypto from 'node:crypto'
-import { hashFile, hash } from 'hasha'
-import { stat, readFile, writeFile, mkdir, unlink } from 'node:fs/promises'
+import { createHash } from 'node:crypto'
+import { hashFile } from 'hasha'
+import { stat, readFile, writeFile, mkdir, unlink, open } from 'node:fs/promises'
 import { createRequire } from 'node:module'
-import TruncateStream from 'truncate-stream'
-import { createReadStream } from 'node:fs'
 import _ from 'lodash'
 import { minimatch } from 'minimatch'
 import path from 'path'
@@ -27,6 +26,18 @@ export function inputHashOf(entity) {
     return crypto.createHash('sha1').update(JSON.stringify({
         meta: entity.meta ?? null,
         content: entity.content ?? null,
+        // `inputs` is how a plugin declares bytes that are NOT part of the
+        // entity's own content but that its output depends on. Whatever is
+        // put here participates in the hash, so a change to it invalidates
+        // every consumer through the normal refClosure path.
+        //
+        // The case that needed it: a layout's `.js` sidecar. It is the
+        // entity's data layer, it is not its content, and it was in no hash
+        // at all — so editing it re-rendered nothing, silently, in a fresh
+        // build. Moving the layout's gate `checksum` was not enough, because
+        // an entity that HAS content is hashed on {meta, content} and its
+        // checksum is ignored. This is the seam that was missing.
+        inputs: entity.inputs ?? null,
     })).digest('hex')
 }
 
@@ -301,17 +312,68 @@ export class AbortError extends Error {
     }
 }
 
+const CHECKSUM_MAX_BYTES = 300 * 1024
+
+// Checksum from bytes the CALLER ALREADY HAS — no I/O, and therefore no
+// second read to disagree with the first.
+//
+// The hazard this exists to remove: a plugin that stores content does
+//
+//     meta: { body: await readFile(source, 'utf8') },
+//     checksum: await checksum(source),
+//
+// which is two independent reads of the same file. A watcher firing on the
+// truncate half of a write gets '' from readFile while checksum(), a moment
+// later, sees the finished file. The entity is stored with an empty body and
+// a checksum that is CORRECT FOR THE FINAL CONTENT — so every later sync
+// short-circuits on "unchanged" and the empty body is permanent. Only
+// --clear recovers it, and nothing anywhere reports a problem.
+//
+// Byte-compatible with checksum(uri) below, deliberately: the two must be
+// interchangeable or swapping a caller over would invalidate its catalog.
+export function checksumOf(content) {
+    const buf = Buffer.isBuffer(content) ? content : Buffer.from(String(content), 'utf8')
+    if (buf.length < CHECKSUM_MAX_BYTES) {
+        return createHash('md5').update(buf).digest('hex')
+    }
+    const head = createHash('md5').update(buf.subarray(0, CHECKSUM_MAX_BYTES)).digest('hex')
+    const tail = createHash('md5').update(buf.subarray(buf.length - CHECKSUM_MAX_BYTES)).digest('hex')
+    return `${buf.length}:${head}:${tail}`
+}
+
+// Checksum a file by path.
+//
+// Files under 300 KB are hashed whole. Larger ones are hashed at both ends
+// plus their length, which reads 600 KB instead of gigabytes — the point of
+// the truncation is that a 1.4 GB video must not be streamed on every cycle.
+//
+// The TAIL is new. The previous form was `size + md5(first 300KB)`, which
+// silently misses any change beyond byte 307200 that preserves the file's
+// length: the checksum matches, the sync reports "unchanged", and the edit
+// is dropped exactly as permanently as the torn-read case above. Hashing
+// both ends does not make this collision-proof — nothing short of a full
+// hash does — but it turns "any late edit of the same length" into
+// "a late edit that also collides on 128 bits".
 export async function checksum(uri) {
-    const maxBytes = 300 * 1024
     const { size } = await stat(uri)
-    if (size < maxBytes) {
+    if (size < CHECKSUM_MAX_BYTES) {
         return await hashFile(uri, { algorithm: 'md5' })
-    } else {
-        const truncate = new TruncateStream({ maxBytes })
-        const fileStream = createReadStream(uri)
-        fileStream.pipe(truncate)
-        const checksum = size.toString() + ':' + await hash(truncate, { algorithm: 'md5' })
-        return checksum
+    }
+    const head = await hashRange(uri, 0, CHECKSUM_MAX_BYTES)
+    const tail = await hashRange(uri, size - CHECKSUM_MAX_BYTES, CHECKSUM_MAX_BYTES)
+    return `${size}:${head}:${tail}`
+}
+
+// md5 of `length` bytes starting at `start`. A positional read, so the
+// bytes in between are never touched.
+async function hashRange(uri, start, length) {
+    const handle = await open(uri, 'r')
+    try {
+        const buf = Buffer.allocUnsafe(length)
+        const { bytesRead } = await handle.read(buf, 0, length, start)
+        return createHash('md5').update(buf.subarray(0, bytesRead)).digest('hex')
+    } finally {
+        await handle.close()
     }
 }