mikser-io 10.5.0 → 10.7.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mikser-io",
-  "version": "10.5.0",
+  "version": "10.7.0",
   "files": [
     "app.js",
     "index.js",

package/src/engine.js

@@ -25,8 +25,8 @@ import Queue from 'p-queue'
 import packageInfo from '../package.json' with { type: 'json' }
 import { attachServerCliOptions, setupServer } from './server.js'
 import {
-    createMikserLogger, setLogLevel, rememberBaseLevel, installLogLevel, resetLogLevel,
-    applyInstalledLogLevel, installedLogLevel, LOG_LEVELS, INSTALLED_LOG_TTL_MS,
+    createMikserLogger, rememberBaseLevel, applyLogRequest,
+    LOG_LEVELS, INSTALLED_LOG_TTL_MS,
 } from './logger.js'
 import { inputHashOf } from './utils.js'
 import { createTrack, mergeTrack } from './track.js'
@@ -685,14 +685,14 @@ The full version, with what each code means: docs/diagnostics.md`)
         // logger — the whole reason --debug did nothing.
         runtime.options.info = true
         const asked = runtime.options.log
-        if (asked) {
-            if (!LOG_LEVELS.includes(asked)) {
-                throw new Error(`--log ${asked}: no such level. Levels: ${LOG_LEVELS.join(', ')}`)
-            }
-            setLogLevel(asked)
-            // A bar on top of debug output is noise, and silent means silent.
-            if (asked === 'trace' || asked === 'debug' || asked === 'silent') runtime.options.info = false
-        }
+        // The same call the instance makes for a forwarded request — see
+        // applyLogRequest. Two implementations drifted within one commit.
+        const logRefusal = applyLogRequest({
+            log: asked,
+            logInstall: runtime.options.logInstall,
+            logReset: runtime.options.logReset,
+        })
+        if (logRefusal) throw new Error(logRefusal)
         rememberBaseLevel(asked && LOG_LEVELS.includes(asked) ? asked : 'info')
 
         // Resolve folders inside onInitialize so journal.js and
@@ -1593,6 +1593,16 @@ The full version, with what each code means: docs/diagnostics.md`)
     // undefined here. The logger has no such problem — it writes during the
     // run, by which time options exist.
     const quietStdout = ['--json', '--tool', '--tools'].some(flag => process.argv.includes(flag))
+    // The banner is an info record, so a level above info must not print it.
+    //
+    // `--log silent` printed the version line and nothing else — the one line
+    // the flag most obviously promises to remove — because this runs before
+    // the level is applied. Only the FORWARDED path was ever silent, and that
+    // is the path the test exercises, so the test could not see it.
+    //
+    // argv again, for the reason quietStdout reads it: commander parses in a
+    // lifecycle hook, which runs after setup() returns.
+    if (!argvWantsInfo()) return runtime
     // Through the logger when nobody is watching, so it gets a timestamp.
     //
     // This line marks a process start, which in a supervisor's log is the
@@ -1625,6 +1635,26 @@ The full version, with what each code means: docs/diagnostics.md`)
     return runtime
 }
 
+// The level asked for on the command line, before commander has parsed.
+// Both spellings, both flags: `--log warn` and `--log=warn`, and
+// `--log-install`, which on a run with no instance to forward to sets the
+// level for this process.
+function argvLogLevel() {
+    for (const flag of ['--log', '--log-install', '-l']) {
+        const at = process.argv.indexOf(flag)
+        if (at >= 0 && process.argv[at + 1]) return process.argv[at + 1]
+        const inline = process.argv.find(arg => arg.startsWith(`${flag}=`))
+        if (inline) return inline.slice(flag.length + 1)
+    }
+    return null
+}
+
+function argvWantsInfo() {
+    const level = argvLogLevel()
+    if (!level || !LOG_LEVELS.includes(level)) return true
+    return LOG_LEVELS.indexOf(level) <= LOG_LEVELS.indexOf('info')
+}
+
 export function useLogger() {
     return runtime.engine?.logger
 }

package/src/instance.js

@@ -38,7 +38,7 @@ import { onLoaded } from './lifecycle.js'
 import { renderErrorCount } from './report.js'
 import { runReportOnly } from './engine.js'
 import {
-    setLogLevel, installLogLevel, resetLogLevel, restingLogLevel,
+    setLogLevel, restingLogLevel, applyLogRequest, LOG_LEVELS,
 } from './logger.js'
 import { emitReport } from './report.js'
 import { pluginOptionsFrom } from './cli.js'
@@ -218,10 +218,29 @@ let server = null
 // both into one undifferentiated stream throws that away, and the client can
 // only guess — it guessed stderr, so every forwarded document landed where no
 // consumer looks while the command exited 0.
-function captureOutput(onChunk) {
+// `echoStdout: false` frames the chunk to the client and writes NOTHING
+// locally.
+//
+// The instance wears the client's output so an operator watching it sees what
+// a forwarded request did — right for log lines, wrong for a DOCUMENT. A
+// --json request put its 364-line report into the instance's own console as
+// well as onto the client's stdout, and under pm2 that is 364 lines in the out
+// log for every report an agent asks for.
+//
+// The seam is exact rather than a guess: under --json / --tool the logger
+// writes to stderr and stdout carries only the document (see logger.js), so
+// stdout chunks during a document-carrying request ARE the document.
+function captureOutput(onChunk, { echoStdout = true } = {}) {
     const originals = [process.stdout.write, process.stderr.write]
     const patch = (stream, original, name) => function (chunk, encoding, callback) {
         try { onChunk(typeof chunk === 'string' ? chunk : chunk.toString(), name) } catch { /* client gone */ }
+        if (!echoStdout && name === 'stdout') {
+            // write()'s callback is the second argument when encoding is
+            // omitted. Dropped, a caller awaiting the drain never resumes.
+            const done = typeof encoding === 'function' ? encoding : callback
+            if (typeof done === 'function') process.nextTick(done)
+            return true
+        }
         return original.call(stream, chunk, encoding, callback)
     }
     process.stdout.write = patch(process.stdout, originals[0], 'stdout')
@@ -240,6 +259,23 @@ function captureOutput(onChunk) {
 // than by content hash — the hash would require the client to import its
 // config, which is most of the startup forwarding exists to skip, and it is
 // not what tells the two apart. Different configs are different files.
+// Does this request name a level that does not exist? Pure, so the refusal
+// can be decided before anything is applied.
+// Does this request's answer go on stdout as a document?
+// The same trio logger.js gates the progress bar on, for the same reason.
+function carriesDocument(request) {
+    return Boolean(request?.json || request?.tool || request?.tools)
+}
+
+function logRequestRefusal(request) {
+    for (const [flag, level] of [['--log', request?.log], ['--log-install', request?.logInstall]]) {
+        if (level !== undefined && level !== null && !LOG_LEVELS.includes(level)) {
+            return `${flag} ${level}: no such level. Levels: ${LOG_LEVELS.join(', ')}`
+        }
+    }
+    return null
+}
+
 function configMismatch(theirs) {
     if (!theirs) return null
     const mine = path.resolve(runtime.options.config ?? 'mikser.config.js')
@@ -285,7 +321,9 @@ async function configStale() {
 // the only process that can answer correctly. Same guards as a build: wrong
 // config refuses, drifted config refuses.
 async function serveReport(socket, request, logger) {
-    const restore = captureOutput((chunk, stream) => frame(socket, { type: 'log', chunk, stream }))
+    const restore = captureOutput(
+        (chunk, stream) => frame(socket, { type: 'log', chunk, stream }),
+        { echoStdout: !carriesDocument(request) })
     let code = 0
     try {
         code = await withRequestOutput(request, () => runReportOnly(request)) ?? 0
@@ -369,6 +407,26 @@ function refuseClear(socket, request) {
     return true
 }
 
+// A level the instance cannot use.
+//
+// Validated where the request ARRIVES rather than where it is applied, because
+// applying it is exactly what a bad level cannot do: setLogLevel returns false
+// and, before this, nobody read it — so `--log chatty` exited 1 locally and
+// built normally with a watcher up. The refusal frame is how the instance says
+// no to everything else, and a client that mistypes a level deserves the same
+// answer whether or not something happens to be listening.
+function refuseLogLevel(socket, request) {
+    const refusal = logRequestRefusal(request)
+    if (!refusal) return false
+    frame(socket, {
+        type: 'refused',
+        reason: refusal,
+        detail: 'The instance would otherwise have built normally and said nothing, which is the '
+            + 'forwarded-versus-local split this surface exists to remove.',
+    })
+    return true
+}
+
 // Which process to restart.
 //
 // "Restart it" is only actionable if you know which `it` — and the machine
@@ -428,15 +486,25 @@ async function withRequestOutput(request, run) {
     // a watcher's own cycle before or after keeps the level it was running at.
     // An installed level is different and deliberately not touched here — it
     // outlives the request, which is its whole point.
-    if (request.logReset) resetLogLevel()
-    if (request.logInstall) installLogLevel(request.logInstall)
-    if (request.log) setLogLevel(request.log)
-
-    for (const key of ['json', 'tool', 'tools', 'requested']) {
+    // Captured BEFORE applyLogRequest, which writes `info`.
+    //
+    // `info` is here because --log silent turns the progress bar off through
+    // it, and without a restore one silent request left the instance with no
+    // bar for the rest of its life. Capturing after the call would have
+    // recorded the value the call just wrote and restored nothing — the same
+    // shape of mistake, one line further on.
+    for (const key of ['json', 'tool', 'tools', 'requested', 'info']) {
         prior[key] = runtime.options[key]
         if (request[key]) runtime.options[key] = request[key]
     }
 
+    // The same call the argv path makes — see applyLogRequest. The level was
+    // validated at arrival by refuseLogLevel, so a refusal here is impossible;
+    // it is asserted rather than ignored, because "cannot happen" is how the
+    // first version of this silently accepted a bad level.
+    const refusal = applyLogRequest(request)
+    if (refusal) throw new Error(refusal)
+
     // Whatever the client's argv said about a PLUGIN's options.
     //
     // The instance parsed its own argv and never saw the client's, so a flag a
@@ -460,7 +528,9 @@ async function withRequestOutput(request, run) {
 }
 
 async function serveBuild(socket, request, logger) {
-    const restore = captureOutput((chunk, stream) => frame(socket, { type: 'log', chunk, stream }))
+    const restore = captureOutput(
+        (chunk, stream) => frame(socket, { type: 'log', chunk, stream }),
+        { echoStdout: !carriesDocument(request) })
     let code = 0
     try {
         // Fire the pending debounce rather than waiting it out.
@@ -558,6 +628,7 @@ export function serveInstance() {
                     // writes anything.
                     if (refuseUnknownFlags(socket, request)) return
                     if (refuseClear(socket, request)) return
+                    if (refuseLogLevel(socket, request)) return
                     const wrongConfig = configMismatch(request.config)
                     if (wrongConfig) return refuseConfig(socket, request, wrongConfig)
                     const movedFile = await configStale()

package/src/journal.js

@@ -227,9 +227,8 @@ export async function* useJournal(name, operations, signal) {
                 stopProgress()
                 throw new AbortError()
             }
-            updateProgress()
-
             const entry = rowToEntry(row)
+            updateProgress(entry.entity?.id)
             const originalEntity = row.entity   // already a JSON string
 
             yield entry

package/src/logger.js

@@ -28,6 +28,7 @@ import pino from 'pino'
 import pretty from 'pino-pretty'
 import Gauge from 'gauge'
 import { Writable } from 'node:stream'
+import path from 'node:path'
 import runtime from './runtime.js'
 import { useLogger } from './engine.js'
 import { onLoad, onFinalized } from './lifecycle.js'
@@ -482,25 +483,103 @@ onLoad(() => {
     runtime.engine.logger = createMikserLogger(level)
 })
 
-// Progress API. Surface preserved: trackProgress / updateProgress /
-// stopProgress / updateProgressDetails. Gauge-backed; no-op in non-TTY
-// contexts or when runtime.options.info is false (i.e. --debug / --trace
-// modes, where logs are voluminous and a bar would just be noise).
+// Progress API: trackProgress / updateProgress / stopProgress /
+// updateProgressDetails. A gauge where a terminal is watching, coded records
+// where one is not, and nothing at all for a phase that took no time.
+
+// Long phases only, and long means TIME.
+//
+// A record per quartile is right for 800 documents and absurd for 5. Eleven
+// of a plain build's thirteen phases carry exactly five items, so reporting
+// them the way a bar counts took a piped no-op build from 32 lines to 97 —
+// sixty-five progress records, five in the same millisecond to say a phase
+// did nothing. That is written into the log a deployment keeps, which is the
+// log the installed-level expiry exists to protect.
+//
+// A minimum TOTAL would fix the count and get the other half wrong: four PDFs
+// through Chrome is a small phase and a slow one, and it is exactly the phase
+// worth narrating. Elapsed time is what "long" means, it needs no per-phase
+// tuning, and it applies to the drawn path too — `finished: 5 0s` is the same
+// non-information in a terminal, and its own `0s` says so.
+export const PROGRESS_MIN_MS = 1000
+
+// Which item, not just how far. A phase name alone says a build is doing
+// something; the thing a stuck build needs to say is WHAT it is stuck on.
+function progressDetail(detail) {
+    if (detail === null || detail === undefined || detail === '') return null
+    const text = String(detail)
+    const root = runtime.options?.workingFolder
+    if (!root || !path.isAbsolute(text)) return text
+    // An entity id LOOKS absolute and is not a filesystem path.
+    // `/documents/p3706.md` relativised against the working folder becomes
+    // `../../../documents/p3706.md`, which names nothing anyone can act on.
+    // So shorten only what is genuinely INSIDE the folder, and leave every
+    // other string exactly as the call site handed it over.
+    const relative = path.relative(root, text)
+    return relative.startsWith('..') ? text : relative
+}
+
+function emitProgress({ name, total, value, detail }) {
+    const at = progressDetail(detail)
+    const fields = { code: 'progress', phase: name, total, value }
+    if (at) useLogger()?.info({ ...fields, detail: at }, '%s: %d/%d — %s', name, value, total, at)
+    else useLogger()?.info(fields, '%s: %d/%d', name, value, total)
+}
 
 export function trackProgress(name, total) {
     if (!name || !total) return
     const logger = useLogger()
     logger.debug('%s started: %d', name, total)
-    if (!process.stdout.isTTY || !runtime.options.info) return
-    currentBar = { name, total, value: 0, started: Date.now() }
-    ensureGauge().show({ section: name, subsection: `0/${total}` }, 0)
+    // The bar writes to stdout, and stdout is where --json and --tool put
+    // their DOCUMENT. Forwarded, those writes are captured and framed to the
+    // client, so the gauge landed inside the JSON:
+    // `^[[?25lDocuments import: >416/800` at byte 0, and JSON.parse threw —
+    // 4 runs in 10 at the default level on an 800-document corpus.
+    //
+    // Checked HERE rather than through runtime.options.info, because this is
+    // the actual invariant and info is a preference. A preference can be
+    // forgotten on a path; an invariant stated at the one place a bar starts
+    // cannot.
+    const carriesDocument = runtime.options?.json || runtime.options?.tool || runtime.options?.tools
+
+    // TRACKED always, DRAWN only where a bar belongs.
+    //
+    // The two used to be one decision, so anything that could not draw also
+    // stopped counting — and `stopProgress` returns early without a bar, which
+    // is where the "finished: N in Ns" line comes from. A piped build
+    // therefore reported no phase timings at all, and suppressing the bar for
+    // --json would have extended that to every machine reading the output.
+    // Losing the graphics is the point; losing the information is not.
+    const drawn = !carriesDocument && Boolean(process.stdout.isTTY) && Boolean(runtime.options.info)
+    currentBar = { name, total, value: 0, started: Date.now(), drawn, milestone: 0, detail: null }
+    if (drawn) ensureGauge().show({ section: name, subsection: `0/${total}` }, 0)
 }
 
-export function updateProgress() {
+export function updateProgress(detail) {
     if (!currentBar) return
     currentBar.value++
-    const { name, total, value } = currentBar
-    gauge?.show({ section: name, subsection: `${value}/${total}` }, value / total)
+    // The call site hands over whatever identifies the item it is on — a
+    // path, an id — and pays one assignment for it. Formatting happens at
+    // emit time, at most four times a phase, so naming the work costs the
+    // 14k-entity import nothing.
+    if (detail !== undefined) currentBar.detail = detail
+    const { name, total, value, drawn } = currentBar
+    if (drawn) {
+        gauge?.show({ section: name, subsection: `${value}/${total}` }, value / total)
+    } else {
+        // Quartiles, not every item: a bar redraws in place and costs one
+        // line, a log record does not. 800 documents is 800 lines of noise if
+        // this counts the way the bar does.
+        //
+        // The milestone advances whether or not the record is emitted. Held
+        // back, every quartile the phase passed under the threshold would fire
+        // in a burst the moment it crossed.
+        const reached = Math.floor((value / total) * 4)
+        if (reached > currentBar.milestone && value < total) {
+            currentBar.milestone = reached
+            if (Date.now() - currentBar.started >= PROGRESS_MIN_MS) emitProgress(currentBar)
+        }
+    }
     if (value >= total) stopProgress()
 }
 
@@ -508,12 +587,19 @@ export function stopProgress() {
     if (!currentBar) return
     const logger = useLogger()
     const { name, total, value, started } = currentBar
+    const ms = Date.now() - started
     gauge?.hide()
+    // Structured either way, so a machine reading --json's stderr gets the
+    // same facts a person reads off the bar.
+    //
+    // A phase that did not finish is worth a line at any duration — it is a
+    // warning, not progress, and the whole point is that it is unexpected.
     if (value < total) {
-        logger.warn('%s unfinished: %d', name, total - value)
-    } else {
-        const elapsed = Math.round((Date.now() - started) / 1000)
-        logger.info('%s finished: %d %ds', name, total, elapsed)
+        logger.warn({ code: 'progress-unfinished', phase: name, total, value, missing: total - value },
+            '%s unfinished: %d', name, total - value)
+    } else if (ms >= PROGRESS_MIN_MS) {
+        logger.info({ code: 'progress-finished', phase: name, total, ms },
+            '%s finished: %d %ds', name, total, Math.round(ms / 1000))
     }
     currentBar = null
 }
@@ -528,6 +614,38 @@ export function updateProgressDetails(details) {
     )
 }
 
+// What a caller asked for about logging, applied the same way from argv and
+// from a forwarded request.
+//
+// It was written twice and the copies drifted immediately: the argv path threw
+// on an unknown level and set `info`, the forwarded path called setLogLevel and
+// ignored the false it returns. So `--log chatty` exited 1 locally and built
+// normally with a watcher up, and `--log silent` left the progress bar running
+// on an instance. The same forwarded/local split --json and --force each had,
+// and this feature's own argument against itself: a flag that lies is worse
+// than a flag that is missing.
+//
+// Returns an error STRING rather than throwing, because the two callers need
+// different things from a failure — argv throws, the instance refuses over the
+// socket — and a shared implementation should not decide that for them.
+export function applyLogRequest({ log, logInstall, logReset } = {}) {
+    for (const [flag, level] of [['--log', log], ['--log-install', logInstall]]) {
+        if (level !== undefined && level !== null && !LOG_LEVELS.includes(level)) {
+            return `${flag} ${level}: no such level. Levels: ${LOG_LEVELS.join(', ')}`
+        }
+    }
+    if (logReset) resetLogLevel()
+    if (logInstall) installLogLevel(logInstall)
+    if (log) setLogLevel(log)
+
+    // A bar on top of debug output is noise, and silent means silent.
+    const level = log || logInstall
+    if (level === 'trace' || level === 'debug' || level === 'silent') {
+        runtime.options.info = false
+    }
+    return null
+}
+
 // An installed level, disclosed and expired, once per cycle.
 //
 // Disclosed for the reason an installed command is: it is state left on a live

package/src/plugins/files.js

@@ -228,7 +228,7 @@ export function files(options = {}) {
                 // nothing to emit. Progress ticks either way — a gated file
                 // was still looked at.
                 const newChecksum = await gateChecksum(source, id, { priorChecksums })
-                updateProgress()
+                updateProgress(id)
                 if (newChecksum === null) return
                 await createEntity({
                     id,

package/src/plugins/observer.js

@@ -100,7 +100,7 @@ export function observer(options = {}) {
                         synced++
                     }
                 }
-                updateProgress()
+                updateProgress(meta.id)
             }
 
             const entitiesToRemove = await findEntities({
@@ -114,7 +114,7 @@ export function observer(options = {}) {
             for (let entity of entitiesToRemove) {
                 deleteEntity(entity)
                 removed++
-                updateProgress()
+                updateProgress(entity.id)
             }
             if (synced || removed) {
                 logger.debug('Syncing api [%s] synced: %d, removed: %d', collection, synced, removed)

package/src/plugins/resources.js

@@ -145,7 +145,7 @@ export function resources(options = {}) {
                     logger.error('Resource error: %s %s %s', entity.id, resource, err.message)
                 }
             }
-            updateProgress()
+            updateProgress(resource)
         }
 
         const resourceFiles = await globby('**/*', { cwd: runtime.options.resourcesFolder })
@@ -226,7 +226,7 @@ export function resources(options = {}) {
                         checksum: await checksum(resource)
                     })
                 }
-                updateProgress()
+                updateProgress(resource)
             }, { concurrency: 10, signal })
             count && logger.info('Downloaded: %d', count)
         }

package/src/source.js

@@ -431,7 +431,7 @@ export function useSource(core, options) {
         // is never the bottleneck.
         await pMap(files, async (file) => {
             await registerFile(file, { logger, scanned, stats: scanStats, priorChecksums })
-            if (phase === 'import') updateProgress()
+            if (phase === 'import') updateProgress(file)
         }, { concurrency: SCAN_CONCURRENCY })
 
         scanStats.deleted = await sweepDeleted(collection, scanned, async (e) => {