mikser-io 10.6.0 → 10.8.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mikser-io",
-  "version": "10.6.0",
+  "version": "10.8.0",
   "files": [
     "app.js",
     "index.js",

package/src/engine.js

@@ -1593,6 +1593,16 @@ The full version, with what each code means: docs/diagnostics.md`)
     // undefined here. The logger has no such problem — it writes during the
     // run, by which time options exist.
     const quietStdout = ['--json', '--tool', '--tools'].some(flag => process.argv.includes(flag))
+    // The banner is an info record, so a level above info must not print it.
+    //
+    // `--log silent` printed the version line and nothing else — the one line
+    // the flag most obviously promises to remove — because this runs before
+    // the level is applied. Only the FORWARDED path was ever silent, and that
+    // is the path the test exercises, so the test could not see it.
+    //
+    // argv again, for the reason quietStdout reads it: commander parses in a
+    // lifecycle hook, which runs after setup() returns.
+    if (!argvWantsInfo()) return runtime
     // Through the logger when nobody is watching, so it gets a timestamp.
     //
     // This line marks a process start, which in a supervisor's log is the
@@ -1625,6 +1635,26 @@ The full version, with what each code means: docs/diagnostics.md`)
     return runtime
 }
 
+// The level asked for on the command line, before commander has parsed.
+// Both spellings, both flags: `--log warn` and `--log=warn`, and
+// `--log-install`, which on a run with no instance to forward to sets the
+// level for this process.
+function argvLogLevel() {
+    for (const flag of ['--log', '--log-install', '-l']) {
+        const at = process.argv.indexOf(flag)
+        if (at >= 0 && process.argv[at + 1]) return process.argv[at + 1]
+        const inline = process.argv.find(arg => arg.startsWith(`${flag}=`))
+        if (inline) return inline.slice(flag.length + 1)
+    }
+    return null
+}
+
+function argvWantsInfo() {
+    const level = argvLogLevel()
+    if (!level || !LOG_LEVELS.includes(level)) return true
+    return LOG_LEVELS.indexOf(level) <= LOG_LEVELS.indexOf('info')
+}
+
 export function useLogger() {
     return runtime.engine?.logger
 }

package/src/instance.js

@@ -218,10 +218,29 @@ let server = null
 // both into one undifferentiated stream throws that away, and the client can
 // only guess — it guessed stderr, so every forwarded document landed where no
 // consumer looks while the command exited 0.
-function captureOutput(onChunk) {
+// `echoStdout: false` frames the chunk to the client and writes NOTHING
+// locally.
+//
+// The instance wears the client's output so an operator watching it sees what
+// a forwarded request did — right for log lines, wrong for a DOCUMENT. A
+// --json request put its 364-line report into the instance's own console as
+// well as onto the client's stdout, and under pm2 that is 364 lines in the out
+// log for every report an agent asks for.
+//
+// The seam is exact rather than a guess: under --json / --tool the logger
+// writes to stderr and stdout carries only the document (see logger.js), so
+// stdout chunks during a document-carrying request ARE the document.
+function captureOutput(onChunk, { echoStdout = true } = {}) {
     const originals = [process.stdout.write, process.stderr.write]
     const patch = (stream, original, name) => function (chunk, encoding, callback) {
         try { onChunk(typeof chunk === 'string' ? chunk : chunk.toString(), name) } catch { /* client gone */ }
+        if (!echoStdout && name === 'stdout') {
+            // write()'s callback is the second argument when encoding is
+            // omitted. Dropped, a caller awaiting the drain never resumes.
+            const done = typeof encoding === 'function' ? encoding : callback
+            if (typeof done === 'function') process.nextTick(done)
+            return true
+        }
         return original.call(stream, chunk, encoding, callback)
     }
     process.stdout.write = patch(process.stdout, originals[0], 'stdout')
@@ -242,6 +261,12 @@ function captureOutput(onChunk) {
 // not what tells the two apart. Different configs are different files.
 // Does this request name a level that does not exist? Pure, so the refusal
 // can be decided before anything is applied.
+// Does this request's answer go on stdout as a document?
+// The same trio logger.js gates the progress bar on, for the same reason.
+function carriesDocument(request) {
+    return Boolean(request?.json || request?.tool || request?.tools)
+}
+
 function logRequestRefusal(request) {
     for (const [flag, level] of [['--log', request?.log], ['--log-install', request?.logInstall]]) {
         if (level !== undefined && level !== null && !LOG_LEVELS.includes(level)) {
@@ -296,7 +321,9 @@ async function configStale() {
 // the only process that can answer correctly. Same guards as a build: wrong
 // config refuses, drifted config refuses.
 async function serveReport(socket, request, logger) {
-    const restore = captureOutput((chunk, stream) => frame(socket, { type: 'log', chunk, stream }))
+    const restore = captureOutput(
+        (chunk, stream) => frame(socket, { type: 'log', chunk, stream }),
+        { echoStdout: !carriesDocument(request) })
     let code = 0
     try {
         code = await withRequestOutput(request, () => runReportOnly(request)) ?? 0
@@ -501,7 +528,9 @@ async function withRequestOutput(request, run) {
 }
 
 async function serveBuild(socket, request, logger) {
-    const restore = captureOutput((chunk, stream) => frame(socket, { type: 'log', chunk, stream }))
+    const restore = captureOutput(
+        (chunk, stream) => frame(socket, { type: 'log', chunk, stream }),
+        { echoStdout: !carriesDocument(request) })
     let code = 0
     try {
         // Fire the pending debounce rather than waiting it out.

package/src/journal.js

@@ -227,9 +227,8 @@ export async function* useJournal(name, operations, signal) {
                 stopProgress()
                 throw new AbortError()
             }
-            updateProgress()
-
             const entry = rowToEntry(row)
+            updateProgress(entry.entity?.id)
             const originalEntity = row.entity   // already a JSON string
 
             yield entry

package/src/logger.js

@@ -28,6 +28,7 @@ import pino from 'pino'
 import pretty from 'pino-pretty'
 import Gauge from 'gauge'
 import { Writable } from 'node:stream'
+import path from 'node:path'
 import runtime from './runtime.js'
 import { useLogger } from './engine.js'
 import { onLoad, onFinalized } from './lifecycle.js'
@@ -482,10 +483,65 @@ onLoad(() => {
     runtime.engine.logger = createMikserLogger(level)
 })
 
-// Progress API. Surface preserved: trackProgress / updateProgress /
-// stopProgress / updateProgressDetails. Gauge-backed; no-op in non-TTY
-// contexts or when runtime.options.info is false (i.e. --debug / --trace
-// modes, where logs are voluminous and a bar would just be noise).
+// Progress API: trackProgress / updateProgress / stopProgress /
+// updateProgressDetails. A gauge where a terminal is watching, coded records
+// where one is not, and nothing at all for a phase that took no time.
+
+// Long phases only, and long means TIME.
+//
+// A record per quartile is right for 800 documents and absurd for 5. Eleven
+// of a plain build's thirteen phases carry exactly five items, so reporting
+// them the way a bar counts took a piped no-op build from 32 lines to 97 —
+// sixty-five progress records, five in the same millisecond to say a phase
+// did nothing. That is written into the log a deployment keeps, which is the
+// log the installed-level expiry exists to protect.
+//
+// A minimum TOTAL would fix the count and get the other half wrong: four PDFs
+// through Chrome is a small phase and a slow one, and it is exactly the phase
+// worth narrating. Elapsed time is what "long" means and it needs no
+// per-phase tuning.
+//
+// It is an INTERVAL rather than a threshold, and quartiles are gone with it.
+// A quartile is a fraction of the WORK, so it says nothing about how often a
+// line appears: four records land in three seconds on a fast phase and four
+// records cover an hour on a slow one. Time is the axis a reader cares about,
+// so one line per interval, however much work passed in between.
+//
+// This gates the RUNNING commentary only. `progress-finished` is not behind
+// it: every phase says that it ran and what it cost, whatever the duration.
+// Gated, a short phase produced no record at all and a build could not say
+// what it had done — and off a TTY that line is the only place phase timings
+// come from.
+export const PROGRESS_INTERVAL_MS = 30_000
+
+// Which item, not just how far. A phase name alone says a build is doing
+// something; the thing a stuck build needs to say is WHAT it is stuck on.
+function progressDetail(detail) {
+    if (detail === null || detail === undefined || detail === '') return null
+    const text = String(detail)
+    const root = runtime.options?.workingFolder
+    if (!root || !path.isAbsolute(text)) return text
+    // An entity id LOOKS absolute and is not a filesystem path.
+    // `/documents/p3706.md` relativised against the working folder becomes
+    // `../../../documents/p3706.md`, which names nothing anyone can act on.
+    // So shorten only what is genuinely INSIDE the folder, and leave every
+    // other string exactly as the call site handed it over.
+    const relative = path.relative(root, text)
+    return relative.startsWith('..') ? text : relative
+}
+
+// `0s` for anything under a second reports the whole phase as nothing. Below
+// a second the number IS the information.
+function formatDuration(ms) {
+    return ms < 1000 ? `${ms}ms` : `${(ms / 1000).toFixed(1)}s`
+}
+
+function emitProgress({ name, total, value, detail }) {
+    const at = progressDetail(detail)
+    const fields = { code: 'progress', phase: name, total, value }
+    if (at) useLogger()?.info({ ...fields, detail: at }, '%s: %d/%d — %s', name, value, total, at)
+    else useLogger()?.info(fields, '%s: %d/%d', name, value, total)
+}
 
 export function trackProgress(name, total) {
     if (!name || !total) return
@@ -512,26 +568,31 @@ export function trackProgress(name, total) {
     // --json would have extended that to every machine reading the output.
     // Losing the graphics is the point; losing the information is not.
     const drawn = !carriesDocument && Boolean(process.stdout.isTTY) && Boolean(runtime.options.info)
-    currentBar = { name, total, value: 0, started: Date.now(), drawn, milestone: 0 }
+    currentBar = { name, total, value: 0, started: Date.now(), drawn, lastReport: Date.now(), detail: null }
     if (drawn) ensureGauge().show({ section: name, subsection: `0/${total}` }, 0)
-    else logger.info({ code: 'progress', phase: name, total, value: 0 }, '%s: 0/%d', name, total)
 }
 
-export function updateProgress() {
+export function updateProgress(detail) {
     if (!currentBar) return
     currentBar.value++
+    // The call site hands over whatever identifies the item it is on — a
+    // path, an id — and pays one assignment for it. Formatting happens at
+    // emit time, at most four times a phase, so naming the work costs the
+    // 14k-entity import nothing.
+    if (detail !== undefined) currentBar.detail = detail
     const { name, total, value, drawn } = currentBar
     if (drawn) {
         gauge?.show({ section: name, subsection: `${value}/${total}` }, value / total)
     } else {
-        // Quartiles, not every item: a bar redraws in place and costs one
-        // line, a log record does not. 800 documents is 800 lines of noise if
-        // this counts the way the bar does.
-        const reached = Math.floor((value / total) * 4)
-        if (reached > currentBar.milestone && value < total) {
-            currentBar.milestone = reached
-            useLogger()?.info({ code: 'progress', phase: name, total, value },
-                '%s: %d/%d', name, value, total)
+        // One line per interval, not per item and not per quartile: a bar
+        // redraws in place and costs one line, a log record does not, so 800
+        // documents is 800 lines of noise if this counts the way the bar does.
+        // A phase that finishes inside the interval says nothing at all while
+        // it runs — its finished line covers it.
+        const now = Date.now()
+        if (now - currentBar.lastReport >= PROGRESS_INTERVAL_MS && value < total) {
+            currentBar.lastReport = now
+            emitProgress(currentBar)
         }
     }
     if (value >= total) stopProgress()
@@ -541,16 +602,31 @@ export function stopProgress() {
     if (!currentBar) return
     const logger = useLogger()
     const { name, total, value, started } = currentBar
+    const ms = Date.now() - started
     gauge?.hide()
     // Structured either way, so a machine reading --json's stderr gets the
     // same facts a person reads off the bar.
+    //
+    // This line has to stand alone. With the running commentary on an
+    // interval it is the ONLY record most phases produce, and `Documents
+    // import finished: 5 0s` said neither what the five were nor how long it
+    // took — a count with no subject and a duration rounded away to nothing.
+    //
+    // The subject is the PHASE, not an item. The last entity a phase happened
+    // to walk is not what the phase was about: seven journal phases in a row
+    // reported the same `/layouts/page.hbs` because that is where the walk
+    // ended, and `Files import finished: 3, last .../social-fb.svg` put a
+    // filename into the build log that nothing had anything to say about — it
+    // broke a test asserting that file is never mentioned, which is exactly
+    // the misreading it invites. `Files import finished: 3 in 3ms` already
+    // says what finished, how much of it, and what it cost. The running
+    // records keep the item, because there it shows MOVEMENT.
     if (value < total) {
         logger.warn({ code: 'progress-unfinished', phase: name, total, value, missing: total - value },
-            '%s unfinished: %d', name, total - value)
+            '%s unfinished: %d of %d after %s', name, total - value, total, formatDuration(ms))
     } else {
-        const ms = Date.now() - started
         logger.info({ code: 'progress-finished', phase: name, total, ms },
-            '%s finished: %d %ds', name, total, Math.round(ms / 1000))
+            '%s finished: %d in %s', name, total, formatDuration(ms))
     }
     currentBar = null
 }

package/src/plugins/files.js

@@ -228,7 +228,7 @@ export function files(options = {}) {
                 // nothing to emit. Progress ticks either way — a gated file
                 // was still looked at.
                 const newChecksum = await gateChecksum(source, id, { priorChecksums })
-                updateProgress()
+                updateProgress(id)
                 if (newChecksum === null) return
                 await createEntity({
                     id,

package/src/plugins/observer.js

@@ -100,7 +100,7 @@ export function observer(options = {}) {
                         synced++
                     }
                 }
-                updateProgress()
+                updateProgress(meta.id)
             }
 
             const entitiesToRemove = await findEntities({
@@ -114,7 +114,7 @@ export function observer(options = {}) {
             for (let entity of entitiesToRemove) {
                 deleteEntity(entity)
                 removed++
-                updateProgress()
+                updateProgress(entity.id)
             }
             if (synced || removed) {
                 logger.debug('Syncing api [%s] synced: %d, removed: %d', collection, synced, removed)

package/src/plugins/resources.js

@@ -145,7 +145,7 @@ export function resources(options = {}) {
                     logger.error('Resource error: %s %s %s', entity.id, resource, err.message)
                 }
             }
-            updateProgress()
+            updateProgress(resource)
         }
 
         const resourceFiles = await globby('**/*', { cwd: runtime.options.resourcesFolder })
@@ -226,7 +226,7 @@ export function resources(options = {}) {
                         checksum: await checksum(resource)
                     })
                 }
-                updateProgress()
+                updateProgress(resource)
             }, { concurrency: 10, signal })
             count && logger.info('Downloaded: %d', count)
         }

package/src/source.js

@@ -431,7 +431,7 @@ export function useSource(core, options) {
         // is never the bottleneck.
         await pMap(files, async (file) => {
             await registerFile(file, { logger, scanned, stats: scanStats, priorChecksums })
-            if (phase === 'import') updateProgress()
+            if (phase === 'import') updateProgress(file)
         }, { concurrency: SCAN_CONCURRENCY })
 
         scanStats.deleted = await sweepDeleted(collection, scanned, async (e) => {