@vltpkg/output 1.6.0 → 1.6.2

package/README.md

@@ -3,3 +3,34 @@
 # @vltpkg/output
 
 ## TODO
+
+## Tracing
+
+`@vltpkg/output/trace` is a low overhead tracer for the install
+pipeline. It is off unless one of these is set, and when off every
+call returns after one boolean check.
+
+- `VLT_TRACE=<file>` appends one JSON line per event to `<file>`.
+  Every thread writes to the same file with the same wall clock, so
+  worker events and main thread events sort onto one timeline by `t`.
+- `VLT_PHASES=1` prints one `PHASES ...` line to stderr at exit: the
+  milliseconds between consecutive phase marks, the event loop delay
+  percentiles (how long the main thread was busy with synchronous work
+  while IO completions waited), and peak memory.
+
+```sh
+VLT_TRACE=trace.jsonl VLT_PHASES=1 vlt install
+./scripts/trace-report.ts trace.jsonl --bucket=250
+```
+
+The report prints the phase timeline, what each span kind cost
+(`manifest`, `request`, `extract`, `tarball:fetch`, `tarball:unpack`,
+`inflate`, `write`), how many of each were in flight per time bucket,
+and for every phase how long each lane sat idle. An idle lane next to
+a busy one is a head-of-line stall.
+
+Event kinds: `phase` (from `mark()`), `<kind>:start`/`<kind>:end`
+(from `span()`, paired by `url`, `spec` or `name`), `request` (the
+registry client's request events), `lane` (scheduler occupancy), `rss`
+(memory samples), `build:level*` (one BFS level of the ideal graph
+build), `open` and `exit`.

package/dist/index.js

@@ -1,4 +1,5 @@
 import EventEmitter from 'node:events';
+import { tracer } from "./trace.js";
 export class OutputEmitter {
     emitter = new EventEmitter();
     emit(eventName, payload) {
@@ -16,8 +17,15 @@ export class OutputEmitter {
 export const emitter = new OutputEmitter();
 export const logRequest = (url, state, details = {}) => {
     emitter.emit('request', { url, state, ...details });
+    if (tracer.file) {
+        tracer.trace('request', { ...details, url: String(url), state });
+    }
 };
 export const graphStep = (step) => {
     emitter.emit('graphStep', { step, state: 'start' });
-    return () => emitter.emit('graphStep', { step, state: 'stop' });
+    tracer.mark(step);
+    return () => {
+        emitter.emit('graphStep', { step, state: 'stop' });
+        tracer.mark(`${step}:done`);
+    };
 };

package/dist/trace.d.ts

@@ -0,0 +1,115 @@
+/**
+ * Extra fields recorded with a trace event. Keep them flat and small:
+ * every event is one JSON line.
+ */
+export type TraceFields = Record<string, string | number | boolean | undefined>;
+/**
+ * One trace event. `t` is an absolute wall clock in milliseconds
+ * (`performance.timeOrigin + performance.now()`), the same clock on every
+ * thread, so events from workers and the main thread can be laid on one
+ * timeline. `tid` is the worker thread id, 0 for the main thread.
+ */
+export type TraceEvent = {
+    t: number;
+    tid: number;
+    ev: string;
+} & TraceFields;
+export type TracerOptions = {
+    /** Path of the JSON-lines file to append events to (`VLT_TRACE`). */
+    file?: string;
+    /** Print a one-line phase summary to stderr at exit (`VLT_PHASES`). */
+    phases?: boolean;
+    /** The thread id to stamp events with. Defaults to the real one. */
+    threadId?: number;
+    /** Absolute clock in ms. Defaults to the shared performance clock. */
+    now?: () => number;
+    /** Appends a chunk of lines to the trace file. */
+    append?: (file: string, data: string) => void;
+    /** Writes the phase summary line. Defaults to stderr. */
+    write?: (line: string) => void;
+    /** Flush after this many buffered lines. */
+    flushLines?: number;
+    /** Flush this many ms after the first buffered line. */
+    flushMs?: number;
+    /** Sample `rss` this often, in ms. 0 disables sampling. */
+    sampleMs?: number;
+    /** Whether this is the main thread, which owns the event loop monitor. */
+    main?: boolean;
+};
+/**
+ * A low overhead tracer for the install pipeline.
+ *
+ * It is off unless `VLT_TRACE=<file>` or `VLT_PHASES=1` is set, and when
+ * off every method returns after one boolean check, so call sites can
+ * stay in the hot path.
+ *
+ * With `VLT_TRACE` every event is appended to the file as one JSON line.
+ * Lines are buffered and flushed every `flushLines` lines or `flushMs`
+ * ms, and at exit. Worker threads
+ * get their own instance from the same env and append to the same file,
+ * so the file interleaves threads; sort by `t` to read it.
+ *
+ * With `VLT_PHASES` the main thread prints one line at exit: the ms
+ * between consecutive {@link Tracer.mark} calls, the event loop delay
+ * percentiles (the direct measure of how long the main thread was busy
+ * with synchronous work while IO completions waited), and peak memory.
+ */
+export declare class Tracer {
+    #private;
+    readonly enabled: boolean;
+    readonly file?: string;
+    readonly phases: boolean;
+    constructor(options?: TracerOptions);
+    /**
+     * Build a tracer from the environment: `VLT_TRACE` names the file,
+     * `VLT_PHASES` (any value but `0` or empty) asks for the summary.
+     */
+    static fromEnv(env?: Record<string, string | undefined>, options?: Omit<TracerOptions, 'file' | 'phases'>): Tracer;
+    /** Record one event. */
+    trace(ev: string, fields?: TraceFields): void;
+    /**
+     * Record a `<ev>:start` event now and return a function that records
+     * `<ev>:end` with the elapsed `ms` (plus any fields it is given). The
+     * same `fields` are stamped on both ends so the two can be paired.
+     */
+    span(ev: string, fields?: TraceFields): (end?: TraceFields) => void;
+    /**
+     * Mark the start of a phase. The summary reports the ms between each
+     * mark and the next one, and the trace records a `phase` event.
+     */
+    mark(name: string, fields?: TraceFields): void;
+    /**
+     * Record how busy a resource lane is: how many jobs it is running and
+     * how many are waiting for a slot. The report derives idle time per
+     * lane from these.
+     */
+    lane(name: string, busy: number, queued: number): void;
+    /** Record a memory sample. */
+    sample(): void;
+    /** Write buffered lines to the file. */
+    flush(): void;
+    /**
+     * The one-line phase summary, or undefined when phases are off. Each
+     * phase is reported as the ms from its mark to the next mark (or to
+     * now, for the last one).
+     */
+    summary(): string | undefined;
+    /**
+     * Stop sampling, flush, and print the summary. Runs at process exit
+     * on the main thread; safe to call more than once.
+     */
+    close(): void;
+}
+/**
+ * The process-wide tracer, built from the environment. Worker threads
+ * inherit the environment, so they trace to the same file.
+ */
+export declare const tracer: Tracer;
+/** Record one event on the process-wide tracer. */
+export declare const trace: (ev: string, fields?: TraceFields) => void;
+/** Open a span on the process-wide tracer. See {@link Tracer.span}. */
+export declare const span: (ev: string, fields?: TraceFields) => ((end?: TraceFields) => void);
+/** Mark a phase on the process-wide tracer. See {@link Tracer.mark}. */
+export declare const mark: (name: string, fields?: TraceFields) => void;
+/** Record lane occupancy on the process-wide tracer. */
+export declare const lane: (name: string, busy: number, queued: number) => void;

package/dist/trace.js

@@ -0,0 +1,233 @@
+import { appendFileSync } from 'node:fs';
+import { monitorEventLoopDelay, performance } from 'node:perf_hooks';
+import { threadId as ownThreadId } from 'node:worker_threads';
+const sharedNow = () => performance.timeOrigin + performance.now();
+// Owner-only: traces record full request and tarball URLs, which can
+// carry tokens or signed query parameters. The mode applies when the file
+// is created; an existing file keeps its own.
+const defaultAppend = (file, data) => appendFileSync(file, data, { mode: 0o600 });
+const defaultWrite = (line) => process.stderr.write(line + '\n');
+/**
+ * A low overhead tracer for the install pipeline.
+ *
+ * It is off unless `VLT_TRACE=<file>` or `VLT_PHASES=1` is set, and when
+ * off every method returns after one boolean check, so call sites can
+ * stay in the hot path.
+ *
+ * With `VLT_TRACE` every event is appended to the file as one JSON line.
+ * Lines are buffered and flushed every `flushLines` lines or `flushMs`
+ * ms, and at exit. Worker threads
+ * get their own instance from the same env and append to the same file,
+ * so the file interleaves threads; sort by `t` to read it.
+ *
+ * With `VLT_PHASES` the main thread prints one line at exit: the ms
+ * between consecutive {@link Tracer.mark} calls, the event loop delay
+ * percentiles (the direct measure of how long the main thread was busy
+ * with synchronous work while IO completions waited), and peak memory.
+ */
+export class Tracer {
+    enabled;
+    file;
+    phases;
+    #tid;
+    #now;
+    #append;
+    #write;
+    #flushLines;
+    #flushMs;
+    #buffer = [];
+    #timer;
+    #sampler;
+    #marks = [];
+    #start;
+    #loop;
+    #closed = false;
+    constructor(options = {}) {
+        this.file = options.file || undefined;
+        this.phases = !!options.phases;
+        this.enabled = !!this.file || this.phases;
+        this.#tid = options.threadId ?? ownThreadId;
+        this.#now = options.now ?? sharedNow;
+        this.#append = options.append ?? defaultAppend;
+        this.#write = options.write ?? defaultWrite;
+        this.#flushLines = options.flushLines ?? 200;
+        this.#flushMs = options.flushMs ?? 50;
+        this.#start = this.#now();
+        if (!this.enabled)
+            return;
+        const main = options.main ?? this.#tid === 0;
+        if (main) {
+            this.#loop = monitorEventLoopDelay({ resolution: 20 });
+            this.#loop.enable();
+            const sampleMs = options.sampleMs ?? 20;
+            if (this.file && sampleMs > 0) {
+                this.#sampler = setInterval(() => this.sample(), sampleMs);
+                this.#sampler.unref();
+            }
+            process.once('exit', () => this.close());
+        }
+        this.trace('open', { pid: process.pid });
+    }
+    /**
+     * Build a tracer from the environment: `VLT_TRACE` names the file,
+     * `VLT_PHASES` (any value but `0` or empty) asks for the summary.
+     */
+    static fromEnv(env = process.env, options = {}) {
+        const phases = env.VLT_PHASES;
+        return new Tracer({
+            ...options,
+            file: env.VLT_TRACE,
+            phases: !!phases && phases !== '0',
+        });
+    }
+    /** Record one event. */
+    trace(ev, fields) {
+        if (!this.file || this.#closed)
+            return;
+        const event = { t: this.#now(), tid: this.#tid, ev };
+        if (fields) {
+            for (const [k, v] of Object.entries(fields)) {
+                if (v !== undefined)
+                    event[k] = v;
+            }
+        }
+        this.#buffer.push(JSON.stringify(event));
+        if (this.#buffer.length >= this.#flushLines)
+            this.flush();
+        else if (!this.#timer) {
+            this.#timer = setTimeout(() => this.flush(), this.#flushMs);
+            this.#timer.unref();
+        }
+    }
+    /**
+     * Record a `<ev>:start` event now and return a function that records
+     * `<ev>:end` with the elapsed `ms` (plus any fields it is given). The
+     * same `fields` are stamped on both ends so the two can be paired.
+     */
+    span(ev, fields) {
+        if (!this.file)
+            return noop;
+        const t0 = this.#now();
+        this.trace(`${ev}:start`, fields);
+        return end => this.trace(`${ev}:end`, {
+            ...fields,
+            ...end,
+            ms: Math.round((this.#now() - t0) * 1000) / 1000,
+        });
+    }
+    /**
+     * Mark the start of a phase. The summary reports the ms between each
+     * mark and the next one, and the trace records a `phase` event.
+     */
+    mark(name, fields) {
+        if (!this.enabled)
+            return;
+        this.#marks.push({ name, t: this.#now() });
+        this.trace('phase', { ...fields, name });
+    }
+    /**
+     * Record how busy a resource lane is: how many jobs it is running and
+     * how many are waiting for a slot. The report derives idle time per
+     * lane from these.
+     */
+    lane(name, busy, queued) {
+        if (!this.file)
+            return;
+        this.trace('lane', { name, busy, queued });
+    }
+    /** Record a memory sample. */
+    sample() {
+        if (!this.file)
+            return;
+        const m = process.memoryUsage();
+        this.trace('rss', {
+            rss: m.rss,
+            heap: m.heapUsed,
+            ext: m.external,
+            ab: m.arrayBuffers,
+        });
+    }
+    /** Write buffered lines to the file. */
+    flush() {
+        if (this.#timer) {
+            clearTimeout(this.#timer);
+            this.#timer = undefined;
+        }
+        if (!this.file || !this.#buffer.length)
+            return;
+        const data = this.#buffer.join('\n') + '\n';
+        this.#buffer = [];
+        try {
+            this.#append(this.file, data);
+        }
+        catch {
+            // a trace that cannot be written must never fail the install
+        }
+    }
+    /**
+     * The one-line phase summary, or undefined when phases are off. Each
+     * phase is reported as the ms from its mark to the next mark (or to
+     * now, for the last one).
+     */
+    summary() {
+        if (!this.phases)
+            return undefined;
+        const now = this.#now();
+        const parts = [];
+        for (const [i, m] of this.#marks.entries()) {
+            const next = this.#marks[i + 1]?.t ?? now;
+            parts.push(`${m.name}=${Math.round(next - m.t)}`);
+        }
+        parts.push(`total=${Math.round(now - this.#start)}`);
+        if (this.#loop) {
+            // the histogram is in nanoseconds
+            const ms = (n) => Math.round(n / 1e6);
+            parts.push(`loop p50=${ms(this.#loop.percentile(50))}` +
+                ` p99=${ms(this.#loop.percentile(99))}` +
+                ` max=${ms(this.#loop.max)}`);
+        }
+        const { maxRSS } = process.resourceUsage();
+        parts.push(`maxrss=${Math.round(maxRSS / 1024)}MB`);
+        return `PHASES ${parts.join(' ')}`;
+    }
+    /**
+     * Stop sampling, flush, and print the summary. Runs at process exit
+     * on the main thread; safe to call more than once.
+     */
+    close() {
+        if (this.#closed)
+            return;
+        if (this.#sampler)
+            clearInterval(this.#sampler);
+        if (this.enabled) {
+            if (this.#loop) {
+                this.#loop.disable();
+                this.trace('exit', {
+                    loopP50: Math.round(this.#loop.percentile(50) / 1e6),
+                    loopP99: Math.round(this.#loop.percentile(99) / 1e6),
+                    loopMax: Math.round(this.#loop.max / 1e6),
+                    maxrss: process.resourceUsage().maxRSS,
+                });
+            }
+            const line = this.summary();
+            this.flush();
+            if (line)
+                this.#write(line);
+        }
+        this.#closed = true;
+    }
+}
+const noop = () => { };
+/**
+ * The process-wide tracer, built from the environment. Worker threads
+ * inherit the environment, so they trace to the same file.
+ */
+export const tracer = Tracer.fromEnv();
+/** Record one event on the process-wide tracer. */
+export const trace = (ev, fields) => tracer.trace(ev, fields);
+/** Open a span on the process-wide tracer. See {@link Tracer.span}. */
+export const span = (ev, fields) => tracer.span(ev, fields);
+/** Mark a phase on the process-wide tracer. See {@link Tracer.mark}. */
+export const mark = (name, fields) => tracer.mark(name, fields);
+/** Record lane occupancy on the process-wide tracer. */
+export const lane = (name, busy, queued) => tracer.lane(name, busy, queued);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@vltpkg/output",
   "description": "Generate output from the vlt CLI",
-  "version": "1.6.0",
+  "version": "1.6.2",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/vltpkg/vltpkg.git",
@@ -13,7 +13,7 @@
     "url": "http://vlt.sh"
   },
   "dependencies": {
-    "@vltpkg/types": "1.6.0"
+    "@vltpkg/types": "1.6.2"
   },
   "devDependencies": {
     "@eslint/js": "^9.39.1",
@@ -57,6 +57,11 @@
       "import": {
         "default": "./dist/error.js"
       }
+    },
+    "./trace": {
+      "import": {
+        "default": "./dist/trace.js"
+      }
     }
   },
   "files": [