jspurefix 5.8.5 → 5.9.2

package/src/benchmark/bench-report.ts

@@ -0,0 +1,101 @@
+import { compare } from './bench-stats'
+import { IBenchResult, IBenchSuiteResult } from './bench-types'
+
+const bytesPerKb = 1024
+const bytesPerMb = bytesPerKb * 1024
+const nanosPerMicro = 1e3
+
+function micros (nanos: number): string {
+  return (nanos / nanosPerMicro).toFixed(3)
+}
+
+function mb (bytes: number): string {
+  return (bytes / bytesPerMb).toFixed(2)
+}
+
+function thousands (n: number): string {
+  return Math.round(n).toLocaleString('en-US')
+}
+
+function signed (n: number, digits: number = 2): string {
+  return `${n >= 0 ? '+' : ''}${n.toFixed(digits)}`
+}
+
+export function formatEnvironment (suite: IBenchSuiteResult): string {
+  const env = suite.environment
+  const lines = [
+    `${suite.label}  ${suite.capturedAt}`,
+    `node ${env.node}  v8 ${env.v8}  ${env.platform} ${env.arch}`,
+    `${env.cpu} x${env.cpuCount}`
+  ]
+  if (!env.forcedGcAvailable) {
+    lines.push('WARNING: no --expose-gc, rounds are not isolated and retained heap is indicative only')
+  }
+  return lines.join('\n')
+}
+
+export function formatResult (result: IBenchResult): string {
+  const t = result.timing
+  const m = result.memory
+  const noise = t.coefficientOfVariation * 100
+  const lines = [
+    `${result.name}`,
+    result.description ? `  ${result.description}` : '',
+    `  rounds        ${t.samples} x ${result.options.operationsPerRound} msgs` +
+      ` (warmup ${result.options.warmupOperations})`,
+    `  per msg       median ${micros(t.median)}us   min ${micros(t.min)}us` +
+      `   p95 ${micros(t.p95)}us   max ${micros(t.max)}us`,
+    `  throughput    ${thousands(result.operationsPerSecond)} msg/sec`,
+    `  noise         ${noise.toFixed(2)}% of median (a change smaller than this means nothing)`,
+    `  gc pass       ${m.operations} msgs uninterrupted: ${m.gc.minor} minor  ${m.gc.major} major  ` +
+      `${m.gc.totalPauseMs.toFixed(1)}ms paused  longest ${m.gc.longestPauseMs.toFixed(2)}ms`,
+    `  gc rate       ${m.minorGcPerMillionOperations.toFixed(1)} minor/M msgs   ` +
+      `${m.majorGcPerMillionOperations.toFixed(1)} major/M msgs   ` +
+      `${m.gcPauseMsPerMillionOperations.toFixed(0)}ms paused/M msgs`,
+    `  retained      ${mb(m.retainedHeapBytes)}MB   ` +
+      `${m.retainedBytesPerOperation.toFixed(1)} bytes/msg   heap at end of pass ${mb(m.heapAtEndOfPassBytes)}MB`
+  ]
+  return lines.filter(l => l.length > 0).join('\n')
+}
+
+export function formatSuite (suite: IBenchSuiteResult): string {
+  const blocks = suite.results.map(formatResult)
+  return [formatEnvironment(suite), '', ...blocks].join('\n\n')
+}
+
+/**
+ * Compare a suite against a stored baseline, matching results by name.
+ *
+ * A difference is only called out as real when it exceeds the run to run
+ * spread both suites displayed. Everything else is reported as noise, because
+ * that is what it is.
+ */
+export function formatComparison (baseline: IBenchSuiteResult, candidate: IBenchSuiteResult): string {
+  const lines: string[] = [`baseline ${baseline.label} (${baseline.capturedAt})`, '']
+  if (baseline.environment.cpu !== candidate.environment.cpu ||
+      baseline.environment.node !== candidate.environment.node) {
+    lines.push('WARNING: baseline was taken on a different machine or node build, timings are not comparable')
+    lines.push('')
+  }
+  for (const result of candidate.results) {
+    const before = baseline.results.find(r => r.name === result.name)
+    if (!before) {
+      lines.push(`${result.name}  no baseline`)
+      continue
+    }
+    const timing = compare(before.timing, result.timing)
+    const verdict = timing.significant
+      ? (timing.deltaPercent < 0 ? 'FASTER' : 'SLOWER')
+      : 'noise'
+    lines.push(`${result.name}`)
+    lines.push(`  per msg   ${micros(before.timing.median)}us -> ${micros(result.timing.median)}us` +
+      `   ${signed(timing.deltaPercent)}%   ${verdict}` +
+      `   (noise floor +/-${timing.noiseThresholdPercent.toFixed(2)}%)`)
+    const gcBefore = before.memory.minorGcPerMillionOperations
+    const gcAfter = result.memory.minorGcPerMillionOperations
+    const gcDelta = gcBefore === 0 ? 0 : ((gcAfter - gcBefore) / gcBefore) * 100
+    lines.push(`  minor gc  ${gcBefore.toFixed(1)} -> ${gcAfter.toFixed(1)} per M msgs   ${signed(gcDelta)}%`)
+    lines.push('')
+  }
+  return lines.join('\n')
+}

package/src/benchmark/bench-runner.ts

@@ -0,0 +1,196 @@
+import * as os from 'os'
+import { GcProbe } from './gc-probe'
+import { summarise } from './bench-stats'
+import {
+  IBenchCase,
+  IBenchOptions,
+  IBenchResult,
+  IBenchSuiteResult,
+  IEnvironment,
+  IMemoryResult,
+  IRoundResult
+} from './bench-types'
+
+const nanosPerSecond = 1e9
+const operationsPerMillion = 1e6
+
+export const defaultBenchOptions: IBenchOptions = {
+  // v8 needs to see a function run a few thousand times before it stops
+  // interpreting it, so an unwarmed first round measures the wrong code
+  warmupOperations: 20000,
+  rounds: 10,
+  operationsPerRound: 20000,
+  gcBetweenRounds: true
+}
+
+export function environment (): IEnvironment {
+  const cpus = os.cpus()
+  return {
+    node: process.version,
+    v8: process.versions.v8,
+    platform: `${os.type()} ${os.release()}`,
+    arch: process.arch,
+    cpu: cpus.length > 0 ? cpus[0].model.trim() : 'unknown',
+    cpuCount: cpus.length,
+    forcedGcAvailable: GcProbe.forcedGcAvailable()
+  }
+}
+
+/** hand the event loop one turn, so queued observer callbacks can run */
+async function tick (): Promise<void> {
+  await new Promise<void>(resolve => { setImmediate(resolve) })
+}
+
+/**
+ * Yield until the probe stops seeing new entries. Node dispatches a gc entry on
+ * the turn after the collection, and a burst takes several turns to come
+ * through, so a single yield loses most of them. Bounded so a pathological case
+ * cannot spin here.
+ */
+async function drain (probe: GcProbe, maxTurns: number = 50): Promise<void> {
+  let quietTurns = 0
+  for (let turn = 0; turn < maxTurns && quietTurns < 3; ++turn) {
+    const before = probe.seen()
+    await tick()
+    quietTurns = probe.seen() === before ? quietTurns + 1 : 0
+  }
+}
+
+/**
+ * Drives a case through a warmup, a series of timed rounds and a separate
+ * diagnostics pass.
+ *
+ * One round is one sample. A single long run cannot tell you whether a number
+ * is repeatable, and a difference smaller than the spread between rounds is not
+ * a difference at all.
+ *
+ * Timing and memory are deliberately not measured together. Isolating rounds
+ * means forcing a collection between them, and a forced collection is itself a
+ * gc event - counting those would report the harness rather than the work. So
+ * the timed rounds collect between them and report no gc figures, and a final
+ * pass runs uninterrupted with the probe attached and reports no timings.
+ */
+export class BenchRunner {
+  constructor (private readonly options: IBenchOptions = defaultBenchOptions) {
+  }
+
+  public async run (benchCase: IBenchCase): Promise<IBenchResult> {
+    const options = this.options
+    benchCase.setup?.()
+    try {
+      this.warmup(benchCase)
+      const rounds: IRoundResult[] = []
+      for (let round = 0; round < options.rounds; ++round) {
+        if (options.gcBetweenRounds) {
+          // a collection provoked by the previous round must not be charged
+          // to the next one
+          GcProbe.forceGc()
+        }
+        rounds.push(this.measureRound(benchCase, round))
+      }
+      const timing = summarise(rounds.map(r => r.nanosPerOperation))
+      return {
+        name: benchCase.name,
+        description: benchCase.description,
+        options,
+        rounds,
+        timing,
+        operationsPerSecond: timing.median === 0 ? 0 : nanosPerSecond / timing.median,
+        memory: await this.measureMemory(benchCase)
+      }
+    } finally {
+      benchCase.teardown?.()
+    }
+  }
+
+  public async runAll (label: string, cases: IBenchCase[]): Promise<IBenchSuiteResult> {
+    const results: IBenchResult[] = []
+    for (const benchCase of cases) {
+      results.push(await this.run(benchCase))
+    }
+    return {
+      label,
+      // stamped from the clock rather than passed in, a suite is a record of
+      // when it was taken
+      capturedAt: new Date().toISOString(),
+      environment: environment(),
+      results
+    }
+  }
+
+  /**
+   * One uninterrupted pass with the gc probe attached.
+   *
+   * Nothing is forced while it runs, so every collection counted is one the
+   * work itself provoked. The heap readings either side are taken after a
+   * forced collection, making them what the pass retained rather than what it
+   * churned - churn is what the collection counts are for.
+   */
+  private async measureMemory (benchCase: IBenchCase): Promise<IMemoryResult> {
+    // settle, then let any delivery from that settling drain before the probe
+    // exists - entries are dispatched a turn late and would otherwise land in
+    // the middle of the measured pass
+    GcProbe.forceGc()
+    await tick()
+
+    const heapBefore = process.memoryUsage().heapUsed
+    const probe = new GcProbe()
+    probe.start()
+    const operations = benchCase.run(this.options.operationsPerRound)
+    const heapAtEndOfPassBytes = process.memoryUsage().heapUsed
+    await drain(probe)
+    const gc = probe.stop()
+
+    GcProbe.forceGc()
+    const heapAfter = process.memoryUsage().heapUsed
+
+    const retainedHeapBytes = heapAfter - heapBefore
+    const perMillion = operations === 0 ? 0 : operationsPerMillion / operations
+    return {
+      operations,
+      retainedHeapBytes,
+      retainedBytesPerOperation: operations === 0 ? 0 : retainedHeapBytes / operations,
+      heapAtEndOfPassBytes,
+      gc,
+      minorGcPerMillionOperations: gc.minor * perMillion,
+      majorGcPerMillionOperations: gc.major * perMillion,
+      gcPauseMsPerMillionOperations: gc.totalPauseMs * perMillion,
+      forcedGcAvailable: GcProbe.forcedGcAvailable()
+    }
+  }
+
+  private warmup (benchCase: IBenchCase): void {
+    const target = this.options.warmupOperations
+    if (target <= 0) return
+    let done = 0
+    // in chunks the size of a real round, so warmup exercises the same shape of
+    // work rather than one long unbroken call
+    const chunk = Math.max(1, Math.min(target, this.options.operationsPerRound))
+    while (done < target) {
+      const requested = Math.min(chunk, target - done)
+      const completed = benchCase.run(requested)
+      // a case that reports less than it was asked for would never let this
+      // loop reach its target, so refuse it here rather than spin
+      if (completed !== requested) {
+        throw new Error(`${benchCase.name}: asked for ${requested} operations during warmup but completed ${completed}`)
+      }
+      done += completed
+    }
+  }
+
+  private measureRound (benchCase: IBenchCase, round: number): IRoundResult {
+    const requested = this.options.operationsPerRound
+    const startedAt = process.hrtime.bigint()
+    const operations = benchCase.run(requested)
+    const elapsedNanos = Number(process.hrtime.bigint() - startedAt)
+    if (operations !== requested) {
+      throw new Error(`${benchCase.name}: asked for ${requested} operations in round ${round} but completed ${operations}`)
+    }
+    return {
+      round,
+      operations,
+      elapsedNanos,
+      nanosPerOperation: operations === 0 ? 0 : elapsedNanos / operations
+    }
+  }
+}

package/src/benchmark/bench-stats.ts

@@ -0,0 +1,86 @@
+import { ISampleStats } from './bench-types'
+
+/**
+ * Descriptive statistics over a set of measured rounds.
+ *
+ * min matters as much as median here. A round can only ever be made slower by
+ * interference - a collection landing mid round, the scheduler taking the core
+ * away - so the fastest round is the closest reading to the true cost of the
+ * work, while the median tells you what to expect in practice.
+ */
+export function summarise (values: number[]): ISampleStats {
+  if (values.length === 0) {
+    throw new Error('cannot summarise an empty sample set')
+  }
+  const sorted = [...values].sort((a, b) => a - b)
+  const n = sorted.length
+  const mean = sorted.reduce((a, v) => a + v, 0) / n
+  const variance = sorted.reduce((a, v) => a + (v - mean) * (v - mean), 0) / n
+  const stdDev = Math.sqrt(variance)
+  return {
+    samples: n,
+    min: sorted[0],
+    max: sorted[n - 1],
+    mean,
+    median: median(sorted),
+    p95: percentile(sorted, 95),
+    p99: percentile(sorted, 99),
+    stdDev,
+    coefficientOfVariation: mean === 0 ? 0 : stdDev / mean
+  }
+}
+
+/**
+ * @param sorted values in ascending order
+ */
+export function median (sorted: number[]): number {
+  const n = sorted.length
+  if (n === 0) return 0
+  const mid = n >> 1
+  return n % 2 === 1
+    ? sorted[mid]
+    : (sorted[mid - 1] + sorted[mid]) / 2
+}
+
+/**
+ * Nearest rank percentile - no interpolation, so every value reported is a
+ * reading that actually occurred.
+ *
+ * @param sorted values in ascending order
+ * @param p percentile in the range 0 to 100
+ */
+export function percentile (sorted: number[], p: number): number {
+  const n = sorted.length
+  if (n === 0) return 0
+  const rank = Math.ceil((p / 100) * n)
+  const index = Math.min(Math.max(rank - 1, 0), n - 1)
+  return sorted[index]
+}
+
+export interface IComparison {
+  /** positive means the candidate took longer than the baseline */
+  readonly deltaPercent: number
+  /**
+   * true when the change is larger than the noise both runs displayed. Below
+   * this threshold a difference is not evidence of anything.
+   */
+  readonly significant: boolean
+  readonly noiseThresholdPercent: number
+}
+
+/**
+ * Compare a candidate timing against a baseline, using the run to run spread
+ * each of them showed to decide whether the difference is real.
+ */
+export function compare (baseline: ISampleStats, candidate: ISampleStats): IComparison {
+  const deltaPercent = baseline.median === 0
+    ? 0
+    : ((candidate.median - baseline.median) / baseline.median) * 100
+  // both runs contribute their own noise to any difference between them
+  const noiseThresholdPercent = (baseline.coefficientOfVariation + candidate.coefficientOfVariation) * 100
+  return {
+    deltaPercent,
+    significant: Math.abs(deltaPercent) > noiseThresholdPercent,
+    noiseThresholdPercent
+  }
+}

package/src/benchmark/bench-types.ts

@@ -0,0 +1,110 @@
+/**
+ * A unit of work the runner can drive. run is handed the number of operations
+ * to perform and returns how many it actually completed, which the runner
+ * checks - a case that silently does less work than asked would otherwise look
+ * like a large improvement.
+ */
+export interface IBenchCase {
+  readonly name: string
+  readonly description?: string
+  /** called once before warmup, outside all timing */
+  setup?: () => void
+  run: (operations: number) => number
+  teardown?: () => void
+}
+
+export interface IBenchOptions {
+  /** operations run and discarded before measuring, to let v8 tier the code up */
+  readonly warmupOperations: number
+  /** measured rounds - each contributes one sample */
+  readonly rounds: number
+  readonly operationsPerRound: number
+  /** collect between rounds so one round's garbage is not charged to the next */
+  readonly gcBetweenRounds: boolean
+}
+
+export interface ISampleStats {
+  readonly samples: number
+  readonly min: number
+  readonly max: number
+  readonly mean: number
+  readonly median: number
+  readonly p95: number
+  readonly p99: number
+  readonly stdDev: number
+  /** stdDev as a fraction of the mean - the run to run noise floor */
+  readonly coefficientOfVariation: number
+}
+
+export interface IGcTotals {
+  readonly minor: number
+  readonly major: number
+  readonly incremental: number
+  readonly weakCallback: number
+  readonly total: number
+  readonly totalPauseMs: number
+  readonly longestPauseMs: number
+}
+
+export interface IMemoryResult {
+  /** operations in the diagnostics pass these figures were taken over */
+  readonly operations: number
+  /**
+   * heapUsed after a forced collection, less the same reading taken before the
+   * measured rounds - what the work left behind rather than what it churned
+   */
+  readonly retainedHeapBytes: number
+  readonly retainedBytesPerOperation: number
+  /**
+   * heapUsed sampled the instant the pass finished. Not a high water mark -
+   * it is wherever the sawtooth happened to be - so read it as a rough sense of
+   * the working set, not as a peak.
+   */
+  readonly heapAtEndOfPassBytes: number
+  readonly gc: IGcTotals
+  /**
+   * collections per million operations. allocation volume is not directly
+   * observable from node, so this is the comparable proxy for it - the same
+   * work triggering fewer scavenges is the same work allocating less.
+   */
+  readonly minorGcPerMillionOperations: number
+  readonly majorGcPerMillionOperations: number
+  readonly gcPauseMsPerMillionOperations: number
+  /** false when the process was started without --expose-gc */
+  readonly forcedGcAvailable: boolean
+}
+
+export interface IRoundResult {
+  readonly round: number
+  readonly operations: number
+  readonly elapsedNanos: number
+  readonly nanosPerOperation: number
+}
+
+export interface IEnvironment {
+  readonly node: string
+  readonly v8: string
+  readonly platform: string
+  readonly arch: string
+  readonly cpu: string
+  readonly cpuCount: number
+  readonly forcedGcAvailable: boolean
+}
+
+export interface IBenchResult {
+  readonly name: string
+  readonly description?: string
+  readonly options: IBenchOptions
+  readonly rounds: IRoundResult[]
+  /** statistics over the per round nanosPerOperation samples */
+  readonly timing: ISampleStats
+  readonly operationsPerSecond: number
+  readonly memory: IMemoryResult
+}
+
+export interface IBenchSuiteResult {
+  readonly label: string
+  readonly capturedAt: string
+  readonly environment: IEnvironment
+  readonly results: IBenchResult[]
+}

package/src/benchmark/gc-probe.ts

@@ -0,0 +1,145 @@
+import { PerformanceObserver, constants } from 'perf_hooks'
+import { IGcTotals } from './bench-types'
+
+interface IGcDetail {
+  kind: number
+  flags: number
+}
+
+/**
+ * a gc performance entry. detail is not on the PerformanceEntry the node types
+ * describe, it is only present on entries of this type
+ */
+interface IGcEntry {
+  duration: number
+  detail?: IGcDetail
+}
+
+const emptyTotals: IGcTotals = {
+  minor: 0,
+  major: 0,
+  incremental: 0,
+  weakCallback: 0,
+  total: 0,
+  totalPauseMs: 0,
+  longestPauseMs: 0
+}
+
+/**
+ * Counts collections and the pause they cost while a benchmark runs.
+ *
+ * Node does not expose how many bytes a stretch of code allocated, so the
+ * number of scavenges is the practical stand in - the same work provoking
+ * fewer minor collections is the same work producing less garbage, and that
+ * is the number worth moving for a drop copy engine where the cost of
+ * allocation is felt as pause rather than as throughput.
+ */
+export class GcProbe {
+  private observer: PerformanceObserver | null = null
+  private minor: number = 0
+  private major: number = 0
+  private incremental: number = 0
+  private weakCallback: number = 0
+  private totalPauseMs: number = 0
+  private longestPauseMs: number = 0
+
+  /** whether the process was started with --expose-gc */
+  public static forcedGcAvailable (): boolean {
+    return typeof global.gc === 'function'
+  }
+
+  /**
+   * Collect until the heap settles, so a reading taken afterwards reflects what
+   * is genuinely retained. Two passes because the first can promote survivors
+   * into the old space that the second is then able to reclaim. A no-op without
+   * --expose-gc, in which case retained figures are indicative only.
+   */
+  public static forceGc (): void {
+    const gc = global.gc
+    if (typeof gc !== 'function') return
+    gc()
+    gc()
+  }
+
+  /** entries recorded so far - lets a caller wait for delivery to go quiet */
+  public seen (): number {
+    return this.minor + this.major + this.incremental + this.weakCallback
+  }
+
+  public start (): void {
+    this.reset()
+    const observer = new PerformanceObserver((list) => {
+      for (const entry of list.getEntries()) {
+        const gcEntry = entry as unknown as IGcEntry
+        this.record(gcEntry.duration, gcEntry.detail?.kind ?? 0)
+      }
+    })
+    // buffered so collections are not missed while the loop is busy - the
+    // callback cannot run until the measured work yields
+    observer.observe({ entryTypes: ['gc'], buffered: true })
+    this.observer = observer
+  }
+
+  /**
+   * Stop observing and return what was seen. Any entries still queued are
+   * drained first - the observer callback is a macrotask and the benchmark
+   * loop never yields, so without this every collection would be lost.
+   */
+  public stop (): IGcTotals {
+    const observer = this.observer
+    if (!observer) return emptyTotals
+    for (const entry of observer.takeRecords()) {
+      const gcEntry = entry as unknown as IGcEntry
+      this.record(gcEntry.duration, gcEntry.detail?.kind ?? 0)
+    }
+    observer.disconnect()
+    this.observer = null
+    return this.totals()
+  }
+
+  private reset (): void {
+    this.minor = 0
+    this.major = 0
+    this.incremental = 0
+    this.weakCallback = 0
+    this.totalPauseMs = 0
+    this.longestPauseMs = 0
+  }
+
+  private record (durationMs: number, kind: number): void {
+    this.totalPauseMs += durationMs
+    this.longestPauseMs = Math.max(this.longestPauseMs, durationMs)
+    switch (kind) {
+      case constants.NODE_PERFORMANCE_GC_MINOR:
+        ++this.minor
+        break
+
+      case constants.NODE_PERFORMANCE_GC_MAJOR:
+        ++this.major
+        break
+
+      case constants.NODE_PERFORMANCE_GC_INCREMENTAL:
+        ++this.incremental
+        break
+
+      case constants.NODE_PERFORMANCE_GC_WEAKCB:
+        ++this.weakCallback
+        break
+
+      default:
+        break
+    }
+  }
+
+  private totals (): IGcTotals {
+    return {
+      minor: this.minor,
+      major: this.major,
+      incremental: this.incremental,
+      weakCallback: this.weakCallback,
+      total: this.minor + this.major + this.incremental + this.weakCallback,
+      totalPauseMs: this.totalPauseMs,
+      longestPauseMs: this.longestPauseMs
+    }
+  }
+}

package/src/benchmark/index.ts

@@ -0,0 +1,6 @@
+export * from './bench-types'
+export * from './bench-stats'
+export * from './bench-runner'
+export * from './bench-report'
+export * from './gc-probe'
+export * from './parse-bench'