jspurefix 5.9.1 → 5.9.2

package/src/benchmark/bench-types.ts

@@ -0,0 +1,110 @@
+/**
+ * A unit of work the runner can drive. run is handed the number of operations
+ * to perform and returns how many it actually completed, which the runner
+ * checks - a case that silently does less work than asked would otherwise look
+ * like a large improvement.
+ */
+export interface IBenchCase {
+  readonly name: string
+  readonly description?: string
+  /** called once before warmup, outside all timing */
+  setup?: () => void
+  run: (operations: number) => number
+  teardown?: () => void
+}
+
+export interface IBenchOptions {
+  /** operations run and discarded before measuring, to let v8 tier the code up */
+  readonly warmupOperations: number
+  /** measured rounds - each contributes one sample */
+  readonly rounds: number
+  readonly operationsPerRound: number
+  /** collect between rounds so one round's garbage is not charged to the next */
+  readonly gcBetweenRounds: boolean
+}
+
+export interface ISampleStats {
+  readonly samples: number
+  readonly min: number
+  readonly max: number
+  readonly mean: number
+  readonly median: number
+  readonly p95: number
+  readonly p99: number
+  readonly stdDev: number
+  /** stdDev as a fraction of the mean - the run to run noise floor */
+  readonly coefficientOfVariation: number
+}
+
+export interface IGcTotals {
+  readonly minor: number
+  readonly major: number
+  readonly incremental: number
+  readonly weakCallback: number
+  readonly total: number
+  readonly totalPauseMs: number
+  readonly longestPauseMs: number
+}
+
+export interface IMemoryResult {
+  /** operations in the diagnostics pass these figures were taken over */
+  readonly operations: number
+  /**
+   * heapUsed after a forced collection, less the same reading taken before the
+   * measured rounds - what the work left behind rather than what it churned
+   */
+  readonly retainedHeapBytes: number
+  readonly retainedBytesPerOperation: number
+  /**
+   * heapUsed sampled the instant the pass finished. Not a high water mark -
+   * it is wherever the sawtooth happened to be - so read it as a rough sense of
+   * the working set, not as a peak.
+   */
+  readonly heapAtEndOfPassBytes: number
+  readonly gc: IGcTotals
+  /**
+   * collections per million operations. allocation volume is not directly
+   * observable from node, so this is the comparable proxy for it - the same
+   * work triggering fewer scavenges is the same work allocating less.
+   */
+  readonly minorGcPerMillionOperations: number
+  readonly majorGcPerMillionOperations: number
+  readonly gcPauseMsPerMillionOperations: number
+  /** false when the process was started without --expose-gc */
+  readonly forcedGcAvailable: boolean
+}
+
+export interface IRoundResult {
+  readonly round: number
+  readonly operations: number
+  readonly elapsedNanos: number
+  readonly nanosPerOperation: number
+}
+
+export interface IEnvironment {
+  readonly node: string
+  readonly v8: string
+  readonly platform: string
+  readonly arch: string
+  readonly cpu: string
+  readonly cpuCount: number
+  readonly forcedGcAvailable: boolean
+}
+
+export interface IBenchResult {
+  readonly name: string
+  readonly description?: string
+  readonly options: IBenchOptions
+  readonly rounds: IRoundResult[]
+  /** statistics over the per round nanosPerOperation samples */
+  readonly timing: ISampleStats
+  readonly operationsPerSecond: number
+  readonly memory: IMemoryResult
+}
+
+export interface IBenchSuiteResult {
+  readonly label: string
+  readonly capturedAt: string
+  readonly environment: IEnvironment
+  readonly results: IBenchResult[]
+}

package/src/benchmark/gc-probe.ts

@@ -0,0 +1,145 @@
+import { PerformanceObserver, constants } from 'perf_hooks'
+import { IGcTotals } from './bench-types'
+
+interface IGcDetail {
+  kind: number
+  flags: number
+}
+
+/**
+ * a gc performance entry. detail is not on the PerformanceEntry the node types
+ * describe, it is only present on entries of this type
+ */
+interface IGcEntry {
+  duration: number
+  detail?: IGcDetail
+}
+
+const emptyTotals: IGcTotals = {
+  minor: 0,
+  major: 0,
+  incremental: 0,
+  weakCallback: 0,
+  total: 0,
+  totalPauseMs: 0,
+  longestPauseMs: 0
+}
+
+/**
+ * Counts collections and the pause they cost while a benchmark runs.
+ *
+ * Node does not expose how many bytes a stretch of code allocated, so the
+ * number of scavenges is the practical stand in - the same work provoking
+ * fewer minor collections is the same work producing less garbage, and that
+ * is the number worth moving for a drop copy engine where the cost of
+ * allocation is felt as pause rather than as throughput.
+ */
+export class GcProbe {
+  private observer: PerformanceObserver | null = null
+  private minor: number = 0
+  private major: number = 0
+  private incremental: number = 0
+  private weakCallback: number = 0
+  private totalPauseMs: number = 0
+  private longestPauseMs: number = 0
+
+  /** whether the process was started with --expose-gc */
+  public static forcedGcAvailable (): boolean {
+    return typeof global.gc === 'function'
+  }
+
+  /**
+   * Collect until the heap settles, so a reading taken afterwards reflects what
+   * is genuinely retained. Two passes because the first can promote survivors
+   * into the old space that the second is then able to reclaim. A no-op without
+   * --expose-gc, in which case retained figures are indicative only.
+   */
+  public static forceGc (): void {
+    const gc = global.gc
+    if (typeof gc !== 'function') return
+    gc()
+    gc()
+  }
+
+  /** entries recorded so far - lets a caller wait for delivery to go quiet */
+  public seen (): number {
+    return this.minor + this.major + this.incremental + this.weakCallback
+  }
+
+  public start (): void {
+    this.reset()
+    const observer = new PerformanceObserver((list) => {
+      for (const entry of list.getEntries()) {
+        const gcEntry = entry as unknown as IGcEntry
+        this.record(gcEntry.duration, gcEntry.detail?.kind ?? 0)
+      }
+    })
+    // buffered so collections are not missed while the loop is busy - the
+    // callback cannot run until the measured work yields
+    observer.observe({ entryTypes: ['gc'], buffered: true })
+    this.observer = observer
+  }
+
+  /**
+   * Stop observing and return what was seen. Any entries still queued are
+   * drained first - the observer callback is a macrotask and the benchmark
+   * loop never yields, so without this every collection would be lost.
+   */
+  public stop (): IGcTotals {
+    const observer = this.observer
+    if (!observer) return emptyTotals
+    for (const entry of observer.takeRecords()) {
+      const gcEntry = entry as unknown as IGcEntry
+      this.record(gcEntry.duration, gcEntry.detail?.kind ?? 0)
+    }
+    observer.disconnect()
+    this.observer = null
+    return this.totals()
+  }
+
+  private reset (): void {
+    this.minor = 0
+    this.major = 0
+    this.incremental = 0
+    this.weakCallback = 0
+    this.totalPauseMs = 0
+    this.longestPauseMs = 0
+  }
+
+  private record (durationMs: number, kind: number): void {
+    this.totalPauseMs += durationMs
+    this.longestPauseMs = Math.max(this.longestPauseMs, durationMs)
+    switch (kind) {
+      case constants.NODE_PERFORMANCE_GC_MINOR:
+        ++this.minor
+        break
+
+      case constants.NODE_PERFORMANCE_GC_MAJOR:
+        ++this.major
+        break
+
+      case constants.NODE_PERFORMANCE_GC_INCREMENTAL:
+        ++this.incremental
+        break
+
+      case constants.NODE_PERFORMANCE_GC_WEAKCB:
+        ++this.weakCallback
+        break
+
+      default:
+        break
+    }
+  }
+
+  private totals (): IGcTotals {
+    return {
+      minor: this.minor,
+      major: this.major,
+      incremental: this.incremental,
+      weakCallback: this.weakCallback,
+      total: this.minor + this.major + this.incremental + this.weakCallback,
+      totalPauseMs: this.totalPauseMs,
+      longestPauseMs: this.longestPauseMs
+    }
+  }
+}

package/src/benchmark/index.ts

@@ -0,0 +1,6 @@
+export * from './bench-types'
+export * from './bench-stats'
+export * from './bench-runner'
+export * from './bench-report'
+export * from './gc-probe'
+export * from './parse-bench'

package/src/benchmark/parse-bench.ts

@@ -0,0 +1,144 @@
+import { AsciiParser } from '../buffer/ascii/ascii-parser'
+import { ElasticBuffer } from '../buffer/elastic-buffer'
+import { MsgView } from '../buffer/msg-view'
+import { IJsFixConfig } from '../config/js-fix-config'
+import { DITokens } from '../runtime/di-tokens'
+import { makeSessionScope } from '../runtime/session-scope'
+import { IBenchCase } from './bench-types'
+
+/**
+ * what a decoded message is taken through once the parser has produced a view
+ */
+export enum ParseDepth {
+  /** stop at the MsgView - tokenise, discover the structure, build the view */
+  View = 'view',
+  /** materialise the whole object graph, which is what most consumers do */
+  Object = 'object'
+}
+
+export interface IParseCaseOptions {
+  readonly config: IJsFixConfig
+  /** the raw text of a fix log, one or more complete messages */
+  readonly contents: string
+  readonly depth: ParseDepth
+}
+
+/**
+ * Drives fix bytes through the real parser and counts messages out of it.
+ *
+ * The bytes are fed as one buffer per invocation, the same way a socket read
+ * reaches parse, rather than by concatenating the file thousands of times into
+ * one enormous string. That concatenation is itself a large allocation, and
+ * doing it before the timer starts leaves a heap that no memory reading taken
+ * afterwards can be trusted through.
+ */
+export class ParseCase implements IBenchCase {
+  public readonly name: string
+  public readonly description: string
+  /** messages produced by one pass over the input */
+  public readonly operationsPerInvocation: number
+
+  private readonly source: Buffer
+  private readonly depth: ParseDepth
+  private readonly options: IParseCaseOptions
+  private parser: AsciiParser | null = null
+  private decoded: number = 0
+
+  constructor (name: string, options: IParseCaseOptions) {
+    this.name = name
+    this.options = options
+    this.depth = options.depth
+    this.source = Buffer.from(options.contents)
+    this.description = options.depth === ParseDepth.Object
+      ? 'tokenise, discover structure, build view, materialise object'
+      : 'tokenise, discover structure, build view'
+    this.operationsPerInvocation = this.countMessages()
+    if (this.operationsPerInvocation === 0) {
+      throw new Error(`${name}: input yielded no messages, check the dictionary and the delimiter`)
+    }
+  }
+
+  /**
+   * Round a requested round size up to a whole number of passes over the input.
+   * A pass cannot be stopped part way, so asking for a count that is not a
+   * multiple of it would silently measure more work than it reported.
+   */
+  public alignOperations (requested: number): number {
+    const per = this.operationsPerInvocation
+    return Math.max(per, Math.ceil(requested / per) * per)
+  }
+
+  public setup (): void {
+    this.parser = this.makeParser(true)
+  }
+
+  public teardown (): void {
+    this.parser?.removeAllListeners()
+    this.parser = null
+  }
+
+  public run (operations: number): number {
+    const parser = this.parser
+    if (!parser) {
+      throw new Error(`${this.name}: run called before setup`)
+    }
+    const source = this.source
+    const passes = operations / this.operationsPerInvocation
+    if (!Number.isInteger(passes)) {
+      throw new Error(`${this.name}: ${operations} operations is not a whole number of passes over the input`)
+    }
+    this.decoded = 0
+    for (let i = 0; i < passes; ++i) {
+      parser.parseBuffer(source)
+    }
+    return this.decoded
+  }
+
+  /**
+   * A parser on a session scope of its own.
+   *
+   * It has to be a scope rather than a hand built AsciiParser: AsciiParserState
+   * takes the ParseBuffer from the container, so a parser handed some other
+   * buffer writes bytes the state never reads and the parse walks off into
+   * nonsense. The scope re-registers the buffer and hands the same instance to
+   * both, which is the same thing an accepted connection gets.
+   */
+  private makeParser (subscribe: boolean): AsciiParser {
+    const scoped = makeSessionScope(this.options.config)
+    const container = scoped.sessionContainer
+    // constructed rather than resolved: DITokens.readStream has no registration
+    // outside a live transport, and there is no stream here - the harness hands
+    // the parser its bytes. Everything else comes from the scope, so the buffer
+    // below is the same instance AsciiParserState will be injected with.
+    const buffer = container.resolve<ElasticBuffer>(DITokens.ParseBuffer)
+    const maxMessageLen = container.resolve<number>(DITokens.maxMessageLen)
+    const parser = new AsciiParser(scoped, null, buffer, maxMessageLen)
+    const object = subscribe && this.depth === ParseDepth.Object
+    parser.on('msg', (_: string, view: MsgView) => {
+      ++this.decoded
+      if (object) {
+        // read the result back so the materialisation cannot be treated as
+        // dead and folded away
+        const materialised = view.toObject()
+        if (materialised === null) {
+          throw new Error('view produced no object')
+        }
+      }
+    })
+    return parser
+  }
+
+  /**
+   * How many messages one pass over the input yields. Counted on a throwaway
+   * parser during construction so the measured run never has to ask.
+   */
+  private countMessages (): number {
+    const parser = this.makeParser(false)
+    this.decoded = 0
+    parser.parseBuffer(this.source)
+    parser.removeAllListeners()
+    const count = this.decoded
+    this.decoded = 0
+    return count
+  }
+}

package/src/buffer/ascii/ascii-parser.ts

@@ -39,7 +39,9 @@ export class AsciiParser extends MsgParser {
     this.id = AsciiParser.nextId++
     this.segmentParser = config.sessionContainer.resolve<AsciiSegmentParser>(AsciiSegmentParser)
     this.state = config.sessionContainer.resolve<AsciiParserState>(AsciiParserState)
-    this.state.locations = new Tags(this.receivingBuffer.size / 10, this.maxMessageLen)
+    // floored - Tags hands the starting length to new Array, which rejects a
+    // fraction, so any buffer size not divisible by 10 threw from here
+    this.state.locations = new Tags(Math.floor(this.receivingBuffer.size / 10), this.maxMessageLen)
     this.state.definitions = this.config.definitions
     this.state.beginMessage()
     if (readStream !== null) {
@@ -101,6 +103,19 @@ export class AsciiParser extends MsgParser {
     this.parse(buff, buff.length)
   }
 
+  /**
+   * feed bytes as though they had arrived from the transport. This is the path
+   * the socket takes - subscribe hands each chunk it receives straight to parse -
+   * so a caller driving the parser directly, a benchmark or a replay harness,
+   * measures the real thing rather than a copy of it.
+   *
+   * @param buff bytes to parse
+   * @param end how much of buff to consume, all of it by default
+   */
+  public parseBuffer (buff: Buffer, end: number = buff.length): void {
+    this.parse(buff, end)
+  }
+
   private parse (readBuffer: Buffer, end: number): void {
     const state = this.state
     const eq: number = AsciiChars.Eq

package/src/buffer/ascii/ascii-segment-parser.ts

@@ -29,10 +29,13 @@ export class AsciiSegmentParser {
   public parse (msgType: string, tags: Tags, last: number): Structure | null {
     // completed segments in that they are fully parsed
     const segments: SegmentDescription[] = []
-    const msgDefinition: MessageDefinition | undefined = this.definitions.message.get(msgType)
-    if (!msgDefinition) {
+    const resolved: MessageDefinition | undefined = this.definitions.message.get(msgType)
+    if (!resolved) {
       return null
     }
+    // narrowing of resolved is not visible inside the hoisted helpers below, so
+    // carry the definition on a const the compiler already sees as non-optional
+    const msgDefinition: MessageDefinition = resolved
     // in process of being discovered and may have any amount of depth
     // i.e. a component containing a repeated group of components
     // with sub-groups of components
@@ -41,8 +44,8 @@ export class AsciiSegmentParser {
     let peek: SegmentDescription
 
     // track depth-1 components that have been exited for fragmentation detection
-    const exitedDepth1Components: Set<string> = new Set()
-    const fragmentedComponents: Set<string> = new Set()
+    const exitedDepth1Components = new Set<string>()
+    const fragmentedComponents = new Set<string>()
 
     // having finished one segments keep unwinding until tag matches further up stack
     function unwind (tag: number): void {
@@ -189,8 +192,8 @@ export class AsciiSegmentParser {
       // only build SegmentViews for components detected as fragmented during discover
       // non-fragmented components use their position ranges directly (zero overhead)
       if (fragmentedComponents.size === 0) return
-      const ti = new TagIndex(msgDefinition!, tags, last + 1)
-      const seen: Set<string> = new Set()
+      const ti = new TagIndex(msgDefinition, tags, last + 1)
+      const seen = new Set<string>()
       for (let i = 1; i < segments.length - 1; i++) {
         const seg = segments[i]
         if (seg.depth !== 1) continue

package/src/buffer/ascii/tag-index.ts

@@ -10,9 +10,9 @@ export interface TagSpan {
 
 export class TagIndex {
   private readonly sortedTagPos: TagPos[]
-  private readonly tagSpans: Map<number, TagSpan> = new Map()
-  private readonly componentGroupWrappers: Set<string> = new Set()
-  private readonly cache: Map<string, SegmentView | null> = new Map()
+  private readonly tagSpans = new Map<number, TagSpan>()
+  private readonly componentGroupWrappers = new Set<string>()
+  private readonly cache = new Map<string, SegmentView | null>()
 
   constructor (
     public readonly set: IContainedSet,

package/src/buffer/msg-view.ts

@@ -263,29 +263,46 @@ export abstract class MsgView {
     return JSON.stringify(this.toObject(), null, 4)
   }
 
+  /**
+   * fetch a nested view, either by a single name - getView('Instrument') - or by a
+   * dotted path walked a step at a time - getView('Parties.PtysSubGrp')
+   */
   public getView (name: string): MsgView | null {
-    const parts: string[] = name.split('.')
-    const reducer = (a: MsgView | null, current: string): MsgView | null => {
-      if (!a) {
-        return a
+    // the dotted form is the exception. resolve a plain name without paying for
+    // the parts array and the closure a split and reduce would allocate - the
+    // component walk in toObject and validate comes through here per component.
+    if (!name.includes('.')) {
+      return this.resolveChild(name)
+    }
+    let view: MsgView | null = this
+    for (const part of name.split('.')) {
+      view = view.resolveChild(part)
+      if (!view) {
+        return null
       }
-      const structure = a.structure
-      const singleton: SegmentDescription | null = structure?.firstContainedWithin(current, a.segment) ?? null
-      if (singleton) {
-        return a.create(singleton)
-      }
-      // is this a full name where abbreviation exists
-      const component: ContainedField | null = a.segment.set?.localNameToField.get(current) ?? null
+    }
+    return view
+  }
 
-      if (component) {
-        const abbreviated: SegmentDescription | null = structure?.firstContainedWithin(component.name, a.segment) ?? null
-        if (abbreviated) {
-          return a.create(abbreviated)
-        }
+  /**
+   * resolve one step of a view path against this view
+   */
+  private resolveChild (name: string): MsgView | null {
+    const structure = this.structure
+    const singleton: SegmentDescription | null = structure?.firstContainedWithin(name, this.segment) ?? null
+    if (singleton) {
+      return this.create(singleton)
+    }
+    // is this a full name where abbreviation exists
+    const component: ContainedField | null = this.segment.set?.localNameToField.get(name) ?? null
+
+    if (component) {
+      const abbreviated: SegmentDescription | null = structure?.firstContainedWithin(component.name, this.segment) ?? null
+      if (abbreviated) {
+        return this.create(abbreviated)
       }
-      return null
     }
-    return parts.reduce<MsgView | null>(reducer, this)
+    return null
   }
 
   public abstract checksum (): number

package/src/dictionary/parser/quickfix/dictionary-validator.ts

@@ -49,7 +49,7 @@ export class DictionaryValidator {
   private readonly allFieldNames: string[] = []
   private readonly allComponentNames: string[] = []
 
-  get errors (): ReadonlyArray<ValidationError> {
+  get errors (): readonly ValidationError[] {
     return this._errors
   }
 

package/src/dictionary/parser/quickfix/quick-fix-graph-parser.ts

@@ -51,7 +51,7 @@ export class GraphNode {
     public readonly element: XNode
   ) {}
 
-  get edges (): ReadonlyArray<Edge> {
+  get edges (): readonly Edge[] {
     return this._edges
   }
 
@@ -118,8 +118,7 @@ export class QuickFixGraphParser {
     this.parseTrailer(doc)
     this.parseMessages(doc)
 
-    while (this.queue.length > 0) {
-      const node = this.queue.shift()!
+    for (let node = this.queue.shift(); node !== undefined; node = this.queue.shift()) {
       this.work(node)
     }
 

package/src/dictionary/parser/quickfix/validation-error.ts

@@ -14,7 +14,7 @@ export interface ValidationError {
 }
 
 export class DictionaryValidationException extends Error {
-  constructor (public readonly errors: ReadonlyArray<ValidationError>) {
+  constructor (public readonly errors: readonly ValidationError[]) {
     const errorCount = errors.filter(e => e.severity === ValidationSeverity.Error).length
     const warningCount = errors.filter(e => e.severity === ValidationSeverity.Warning).length
     const header = `FIX dictionary validation failed with ${errorCount} error(s) and ${warningCount} warning(s):`