jspurefix 5.9.1 → 5.10.0

package/src/jsfix-bench.ts

@@ -0,0 +1,204 @@
+import 'reflect-metadata'
+
+import * as path from 'path'
+import * as fs from 'fs'
+import minimist = require('minimist')
+import buildOptions from 'minimist-options'
+
+import { AsciiChars } from './buffer/ascii'
+import { IJsFixConfig } from './config'
+import { DefinitionFactory } from './util'
+import { SessionContainer } from './runtime'
+import { DITokens } from './runtime/di-tokens'
+import {
+  BenchRunner,
+  IBenchCase,
+  IBenchOptions,
+  IBenchSuiteResult,
+  ParseCase,
+  ParseDepth,
+  defaultBenchOptions,
+  formatComparison,
+  formatSuite
+} from './benchmark'
+
+const options = buildOptions({
+  dict: {
+    type: 'string',
+    alias: 'd',
+    default: 'qf44'
+  },
+
+  fix: {
+    type: 'string',
+    alias: 'f',
+    default: 'data/examples/FIX.4.4/quickfix/execution-report/fix.txt'
+  },
+
+  session: {
+    type: 'string',
+    alias: 's',
+    default: 'data/session/test-initiator.json'
+  },
+
+  delimiter: {
+    type: 'string',
+    alias: 'l',
+    default: '|'
+  },
+
+  depth: {
+    type: 'string',
+    default: 'view,object'
+  },
+
+  rounds: {
+    type: 'number',
+    default: defaultBenchOptions.rounds
+  },
+
+  msgs: {
+    type: 'number',
+    default: defaultBenchOptions.operationsPerRound
+  },
+
+  warmup: {
+    type: 'number',
+    default: defaultBenchOptions.warmupOperations
+  },
+
+  label: {
+    type: 'string',
+    default: 'jspurefix'
+  },
+
+  save: {
+    type: 'string'
+  },
+
+  baseline: {
+    type: 'string'
+  },
+
+  json: {
+    type: 'boolean',
+    default: false
+  },
+
+  help: {
+    type: 'boolean',
+    alias: 'h',
+    default: false
+  }
+})
+
+const argv: any = minimist(process.argv.slice(2), options)
+
+const root: string = path.join(__dirname, '../')
+
+function norm (p: string): string {
+  return path.isAbsolute(p) ? p : path.join(root, p)
+}
+
+function usage (): void {
+  console.log('measure the parse path and what it costs in time and in garbage')
+  console.log()
+  console.log('  --fix=<path>        fix log to parse, defaults to a 4.4 execution report')
+  console.log('  --dict=<name>       dictionary, e.g. qf44, repo44, qf50sp2')
+  console.log('  --delimiter=<char>  field delimiter in the log, defaults to |')
+  console.log('  --depth=view,object which stages to measure, comma separated')
+  console.log('  --rounds=<n>        measured rounds, one sample each')
+  console.log('  --msgs=<n>          messages per round')
+  console.log('  --warmup=<n>        messages discarded before measuring')
+  console.log('  --save=<path>       write the suite to json')
+  console.log('  --baseline=<path>   compare against a previously saved suite')
+  console.log('  --json              print the suite as json instead of a report')
+  console.log()
+  console.log('run under --expose-gc so rounds can be isolated and heap read cleanly:')
+  console.log('  node --expose-gc dist/jsfix-bench --fix=data/examples/FIX.4.4/quickfix/execution-report/fix.txt')
+  console.log('  npm run bench')
+}
+
+async function makeConfig (): Promise<IJsFixConfig> {
+  const sys = new SessionContainer()
+  sys.registerGlobal('error')
+  const sessionDescription = require(norm(argv.session))
+  const container = await sys.makeSystem(sessionDescription)
+  const config = container.resolve<IJsFixConfig>(DITokens.IJsFixConfig)
+  config.definitions = await new DefinitionFactory().getDefinitions(argv.dict)
+  // the parser reads the delimiter when it is constructed, so this has to be
+  // settled before any case is built
+  config.delimiter = AsciiChars.firstChar(argv.delimiter)
+  return config
+}
+
+function requestedDepths (): ParseDepth[] {
+  const known = new Map<string, ParseDepth>([
+    ['view', ParseDepth.View],
+    ['object', ParseDepth.Object]
+  ])
+  const asked = String(argv.depth).split(',').map(d => d.trim().toLowerCase()).filter(d => d.length > 0)
+  return asked.map(d => {
+    const depth = known.get(d)
+    if (!depth) {
+      throw new Error(`unknown depth ${d}, expected view or object`)
+    }
+    return depth
+  })
+}
+
+function readSuite (p: string): IBenchSuiteResult {
+  return JSON.parse(fs.readFileSync(norm(p), 'utf8'))
+}
+
+async function main (): Promise<void> {
+  if (argv.help) {
+    usage()
+    return
+  }
+  const fixPath = norm(argv.fix)
+  if (!fs.existsSync(fixPath)) {
+    throw new Error(`no such fix log ${fixPath}`)
+  }
+  const contents = fs.readFileSync(fixPath, 'utf8')
+  const config = await makeConfig()
+
+  const cases: ParseCase[] = requestedDepths().map(depth =>
+    new ParseCase(`parse:${depth}`, { config, contents, depth }))
+
+  // every case walks the same input, so one alignment covers the suite and all
+  // of them stay directly comparable
+  const perInvocation = cases[0].operationsPerInvocation
+  const operationsPerRound = cases[0].alignOperations(argv.msgs)
+  const benchOptions: IBenchOptions = {
+    warmupOperations: cases[0].alignOperations(argv.warmup),
+    rounds: argv.rounds,
+    operationsPerRound,
+    gcBetweenRounds: true
+  }
+
+  console.log(`${path.relative(root, fixPath)}  ${perInvocation} msg/pass  dict ${argv.dict}`)
+  const suite = await new BenchRunner(benchOptions).runAll(argv.label, cases as IBenchCase[])
+
+  if (argv.json) {
+    console.log(JSON.stringify(suite, null, 2))
+  } else {
+    console.log()
+    console.log(formatSuite(suite))
+  }
+  if (argv.baseline) {
+    console.log()
+    console.log(formatComparison(readSuite(argv.baseline), suite))
+  }
+  if (argv.save) {
+    const target = norm(argv.save)
+    fs.mkdirSync(path.dirname(target), { recursive: true })
+    fs.writeFileSync(target, JSON.stringify(suite, null, 2), 'utf8')
+    console.log(`saved ${path.relative(root, target)}`)
+  }
+}
+
+main().catch(e => {
+  console.error(e instanceof Error ? e.message : e)
+  process.exit(1)
+})

package/src/runtime/session-container.ts

@@ -20,7 +20,23 @@ import { FixmlEncoder, FiXmlParser } from '../buffer/fixml'
 import { FixEntity } from '../transport/fix-entity'
 import { IHttpAdapter } from '../transport/http/http-adapter'
 
+/**
+ * Supplies the session message factory for a description.  Return null to accept the
+ * stock factory for the protocol.
+ */
+export type SessionMsgFactoryProvider = (description: ISessionDescription) => ISessionMsgFactory | null
+
 export class SessionContainer {
+  /**
+   * @param msgFactoryProvider an application that needs its own session messages -
+   * typically a Logon carrying tags the standard message does not - can pass one
+   * here rather than subclassing to override makeSessionFactory:
+   *
+   *   new SessionContainer(d => new MyFactory(d))
+   */
+  constructor (public readonly msgFactoryProvider: SessionMsgFactoryProvider | null = null) {
+  }
+
   public reset (): void {
     container.reset()
   }
@@ -45,6 +61,10 @@ export class SessionContainer {
   }
 
   protected makeSessionFactory (description: ISessionDescription): ISessionMsgFactory {
+    const supplied = this.msgFactoryProvider?.(description)
+    if (supplied) {
+      return supplied
+    }
     const fixml = !this.isAscii(description)
     return fixml
       ? new FixmlSessionMsgFactory(description)

package/src/runtime/session-launcher.ts

@@ -1,6 +1,6 @@
 import * as path from 'path'
 import { IJsFixConfig, IJsFixLogger, JsFixLoggerFactory, JsFixWinstonLogFactory, WinstonLogger } from '../config'
-import { FixEntity, FixSession, ISessionDescription } from '../transport'
+import { FixEntity, FixSession, ISessionDescription, ISessionMsgFactory } from '../transport'
 import { DependencyContainer } from 'tsyringe'
 import { EngineFactory } from './engine-factory'
 import { SessionContainer } from './session-container'
@@ -24,7 +24,24 @@ export abstract class SessionLauncher {
     this.acceptorConfig = acceptorConfig ? this.loadConfig(acceptorConfig) : null
   }
 
-  protected sessionContainer: SessionContainer = new SessionContainer()
+  // the arrow defers to makeSessionMsgFactory at resolve time, so a subclass need
+  // only override that one method - no SessionContainer subclass required
+  protected sessionContainer: SessionContainer =
+    new SessionContainer((d: ISessionDescription) => this.makeSessionMsgFactory(d))
+
+  /**
+   * Override to supply this application's own session message factory - the place to
+   * put a Logon carrying tags the standard message does not, or a header a
+   * counterparty stamps differently.  Return null (the default) for the stock
+   * factory.
+   *
+   * For a Logon that only needs extra fields, prefer the "Logon" block in the session
+   * description - it needs no code at all.  Reach for a factory when the values are
+   * computed at run time (a signature, a nonce, a token fetched at start up).
+   */
+  protected makeSessionMsgFactory (_description: ISessionDescription): ISessionMsgFactory | null {
+    return null
+  }
 
   private async empty (): Promise<any> {
     return await new Promise((resolve, reject) => {

package/src/store/file-session-store.ts

@@ -22,7 +22,7 @@ export class FileSessionStore implements IFixSessionStore {
   private creationTimeValue: Date = new Date()
 
   // In-memory index: seqnum -> { offset, length }
-  private readonly headerIndex: Map<number, { offset: number, length: number }> = new Map()
+  private readonly headerIndex = new Map<number, { offset: number, length: number }>()
 
   /**
    * Creates a FileSessionStore with the default file-based stream provider.
@@ -285,7 +285,7 @@ export class FileSessionStore implements IFixSessionStore {
    */
   static parseSessionTime (str: string): Date | null {
     // Format: YYYYMMDD-HH:MM:SS.ffffff
-    const match = str.match(/^(\d{4})(\d{2})(\d{2})-(\d{2}):(\d{2}):(\d{2})\.(\d{3,6})$/)
+    const match = /^(\d{4})(\d{2})(\d{2})-(\d{2}):(\d{2}):(\d{2})\.(\d{3,6})$/.exec(str)
     if (!match) return null
     const [, y, M, d, h, m, s, frac] = match
     const ms = parseInt(frac.substring(0, 3), 10)

package/src/store/file-session-stream-provider.ts

@@ -32,28 +32,34 @@ export class FileSessionStreamProvider implements ISessionStreamProvider {
   }
 
   openBody (): void {
-    if (this.bodyFd !== null) return
-    this.bodyFd = fs.openSync(this.bodyPath, 'a+')
-    const stat = fs.fstatSync(this.bodyFd)
-    this.bodySize = stat.size
+    this.openBodyFd()
   }
 
-  async appendBody (data: Buffer): Promise<number> {
+  /**
+   * open the body file on first use and hand back the descriptor - returning it
+   * keeps callers free of a null check the compiler cannot see through.
+   */
+  private openBodyFd (): number {
     if (this.bodyFd === null) {
-      this.openBody()
+      this.bodyFd = fs.openSync(this.bodyPath, 'a+')
+      const stat = fs.fstatSync(this.bodyFd)
+      this.bodySize = stat.size
     }
+    return this.bodyFd
+  }
+
+  async appendBody (data: Buffer): Promise<number> {
+    const fd = this.openBodyFd()
     const offset = this.bodySize
-    fs.writeSync(this.bodyFd!, data, 0, data.length, offset)
+    fs.writeSync(fd, data, 0, data.length, offset)
     this.bodySize += data.length
     return offset
   }
 
   async readBody (offset: number, length: number): Promise<Buffer> {
-    if (this.bodyFd === null) {
-      this.openBody()
-    }
+    const fd = this.openBodyFd()
     const buf = Buffer.alloc(length)
-    fs.readSync(this.bodyFd!, buf, 0, length, offset)
+    fs.readSync(fd, buf, 0, length, offset)
     return buf
   }
 

package/src/store/fix-session-store-factory.ts

@@ -7,7 +7,7 @@ import { FileSessionStore } from './file-session-store'
  * Factory for creating session stores.
  */
 export interface IFixSessionStoreFactory {
-  create (sessionId: SessionId): IFixSessionStore
+  create: (sessionId: SessionId) => IFixSessionStore
 }
 
 /**

package/src/store/fix-session-store.ts

@@ -14,24 +14,24 @@ export interface IFixSessionStore {
   readonly sessionId: SessionId
 
   // Message Operations
-  put (record: IFixMsgStoreRecord): Promise<void>
-  get (seqNum: number): Promise<IFixMsgStoreRecord | null>
-  getRange (fromSeqNum: number, toSeqNum: number): Promise<IFixMsgStoreRecord[]>
+  put: (record: IFixMsgStoreRecord) => Promise<void>
+  get: (seqNum: number) => Promise<IFixMsgStoreRecord | null>
+  getRange: (fromSeqNum: number, toSeqNum: number) => Promise<IFixMsgStoreRecord[]>
 
   // Sequence Number Operations
   senderSeqNum: number
   targetSeqNum: number
-  setSenderSeqNum (value: number): Promise<void>
-  setTargetSeqNum (value: number): Promise<void>
-  nextSenderSeqNum (): Promise<number>
-  nextTargetSeqNum (): Promise<number>
+  setSenderSeqNum: (value: number) => Promise<void>
+  setTargetSeqNum: (value: number) => Promise<void>
+  nextSenderSeqNum: () => Promise<number>
+  nextTargetSeqNum: () => Promise<number>
 
   // Session Operations
   readonly creationTime: Date
-  reset (): Promise<void>
+  reset: () => Promise<void>
 
   // Lifecycle
-  initialize (): Promise<void>
-  flush (): Promise<void>
-  dispose (): Promise<void>
+  initialize: () => Promise<void>
+  flush: () => Promise<void>
+  dispose: () => Promise<void>
 }

package/src/store/memory-session-store.ts

@@ -7,7 +7,7 @@ import { IFixMsgStoreRecord } from './fix-msg-store-record'
  * Not persistent - all data lost on dispose.
  */
 export class MemorySessionStore implements IFixSessionStore {
-  private readonly messages: Map<number, IFixMsgStoreRecord> = new Map()
+  private readonly messages = new Map<number, IFixMsgStoreRecord>()
   private senderSeqNumValue: number = 1
   private targetSeqNumValue: number = 1
   private creationTimeValue: Date = new Date()

package/src/store/session-stream-provider.ts

@@ -7,68 +7,68 @@ export interface ISessionStreamProvider {
    * Opens or creates a read-write buffer for the message body file.
    * Must support random-access reads (by offset+length).
    */
-  openBody (): void
+  openBody: () => void
 
   /**
    * Appends data to the body. Returns the offset at which data was written.
    */
-  appendBody (data: Buffer): Promise<number>
+  appendBody: (data: Buffer) => Promise<number>
 
   /**
    * Reads data from the body at the given offset and length.
    */
-  readBody (offset: number, length: number): Promise<Buffer>
+  readBody: (offset: number, length: number) => Promise<Buffer>
 
   /**
    * Gets the current body size (for calculating offsets).
    */
-  getBodySize (): number
+  getBodySize: () => number
 
   /**
    * Appends a line to the header index file.
    */
-  appendHeaderLine (line: string): Promise<void>
+  appendHeaderLine: (line: string) => Promise<void>
 
   /**
    * Reads all lines from the header index file.
    * Returns empty array if no data exists.
    */
-  readHeaderLines (): Promise<string[]>
+  readHeaderLines: () => Promise<string[]>
 
   /**
    * Reads the sequence numbers string.
    * Returns null if no data exists.
    */
-  readSeqNums (): Promise<string | null>
+  readSeqNums: () => Promise<string | null>
 
   /**
    * Writes the sequence numbers string.
    */
-  writeSeqNums (content: string): Promise<void>
+  writeSeqNums: (content: string) => Promise<void>
 
   /**
    * Reads the session time string.
    * Returns null if no data exists.
    */
-  readSessionTime (): Promise<string | null>
+  readSessionTime: () => Promise<string | null>
 
   /**
    * Writes the session time string.
    */
-  writeSessionTime (content: string): Promise<void>
+  writeSessionTime: (content: string) => Promise<void>
 
   /**
    * Resets all streams/files for a new session.
    */
-  reset (): Promise<void>
+  reset: () => Promise<void>
 
   /**
    * Flushes any pending writes.
    */
-  flush (): Promise<void>
+  flush: () => Promise<void>
 
   /**
    * Disposes of all resources.
    */
-  dispose (): Promise<void>
+  dispose: () => Promise<void>
 }

package/src/transport/ascii/ascii-msg-transmitter.ts

@@ -28,6 +28,26 @@ export class AsciiMsgTransmitter extends MsgTransmitter {
     const components = config.definitions.component
     this.header = components.get('StandardHeader')
     this.trailer = components.get('StandardTrailer')
+    this.reportUnknownFields(config)
+  }
+
+  /**
+   * A field the dictionary does not declare on the message being encoded has no tag,
+   * so it cannot be written and is dropped.  Say so once per offending field - the
+   * alternative, and what this library did for years, is that a bespoke Logon tag
+   * simply never appears on the wire and nothing anywhere explains why.
+   */
+  private reportUnknownFields (config: IJsFixConfig): void {
+    const name = config.description.application?.name ?? 'na'
+    const logger = config.logFactory.logger(`${name}:encoder`)
+    const reported = new Set<string>()
+    this.encoder.onUnknownField = (fieldName: string, setName: string) => {
+      const key = `${setName}.${fieldName}`
+      if (reported.has(key)) return
+      reported.add(key)
+      logger.warning(`field '${fieldName}' is not declared on ${setName} in this dictionary - it will not be sent`)
+      this.emit('unknown-field', fieldName, setName)
+    }
   }
 
   private checksum (): number {

package/src/transport/ascii/ascii-session-msg-factory.ts

@@ -23,7 +23,12 @@ export class AsciiSessionMsgFactory extends ASessionMsgFactory {
       Password: description.Password,
       HeartBtInt: description.HeartBtInt,
       ResetSeqNumFlag: description.ResetSeqNumFlag,
-      EncryptMethod: EncryptMethod.None
+      EncryptMethod: EncryptMethod.None,
+      // a counterparty demanding a tag the standard Logon does not carry needs no
+      // code at all - name it under "Logon" in the session description.  Spread last
+      // so an application can also override what the engine derived, or null out a
+      // field it does not want sent.  See issue #93.
+      ...description.Logon
     } as ILogon
     return this.mutate(o, MsgType.Logon)
   }

package/src/transport/fixml/fixml-session-msg-factory.ts

@@ -42,7 +42,9 @@ export class FixmlSessionMsgFactory extends ASessionMsgFactory {
         Username: description.Username,
         Password: description.Password,
         UserRequestID: userRequestId,
-        UserRequestType: UserRequestType.LogOnUser
+        UserRequestType: UserRequestType.LogOnUser,
+        // same escape hatch as the ascii factory - see ISessionDescription.Logon
+        ...description.Logon
       } as IUserRequest
       return this.mutate(o, MsgType.Logon)
     } else {

package/src/transport/index.ts

@@ -4,7 +4,12 @@ export * from './fix-acceptor'
 export * from './fix-initiator'
 export * from './msg-transmitter'
 export * from './msg-payload'
+export * from './send-callback'
 export * from './tick-action'
 export * from './msg-application'
 export * from './ascii/ascii-session'
+// the factory an application extends to customise its Logon or header - it belongs
+// at the root alongside ASessionMsgFactory, not behind a dist/ path
+export * from './ascii/ascii-session-msg-factory'
+export * from './fixml/fixml-session-msg-factory'
 export * from './session'

package/src/transport/msg-payload.ts

@@ -1,7 +1,11 @@
 import { ILooseObject } from '../collections/collection'
+import { SendCallback } from './send-callback'
 
 export class MsgPayload {
   public encoded: Buffer
-  constructor (public readonly msgType: string, public readonly obj: ILooseObject) {
+  constructor (public readonly msgType: string,
+    public readonly obj: ILooseObject,
+    /** invoked once this payload has been encoded, or has failed to encode */
+    public readonly callback: SendCallback | null = null) {
   }
 }

package/src/transport/msg-transmitter.ts

@@ -4,6 +4,7 @@ import { Transform } from 'stream'
 import { MsgPayload } from './msg-payload'
 import { ILooseObject } from '../collections/collection'
 import { ISessionDescription } from './session/session-description'
+import { SendCallback } from './send-callback'
 import * as events from 'events'
 
 export abstract class MsgTransmitter extends events.EventEmitter {
@@ -24,8 +25,23 @@ export abstract class MsgTransmitter extends events.EventEmitter {
   }
 
   // messages at front, byte stream at back
-  public send (msgType: string, obj: ILooseObject): void {
-    this.encodeStream.write(new MsgPayload(msgType, obj))
+  public send (msgType: string, obj: ILooseObject, callback: SendCallback | null = null): void {
+    this.encodeStream.write(new MsgPayload(msgType, obj, callback))
+  }
+
+  /**
+   * Report a send outcome to whoever asked for one, without letting their code break
+   * the encode stream - a throwing callback would otherwise surface as a transform
+   * failure and be indistinguishable from an encoding fault.
+   */
+  protected settle (payload: MsgPayload, error: Error | null, header: ILooseObject | null): void {
+    const callback = payload.callback
+    if (!callback) return
+    try {
+      callback(error, { msgType: payload.msgType, header, encoded: payload.encoded ?? null })
+    } catch (e) {
+      this.emit('error', e)
+    }
   }
 
   public abstract encodeMessage (msgType: string, obj: ILooseObject): (ILooseObject | null)
@@ -46,8 +62,21 @@ export abstract class MsgTransmitter extends events.EventEmitter {
           this.push(payload.encoded)
           const encodedTxt = transmitter.buffer.toString()
           transmitter.emit('encoded', msgType, encodedTxt, state)
+          // encodeMessage returns null when it could not build the message - an
+          // unknown msgType, or a factory that produced no header.  It reports that
+          // on the error channel and carries on, so without this the callback would
+          // announce success for a message that never formed.
+          state
+            ? transmitter.settle(payload, null, state)
+            : transmitter.settle(payload, new Error(`could not encode ${msgType}`), null)
+          // note: the second argument to a Transform callback is *pushed to the
+          // readable side*, and this readable side is piped to the socket - so a
+          // result cannot be returned that way without corrupting the byte stream.
+          // Hence the per-payload callback above.
           done()
         } catch (e) {
+          transmitter.settle(payload, e as Error, null)
+          // still fail the stream, so the session's error handling is unchanged
           done(e)
         }
       }

package/src/transport/send-callback.ts

@@ -0,0 +1,33 @@
+import { ILooseObject } from '../collections/collection'
+
+/**
+ * What a send produced.  `header` is the StandardHeader the engine stamped on the
+ * message - the sequence number it went out under, the sending time, the comp ids -
+ * which is the piece applications most often need to correlate an outbound message
+ * with an acknowledgement, a database row, or a resend.
+ */
+export interface SendResult {
+  msgType: string
+  /** the stamped StandardHeader, or null if the message never got that far */
+  header: ILooseObject | null
+  /** the bytes written to the transport */
+  encoded: Buffer | null
+}
+
+/**
+ * Called once per `send`, after the message has been encoded and pushed to the
+ * transport, or as soon as encoding fails.
+ *
+ * `send` is otherwise fire and forget: it writes into a stream and returns, so
+ * nothing tells the caller which sequence number their message went out under, and a
+ * failure surfaces on the session's error channel with no way to tell which call
+ * caused it.  See https://github.com/TimelordUK/jspurefix/issues/86
+ *
+ * Note this fires when the message has been *encoded and handed to the transport*,
+ * not when the socket has flushed it - the FIX session layer has no acknowledgement
+ * of the latter either.
+ *
+ * A callback that throws is reported on the transmitter's error channel rather than
+ * being allowed to tear down the encode stream.
+ */
+export type SendCallback = (error: Error | null, result: SendResult) => void

package/src/transport/session/fix-clock.ts

@@ -1,5 +1,5 @@
 export interface IFixClock {
-  now (): Date
+  now: () => Date
 }
 
 export class DefaultFixClock implements IFixClock {