jspurefix 5.11.2 → 5.11.4

package/src/generator/layout.ts

@@ -0,0 +1,279 @@
+import { ILooseObject } from '../collections/collection'
+import {
+  ContainedComponentField,
+  ContainedField,
+  ContainedFieldType,
+  ContainedGroupField,
+  ContainedSimpleField,
+  IContainedSet
+} from '../dictionary/contained'
+import { TagType } from '../buffer/tag/tag-type'
+
+/**
+ * Where every tag on the wire came from in the dictionary.
+ *
+ * The parser answers this question from position, which is exactly the assumption the
+ * scattered component work is about, so it cannot be the source of truth here.  Instead
+ * the layout is built by walking the object and the dictionary in lock step with
+ * `AsciiEncoder.encodeObject` - same field ordering, same treatment of components,
+ * groups and raw data - and consuming one token per emission.  The walk is not a parse:
+ * it knows what the encoder was about to write before it wrote it.
+ *
+ * If the two ever disagree the token stream will not line up and `Layout.build` throws,
+ * which makes this a standing check that the mirror is still faithful.
+ */
+
+export interface IToken {
+  readonly tag: number
+  /** the `tag=value` text exactly as encoded, without the delimiter */
+  readonly text: string
+}
+
+export enum LayoutKind {
+  Message = 'message',
+  Component = 'component',
+  Group = 'group',
+  Instance = 'instance',
+  Field = 'field'
+}
+
+export class LayoutNode {
+  public readonly children: LayoutNode[] = []
+
+  constructor (
+    public readonly kind: LayoutKind,
+    public readonly name: string,
+    public readonly path: string,
+    /** for a field, its token; for a group, the token carrying NoXXX */
+    public readonly tokenIndex: number = -1,
+    public readonly set: IContainedSet | null = null,
+    /** a raw data field is written as a Length token and then the data token */
+    public readonly extraTokenIndex: number = -1) {
+  }
+
+  /** token indices below this node, in the order the encoder wrote them */
+  public canonical (into: number[] = []): number[] {
+    if (this.tokenIndex >= 0) into.push(this.tokenIndex)
+    if (this.extraTokenIndex >= 0) into.push(this.extraTokenIndex)
+    for (const c of this.children) {
+      c.canonical(into)
+    }
+    return into
+  }
+
+  public get isLevel (): boolean {
+    return this.kind === LayoutKind.Message || this.kind === LayoutKind.Instance
+  }
+}
+
+/**
+ * split a wire message into tokens.  Values containing the delimiter would defeat this,
+ * which is why the generator never produces one.
+ */
+export function tokenise (wire: string, delimiter: string): IToken[] {
+  const parts = wire.split(delimiter)
+  const tokens: IToken[] = []
+  for (const p of parts) {
+    if (p.length === 0) continue
+    const eq = p.indexOf('=')
+    if (eq <= 0) continue
+    tokens.push({ tag: parseInt(p.substring(0, eq), 10), text: p })
+  }
+  return tokens
+}
+
+export function assemble (tokens: readonly IToken[], order: readonly number[], delimiter: string): string {
+  let out = ''
+  for (const i of order) {
+    out += tokens[i].text
+    out += delimiter
+  }
+  return out
+}
+
+export interface ILayout {
+  readonly root: LayoutNode
+  readonly tokens: readonly IToken[]
+  /** index of the first body token; everything before it is the standard header */
+  readonly bodyStart: number
+  /** index one past the last body token; everything from here is the trailer */
+  readonly bodyEnd: number
+}
+
+export class Layout {
+  private index: number = 0
+
+  private constructor (start: number) {
+    this.index = start
+  }
+
+  /**
+   * Build the layout of `object` encoded as `wire`.
+   *
+   * The header and trailer are written by the transmitter rather than from the object,
+   * so their extent is discovered rather than walked: the body is located by matching
+   * the tag sequence the walk predicts against the token stream.
+   */
+  public static build (
+    wire: string,
+    delimiter: string,
+    object: ILooseObject,
+    set: IContainedSet): ILayout {
+    const tokens = tokenise(wire, delimiter)
+    const expected = Layout.expectedTags(object, set)
+    const bodyStart = Layout.locate(tokens, expected)
+    const builder = new Layout(bodyStart)
+    const root = new LayoutNode(LayoutKind.Message, set.name, set.name, -1, set)
+    builder.walk(object, set, root, set.name)
+    return {
+      root,
+      tokens,
+      bodyStart,
+      bodyEnd: builder.index
+    }
+  }
+
+  /**
+   * the tag sequence the encoder will write for this object, used only to find where
+   * the body begins
+   */
+  private static expectedTags (object: ILooseObject, set: IContainedSet): number[] {
+    const out: number[] = []
+    const collect = (o: ILooseObject, s: IContainedSet): void => {
+      for (const field of Layout.presentFields(o, s)) {
+        switch (field.type) {
+          case ContainedFieldType.Simple: {
+            const sf = field as ContainedSimpleField
+            if (sf.definition.tagType === TagType.RawData && o[Layout.lengthName(s, sf)] == null) {
+              out.push(Layout.lengthTag(s, sf))
+            }
+            out.push(sf.definition.tag)
+            break
+          }
+          case ContainedFieldType.Component: {
+            const cf = field as ContainedComponentField
+            collect(o[cf.name] as ILooseObject, cf.definition)
+            break
+          }
+          case ContainedFieldType.Group: {
+            const gf = field as ContainedGroupField
+            const nof = gf.definition.noOfField
+            if (!nof) break
+            out.push(nof.tag)
+            const instances = Layout.instancesOf(o, gf)
+            for (const instance of instances) {
+              collect(instance, gf.definition)
+            }
+            break
+          }
+          default:
+            break
+        }
+      }
+    }
+    collect(object, set)
+    return out
+  }
+
+  private static locate (tokens: readonly IToken[], expected: readonly number[]): number {
+    if (expected.length === 0) return 0
+    for (let start = 0; start + expected.length <= tokens.length; ++start) {
+      let ok = true
+      for (let i = 0; i < expected.length; ++i) {
+        if (tokens[start + i].tag !== expected[i]) {
+          ok = false
+          break
+        }
+      }
+      if (ok) return start
+    }
+    throw new Error('layout walk does not match the encoded message - the mirror of AsciiEncoder has drifted')
+  }
+
+  /**
+   * the fields of `set` that `o` actually carries, in the order the encoder writes them.
+   * This is `AsciiEncoder.getFields`, deliberately duplicated rather than shared, because
+   * the encoder is a hot path and should not grow a seam for a test tool.
+   */
+  private static presentFields (o: ILooseObject, set: IContainedSet): ContainedField[] {
+    const fields: ContainedField[] = []
+    for (const key of Object.keys(o)) {
+      const field = set.localNameToField.get(key)
+      if (field) fields.push(field)
+    }
+    fields.sort((a, b) => a.position - b.position)
+    return fields
+  }
+
+  private static instancesOf (o: ILooseObject, gf: ContainedGroupField): ILooseObject[] {
+    const nof = gf.definition.noOfField
+    const instances = o[gf.name] ?? (nof ? o[nof.name] : null)
+    return Array.isArray(instances) ? instances : []
+  }
+
+  private static lengthField (set: IContainedSet, sf: ContainedSimpleField): ContainedSimpleField | null {
+    const prior = set.fields[sf.position - 1]
+    if (!prior || prior.type !== ContainedFieldType.Simple) return null
+    return prior as ContainedSimpleField
+  }
+
+  private static lengthName (set: IContainedSet, sf: ContainedSimpleField): string {
+    return Layout.lengthField(set, sf)?.name ?? ''
+  }
+
+  private static lengthTag (set: IContainedSet, sf: ContainedSimpleField): number {
+    return Layout.lengthField(set, sf)?.definition.tag ?? 0
+  }
+
+  private walk (o: ILooseObject, set: IContainedSet, parent: LayoutNode, path: string): void {
+    for (const field of Layout.presentFields(o, set)) {
+      switch (field.type) {
+        case ContainedFieldType.Simple: {
+          const sf = field as ContainedSimpleField
+          let lengthIndex = -1
+          if (sf.definition.tagType === TagType.RawData && o[Layout.lengthName(set, sf)] == null) {
+            lengthIndex = this.index++
+          }
+          const dataIndex = this.index++
+          // the length and its data are one indivisible unit - separating them would
+          // produce a message that is not merely scattered but unparseable
+          parent.children.push(new LayoutNode(
+            LayoutKind.Field,
+            sf.name,
+            `${path}.${sf.name}`,
+            lengthIndex >= 0 ? lengthIndex : dataIndex,
+            null,
+            lengthIndex >= 0 ? dataIndex : -1))
+          break
+        }
+        case ContainedFieldType.Component: {
+          const cf = field as ContainedComponentField
+          const node = new LayoutNode(
+            LayoutKind.Component, cf.name, `${path}.${cf.name}`, -1, cf.definition)
+          parent.children.push(node)
+          this.walk(o[cf.name] as ILooseObject, cf.definition, node, node.path)
+          break
+        }
+        case ContainedFieldType.Group: {
+          const gf = field as ContainedGroupField
+          const nof = gf.definition.noOfField
+          if (!nof) break
+          const countIndex = this.index++
+          const node = new LayoutNode(
+            LayoutKind.Group, gf.name, `${path}.${gf.name}`, countIndex, gf.definition)
+          parent.children.push(node)
+          const instances = Layout.instancesOf(o, gf)
+          for (let i = 0; i < instances.length; ++i) {
+            const instance = new LayoutNode(
+              LayoutKind.Instance, `${gf.name}[${i}]`, `${node.path}[${i}]`, -1, gf.definition)
+            node.children.push(instance)
+            this.walk(instances[i], gf.definition, instance, instance.path)
+          }
+          break
+        }
+        default:
+          break
+      }
+    }
+  }
+}

package/src/generator/market-context.ts

@@ -0,0 +1,207 @@
+import { Rng } from './rng'
+import { AssetClass, IInstrumentSpec, ILegSpec, InstrumentUniverse } from './instrument-universe'
+
+/**
+ * Everything one generated message agrees about.
+ *
+ * The old generator drew each field independently, which is why its output never added
+ * up - an OrderQty of -483.04 filled at a price of 97319 on a security called `Nunc`.
+ * A message is a statement about a single trade, so the trade is decided once, up front,
+ * and every field is then read off it.
+ *
+ * The walker moves this context as it descends: entering a leg group instance sets
+ * `leg`, so `LegPrice` is drawn from that leg rather than from the parent.
+ */
+
+export interface IParty {
+  readonly id: string
+  readonly idSource: string
+  readonly role: number
+  readonly desc: string
+}
+
+export interface IMarketContextOptions {
+  readonly rng: Rng
+  readonly instrument?: IInstrumentSpec
+  readonly assetClass?: AssetClass
+  /** anchor for every date in the message; defaults to today */
+  readonly asOf?: Date
+  readonly senderCompId?: string
+  readonly targetCompId?: string
+}
+
+const firms: readonly IParty[] = [
+  { id: 'MGSTGB2L', idSource: 'C', role: 1, desc: 'executing firm' },
+  { id: 'CHASGB2L', idSource: 'C', role: 4, desc: 'clearing firm' },
+  { id: 'BARCGB22', idSource: 'C', role: 17, desc: 'contra firm' },
+  { id: 'ABNANL2A', idSource: 'C', role: 13, desc: 'order origination firm' }
+]
+
+const traders: readonly IParty[] = [
+  { id: 'jhalliday', idSource: 'D', role: 12, desc: 'executing trader' },
+  { id: 'rmcdowall', idSource: 'D', role: 11, desc: 'order origination trader' },
+  { id: 'kbannerjee', idSource: 'D', role: 12, desc: 'executing trader' }
+]
+
+const accounts: readonly string[] = [
+  'ACCT-00417', 'ACCT-01925', 'OMNI-4471', 'SEGR-2210'
+]
+
+const desks: readonly string[] = ['ENERGY', 'MACRO', 'CASH-EQ', 'FICC']
+
+const freeText: readonly string[] = [
+  'give up to clearer',
+  'client instruction on file',
+  'worked over the close',
+  'cross with house book',
+  'blocked - settle T+2'
+]
+
+export class MarketContext {
+  public readonly rng: Rng
+  public readonly instrument: IInstrumentSpec
+  public readonly asOf: Date
+
+  /** the trade this message describes */
+  public readonly side: string
+  public readonly orderQty: number
+  public readonly price: number
+  public readonly lastPx: number
+  public readonly lastQty: number
+  public readonly cumQty: number
+  public readonly leavesQty: number
+  public readonly avgPx: number
+
+  public readonly tradeDate: Date
+  public readonly transactTime: Date
+  public readonly settlDate: Date
+
+  public readonly clOrdId: string
+  public readonly origClOrdId: string
+  public readonly orderId: string
+  public readonly execId: string
+  public readonly tradeId: string
+  public readonly account: string
+  public readonly desk: string
+  public readonly senderCompId: string
+  public readonly targetCompId: string
+
+  public readonly firms: readonly IParty[]
+  public readonly traders: readonly IParty[]
+
+  /** set while the walker is inside a leg, cleared on the way out */
+  public leg: ILegSpec | null = null
+  /** set while the walker is inside an underlying */
+  public underlying: IInstrumentSpec | null = null
+
+  private sequence: number = 0
+
+  constructor (options: IMarketContextOptions) {
+    const rng = this.rng = options.rng
+    this.instrument = options.instrument ?? InstrumentUniverse.choose(rng, options.assetClass)
+    const inst = this.instrument
+    this.asOf = options.asOf ?? new Date()
+
+    this.side = rng.bool(0.5) ? '1' : '2'
+    const lots = rng.int(1, 40)
+    this.orderQty = lots * inst.lotSize
+    this.price = rng.around(inst.refPx, 0.004, inst.tick)
+    this.lastPx = rng.around(this.price, 0.002, inst.tick)
+    // a partial fill more often than not, which is what a real execution stream looks like
+    const filledLots = rng.bool(0.35) ? lots : rng.int(1, Math.max(1, lots - 1))
+    this.cumQty = filledLots * inst.lotSize
+    this.leavesQty = this.orderQty - this.cumQty
+    this.lastQty = this.cumQty
+    this.avgPx = this.lastPx
+
+    this.tradeDate = MarketContext.midnightUtc(this.asOf)
+    this.transactTime = new Date(this.asOf.getTime() - rng.int(0, 90) * 1000)
+    this.settlDate = MarketContext.addBusinessDays(this.tradeDate, inst.assetClass === AssetClass.Fx ? 2 : 2)
+
+    const stamp = MarketContext.yyyymmdd(this.tradeDate)
+    this.clOrdId = `CLORD-${stamp}-${rng.int(10000, 99999)}`
+    this.origClOrdId = `CLORD-${stamp}-${rng.int(10000, 99999)}`
+    this.orderId = `ORD-${rng.int(1000000, 9999999)}`
+    this.execId = `EXE-${rng.int(1000000, 9999999)}`
+    this.tradeId = `TRD-${stamp}-${rng.int(1000, 9999)}`
+    this.account = rng.pick(accounts)
+    this.desk = rng.pick(desks)
+    this.senderCompId = options.senderCompId ?? 'init-comp'
+    this.targetCompId = options.targetCompId ?? 'accept-comp'
+
+    this.firms = rng.shuffled(firms)
+    this.traders = rng.shuffled(traders)
+  }
+
+  /**
+   * the instrument whose fields should be read right now - a leg is described by its own
+   * contract, an underlying by the underlying, and anything else by the parent
+   */
+  public current (): { symbol: string, securityId: string, securityIdSource: string, securityType: string, refPx: number, maturityMonthYear?: string } {
+    const leg = this.leg
+    if (leg) return leg
+    const under = this.underlying
+    if (under) return under
+    return this.instrument
+  }
+
+  public nextSeq (): number {
+    return ++this.sequence
+  }
+
+  public text (): string {
+    return this.rng.pick(freeText)
+  }
+
+  public party (index: number): IParty {
+    const all = [...this.firms, ...this.traders]
+    return all[index % all.length]
+  }
+
+  /**
+   * a quantity that fits the contract - always a whole number of lots, so a message
+   * never claims a fraction of a future
+   */
+  public quantity (max?: number): number {
+    const lot = this.instrument.lotSize
+    const cap = max != null ? Math.max(1, Math.floor(max / lot)) : 40
+    return this.rng.int(1, cap) * lot
+  }
+
+  /**
+   * what the trade is worth - quantity by price, and by the contract multiplier where
+   * there is one, so a Brent fill does not report a notional of six dollars
+   */
+  public grossAmount (): number {
+    const multiplier = this.instrument.contractMultiplier ?? 1
+    return Number((this.lastQty * this.lastPx * multiplier).toFixed(2))
+  }
+
+  public priceNear (centre?: number): number {
+    const inst = this.instrument
+    const leg = this.leg
+    const base = centre ?? (leg ? leg.refPx : inst.refPx)
+    return this.rng.around(base, 0.006, leg ? 0.01 : inst.tick)
+  }
+
+  public static midnightUtc (d: Date): Date {
+    return new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate(), 0, 0, 0, 0))
+  }
+
+  public static addBusinessDays (from: Date, days: number): Date {
+    const d = new Date(from.getTime())
+    let left = days
+    while (left > 0) {
+      d.setUTCDate(d.getUTCDate() + 1)
+      const day = d.getUTCDay()
+      if (day !== 0 && day !== 6) left--
+    }
+    return d
+  }
+
+  public static yyyymmdd (d: Date): string {
+    const m = d.getUTCMonth() + 1
+    const day = d.getUTCDate()
+    return `${d.getUTCFullYear()}${m < 10 ? '0' : ''}${m}${day < 10 ? '0' : ''}${day}`
+  }
+}

package/src/generator/outcome.ts

@@ -0,0 +1,102 @@
+import { ILooseObject } from '../collections/collection'
+
+/**
+ * What the engine did with a scattered message, stated more precisely than "the objects
+ * differ".
+ *
+ * The distinction earns its place because the two failures found so far are not equally
+ * serious, and an application can defend against only one of them:
+ *
+ *  - **loses fields** — the scattered parse is a strict subset of the canonical one.
+ *    Data is missing, but nothing present is false.  An application checking for the
+ *    fields it needs will notice.
+ *  - **mis-attributes** — the scattered parse contains something the canonical one does
+ *    not: a value moved into a component that never carried it, or a value changed.
+ *    Nothing looks wrong, and no amount of care at the application catches it.
+ *
+ * Keeping them apart is also what lets a corpus entry record an expectation sharp enough
+ * to be worth freezing.  "Differs" would pass for the wrong reason.
+ */
+
+export enum CaseOutcome {
+  /** the scattered ordering parses to the same object - the engine handled it */
+  RoundTrips = 'round-trips',
+  /** parses, but with fields dropped; every value present is still correct */
+  LosesFields = 'loses-fields',
+  /** parses, and carries a value the message did not say - the dangerous one */
+  MisAttributes = 'mis-attributes',
+  /** refused outright, which for a legal ordering is a defect, but a loud one */
+  Rejected = 'rejected'
+}
+
+export interface IOutcome {
+  readonly outcome: CaseOutcome
+  /** leaf paths the canonical parse has and the scattered one does not */
+  readonly missing: string[]
+  /** leaf paths the scattered parse has and the canonical one does not */
+  readonly added: string[]
+  /** leaf paths in both, disagreeing */
+  readonly changed: string[]
+}
+
+/**
+ * every leaf of an object as `path -> json`, so two parses can be compared without
+ * caring how the tree is shaped
+ */
+export function leaves (o: any, path: string = '', into: Map<string, string> = new Map<string, string>()): Map<string, string> {
+  if (o === null || o === undefined || typeof o !== 'object' || o instanceof Date || Buffer.isBuffer(o)) {
+    into.set(path, JSON.stringify(o))
+    return into
+  }
+  for (const key of Object.keys(o)) {
+    leaves(o[key], `${path}.${key}`, into)
+  }
+  return into
+}
+
+export function classify (canonical: ILooseObject | null, scattered: ILooseObject | null): IOutcome {
+  if (!scattered) {
+    return { outcome: CaseOutcome.Rejected, missing: [], added: [], changed: [] }
+  }
+  const a = leaves(canonical)
+  const b = leaves(scattered)
+  const missing: string[] = []
+  const added: string[] = []
+  const changed: string[] = []
+  for (const [k, v] of a) {
+    const other = b.get(k)
+    if (other === undefined) {
+      missing.push(k)
+    } else if (other !== v) {
+      changed.push(k)
+    }
+  }
+  for (const k of b.keys()) {
+    if (!a.has(k)) added.push(k)
+  }
+  missing.sort()
+  added.sort()
+  changed.sort()
+  if (missing.length === 0 && added.length === 0 && changed.length === 0) {
+    return { outcome: CaseOutcome.RoundTrips, missing, added, changed }
+  }
+  if (added.length > 0 || changed.length > 0) {
+    return { outcome: CaseOutcome.MisAttributes, missing, added, changed }
+  }
+  return { outcome: CaseOutcome.LosesFields, missing, added, changed }
+}
+
+export function describeOutcome (o: IOutcome): string {
+  switch (o.outcome) {
+    case CaseOutcome.RoundTrips:
+      return 'round trips - the scattered ordering parses to the same object'
+    case CaseOutcome.LosesFields:
+      return `loses ${o.missing.length} field(s): ${o.missing.join(', ')}`
+    case CaseOutcome.MisAttributes:
+      return `mis-attributes ${o.added.length + o.changed.length} value(s): ${[...o.added, ...o.changed].join(', ')}`
+    case CaseOutcome.Rejected:
+      return 'rejected - the parser refused a legal ordering'
+    default:
+      return 'unknown'
+  }
+}