@mojaloop/central-ledger 19.14.0 → 20.1.0

package/src/testing/kafka.ts

@@ -0,0 +1,221 @@
+/*****
+ License
+ --------------
+ Copyright © 2020-2026 Mojaloop Foundation
+ The Mojaloop files are made available by the Mojaloop Foundation under the Apache License, Version 2.0 (the "License") and you may not use these files except in compliance with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, the Mojaloop files are distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+
+ Contributors
+ --------------
+ This is the official list of the Mojaloop project contributors for this file.
+ Names of the original copyright holders (individuals or organizations)
+ should be listed with a '*' in the first column. People who have
+ contributed from an organization can be listed under the organization
+ that actually holds the copyright for their contributions (see the
+ Mojaloop Foundation for an example). Those individuals should have
+ their names indented and be marked with a '-'. Email address can be added
+ optionally within square brackets <email>.
+
+ * TigerBeetle
+ - Lewis Daly <lewis@tigerbeetle.com>
+ --------------
+
+ ******/
+
+import Kafka from 'node-rdkafka'
+import assert from "node:assert"
+import Logger from "@mojaloop/central-services-logger"
+
+const logger = Logger.child({ scope: 'testing/kafka' })
+
+export class Producer {
+  private _producer: Kafka.HighLevelProducer
+  private _status: 'SETUP' | 'READY' | 'ERROR' | 'DISCONNECTED' = 'SETUP'
+
+  constructor(broker: string) {
+    this._producer = new Kafka.HighLevelProducer({
+      'metadata.broker.list': broker
+    })
+    // Seemed to be a good number in our local benchmarks.
+    this._producer.setPollInterval(50)
+  }
+
+  async connect(onError: (err: Kafka.LibrdKafkaError) => void): Promise<void> {
+    return new Promise((resolve, reject) => {
+      this._producer.connect({}, (err, data) => {
+        if (err) {
+          this._status = 'ERROR'
+          logger.error(`Producer.connect() failed with error: ${err.message}.`)
+          return reject(err)
+        }
+      })
+
+      this._producer.on('ready', () => {
+        this._status = 'READY'
+        return resolve()
+      })
+
+      this._producer.on('event.error', (err) => {
+        logger.error(`Producer on 'event.error': ${err.message}`)
+        onError(err)
+      })
+
+      this._producer.setValueSerializer((value) => Buffer.from(value))
+    })
+  }
+
+  async disconnect(): Promise<void> {
+    assert.equal(this._status, 'READY', 'Producer is not ready. Did you call `connect()`?')
+
+    return new Promise((resolve, reject) => {
+      this._producer.on('disconnected', () => {
+        resolve()
+      })
+      this._producer.flush()
+      this._producer.disconnect()
+    })
+  }
+
+  /**
+   * Produce a set of messages to a topic.
+   */
+  async produce(topic: string, messages: Array<any>): Promise<void> {
+    assert.equal(this._status, 'READY', 'Producer is not ready. Did you call `connect()`?')
+
+    const producePromise = (message: any): Promise<void> => new Promise((resolve, reject) => {
+      // Adding this into the hot path to more closely mimick the legacy kafka behaviour.
+      const messageStr = JSON.stringify(message)
+      this._producer.produce(topic, null, messageStr, null, Date.now(), (err, offset) => {
+
+        if (err) {
+          return reject(err)
+        }
+        return resolve()
+      })
+    })
+    const messagePromises = Promise.all(messages.map(message => producePromise(message)))
+    await messagePromises
+  }
+}
+
+export class Consumer {
+  private _consumer: Kafka.KafkaConsumer
+  private _status: 'CREATED' | 'SUBSCRIBING' | 'CONSUMING' | 'ERRORED' | 'DISCONNECTED' = 'CREATED'
+
+  constructor(group: string, broker: string) {
+    assert(group)
+    assert(broker)
+
+    this._consumer = new Kafka.KafkaConsumer({
+      'group.id': group,
+      'enable.auto.commit': false,
+      'metadata.broker.list': broker,
+      'session.timeout.ms': 6000,
+      'heartbeat.interval.ms': 2000,
+      'socket.timeout.ms': 5000,
+      'fetch.wait.max.ms': 500,
+    }, {
+      'auto.offset.reset': 'earliest'
+    })
+  }
+
+  public async subscribe(topics: Array<string | RegExp>, cb: (msg: Kafka.Message) => void): Promise<void> {
+    logger.debug(`Consumer.subscribe()`)
+
+    if (this._status !== 'CREATED') {
+      throw new Error(`Consumer.subscribe() - tried to subscribe but status was: ${this._status}.\
+        Expectected status: 'CREATED'.`)
+    }
+    this._status = 'SUBSCRIBING'
+    return new Promise((resolve, reject) => {
+      this._consumer.connect()
+
+      this._consumer.on('ready', (info, metadata) => {
+        this._consumer.subscribe(topics)
+        this._consumer.consume()
+        this._status = 'CONSUMING'
+        logger.debug(`Consumer subscribed and ready.`)
+        return resolve()
+      })
+
+      this._consumer.on('data', (msg) => {
+        return cb(msg)
+      })
+
+      this._consumer.on('event.error', (error: Kafka.LibrdKafkaError) => {
+        logger.error(`Consumer.on('event.error'): ${error.message}.`)
+      })
+
+      this._consumer.on('connection.failure', (err) => {
+        this._status = 'ERRORED'
+        logger.error('connection failure', err);
+      });
+
+      this._consumer.on('event.log', (log) => {
+        logger.info(log);
+      });
+
+      this._consumer.on('disconnected', (msg) => {
+        this._status = 'DISCONNECTED'
+        logger.warn('disconnected', msg);
+      })
+      
+      this._consumer.on('rebalance', (err, assignments) => {
+        logger.error('rebalance', err);
+      })
+    })
+  }
+
+  /**
+   * Disconnect the consumer. Noop if not already consuming.
+   */
+  public async disconnect(): Promise<void> {
+    logger.warn(`Consumer.disconnect()`)
+    const connected = this._consumer.isConnected()
+    
+    if (!connected) {
+      logger.warn('Consumer already disconnected')
+      this._status = 'DISCONNECTED'
+      return
+    }
+
+    if (this._status !== 'CONSUMING') {
+      logger.warn(`Consumer.disconnect() called but status was: ${this._status}. Treating as a noop.`)
+      return
+    }
+
+    const assignments = this._consumer.assignments()
+    if (assignments.length > 0) {
+      this._consumer.pause(assignments)
+    }
+
+    this._consumer.unsubscribe()
+
+    const timeoutMs = 10000
+    return new Promise((resolve, reject) => {
+      const timeout = setTimeout(() => {
+        const error = new Error(`Consumer.disconnect() timed out after: ${timeoutMs} ms`)
+        logger.error(error.message)
+        reject(error)
+      }, timeoutMs)
+      logger.info('calling _consumer.disconnect()')
+      
+      this._consumer.disconnect((err: any, data: Kafka.ClientMetrics) => {
+
+        logger.info('_consumer.disconnect - callback!')
+        clearTimeout(timeout)
+        if (err) {
+          this._status = 'ERRORED'
+          return reject(err)
+        }
+
+        this._status = 'DISCONNECTED'
+        return resolve()
+      })
+    })
+  }
+}
+

package/src/testing/prng.ts

@@ -0,0 +1,82 @@
+import assert from 'node:assert'
+
+/**
+ * @class PRNG
+ * @description A seeded Progressive Random Number Generator. Useful for setting up fuzz tests etc.
+ *   Reference: https://stackoverflow.com/questions/521295/seeding-the-random-number-generator-in-javascript
+ */
+export default class PRNG {
+  prng: () => number
+
+  constructor(seed: number) {
+    this.prng = splitmix32(seed)
+  }
+
+  public randomElementFrom<T>(array: Array<T>): T {
+    assert(array.length > 0, 'Expected array to have at least 1 element.')
+    const index = this.intExclusive(array.length)
+    return array[index]
+  }
+
+  public intExclusive(bound: number): number {
+    assert(typeof bound === 'number')
+    assert(bound > 0)
+    
+    const max = 0xFFFFFFFF
+    const threshold = max - (max % bound)
+
+    while (true) {
+      const r = Math.floor(this.prng() * 0x100000000)
+      if (r < threshold) {
+        return r % bound
+      }
+    }
+  }
+
+  public intInRange(min: number, max: number): number {
+    assert(max > min, 'Max must be greater than min.')
+    const range = max - min
+    return min + this.intExclusive(range)
+  }
+
+  public randomBytes(size: number): Buffer {
+    const buffer = Buffer.alloc(size)
+    for (let i = 0; i < size; i++) {
+      // Convert float [0,1) to int [0,255].
+      buffer[i] = Math.floor(this.prng() * 256)
+    }
+    return buffer
+  }
+
+  public uuidv4(): string {
+    const buf = this.randomBytes(16)
+    // Set version (4) bits: byte 6 (index 6), bits 4 high bits.
+    buf[6] = (buf[6] & 0x0f) | 0x40
+
+    // Set variant bits: byte 8 (index 8), high 2 bits to 10.
+    buf[8] = (buf[8] & 0x3f) | 0x80
+
+    // Convert to UUID string.
+    const hex = buf.toString("hex")
+
+    return [
+      hex.substring(0, 8),
+      hex.substring(8, 12),
+      hex.substring(12, 16),
+      hex.substring(16, 20),
+      hex.substring(20, 32),
+    ].join("-")
+  }
+}
+
+function splitmix32(a: number) {
+  return function () {
+    a = Math.trunc(a)
+    a = Math.trunc(a + 0x9e3779b9)
+    let t = a ^ a >>> 16
+    t = Math.imul(t, 0x21f0aaad)
+    t = t ^ t >>> 15
+    t = Math.imul(t, 0x735a2d97)
+    return ((t = t ^ t >>> 15) >>> 0) / 4294967296
+  }
+}

package/src/testing/rdkafka.test.ts

@@ -0,0 +1,263 @@
+import { describe, it } from "node:test";
+import Harness from "./harness"
+import { sleepSeconds } from "./util";
+import { Consumer, Producer } from "./kafka";
+const ProducerLegacy = require('@mojaloop/central-services-stream').Util.Producer
+const ConsumerLegacy = require('@mojaloop/central-services-stream').Kafka.Consumer
+
+const harness = Harness.getInstance()
+
+const NUM_MESSAGES = 1_000_000
+const NUM_BATCHES = 10
+const MESSAGES_PER_BATCH = Math.floor(NUM_MESSAGES / NUM_BATCHES)
+
+const formatMs = (ms: number): string => {
+  return ms.toLocaleString().split('.')[0]
+}
+
+const formatInt = (ms: number): string => {
+  return ms.toLocaleString()
+}
+
+/**
+ * This is a benchmark that compares the plain node-rdkafka performance to central-services-stream.
+ */
+describe('producer benchmark', () => {
+  it('[node-rdkafka] connects and produces poll interval', async () => {
+    await harness.up()
+
+    const broker = `localhost:${harness.redpandaConnectionOptions.port}`
+    const producer = new Producer(broker)
+
+    try {
+      await producer.connect(err => {
+        console.error('producer error', err.message)
+      })
+      const testStart = performance.now()
+
+      console.log(`[node-rdkafka] Producing: ${formatInt(NUM_MESSAGES)} messages (pollInterval = 50ms)`)
+
+      for (let batchNum = 0; batchNum < NUM_BATCHES; batchNum++) {
+        const batchStart = performance.now()
+        let batch: Array<LegacyMessage> = []
+        for (let messageNum = 0; messageNum < MESSAGES_PER_BATCH; messageNum++) {
+          batch.push({
+            from: "lewis",
+            to: "lewis",
+            id: 0,
+            content: { "hello": true },
+            type: "",
+            metadata: undefined,
+            pp: undefined
+          })
+        }
+
+        await producer.produce('thingo', batch)
+        const batchEnd = performance.now()
+        console.log(`  - Produced: ${formatInt(MESSAGES_PER_BATCH)} messages in ${formatMs(batchEnd - batchStart)} ms`)
+      }
+      const testEnd = performance.now()
+      console.log(`[node-rdkafka] produced ${formatInt(NUM_MESSAGES)} messages across ${formatInt(NUM_BATCHES)} batches took: ${formatMs(testEnd - testStart)} ms`)
+    } finally {
+      await producer.disconnect()
+      await harness.down()
+    }
+  })
+
+  it('[central-services-stream] connects and produces', async () => {
+    await harness.up()
+    const configs = [
+      {
+        topicConfig: {
+          topicName: 'thingo',
+        },
+        kafkaConfig: {
+          options: {
+            messageCharset: "utf8"
+          },
+          topicConfig: {
+            topicName: 'thingo',
+          },
+          rdkafkaConf: {
+            "metadata.broker.list": `localhost:${harness.redpandaConnectionOptions.port}`,
+            "client.id": "test-client",
+            "event_cb": true,
+            "dr_cb": true,
+            "socket.keepalive.enable": true,
+            "queue.buffering.max.messages": 10000000
+          }
+        }
+      }
+    ]
+
+    try {
+      await ProducerLegacy.connectAll(configs)
+      const testStart = performance.now()
+
+      console.log(`[central-services-stream] Producing: ${formatInt(NUM_MESSAGES)} messages`)
+
+      for (let batchNum = 0; batchNum < NUM_BATCHES; batchNum++) {
+        const batchStart = performance.now()
+        let batch: Array<LegacyMessage> = []
+        for (let messageNum = 0; messageNum < MESSAGES_PER_BATCH; messageNum++) {
+          batch.push({
+            from: "lewis",
+            to: "lewis",
+            id: 0,
+            content: { "hello": true },
+            type: "",
+            metadata: undefined,
+            pp: undefined
+          })
+        }
+
+        await produceLegacyMessage('thingo', batch)
+
+        const batchEnd = performance.now()
+        console.log(`  - Produced: ${formatInt(MESSAGES_PER_BATCH)} messages in ${formatMs(batchEnd - batchStart)} ms`)
+      }
+      const testEnd = performance.now()
+      console.log(`[central-services-stream] produced ${formatInt(NUM_MESSAGES)} messages across ${formatInt(NUM_BATCHES)} batches took: ${formatMs(testEnd - testStart)} ms`)
+    } finally {
+      await ProducerLegacy.disconnect()
+      await harness.down()
+    }
+  })
+})
+
+describe('consumer benchmark', () => {
+  const produceMessages = async () => {
+    const broker = `localhost:${harness.redpandaConnectionOptions.port}`
+    const producer = new Producer(broker)
+
+    try {
+      await producer.connect(err => {
+        console.error('producer error', err.message)
+      });
+
+      console.log(`[setup] Producing: ${formatInt(NUM_MESSAGES)} messages.`)
+
+      for (let batchNum = 0; batchNum < NUM_BATCHES; batchNum++) {
+        let batch: Array<LegacyMessage> = []
+        for (let messageNum = 0; messageNum < MESSAGES_PER_BATCH; messageNum++) {
+          batch.push({
+            from: "lewis",
+            to: "lewis",
+            id: 0,
+            content: { "hello": true },
+            type: "",
+            metadata: undefined,
+            pp: undefined
+          })
+        }
+
+        await producer.produce('thingo', batch)
+      }
+    } finally {
+      await producer.disconnect()
+    }
+  }
+
+  it('[node-rdkafka] consumes all of the messages', async () => {
+    await harness.up()
+    await produceMessages()
+    const broker = `localhost:${harness.redpandaConnectionOptions.port}`
+
+    let countConsumed = 0
+    let remainingRetries = 10
+    const consumer = new Consumer('test-group', broker)
+    try {
+      console.log(`[node-rdkafka] consuming: ${formatInt(NUM_MESSAGES)} messages.`)
+      const consumeStart = performance.now()
+
+      await consumer.subscribe(['thingo'], (message) => {
+        countConsumed += 1
+      })
+
+      while (countConsumed < NUM_MESSAGES) {
+        if (remainingRetries === 0) {
+          throw new Error(`Timed out waiting to consume ${formatInt(NUM_MESSAGES)}.`)
+        }
+
+        await sleepSeconds(1)
+        console.log(`  -      consumed ${formatInt(countConsumed)} took: ${formatMs(performance.now() - consumeStart)} ms`)
+        remainingRetries -= 1;
+      }
+
+      const consumeEnd = performance.now()
+      console.log(`[node-rdkafka] consumed ${formatInt(NUM_MESSAGES)} took: ${formatMs(consumeEnd - consumeStart)} ms`)
+    } finally {
+      consumer.disconnect()
+      harness.down()
+    }
+  })
+
+  it('[central-services-stream] consumes all of the messages', async () => {
+    await harness.up()
+    await produceMessages()
+
+    const consumerConfig = {
+      topicConf: {
+        "auto.offset.reset": 'earliest',
+      },
+      rdkafkaConf: {
+        "group.id": 'test-consumer-group',
+        "metadata.broker.list": `localhost:${harness.redpandaConnectionOptions.port}`,
+        "client.id": "test-client",
+        "socket.keepalive.enable": true,
+        "queue.buffering.max.messages": 10000000
+      },
+    }
+    const consumer = new ConsumerLegacy(['thingo'], consumerConfig)
+    let countConsumed = 0
+    let remainingRetries = 20
+    try {
+      console.log(`[central-services-stream] consuming: ${formatInt(NUM_MESSAGES)} messages.`)
+      const consumeStart = performance.now()
+
+      await consumer.connect()
+      await consumer.consume((err: any, messages: Array<any>) => {
+        countConsumed += 1
+      })
+
+      while (countConsumed < NUM_MESSAGES) {
+        if (remainingRetries === 0) {
+          throw new Error(`Timed out waiting to consume ${formatInt(NUM_MESSAGES)}.`)
+        }
+
+        await sleepSeconds(1)
+        console.log(`  -      consumed ${formatInt(countConsumed)} took: ${formatMs(performance.now() - consumeStart)} ms`)
+        remainingRetries -= 1;
+      }
+
+      const consumeEnd = performance.now()
+      console.log(`[central-services-stream] consumed ${formatInt(NUM_MESSAGES)} took: ${formatMs(consumeEnd - consumeStart)} ms`)
+    } finally {
+      await new Promise((resolve, reject) => consumer.disconnect(resolve))
+      harness.down()
+    }
+  })
+})
+
+type LegacyMessage = {
+  from: string,
+  to: string,
+  id: number,
+  content: any,
+  type: string,
+  metadata: any,
+  pp: any
+}
+
+const produceLegacyMessage = async (topic: string, messages: Array<LegacyMessage>): Promise<void> => {
+  const topicConf = {
+    topicName: topic,
+  }
+  const config = {
+
+  }
+  const promises = Promise.all(messages.map(message => {
+    return ProducerLegacy.produceMessage(message, topicConf, config)
+  }))
+  await promises
+}