@nxtedition/lib 28.0.5 → 28.0.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nxtedition/lib",
-  "version": "28.0.5",
+  "version": "28.0.7",
   "license": "UNLICENSED",
   "author": "Robert Nagy <robert.nagy@boffins.se>",
   "type": "module",
@@ -92,5 +92,5 @@
     "pino": ">=7.0.0",
     "rxjs": "^7.0.0"
   },
-  "gitHead": "f778b4c525a240411f681137d8dd0fe19031c7b0"
+  "gitHead": "545d4052432b61ff13ba81fa5a69644cf4901103"
 }

package/sequence.js

@@ -26,14 +26,14 @@ export class Sequence {
   #identity
   #count = 0
 
-  // TODO (perf): Optimize
   /**
    *
    * @param {string|Sequence|null|undefined} a
    * @param {string|Sequence|null|undefined} b
    * @param {boolean} [strict=true]
-   * @returns
+   * @returns {-1|0|1}
    */
+  /** @deprecated */
   static compare(a, b, strict) {
     if (!a && !b) {
       return 0
@@ -58,6 +58,17 @@ export class Sequence {
     return a.compare(b, strict)
   }
 
+  /**
+   *
+   * @param {string|Sequence|null|undefined} a
+   * @param {string|Sequence|null|undefined} b
+   * @param {boolean} [strict=true]
+   * @returns {boolean}
+   */
+  static has(a, b, strict) {
+    return Sequence.compare(a, b, strict) >= 0
+  }
+
   /**
    *
    * @param {string} seq
@@ -75,13 +86,16 @@ export class Sequence {
     }
   }
 
+  /**
+   * @param {null|undefined|string|Sequence|Array<string|number|{id:string,sequence:number}>>} value
+   * @param {null|undefined|number|Array<string|{id:string}>>} [identity]
+   */
   constructor(value, identity) {
     try {
       if (!value) {
         this.#value = '0'
       } else if (Array.isArray(value)) {
         this.#identity = identity
-        this.#count = 0
         for (const part of value) {
           if (typeof part === 'string') {
             const [sequenceStr, id] = part.split(ID_SEP)
@@ -103,7 +117,6 @@ export class Sequence {
         assert(this.#identity === 0 || this.#parts.length > 0)
       } else if (typeof value === 'string') {
         const [countStr, token] = value.split('-')
-        const count = parseInt(countStr)
         if (token) {
           for (const str of token.split('_')) {
             const [sequenceStr, id] = str.split(ID_SEP)
@@ -111,7 +124,7 @@ export class Sequence {
             this.#parts.push(id, sequence)
           }
         }
-        this.#count = count
+        this.#count = parseInt(countStr)
         this.#value = value
       } else if (value instanceof Sequence) {
         this.#count = value.#count
@@ -174,6 +187,9 @@ export class Sequence {
     }
   }
 
+  /**
+   * @returns {number}
+   */
   get identity() {
     if (this.#identity == null) {
       if (this.#parts.length === 0) {
@@ -190,6 +206,9 @@ export class Sequence {
     return this.#identity
   }
 
+  /**
+   * @returns {number}
+   */
   get count() {
     if (this.#count == null) {
       let count = 0
@@ -201,10 +220,17 @@ export class Sequence {
     return this.#count
   }
 
+  /**
+   * @returns {number}
+   */
   get length() {
     return this.#parts.length / 2
   }
 
+  /**
+   * @param {number} index
+   * @returns {number}
+   */
   at(index) {
     if (!Number.isInteger(index)) {
       throw new TypeError('index must be an integer')
@@ -216,6 +242,10 @@ export class Sequence {
     return this.#parts[index * 2 + 1]
   }
 
+  /**
+   * @param {number} index
+   * @param {number} sequence
+   */
   set(index, sequence) {
     if (!Number.isInteger(index)) {
       throw new TypeError('index must be an integer')
@@ -236,6 +266,7 @@ export class Sequence {
     }
   }
 
+  /** @deprecated */
   compare(other, strict) {
     if (strict === undefined) {
       strict = true
@@ -256,7 +287,13 @@ export class Sequence {
       throw new TypeError('strict must be a boolean')
     }
 
-    if (strict && other.identity && this.identity && other.identity !== this.identity) {
+    if (
+      strict &&
+      (other.identity || this.identity) &&
+      other.identity !== this.identity &&
+      this.#value !== '0' &&
+      other.#value !== '0'
+    ) {
       throw new Error('Cannot compare sequences with different identities')
     }
 
@@ -273,6 +310,19 @@ export class Sequence {
     return 0
   }
 
+  /**
+   *
+   * @param {string|Sequence|null|undefined} other
+   * @param {boolean} [strict=true]
+   * @returns {boolean}
+   */
+  has(other, strict = true) {
+    return this.compare(other, strict) >= 0
+  }
+
+  /**
+   * @returns {string}
+   */
   toString() {
     if (!this.#value) {
       let count = 0
@@ -286,10 +336,16 @@ export class Sequence {
     return this.#value
   }
 
+  /**
+   * @returns {string}
+   */
   [Symbol.toStringTag]() {
     return this.toString()
   }
 
+  /**
+   * @returns {string}
+   */
   [util.inspect.custom](depth, options, inspect) {
     return `Sequence: "${this.toString()}"`
   }

package/shared.js

@@ -1,6 +1,3 @@
-import stream from 'node:stream'
-import assert from 'node:assert'
-
 // By placing the read and write indices far apart (multiples of a common
 // cache line size, 64 bytes), we prevent "false sharing". This is a
 // low-level CPU optimization where two cores writing to different variables
@@ -35,6 +32,16 @@ export function alloc(size) {
  * @returns {{ readSome: function }}
  */
 export function reader({ sharedState, sharedBuffer }) {
+  if (!(sharedState instanceof SharedArrayBuffer)) {
+    throw new TypeError('sharedState must be a SharedArrayBuffer')
+  }
+  if (!(sharedBuffer instanceof SharedArrayBuffer)) {
+    throw new TypeError('sharedBuffer must be a SharedArrayBuffer')
+  }
+  if (sharedBuffer.byteLength >= 2 ** 31) {
+    throw new RangeError('Shared buffer size exceeds maximum of 2GB')
+  }
+
   const state = new Int32Array(sharedState)
   const size = sharedBuffer.byteLength
   const buffer = Buffer.from(sharedBuffer)
@@ -52,7 +59,7 @@ export function reader({ sharedState, sharedBuffer }) {
 
   /**
    * Reads a batch of messages from the buffer.
-   * @param {(data: {buffer: Buffer, view: DataView, offset: number, length: number}) => void} next Callback to process a message.
+   * @param {(data: {buffer: Buffer, view: DataView, offset: number, length: number}) => void|boolean} next Callback to process a message.
    * @returns {number} The number of messages read.
    */
   function readSome(next) {
@@ -70,10 +77,11 @@ export function reader({ sharedState, sharedBuffer }) {
       const dataPos = readPos + 4
       const dataLen = view.getInt32(dataPos - 4, true) | 0
 
+      bytes += 4
+
       // A length of -1 is a special marker indicating the writer has
       // wrapped around to the beginning of the buffer.
       if (dataLen === -1) {
-        bytes += 4
         readPos = 0
         // After wrapping, we must re-check against the writer's position.
         // It's possible the writer is now at a position > 0.
@@ -86,22 +94,17 @@ export function reader({ sharedState, sharedBuffer }) {
           throw new Error('Data exceeds buffer size')
         }
 
-        bytes += dataLen
         readPos += 4 + dataLen
+
+        bytes += dataLen
         count += 1
 
         // This is a "zero-copy" operation. We don't copy the data out.
         // Instead, we pass a "view" into the shared buffer.
         data.offset = dataPos
         data.length = dataLen
-        const cont = next(data)
-
-        if (readPos === writePos) {
-          // If we reach the end of the buffer, we must re-check the writer's position.
-          writePos = Atomics.load(state, WRITE_INDEX) | 0
-        }
 
-        if (cont === false) {
+        if (next(data) === false) {
           break
         }
       }
@@ -125,6 +128,16 @@ export function reader({ sharedState, sharedBuffer }) {
  * @returns {{ write: function, cork: function(function) }}
  */
 export function writer({ sharedState, sharedBuffer }, { yield: onYield, logger } = {}) {
+  if (!(sharedState instanceof SharedArrayBuffer)) {
+    throw new TypeError('sharedState must be a SharedArrayBuffer')
+  }
+  if (!(sharedBuffer instanceof SharedArrayBuffer)) {
+    throw new TypeError('sharedBuffer must be a SharedArrayBuffer')
+  }
+  if (sharedBuffer.byteLength >= 2 ** 31) {
+    throw new RangeError('Shared buffer size exceeds maximum of 2GB')
+  }
+
   const state = new Int32Array(sharedState)
   const size = sharedBuffer.byteLength
   const buffer = Buffer.from(sharedBuffer)
@@ -140,7 +153,7 @@ export function writer({ sharedState, sharedBuffer }, { yield: onYield, logger }
   let readPos = Atomics.load(state, READ_INDEX) | 0
   let writePos = Atomics.load(state, WRITE_INDEX) | 0
 
-  let yielding = false
+  let yielding = 0
   let corked = 0
   let pending = 0
 
@@ -153,22 +166,22 @@ export function writer({ sharedState, sharedBuffer }, { yield: onYield, logger }
    * @param {number} delay The timeout for Atomics.wait.
    */
   function _yield(delay) {
-    if (yielding) {
-      throw new Error('Cannot yield while yielding')
+    if (yielding > 128) {
+      throw new Error('Detected possible deadlock: writer yielding too many times')
     }
 
     // First, ensure the very latest write position is visible to the reader.
     _flush()
 
     if (onYield) {
-      yielding = true
+      yielding += 1
       try {
         // Call the user-provided yield function, if any. This can be important
         // if the writer is waiting for the reader to process data which would
         // otherwise deadlock.
         onYield()
       } finally {
-        yielding = false
+        yielding -= 1
       }
     }
 
@@ -176,6 +189,8 @@ export function writer({ sharedState, sharedBuffer }, { yield: onYield, logger }
     // to sleep, consuming no CPU, until the reader changes the READ_INDEX.
     if (delay > 0) {
       Atomics.wait(state, READ_INDEX, readPos, delay)
+    } else {
+      Atomics.pause()
     }
 
     // After waking up, refresh the local view of the reader's position.
@@ -192,24 +207,22 @@ export function writer({ sharedState, sharedBuffer }, { yield: onYield, logger }
     // 4-byte header for the *next* message (for wrap-around check).
     const required = len + 4 + 4
 
-    if (required < 0) {
-      throw new Error(`Required length ${required} is negative, expected at least 0`)
-    }
-    if (required > size) {
-      throw new Error(`Required length ${required} exceeds buffer size ${size}`)
-    }
-
     if (writePos >= readPos) {
-      // Case 1: The writer is ahead of the reader. [ 0 ---- R ... W ---- S ]
-      // There is free space from W to the end (S) and from 0 to R.
+      // Case 1: The writer is ahead of the reader. [ 0 - R ... W - size ]
+      // There is free space from W to the end (s) and from 0 to R.
+
       if (size - writePos >= required) {
         // Enough space at the end of the buffer.
         return true
       }
 
+      if (readPos === 0) {
+        readPos = Atomics.load(state, READ_INDEX) | 0
+      }
+
       // Not enough space at the end. Check if there's space at the beginning.
       if (readPos === 0) {
-        // Reader is at the very beginning, so no space to wrap around into.
+        // Reader is at the beginning, so no space to wrap around into.
         return false
       }
 
@@ -220,17 +233,24 @@ export function writer({ sharedState, sharedBuffer }, { yield: onYield, logger }
       writePos = 0
 
       if (writePos + 4 > size) {
+        // assertion
         throw new Error(`Write position ${writePos} with next header exceeds buffer size ${size}`)
       }
       if (writePos === readPos) {
+        // assertion
         throw new Error(`Write position ${writePos} cannot equal read position ${readPos}`)
       }
 
       Atomics.store(state, WRITE_INDEX, writePos)
     }
 
-    // Case 2: The writer has wrapped around. [ 0 ... W ---- R ... S ]
+    // Case 2: The writer has wrapped around. [ 0 ... W - R ... s ]
     // The only free space is between W and R.
+
+    if (readPos - writePos < required) {
+      readPos = Atomics.load(state, READ_INDEX) | 0
+    }
+
     return readPos - writePos >= required
   }
 
@@ -245,84 +265,114 @@ export function writer({ sharedState, sharedBuffer }, { yield: onYield, logger }
     }
   }
 
+  function _flush() {
+    if (pending > 0) {
+      Atomics.store(state, WRITE_INDEX, writePos)
+      pending = 0
+    }
+  }
+
   /**
    * Performs the actual write into the buffer after space has been acquired.
-   * @param {number} len The exact length of the payload.
+   * @param {number} dataCap Max length of the payload.
    * @param {({ buffer, view, offset, length }) => number} fn The callback that writes the data.
    * @returns {void}
    */
-  function _write(len, fn) {
+  function _write(dataCap, fn) {
     const dataPos = writePos + 4
 
     data.offset = dataPos
-    data.length = len
+    data.length = dataCap
 
     // The user-provided function writes the data and returns the final position.
     // We calculate the actual bytes written from that.
+    // NOTE: This is unsafe as the user function can write beyond the reserved length.
     const dataLen = fn(data) - dataPos
 
+    if (typeof dataLen !== 'number') {
+      throw new TypeError('"fn" must return the number of bytes written')
+    }
     if (dataLen < 0) {
-      throw new Error(`Data length ${dataLen} is negative`)
+      throw new RangeError(`"fn" returned a negative number ${dataLen}`)
     }
-    if (dataLen > len) {
-      throw new Error(`Data length ${dataLen} exceeds expected length ${len}`)
+    if (dataLen > dataCap) {
+      throw new RangeError(`"fn" returned a number ${dataLen} that exceeds capacity ${dataCap}`)
     }
+
     if (dataPos + dataLen > size) {
+      // assertion
       throw new Error(`Data position ${dataPos} with length ${dataLen} exceeds buffer size ${size}`)
     }
 
-    // Write the actual length of the data into the 4-byte header.
-    view.setInt32(writePos, dataLen, true)
-    writePos += 4 + dataLen
+    const nextPos = writePos + 4 + dataLen
 
-    if (writePos + 4 > size) {
-      throw new Error(`Write position ${writePos} with next header exceeds buffer size ${size}`)
+    if (nextPos + 4 > size) {
+      // assertion
+      throw new Error(`Write position ${nextPos} with next header exceeds buffer size ${size}`)
     }
-    if (writePos === readPos) {
-      throw new Error(`Write position ${writePos} cannot equal read position ${readPos}`)
+    if (nextPos === readPos) {
+      // assertion
+      throw new Error(`Write position ${nextPos} cannot equal read position ${readPos}`)
     }
 
+    // Write the actual length of the data into the 4-byte header.
+    view.setInt32(writePos, dataLen, true)
+    writePos += 4 + dataLen
+    pending += 4 + dataLen
+
     // This is the "corking" optimization. Instead of calling Atomics.store
     // on every write, we batch them. We either write when a certain
     // amount of data is pending (HWM_BYTES) or at the end of the current
-    // JS microtask. This drastically reduces atomic operation overhead.
+    // event loop tick. This drastically reduces atomic operation overhead.
     if (pending >= HWM_BYTES) {
       Atomics.store(state, WRITE_INDEX, writePos)
       pending = 0
-    } else {
-      pending += 4 + dataLen
-      if (corked === 0) {
-        corked += 1
-        setImmediate(_uncork)
-      }
+    } else if (corked === 0) {
+      corked += 1
+      setImmediate(_uncork)
     }
   }
 
   /**
-   * Public write method. Acquires space and writes data.
+   * Public write method. Acquires space and synchronously writes data with a timeout. Will
+   * wait until space is available.
+   * Writing more than "len" bytes in the callback will cause undefined behavior.
    * @param {number} len The maximum expected length of the payload.
    * @param {({ buffer, view, offset, length }) => number} fn The callback that writes the data.
+   * @param {number} [timeout=60000] The maximum time to wait for space in milliseconds.
    */
-  function writeSync(len, fn) {
+  function writeSync(len, fn, timeout = 60e3) {
+    if (typeof len !== 'number') {
+      throw new TypeError('"len" must be a non-negative number')
+    }
     if (len < 0) {
-      throw new Error(`Length ${len} is negative`)
+      throw new RangeError(`"len" ${len} is negative`)
     }
     if (len >= 2 ** 31 || len > size - 8) {
-      throw new Error(`Length ${len} exceeds maximum allowed size`)
+      throw new Error(`"len" ${len} exceeds maximum allowed size ${size - 8}`)
+    }
+    if (typeof fn !== 'function') {
+      throw new TypeError('"fn" must be a function')
+    }
+    if (typeof timeout !== 'number') {
+      throw new TypeError('"timeout" must be a non-negative number')
+    }
+    if (timeout < 0) {
+      throw new RangeError('"timeout" must be a non-negative number')
     }
 
     if (!_acquire(len)) {
-      readPos = Atomics.load(state, READ_INDEX) | 0
-      if (!_acquire(len)) {
-        const startTime = performance.now()
-        logger?.warn({ readPos, writePos }, 'yield started')
-        _yield(0)
-        while (!_acquire(len)) {
-          _yield(3)
+      const startTime = performance.now()
+      logger?.warn({ readPos, writePos }, 'yield started')
+      _yield(0)
+      for (let n = 0; !_acquire(len); n++) {
+        if (performance.now() - startTime > timeout) {
+          throw new Error('Timeout while waiting for space in the buffer')
         }
-        const elapsedTime = performance.now() - startTime
-        logger?.warn({ readPos, writePos, elapsedTime }, 'yield completed')
+        _yield(3)
       }
+      const elapsedTime = performance.now() - startTime
+      logger?.warn({ readPos, writePos, elapsedTime }, 'yield completed')
     }
 
     _write(len, fn)
@@ -332,19 +382,28 @@ export function writer({ sharedState, sharedBuffer }, { yield: onYield, logger }
     }
   }
 
+  /**
+   * Public write method. Acquires space and tries to writes data.
+   * Writing more than "len" bytes in the callback will cause undefined behavior.
+   * @param {number} len The maximum expected length of the payload.
+   * @param {({ buffer, view, offset, length }) => number} fn The callback that writes the data.
+   */
   function tryWrite(len, fn) {
+    if (typeof len !== 'number') {
+      throw new TypeError('"len" must be a non-negative number')
+    }
     if (len < 0) {
-      throw new Error(`Length ${len} is negative`)
+      throw new RangeError(`"len" ${len} is negative`)
     }
     if (len >= 2 ** 31 || len > size - 8) {
-      throw new Error(`Length ${len} exceeds maximum allowed size`)
+      throw new Error(`"len" ${len} exceeds maximum allowed size ${size - 8}`)
+    }
+    if (typeof fn !== 'function') {
+      throw new TypeError('"fn" must be a function')
     }
 
     if (!_acquire(len)) {
-      readPos = Atomics.load(state, READ_INDEX) | 0
-      if (!_acquire(len)) {
-        return false
-      }
+      return false
     }
 
     _write(len, fn)
@@ -356,13 +415,6 @@ export function writer({ sharedState, sharedBuffer }, { yield: onYield, logger }
     return true
   }
 
-  function _flush() {
-    if (pending > 0) {
-      Atomics.store(state, WRITE_INDEX, writePos)
-      pending = 0
-    }
-  }
-
   function cork(callback) {
     corked += 1
     try {
@@ -374,118 +426,3 @@ export function writer({ sharedState, sharedBuffer }, { yield: onYield, logger }
 
   return { tryWrite, writeSync, cork }
 }
-
-export class Writable extends stream.Writable {
-  #writer
-  #retries = 0
-  #timeout = null
-
-  #chunk
-  #encoding
-  #callback
-
-  constructor({ state, ...options }) {
-    super({ ...options })
-    this.#writer = writer(state)
-  }
-
-  _write(chunk, encoding, callback) {
-    if (chunk.byteLength === 0) {
-      callback(null)
-      return
-    }
-
-    assert(!this.#timeout)
-
-    this.#chunk = chunk
-    this.#encoding = encoding
-    this.#callback = callback
-    this._writeSome()
-  }
-
-  _final(callback) {
-    this.#chunk = Buffer.allocUnsafe(0)
-    this.#encoding = null
-    this.#callback = callback
-    this._writeSome()
-  }
-
-  _writeSome = () => {
-    this.#timeout = null
-
-    if (this.#writer.tryWrite(this.#chunk.byteLength, this._doWrite)) {
-      const callback = this.#callback
-      this.#retries = 0
-      this.#chunk = null
-      this.#encoding = null
-      this.#callback = null
-      callback(null)
-    } else {
-      this.#retries += 1
-      this.#timeout = setTimeout(this._writeSome, Math.min(10, this.#retries))
-    }
-  }
-
-  _doWrite = (data) => {
-    const written =
-      typeof this.#chunk === 'string'
-        ? data.buffer.write(this.#chunk, data.offset, data.length, this.#encoding)
-        : this.#chunk.copy(data.buffer, data.offset, 0, this.#chunk.length)
-    return data.offset + written
-  }
-
-  _destroy(err, callback) {
-    if (this.#timeout != null) {
-      clearTimeout(this.#timeout)
-      this.#timeout = null
-    }
-    callback(err)
-  }
-}
-
-export class Readable extends stream.Readable {
-  #reader
-  #retries = 0
-  #timeout
-
-  constructor({ state, ...options }) {
-    super(options)
-    this.#reader = reader(state)
-  }
-
-  _read() {
-    assert(!this.#timeout)
-
-    this._readSome()
-  }
-
-  _readSome = () => {
-    this.#timeout = null
-
-    const count = this.#reader.readSome((data) => {
-      if (data.length === 0) {
-        this.push(null)
-        return false
-      }
-
-      const chunk = Buffer.allocUnsafe(data.length)
-      data.buffer.copy(chunk, 0, data.offset, data.offset + data.length)
-      return this.push(chunk)
-    })
-
-    if (count > 0 || this.readableEnded) {
-      this.#retries = 0
-    } else {
-      this.#retries += 1
-      this.#timeout = setTimeout(this._readSome, Math.min(10, this.#retries))
-    }
-  }
-
-  _destroy(err, callback) {
-    if (this.#timeout != null) {
-      clearTimeout(this.#timeout)
-      this.#timeout = null
-    }
-    callback(err)
-  }
-}