cbor-x 1.6.8 → 1.6.9

package/encode.js

@@ -572,6 +572,11 @@ export class Encoder extends Decoder {
 								return encode(json)
 						}
 
+						// not in extensions, so class instances that reach writeObject do not pay another instanceof
+						if (constructor === DataView) {
+							writeBuffer(new Uint8Array(value.buffer, value.byteOffset, value.byteLength), makeRoom)
+							return
+						}
 						// no extension found, write as a plain object
 						writeObject(value)
 					}
@@ -666,8 +671,22 @@ export class Encoder extends Decoder {
 					size++
 				}
 			}
-			target[objectOffset++ + start] = size >> 8
-			target[objectOffset + start] = size & 0xff
+			if (size < 0x10000) {
+				// fits in a map-16 (uint16) count: back-patch the reserved 2 bytes
+				target[objectOffset++ + start] = size >> 8
+				target[objectOffset + start] = size & 0xff
+			} else {
+				// too many entries for a map-16 count: widen the already-written header
+				// to map-32, mirroring writeArrayHeader's 0x9a/setUint32 path. Shift the
+				// encoded entries 2 bytes forward to make room for the wider (uint32) count.
+				if (position + 2 > safeEnd)
+					makeRoom(position + 2)
+				let headerPosition = objectOffset + start
+				target.copyWithin(headerPosition + 4, headerPosition + 2, position)
+				position += 2
+				target[headerPosition - 1] = 0xba // map 32
+				targetView.setUint32(headerPosition, size)
+			}
 		} :
 		(object, skipValues) => {
 			let nextTransition, transition = structures.transitions || (structures.transitions = Object.create(null))

package/index.d.ts

@@ -66,7 +66,7 @@ import { Transform, Readable } from 'stream'
 
 export as namespace CBOR;
 export class DecoderStream extends Transform {
-	constructor(options?: Options | { highWaterMark: number, emitClose: boolean, allowHalfOpen: boolean })
+	constructor(options?: Options & { maxIncompleteBufferSize?: number } | { highWaterMark: number, emitClose: boolean, allowHalfOpen: boolean, maxIncompleteBufferSize?: number })
 }
 export class EncoderStream extends Transform {
 	constructor(options?: Options | { highWaterMark: number, emitClose: boolean, allowHalfOpen: boolean })

package/iterators.js

@@ -1,5 +1,6 @@
 import { Encoder } from './encode.js'
-import { Decoder } from './decode.js'
+import { Decoder, _stream } from './decode.js'
+import { StructureScanner } from './scan.js'
 
 /**
  * Given an Iterable first argument, returns an Iterable where each value is encoded as a Buffer
@@ -38,7 +39,7 @@ async function * encodeIterAsync (objectIterator, options) {
  * Given an Iterable/Iterator input which yields buffers, returns an IterableIterator which yields sync decoded objects
  * Or, given an Async Iterable/Iterator which yields promises resolving in buffers, returns an AsyncIterableIterator.
  * @param {Iterable|Iterator|AsyncIterable|AsyncIterableIterator} bufferIterator
- * @param {object} [options] - Decoder options
+ * @param {object} [options] - Decoder options, and maxIncompleteBufferSize
  * @returns {IterableIterator|Promise.<AsyncIterableIterator}
  */
 export function decodeIter (bufferIterator, options = {}) {
@@ -47,26 +48,15 @@ export function decodeIter (bufferIterator, options = {}) {
   }
 
   const decoder = new Decoder(options)
-  let incomplete, fatalError
+  const chunkDecoder = new ChunkDecoder(decoder, options.maxIncompleteBufferSize)
+  let fatalError
   const parser = (chunk) => {
-    let yields
-    // if there's incomplete data from previous chunk, concatinate and try again
-    if (incomplete) {
-      chunk = Buffer.concat([incomplete, chunk])
-      incomplete = undefined
-    }
-
     try {
-      yields = decoder.decodeMultiple(chunk)
-    } catch (err) {
-      yields = err.values
-      if (err.incomplete) {
-        incomplete = chunk.slice(err.lastPosition)
-      } else {
-        fatalError = err
-      }
+      return chunkDecoder.buffer ? chunkDecoder.decodeAfterIncomplete(chunk) : decoder.decodeMultiple(chunk)
+    } catch (error) {
+      fatalError = chunkDecoder.keepIncomplete(error, chunk)
+      return (error && error.values) || []
     }
-    return yields || []
   }
 
   if (typeof bufferIterator[Symbol.iterator] === 'function') {
@@ -75,6 +65,8 @@ export function decodeIter (bufferIterator, options = {}) {
         yield * parser(value)
         if (fatalError) throw fatalError
       }
+      const endError = chunkDecoder.end()
+      if (endError) throw endError
     })()
   } else if (typeof bufferIterator[Symbol.asyncIterator] === 'function') {
     return (async function * iter () {
@@ -82,6 +74,118 @@ export function decodeIter (bufferIterator, options = {}) {
         yield * parser(value)
         if (fatalError) throw fatalError
       }
+      const endError = chunkDecoder.end()
+      if (endError) throw endError
     })()
   }
 }
+
+/**
+ * Keeps the bytes of a value that a chunk of a stream ends inside of, for DecoderStream and decodeIter, and decodes
+ * them again once enough have arrived to possibly complete it. Callers decode a chunk with decodeAfterIncomplete if
+ * buffer is set, or else directly with decodeMultiple, and pass any error to keepIncomplete, which returns the error
+ * to report, if any. end returns an error if the input ended inside a value.
+ */
+export class ChunkDecoder {
+  constructor (decoder, maxIncompleteBufferSize) {
+    this.decoder = decoder
+    if (maxIncompleteBufferSize === undefined) maxIncompleteBufferSize = 0x4000000
+    if (typeof maxIncompleteBufferSize !== 'number' || !(maxIncompleteBufferSize >= 0)) {
+      throw new TypeError('maxIncompleteBufferSize must be a non-negative number')
+    }
+    this.maxIncompleteBufferSize = Math.floor(maxIncompleteBufferSize)
+    // buffer[0, bufferEnd) is the incomplete value; no decoded value is a view of buffer while it is kept here
+    this.buffer = null
+    this.bufferEnd = 0
+    this.appends = 0
+    this.scanner = null
+    this.scanned = false
+    this.resetScanner = false
+    this.neededLength = 0 // a lower bound on the incomplete value's length
+  }
+
+  decodeAfterIncomplete (chunk) {
+    if (!(ArrayBuffer.isView(chunk) && chunk.BYTES_PER_ELEMENT === 1)) {
+      throw new TypeError('Source must be a Uint8Array or Buffer')
+    }
+    const length = this.bufferEnd + chunk.length
+    if (length < this.neededLength && length > this.maxIncompleteBufferSize) throw maxIncompleteBufferSizeError()
+    if (length > this.buffer.length) {
+      // Values usually complete within two appends, which are sized exactly. After that the buffer grows by at least
+      // the bytes already kept, so that they are copied a bounded number of times however small the chunks are.
+      const size = this.appends < 2 ? length
+        : Math.max(length, Math.min(length + this.bufferEnd, this.maxIncompleteBufferSize))
+      const buffer = Buffer.allocUnsafe(size)
+      if (size > length) buffer.fill(0, length) // decoded values are views of this buffer, so no uninitialized memory
+      buffer.set(this.buffer.subarray(0, this.bufferEnd))
+      this.buffer = buffer
+    }
+    this.buffer.set(chunk, this.bufferEnd)
+    this.bufferEnd = length
+    this.appends++
+    if (length < this.neededLength) return []
+    if (!this.scanner) this.scanner = new StructureScanner(this.decoder)
+    if (this.resetScanner) {
+      this.scanner.reset()
+      this.resetScanner = false
+    }
+    if (!this.scanner.scan(this.buffer, length)) {
+      this.neededLength = this.scanner.neededLength
+      if (length > this.maxIncompleteBufferSize) throw maxIncompleteBufferSizeError()
+      return []
+    }
+    this.scanned = true
+    const values = _stream.decode(this.decoder, this.buffer.subarray(0, length), this.scanner.position)
+    this.scanned = false
+    this.buffer = null
+    return values
+  }
+
+  keepIncomplete (error, chunk) {
+    // an extension may throw any value, and a falsy one would read as success
+    if (!error) return this.fail(new Error('Decoding failed: ' + error))
+    if (!error.incomplete) return this.fail(error)
+    if (this.scanned && error.lastPosition === 0) {
+      error.incomplete = false
+      return this.fail(error)
+    }
+    const source = this.buffer ? this.buffer.subarray(0, this.bufferEnd) : chunk
+    const length = source.length - error.lastPosition
+    if (length > this.maxIncompleteBufferSize) return this.fail(maxIncompleteBufferSizeError())
+    if (!this.buffer || error.lastPosition > 0) {
+      if (length === 0) return
+      try {
+        // copied, so that the remainder does not keep in memory the chunk, or the buffer that the values before it
+        // are views of
+        this.buffer = Buffer.from(source.subarray(error.lastPosition))
+      } catch (copyError) {
+        return this.fail(copyError)
+      }
+      this.bufferEnd = length
+      this.appends = 0
+    }
+    this.scanned = false
+    this.resetScanner = true
+    this.neededLength = Math.max(length + 1, error.neededEnd ? error.neededEnd - error.lastPosition : 0)
+  }
+
+  end () {
+    if (this.buffer) {
+      const error = this.fail(new Error('Unexpected end of CBOR data'))
+      error.incomplete = true
+      return error
+    }
+  }
+
+  fail (error) {
+    this.buffer = null
+    this.scanner = null
+    this.scanned = false
+    this.resetScanner = false
+    return error
+  }
+}
+
+function maxIncompleteBufferSizeError () {
+  return new Error('Maximum incomplete buffer size exceeded')
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "cbor-x",
   "author": "Kris Zyp",
-  "version": "1.6.8",
+  "version": "1.6.9",
   "description": "Ultra-fast and conformant CBOR (RFC 8949) implementation with support for numerous tag extensions including records and structured cloning",
   "license": "MIT",
   "types": "./index.d.ts",
@@ -75,7 +75,7 @@
     "node:buffer": false
   },
   "optionalDependencies": {
-    "cbor-extract": "^2.2.2"
+    "cbor-extract": "^2.2.3"
   },
   "devDependencies": {
     "@rollup/plugin-json": "^5.0.1",

package/scan.js

@@ -0,0 +1,253 @@
+import { _stream } from './decode.js'
+
+// Positions are relative to the retained value, including bundle pointers. Frames and their storage survive appends.
+export class StructureScanner {
+  constructor (decoder) {
+    this.decoder = decoder
+    this.frames = []
+    this.records = new Map()
+    this.readers = new Map()
+    this.reset()
+  }
+
+  reset () {
+    this.position = 0
+    this.depth = 0
+    this.neededLength = 0
+    this.postBundlePosition = 0
+    this.headers = 0
+    this.mapsAsObjects = !!this.decoder.mapsAsObjects
+    this.restoreMap = false
+    this.records.clear()
+    this.readers.clear()
+    this.push('items', 1)
+  }
+
+  push (kind, remaining) {
+    if (this.depth === 16384) throw new RangeError('Maximum CBOR nesting depth exceeded')
+    const frame = this.frames[this.depth] || (this.frames[this.depth] = {})
+    frame.kind = kind
+    frame.remaining = remaining
+    frame.stage = 0
+    frame.id = undefined
+    frame.restoreMap = false
+    frame.legacy = false
+    frame.definition = undefined
+    frame.limitMajor = 0
+    frame.count = 0
+    this.depth++
+    return frame
+  }
+
+  header (source, end, lengthOnly = false) {
+    const start = this.position
+    if (start >= end) {
+      this.neededLength = start + 1
+      return false
+    }
+    const byte = source[start]
+    const major = byte >> 5
+    const info = byte & 31
+    const width = info < 24 || (lengthOnly && info > 26) ? 0 : info === 24 ? 1 : info === 25 ? 2 : info === 26 ? 4 : info === 27 ? 8 : 0
+    if (!lengthOnly && info >= 28 && info <= 30) throw new Error('Unknown token ' + info)
+    if (start + 1 + width > end) {
+      this.neededLength = start + 1 + width
+      return false
+    }
+    let value = info
+    if (width) {
+      value = 0
+      for (let i = start + 1; i <= start + width; i++) value = value * 256 + source[i]
+      if (width === 8 && major > 1 && major !== 7 && value > 0xffffffff) {
+        throw new Error('JavaScript does not support arrays, maps, or strings with length over 4294967295')
+      }
+    }
+    this.major = major
+    this.info = info
+    this.argument = value
+    this.position = start + 1 + width
+    this.headers++
+    return true
+  }
+
+  scan (source, end) {
+    while (this.depth) {
+      if (this.position > end) {
+        this.neededLength = this.position
+        return false
+      }
+      const frame = this.frames[this.depth - 1]
+      if (frame.kind === 'bundle') {
+        if (frame.stage === 0) {
+          if (!this.header(source, end, true)) return false
+          frame.remaining = Math.max(0, this.argument - 2)
+          frame.stage = 1
+        }
+        if (frame.stage === 1) {
+          const offsetStart = this.position
+          if (!this.header(source, end)) return false
+          if (this.major > 1 || this.info === 31) throw new Error('Invalid bundled string offset')
+          frame.target = offsetStart + (this.major === 1 ? -1 - this.argument : this.argument)
+          if (!Number.isSafeInteger(frame.target) || frame.target < 0) throw new Error('Invalid bundled string offset')
+          frame.stage = 2
+        }
+        if (frame.stage === 2) {
+          if (frame.remaining) {
+            if (!this.header(source, end, true)) return false
+            this.position += this.argument
+            frame.remaining--
+            continue
+          }
+          frame.resume = this.position
+          this.position = frame.target
+          frame.remaining = 2
+          frame.stage = 3
+          continue
+        }
+        if (frame.stage === 3) {
+          if (frame.remaining) {
+            if (!this.header(source, end, true)) return false
+            this.position += this.argument
+            frame.remaining--
+            continue
+          }
+          this.postBundlePosition = this.position
+          this.position = frame.resume
+          frame.kind = 'items'
+          frame.remaining = 1
+          continue
+        }
+      }
+      if (frame.kind === 'record') {
+        if (!this.header(source, end, true)) return false
+        if (this.info > 26) throw new Error('Expected array header, but got ' + source[this.position - 1])
+        frame.kind = 'items'
+        frame.remaining = this.argument
+        const reader = frame.reader
+        if (reader.compiled.has(this.argument)) frame.remaining = Math.min(this.argument, reader.length)
+        else if (reader.slowReads++ >= _stream.readThreshold) {
+          reader.compiled.add(this.argument)
+          frame.remaining = Math.min(this.argument, reader.length)
+        }
+        continue
+      }
+      if (frame.kind === 'definition' && frame.stage === 0) {
+        const start = this.position
+        if (!this.header(source, end, !frame.legacy)) return false
+        if (frame.legacy && this.major !== 4) {
+          this.position = start
+          this.headers--
+          frame.kind = 'items'
+          frame.remaining = 1
+          continue
+        }
+        frame.remaining = frame.legacy ? (this.info === 31 ? -1 : this.argument) : Math.max(2, this.argument)
+        if (frame.legacy) {
+          if (frame.remaining >= 0) _stream.checkContainer(4, frame.remaining)
+          else frame.limitMajor = 4
+          frame.index = 0
+        }
+        frame.total = frame.remaining
+        frame.stage = 1
+        continue
+      }
+      if (frame.remaining === 0) {
+        if (frame.definition) this.records.set(frame.id - 0xe000, frame.definition)
+        if (frame.restoreMap && this.restoreMap) {
+          this.mapsAsObjects = true
+          this.restoreMap = false
+        }
+        this.depth--
+        continue
+      }
+      if (!this.header(source, end)) return false
+      let { major, argument, info } = this
+      if (frame.kind === 'definition') {
+        const index = frame.legacy ? frame.index++ : frame.total - frame.remaining
+        if (index === 0 && major === 0) frame.id = argument
+        if ((frame.multiple ? index >= 1 && index < frame.total - 1 : index === 1) && major === 4 && info !== 31 && frame.id !== undefined) {
+          const definition = { length: argument, slowReads: 0, compiled: new Set() }
+          if (frame.legacy) frame.definition = definition
+          else this.records.set(frame.id - 0xe000 + (frame.multiple ? index - 1 : 0), definition)
+        }
+      }
+      if (major === 7 && info === 31 && (frame.remaining === -1 || frame.remaining === -2)) {
+        if (frame.definition) this.records.set(frame.id - 0xe000, frame.definition)
+        this.depth--
+        continue
+      }
+      if (frame.limitMajor && (frame.remaining === -1 || frame.remaining === -2)) {
+        _stream.checkContainer(frame.limitMajor, ++frame.count, frame.mapsAsObjects)
+      }
+      if (frame.remaining > 0) frame.remaining--
+      else if (frame.remaining === -2) frame.remaining = -3
+      else if (frame.remaining === -3) frame.remaining = -2
+      if (info === 31) {
+        if (major === 2 || major === 3) throw new Error('Indefinite length not supported for byte or text strings')
+        if (major === 4 || major === 5) {
+          const container = this.push('items', major === 4 ? -1 : -2)
+          container.limitMajor = major
+          container.mapsAsObjects = this.mapsAsObjects
+          if (major === 5 && this.restoreMap) {
+            this.mapsAsObjects = true
+            this.restoreMap = false
+          }
+        }
+        else if (major === 7) throw new Error('Unexpected break stop code outside of an indefinite-length item')
+        else throw new Error('Invalid major type for indefinite length ' + major)
+        continue
+      }
+      switch (major) {
+        case 2:
+        case 3:
+          this.position += argument
+          break
+        case 4:
+        case 5:
+          _stream.checkContainer(major, argument, this.mapsAsObjects)
+          if (major === 5 && this.restoreMap) {
+            this.mapsAsObjects = true
+            this.restoreMap = false
+          }
+          if (argument) this.push('items', major === 5 ? argument * 2 : argument)
+          break
+        case 6: {
+          const structure = _stream.getTag(argument, this.decoder)
+          let reader = this.records.get(argument & 0x1fff)
+          if (argument >= 0xdff9 && (structure || reader)) {
+            if (!reader) {
+              reader = this.readers.get(structure)
+              if (!reader) {
+                reader = { length: structure.length, slowReads: structure.slowReads || 0, compiled: new Set() }
+                for (let compiled = structure.compiledReader; compiled; compiled = compiled.next) reader.compiled.add(compiled.propertyCount)
+                this.readers.set(structure, reader)
+              }
+            }
+            this.push('record').reader = reader
+          } else if (argument === 0xdff9) {
+            this.push('bundle')
+          } else if (argument === 0xdfff || argument === 0xdffe) {
+            this.push('definition').multiple = argument === 0xdffe
+          } else if (argument === 259 && _stream.isBuiltinTag(argument) && this.mapsAsObjects) {
+            this.push('items', 1).restoreMap = true
+            this.mapsAsObjects = false
+            this.restoreMap = true
+          } else if (argument === 105 && _stream.isBuiltinTag(argument)) {
+            const definition = this.push('definition')
+            definition.multiple = false
+            definition.legacy = true
+          } else {
+            this.push('items', 1)
+          }
+          break
+        }
+      }
+    }
+    if (this.postBundlePosition) {
+      if (this.position >= this.postBundlePosition) throw new Error('Unexpected bundle position')
+      this.position = this.postBundlePosition
+    }
+    if (this.frames.length > 256) this.frames.length = 256
+    return true
+  }
+}

package/stream.js

@@ -1,6 +1,7 @@
 import { Transform } from 'stream'
 import { Encoder } from './encode.js'
 import { checkedRead, getPosition, Decoder, clearSource } from './decode.js'
+import { ChunkDecoder } from './iterators.js'
 var DEFAULT_OPTIONS = {objectMode: true}
 
 export class EncoderStream extends Transform {
@@ -30,21 +31,16 @@ export class DecoderStream extends Transform {
 		super(options)
 		options.structures = []
 		this.decoder = options.decoder || new Decoder(options)
+		this.chunkDecoder = new ChunkDecoder(this.decoder, options.maxIncompleteBufferSize)
 	}
 	_transform(chunk, encoding, callback) {
-		if (this.incompleteBuffer) {
-			chunk = Buffer.concat([this.incompleteBuffer, chunk])
-			this.incompleteBuffer = null
-		}
 		let values, fatalError
 		try {
-			values = this.decoder.decodeMultiple(chunk)
+			// decodeMultiple is called directly, since another call level here measurably slows small writes
+			values = this.chunkDecoder.buffer ? this.chunkDecoder.decodeAfterIncomplete(chunk) : this.decoder.decodeMultiple(chunk)
 		} catch(error) {
-			values = error.values
-			if (error.incomplete)
-				this.incompleteBuffer = chunk.slice(error.lastPosition)
-			else
-				fatalError = error
+			values = error && error.values
+			fatalError = this.chunkDecoder.keepIncomplete(error, chunk)
 		}
 		for (let value of values || []) {
 			if (value === null)
@@ -53,6 +49,9 @@ export class DecoderStream extends Transform {
 		}
 		callback(fatalError)
 	}
+	_flush(callback) {
+		callback(this.chunkDecoder.end())
+	}
 	getNullValue() {
 		return Symbol.for(null)
 	}