msgpack5 6.0.1 → 6.1.0

package/example.js

@@ -8,8 +8,13 @@ const decode = msgpack.decode
 
 msgpack.register(0x42, MyType, mytipeEncode, mytipeDecode)
 
-console.log(encode({ hello: 'world' }).toString('hex'))
+const hex = encode({ hello: 'world' }).toString('hex')
+console.log(hex)
 // 81a568656c6c6fa5776f726c64
+const obj = decode(Buffer.from(hex, 'hex'))
+console.log(obj)
+// { hello: 'world' }
+
 console.log(decode(encode({ hello: 'world' })))
 // { hello: 'world' }
 console.log(encode(a).toString('hex'))

package/index.js

@@ -13,17 +13,25 @@ function msgpack (options) {
   const encodingTypes = []
   const decodingTypes = new Map()
 
-  options = options || {
+  options = Object.assign({
     forceFloat64: false,
     compatibilityMode: false,
     // if true, skips encoding Dates using the msgpack
     // timestamp ext format (-1)
     disableTimestampEncoding: false,
     preferMap: false,
-    // options.protoAction: 'error' (default) / 'remove' / 'ignore'
+    maxDepth: 100,
     protoAction: 'error'
+  }, options || {})
+
+  if (options.protoAction === undefined) options.protoAction = 'error'
+  if (options.protoAction !== 'error' && options.protoAction !== 'remove' && options.protoAction !== 'ignore') {
+    throw new TypeError('protoAction must be "error", "remove", or "ignore"')
   }
 
+  validateMaxLength(options.maxArrayLength, 'maxArrayLength')
+  validateMaxLength(options.maxMapLength, 'maxMapLength')
+
   decodingTypes.set(DateCodec.type, DateCodec.decode)
   if (!options.disableTimestampEncoding) {
     encodingTypes.push(DateCodec)
@@ -88,4 +96,10 @@ function msgpack (options) {
   }
 }
 
+function validateMaxLength (value, name) {
+  if (value !== undefined && (!Number.isSafeInteger(value) || value < 0)) {
+    throw new TypeError(name + ' must be a non-negative safe integer')
+  }
+}
+
 module.exports = msgpack

package/lib/decoder.js

@@ -28,9 +28,10 @@ const SIZES = {
   0xd9: 2,
   0xda: 3,
   0xdb: 5,
-  0xde: 3,
   0xdc: 3,
-  0xdd: 5
+  0xdd: 5,
+  0xde: 3,
+  0xdf: 5
 }
 
 function isValidDataSize (dataLength, bufLength, headerLength) {
@@ -38,30 +39,135 @@ function isValidDataSize (dataLength, bufLength, headerLength) {
 }
 
 module.exports = function buildDecode (decodingTypes, options) {
-  const context = { decodingTypes, options, decode }
+  const maxDepth = options.maxDepth === undefined ? 100 : options.maxDepth
+  if (!Number.isInteger(maxDepth) || maxDepth < 0) {
+    throw new TypeError('maxDepth must be a non-negative integer')
+  }
+
+  const context = { decodingTypes, options, maxDepth, decode }
   return decode
 
-  function decode (buf) {
+  function decode (buf, decodeState) {
     if (!bl.isBufferList(buf)) {
       buf = bl(buf)
     }
 
-    const result = tryDecode(buf, 0, context)
+    let result
+    try {
+      result = decodeState
+        ? tryDecodeIncremental(buf, decodeState, context)
+        : tryDecode(buf, 0, context, 0)
+    } catch (err) {
+      if (decodeState) decodeState.stack.length = 0
+      throw err
+    }
+
     // Handle worst case ASAP and keep code flat
     if (!result) throw new IncompleteBufferError()
 
-    buf.consume(result[1])
+    if (!decodeState) buf.consume(result[1])
     return result[0]
   }
 }
 
-function decodeArray (buf, initialOffset, length, headerLength, context) {
+function tryDecodeIncremental (buf, state, context) {
+  while (buf.length > 0) {
+    const container = decodeContainerHeader(buf)
+    let value
+
+    if (container === null) return null
+
+    if (container) {
+      const depth = state.stack.length + 1
+      if (depth > context.maxDepth) {
+        throw new Error('Maximum decode depth exceeded')
+      }
+      checkCollectionLength(
+        container.type,
+        container.length,
+        container.type === 'map' ? context.options.maxMapLength : context.options.maxArrayLength
+      )
+
+      buf.consume(container.headerLength)
+
+      const itemCount = container.type === 'map'
+        ? 2 * container.length
+        : container.length
+
+      if (itemCount > 0) {
+        state.stack.push({
+          type: container.type,
+          length: container.length,
+          itemCount,
+          result: []
+        })
+        continue
+      }
+
+      value = container.type === 'map'
+        ? buildMap([], 0, context)
+        : []
+    } else {
+      const result = tryDecode(buf, 0, context, state.stack.length)
+      if (!result) return null
+
+      buf.consume(result[1])
+      value = result[0]
+    }
+
+    const completed = completeIncrementalValue(state, value, context)
+    if (completed) return completed
+  }
+
+  return null
+}
+
+function completeIncrementalValue (state, value, context) {
+  while (state.stack.length > 0) {
+    const frame = state.stack[state.stack.length - 1]
+    frame.result.push(value)
+
+    if (frame.result.length < frame.itemCount) return null
+
+    state.stack.pop()
+    value = frame.type === 'map'
+      ? buildMap(frame.result, frame.length, context)
+      : frame.result
+  }
+
+  return [value]
+}
+
+function decodeContainerHeader (buf) {
+  const first = buf.readUInt8(0)
+
+  if ((first & 0xf0) === 0x80) {
+    return { type: 'map', length: first & 0x0f, headerLength: 1 }
+  }
+  if ((first & 0xf0) === 0x90) {
+    return { type: 'array', length: first & 0x0f, headerLength: 1 }
+  }
+
+  let headerLength
+  if (first === 0xdc || first === 0xde) headerLength = 3
+  if (first === 0xdd || first === 0xdf) headerLength = 5
+  if (!headerLength) return false
+  if (buf.length < headerLength) return null
+
+  return {
+    type: first === 0xdc || first === 0xdd ? 'array' : 'map',
+    length: buf.readUIntBE(1, headerLength - 1),
+    headerLength
+  }
+}
+
+function decodeItems (buf, initialOffset, length, headerLength, context, depth) {
   let offset = initialOffset
   const result = []
   let i = 0
 
   while (i++ < length) {
-    const decodeResult = tryDecode(buf, offset, context)
+    const decodeResult = tryDecode(buf, offset, context, depth)
     if (!decodeResult) return null
 
     result.push(decodeResult[0])
@@ -70,11 +176,32 @@ function decodeArray (buf, initialOffset, length, headerLength, context) {
   return [result, headerLength + offset - initialOffset]
 }
 
-function decodeMap (buf, offset, length, headerLength, context) {
-  const _temp = decodeArray(buf, offset, 2 * length, headerLength, context)
+function checkCollectionLength (type, length, maxLength) {
+  if (maxLength !== undefined && length > maxLength) {
+    throw new RangeError(type + ' length ' + length + ' exceeds configured limit of ' + maxLength)
+  }
+}
+
+function decodeArray (buf, initialOffset, length, headerLength, context, depth) {
+  if (depth > context.maxDepth) {
+    throw new Error('Maximum decode depth exceeded')
+  }
+  checkCollectionLength('array', length, context.options.maxArrayLength)
+  return decodeItems(buf, initialOffset, length, headerLength, context, depth)
+}
+
+function decodeMap (buf, offset, length, headerLength, context, depth) {
+  if (depth > context.maxDepth) {
+    throw new Error('Maximum decode depth exceeded')
+  }
+  checkCollectionLength('map', length, context.options.maxMapLength)
+  const _temp = decodeItems(buf, offset, 2 * length, headerLength, context, depth)
   if (!_temp) return null
   const [result, consumedBytes] = _temp
+  return [buildMap(result, length, context), consumedBytes]
+}
 
+function buildMap (result, length, context) {
   let isPlainObject = !context.options.preferMap
 
   if (isPlainObject) {
@@ -104,7 +231,7 @@ function decodeMap (buf, offset, length, headerLength, context) {
 
       object[key] = val
     }
-    return [object, consumedBytes]
+    return object
   } else {
     const mapping = new Map()
     for (let i = 0; i < 2 * length; i += 2) {
@@ -112,11 +239,11 @@ function decodeMap (buf, offset, length, headerLength, context) {
       const val = result[i + 1]
       mapping.set(key, val)
     }
-    return [mapping, consumedBytes]
+    return mapping
   }
 }
 
-function tryDecode (buf, initialOffset, context) {
+function tryDecode (buf, initialOffset, context, depth) {
   if (buf.length <= initialOffset) return null
 
   const bufLength = buf.length - initialOffset
@@ -133,13 +260,13 @@ function tryDecode (buf, initialOffset, context) {
     const length = first & 0x0f
     const headerSize = offset - initialOffset
     // we have a map with less than 15 elements
-    return decodeMap(buf, offset, length, headerSize, context)
+    return decodeMap(buf, offset, length, headerSize, context, depth + 1)
   }
   if ((first & 0xf0) === 0x90) {
     const length = first & 0x0f
     const headerSize = offset - initialOffset
     // we have an array with less than 15 elements
-    return decodeArray(buf, offset, length, headerSize, context)
+    return decodeArray(buf, offset, length, headerSize, context, depth + 1)
   }
 
   if ((first & 0xe0) === 0xa0) {
@@ -149,6 +276,7 @@ function tryDecode (buf, initialOffset, context) {
     const result = buf.toString('utf8', offset, offset + length)
     return [result, length + 1]
   }
+  if (first === 0xc1) throw new Error('0xc1 is a reserved MessagePack byte')
   if (first >= 0xc0 && first <= 0xc3) return decodeConstants(first)
   if (first >= 0xc4 && first <= 0xc6) {
     const length = buf.readUIntBE(offset, size - 1)
@@ -188,7 +316,7 @@ function tryDecode (buf, initialOffset, context) {
   if (first >= 0xdc && first <= 0xdd) {
     const length = buf.readUIntBE(offset, size - 1)
     offset += size - 1
-    return decodeArray(buf, offset, length, size, context)
+    return decodeArray(buf, offset, length, size, context, depth + 1)
   }
   if (first >= 0xde && first <= 0xdf) {
     let length
@@ -198,12 +326,12 @@ function tryDecode (buf, initialOffset, context) {
         length = buf.readUInt16BE(offset)
         offset += 2
         // console.log(offset - initialOffset)
-        return decodeMap(buf, offset, length, 3, context)
+        return decodeMap(buf, offset, length, 3, context, depth + 1)
 
       case 0xdf:
         length = buf.readUInt32BE(offset)
         offset += 4
-        return decodeMap(buf, offset, length, 5, context)
+        return decodeMap(buf, offset, length, 5, context, depth + 1)
     }
   }
   if (first >= 0xe0) return [first - 0x100, 1] // 5 bits negative ints
@@ -216,7 +344,7 @@ function decodeSigned (buf, offset, size) {
   if (size === 1) result = buf.readInt8(offset)
   if (size === 2) result = buf.readInt16BE(offset)
   if (size === 4) result = buf.readInt32BE(offset)
-  if (size === 8) result = readInt64BE(buf.slice(offset, offset + 8), 0)
+  if (size === 8) result = readInt64BE(buf, offset)
   return [result, size + 1]
 }
 
@@ -251,18 +379,14 @@ function decodeFloat (buf, offset, size) {
 }
 
 function readInt64BE (buf, offset) {
-  var negate = (buf[offset] & 0x80) == 0x80; // eslint-disable-line
+  const negate = (buf.readUInt8(offset) & 0x80) === 0x80
+  let hi = buf.readUInt32BE(offset + 0)
+  let lo = buf.readUInt32BE(offset + 4)
 
   if (negate) {
-    let carry = 1
-    for (let i = offset + 7; i >= offset; i--) {
-      const v = (buf[i] ^ 0xff) + carry
-      buf[i] = v & 0xff
-      carry = v >> 8
-    }
+    lo = (~lo + 1) >>> 0
+    hi = (~hi + (lo === 0 ? 1 : 0)) >>> 0
   }
 
-  const hi = buf.readUInt32BE(offset + 0)
-  const lo = buf.readUInt32BE(offset + 4)
   return (hi * 4294967296 + lo) * (negate ? -1 : +1)
 }

package/lib/encoder.js

@@ -22,7 +22,8 @@ module.exports = function buildEncode (encodingTypes, options) {
       }
       // weird hack to support Buffer
       // and Buffer-like objects
-      return bl([getBufferHeader(obj.length), obj])
+      const _getBufferHeader = options.compatibilityMode ? getCompatibleBufferHeader : getBufferHeader
+      return bl([_getBufferHeader(obj.length), obj])
     }
     if (Array.isArray(obj)) return encodeArray(obj, encode)
     if (typeof obj === 'object') return encodeExt(obj, encodingTypes) || encodeObject(obj, options, encode)
@@ -225,6 +226,26 @@ function getBufferHeader (length) {
   return header
 }
 
+function getCompatibleBufferHeader (length) {
+  let header
+  if (length <= 0x1f) {
+    // fix raw header: 101XXXXX
+    header = Buffer.allocUnsafe(1)
+    header[0] = 0xa0 | length
+  } else if (length <= 0xffff) {
+    // raw 16 header: 0xda, XXXXXXXX, XXXXXXXX
+    header = Buffer.allocUnsafe(3)
+    header[0] = 0xda
+    header.writeUInt16BE(length, 1)
+  } else {
+    // raw 32 header: 0xdb, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX
+    header = Buffer.allocUnsafe(5)
+    header[0] = 0xdb
+    header.writeUInt32BE(length, 1)
+  }
+  return header
+}
+
 function encodeNumber (obj, options) {
   let buf
   if (isFloat(obj)) return encodeFloat(obj, options.forceFloat64)

package/lib/streams.js

@@ -54,6 +54,7 @@ function Decoder (opts) {
   Base.call(this, opts)
 
   this._chunks = bl()
+  this._decodeState = { stack: [] }
   this._wrap = ('wrap' in opts) && opts.wrap
 }
 
@@ -64,26 +65,26 @@ Decoder.prototype._transform = function (buf, enc, done) {
     this._chunks.append(buf)
   }
 
-  try {
-    let result = this._msgpack.decode(this._chunks)
-    if (this._wrap) {
-      result = { value: result }
-    }
-    this.push(result)
-  } catch (err) {
-    if (err instanceof this._msgpack.IncompleteBufferError) {
-      done()
-    } else {
-      this.emit('error', err)
+  while (this._chunks.length > 0) {
+    try {
+      let result = this._msgpack.decode(this._chunks, this._decodeState)
+      if (this._wrap) {
+        result = { value: result }
+      }
+      this.push(result)
+    } catch (err) {
+      if (err instanceof this._msgpack.IncompleteBufferError) {
+        done()
+      } else {
+        this._chunks = bl()
+        this.destroy(err)
+        done()
+      }
+      return
     }
-    return
   }
 
-  if (this._chunks.length > 0) {
-    this._transform(null, enc, done)
-  } else {
-    done()
-  }
+  done()
 }
 
 module.exports.decoder = Decoder

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "msgpack5",
-  "version": "6.0.1",
+  "version": "6.1.0",
   "description": "A msgpack v5 implementation for node.js and the browser, with extension points",
   "main": "index.js",
   "scripts": {

package/test/4-bytes-length-maps.js

@@ -0,0 +1,31 @@
+'use strict'
+
+const Buffer = require('safe-buffer').Buffer
+const test = require('tape').test
+const msgpack = require('../')
+const bl = require('bl')
+
+test('decoding incomplete map32 headers', function (t) {
+  const pack = msgpack()
+
+  for (let length = 1; length < 5; length++) {
+    const buf = Buffer.alloc(length)
+    buf[0] = 0xdf
+    const input = bl().append(buf)
+
+    t.throws(function () {
+      pack.decode(input)
+    }, pack.IncompleteBufferError, 'must reject a ' + length + '-byte header as incomplete')
+    t.equal(input.length, length, 'must not consume an incomplete header')
+  }
+
+  t.end()
+})
+
+test('decoding an empty map32', function (t) {
+  const pack = msgpack()
+  const buf = Buffer.from([0xdf, 0x00, 0x00, 0x00, 0x00])
+
+  t.deepEqual(pack.decode(buf), {}, 'must decode a complete map32 header')
+  t.end()
+})

package/test/64-bits-signed-integers.js

@@ -34,6 +34,27 @@ test('encoding/decoding 64-bits big-endian signed integers', function (t) {
   t.end()
 })
 
+test('decoding a negative 64-bits integer does not mutate the input', function (t) {
+  const encoder = msgpack()
+  const encoded = Buffer.from([0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe])
+  const expected = encoded.toString('hex')
+
+  t.equal(encoder.decode(encoded), -2, 'decodes a direct buffer')
+  t.equal(encoded.toString('hex'), expected, 'does not mutate a direct buffer')
+  t.equal(encoder.decode(encoded), -2, 'decodes the same buffer consistently')
+  t.equal(encoded.toString('hex'), expected, 'leaves the buffer unchanged after repeated decoding')
+
+  const singleChunk = Buffer.from(encoded)
+  t.equal(encoder.decode(bl().append(singleChunk)), -2, 'decodes a single-chunk BufferList')
+  t.equal(singleChunk.toString('hex'), expected, 'does not mutate the underlying chunk')
+
+  const firstChunk = Buffer.from(encoded.slice(0, 4))
+  const secondChunk = Buffer.from(encoded.slice(4))
+  t.equal(encoder.decode(bl().append(firstChunk).append(secondChunk)), -2, 'decodes a split-chunk BufferList')
+  t.equal(Buffer.concat([firstChunk, secondChunk]).toString('hex'), expected, 'does not mutate split chunks')
+  t.end()
+})
+
 test('decoding an incomplete 64-bits big-endian signed integer', function (t) {
   const encoder = msgpack()
   let buf = Buffer.allocUnsafe(8)

package/test/collection-length-limits.js

@@ -0,0 +1,64 @@
+'use strict'
+
+const Buffer = require('safe-buffer').Buffer
+const test = require('tape').test
+const msgpack = require('../')
+
+test('maxArrayLength limits decoded arrays', function (t) {
+  const decoder = msgpack({ maxArrayLength: 2 })
+
+  t.deepEqual(decoder.decode(Buffer.from([0x92, 0x01, 0x02])), [1, 2], 'allows the configured limit')
+  t.throws(function () {
+    decoder.decode(Buffer.from([0x93, 0x01, 0x02, 0x03]))
+  }, /array length 3 exceeds configured limit of 2/, 'rejects fixarray over the limit')
+  t.throws(function () {
+    decoder.decode(Buffer.from([0xdc, 0x00, 0x03]))
+  }, /array length 3 exceeds configured limit of 2/, 'rejects array16 before decoding its elements')
+  t.throws(function () {
+    decoder.decode(Buffer.from([0xdd, 0x00, 0x00, 0x00, 0x03]))
+  }, /array length 3 exceeds configured limit of 2/, 'rejects array32 before decoding its elements')
+  t.end()
+})
+
+test('maxMapLength limits decoded maps', function (t) {
+  const decoder = msgpack({ maxMapLength: 1 })
+
+  t.deepEqual(decoder.decode(Buffer.from([0x81, 0xa1, 0x61, 0x01])), { a: 1 }, 'allows the configured limit')
+  t.throws(function () {
+    decoder.decode(Buffer.from([0x82, 0xa1, 0x61, 0x01, 0xa1, 0x62, 0x02]))
+  }, /map length 2 exceeds configured limit of 1/, 'rejects fixmap over the limit')
+  t.throws(function () {
+    decoder.decode(Buffer.from([0xde, 0x00, 0x02]))
+  }, /map length 2 exceeds configured limit of 1/, 'rejects map16 before decoding its entries')
+  t.throws(function () {
+    decoder.decode(Buffer.from([0xdf, 0x00, 0x00, 0x00, 0x02]))
+  }, /map length 2 exceeds configured limit of 1/, 'rejects map32 before decoding its entries')
+  t.end()
+})
+
+test('collection limits apply independently and to nested values', function (t) {
+  const decoder = msgpack({ maxArrayLength: 1, maxMapLength: 1 })
+
+  t.deepEqual(decoder.decode(Buffer.from([0x81, 0xa1, 0x61, 0x91, 0x01])), { a: [1] }, 'allows nested collections at their limits')
+  t.throws(function () {
+    decoder.decode(Buffer.from([0x81, 0xa1, 0x61, 0x92, 0x01, 0x02]))
+  }, /array length 2 exceeds configured limit of 1/, 'rejects a nested array over its limit')
+  t.end()
+})
+
+test('collection limits must be non-negative safe integers', function (t) {
+  const invalid = [-1, 1.5, Infinity, null, '1']
+
+  invalid.forEach(function (value) {
+    t.throws(function () {
+      msgpack({ maxArrayLength: value })
+    }, /maxArrayLength must be a non-negative safe integer/, 'rejects maxArrayLength ' + value)
+    t.throws(function () {
+      msgpack({ maxMapLength: value })
+    }, /maxMapLength must be a non-negative safe integer/, 'rejects maxMapLength ' + value)
+  })
+  t.doesNotThrow(function () {
+    msgpack({ maxArrayLength: 0, maxMapLength: Number.MAX_SAFE_INTEGER })
+  }, 'accepts boundary values')
+  t.end()
+})

package/test/compatibility-mode.js

@@ -3,6 +3,13 @@
 const test = require('tape').test
 const msgpack = require('../')
 
+function buildBuffer (size) {
+  const buf = Buffer.allocUnsafe(size)
+  buf.fill('a')
+
+  return buf
+}
+
 test('encode/compatibility mode', function (t) {
   const compatEncoder = msgpack({
     compatibilityMode: true
@@ -37,4 +44,30 @@ test('encode/compatibility mode', function (t) {
     t.deepEqual(buf1, buf2, 'must be equal for two byte strings')
     t.end()
   })
+
+  const fixRawBuffer = buildBuffer(1)
+  const raw16Buffer = buildBuffer(Math.pow(2, 16) - 1)
+  const raw32Buffer = buildBuffer(Math.pow(2, 16) + 1)
+
+  t.test('compat. encoding a Buffer of length ' + fixRawBuffer.length, function (t) {
+    // fix raw header: 0xa0 | 1 = 0xa1
+    const buf = compatEncoder.encode(fixRawBuffer)
+    t.equal(buf[0], 0xa1, 'must have the proper header (fix raw)')
+    t.equal(buf.toString('utf8', 1, Buffer.byteLength(fixRawBuffer) + 1), fixRawBuffer.toString('utf8'), 'must decode correctly')
+    t.end()
+  })
+
+  t.test('compat. encoding a Buffer of length ' + raw16Buffer.length, function (t) {
+    const buf = compatEncoder.encode(raw16Buffer)
+    t.equal(buf[0], 0xda, 'must have the proper header (raw 16)')
+    t.equal(buf.toString('utf8', 3, Buffer.byteLength(raw16Buffer) + 3), raw16Buffer.toString('utf8'), 'must decode correctly')
+    t.end()
+  })
+
+  t.test('compat. encoding a Buffer of length ' + raw32Buffer.length, function (t) {
+    const buf = compatEncoder.encode(raw32Buffer)
+    t.equal(buf[0], 0xdb, 'must have the proper header (raw 32)')
+    t.equal(buf.toString('utf8', 5, Buffer.byteLength(raw32Buffer) + 5), raw32Buffer.toString('utf8'), 'must decode correctly')
+    t.end()
+  })
 })