compact-encoding 2.6.0 → 2.8.0

package/README.md

@@ -79,13 +79,20 @@ to build others on top. Feel free to PR more that are missing.
 * `cenc.uint16` - Encodes a fixed size uint16. Useful for things like ports.
 * `cenc.uint24` - Encodes a fixed size uint24. Useful for message framing.
 * `cenc.uint32` - Encodes a fixed size uint32. Useful for very large message framing.
+* `cenc.uint40` - Encodes a fixed size uint40.
+* `cenc.uint48` - Encodes a fixed size uint48.
+* `cenc.uint56` - Encodes a fixed size uint56.
 * `cenc.uint64` - Encodes a fixed size uint64.
 * `cenc.int` - Encodes an int using `cenc.uint` with ZigZag encoding.
 * `cenc.int8` - Encodes a fixed size int8 using `cenc.uint8` with ZigZag encoding.
 * `cenc.int16` - Encodes a fixed size int16 using `cenc.uint16` with ZigZag encoding.
 * `cenc.int24` - Encodes a fixed size int24 using `cenc.uint24` with ZigZag encoding.
 * `cenc.int32` - Encodes a fixed size int32 using `cenc.uint32` with ZigZag encoding.
+* `cenc.int40` - Encodes a fixed size int40 using `cenc.uint40` with ZigZag encoding.
+* `cenc.int48` - Encodes a fixed size int48 using `cenc.uint48` with ZigZag encoding.
+* `cenc.int56` - Encodes a fixed size int56 using `cenc.uint56` with ZigZag encoding.
 * `cenc.int64` - Encodes a fixed size int64 using `cenc.uint64` with ZigZag encoding.
+* `cenc.lexint` - Encodes an int using [lexicographic-integer](https://github.com/substack/lexicographic-integer) encoding so that encoded values are lexicographically sorted in ascending numerical order.
 * `cenc.float32` - Encodes a fixed size float32.
 * `cenc.float64` - Encodes a fixed size float64.
 * `cenc.buffer` - Encodes a buffer with its length uint prefixed. When decoding an empty buffer, `null` is returned.
@@ -99,11 +106,17 @@ to build others on top. Feel free to PR more that are missing.
 * `cenc.float32array` - Encodes a float32array with its element length uint prefixed.
 * `cenc.float64array` - Encodes a float64array with its element length uint prefixed.
 * `cenc.bool` - Encodes a boolean as 1 or 0.
-* `cenc.string` - Encodes a utf-8 string, similar to buffer.
+* `cenc.string`, `cenc.utf8` - Encodes a utf-8 string, similar to buffer.
+* `cenc.ascii` - Encodes an ascii string.
+* `cenc.hex` - Encodes a hex string.
+* `cenc.base64` - Encodes a base64 string.
+* `cenc.utf16le`, `cenc.ucs2` - Encodes a utf16le string.
 * `cenc.fixed32` - Encodes a fixed 32 byte buffer.
 * `cenc.fixed64` - Encodes a fixed 64 byte buffer.
 * `cenc.fixed(n)` - Makes a fixed sized encoder.
 * `cenc.array(enc)` - Makes an array encoder from another encoder. Arrays are uint prefixed with their length.
+* `cenc.json` - Encodes a JSON value as utf-8.
+* `cenc.ndjson` - Encodes a JSON value as newline delimited utf-8.
 * `cenc.from(enc)` - Makes a compact encoder from a [codec](https://github.com/mafintosh/codecs) or [abstract-encoding](https://github.com/mafintosh/abstract-encoding).
 
 ## License

package/index.js

@@ -1,6 +1,6 @@
 const b4a = require('b4a')
 
-const LE = (new Uint8Array(new Uint16Array([255]).buffer))[0] === 0xff
+const LE = (new Uint8Array(new Uint16Array([0xff]).buffer))[0] === 0xff
 const BE = !LE
 
 exports.state = function () {
@@ -58,7 +58,7 @@ const uint16 = exports.uint16 = {
     if (state.end - state.start < 2) throw new Error('Out of bounds')
     return (
       state.buffer[state.start++] +
-      state.buffer[state.start++] * 256
+      state.buffer[state.start++] * 0x100
     )
   }
 }
@@ -76,8 +76,8 @@ const uint24 = exports.uint24 = {
     if (state.end - state.start < 3) throw new Error('Out of bounds')
     return (
       state.buffer[state.start++] +
-      state.buffer[state.start++] * 256 +
-      state.buffer[state.start++] * 65536
+      state.buffer[state.start++] * 0x100 +
+      state.buffer[state.start++] * 0x10000
     )
   }
 }
@@ -96,25 +96,70 @@ const uint32 = exports.uint32 = {
     if (state.end - state.start < 4) throw new Error('Out of bounds')
     return (
       state.buffer[state.start++] +
-      state.buffer[state.start++] * 256 +
-      state.buffer[state.start++] * 65536 +
-      state.buffer[state.start++] * 16777216
+      state.buffer[state.start++] * 0x100 +
+      state.buffer[state.start++] * 0x10000 +
+      state.buffer[state.start++] * 0x1000000
     )
   }
 }
 
+const uint40 = exports.uint40 = {
+  preencode (state, n) {
+    state.end += 5
+  },
+  encode (state, n) {
+    const r = Math.floor(n / 0x100)
+    uint8.encode(state, n)
+    uint32.encode(state, r)
+  },
+  decode (state) {
+    if (state.end - state.start < 5) throw new Error('Out of bounds')
+    return uint8.decode(state) + 0x100 * uint32.decode(state)
+  }
+}
+
+const uint48 = exports.uint48 = {
+  preencode (state, n) {
+    state.end += 6
+  },
+  encode (state, n) {
+    const r = Math.floor(n / 0x10000)
+    uint16.encode(state, n)
+    uint32.encode(state, r)
+  },
+  decode (state) {
+    if (state.end - state.start < 6) throw new Error('Out of bounds')
+    return uint16.decode(state) + 0x10000 * uint32.decode(state)
+  }
+}
+
+const uint56 = exports.uint56 = {
+  preencode (state, n) {
+    state.end += 7
+  },
+  encode (state, n) {
+    const r = Math.floor(n / 0x1000000)
+    uint24.encode(state, n)
+    uint32.encode(state, r)
+  },
+  decode (state) {
+    if (state.end - state.start < 7) throw new Error('Out of bounds')
+    return uint24.decode(state) + 0x1000000 * uint32.decode(state)
+  }
+}
+
 const uint64 = exports.uint64 = {
   preencode (state, n) {
     state.end += 8
   },
   encode (state, n) {
-    const r = Math.floor(n / 4294967296)
+    const r = Math.floor(n / 0x100000000)
     uint32.encode(state, n)
     uint32.encode(state, r)
   },
   decode (state) {
     if (state.end - state.start < 8) throw new Error('Out of bounds')
-    return uint32.decode(state) + 4294967296 * uint32.decode(state)
+    return uint32.decode(state) + 0x100000000 * uint32.decode(state)
   }
 }
 
@@ -123,8 +168,13 @@ exports.int8 = zigZag(uint8)
 exports.int16 = zigZag(uint16)
 exports.int24 = zigZag(uint24)
 exports.int32 = zigZag(uint32)
+exports.int40 = zigZag(uint40)
+exports.int48 = zigZag(uint48)
+exports.int56 = zigZag(uint56)
 exports.int64 = zigZag(uint64)
 
+exports.lexint = require('./lexint')
+
 exports.float32 = {
   preencode (state, n) {
     state.end += 4
@@ -173,19 +223,18 @@ exports.buffer = {
   decode (state) {
     const len = uint.decode(state)
     if (len === 0) return null
-    const b = state.buffer.subarray(state.start, state.start += len)
-    if (b.length !== len) throw new Error('Out of bounds')
-    return b
+    if (state.end - state.start < len) throw new Error('Out of bounds')
+    return state.buffer.subarray(state.start, (state.start += len))
   }
 }
 
 const raw = exports.raw = {
   preencode (state, b) {
-    state.end += b.length
+    state.end += b.byteLength
   },
   encode (state, b) {
     state.buffer.set(b, state.start)
-    state.start += b.length
+    state.start += b.byteLength
   },
   decode (state) {
     const b = state.buffer.subarray(state.start, state.end)
@@ -237,26 +286,33 @@ exports.int32array = typedarray(Int32Array, b4a.swap32)
 exports.float32array = typedarray(Float32Array, b4a.swap32)
 exports.float64array = typedarray(Float64Array, b4a.swap64)
 
-exports.string = {
-  preencode (state, s) {
-    const len = b4a.byteLength(s)
-    uint.preencode(state, len)
-    state.end += len
-  },
-  encode (state, s) {
-    const len = b4a.byteLength(s)
-    uint.encode(state, len)
-    b4a.write(state.buffer, s, state.start)
-    state.start += len
-  },
-  decode (state) {
-    const len = uint.decode(state)
-    const s = b4a.toString(state.buffer, 'utf-8', state.start, state.start += len)
-    if (b4a.byteLength(s) !== len || state.start > state.end) throw new Error('Out of bounds')
-    return s
+function string (encoding) {
+  return {
+    preencode (state, s) {
+      const len = b4a.byteLength(s, encoding)
+      uint.preencode(state, len)
+      state.end += len
+    },
+    encode (state, s) {
+      const len = b4a.byteLength(s, encoding)
+      uint.encode(state, len)
+      b4a.write(state.buffer, s, state.start, encoding)
+      state.start += len
+    },
+    decode (state) {
+      const len = uint.decode(state)
+      if (state.end - state.start < len) throw new Error('Out of bounds')
+      return b4a.toString(state.buffer, encoding, state.start, (state.start += len))
+    }
   }
 }
 
+const utf8 = exports.string = exports.utf8 = string('utf-8')
+exports.ascii = string('ascii')
+exports.hex = string('hex')
+exports.base64 = string('base64')
+exports.ucs2 = exports.utf16le = string('utf16le')
+
 exports.bool = {
   preencode (state, b) {
     state.end++
@@ -280,9 +336,8 @@ const fixed = exports.fixed = function fixed (n) {
       state.start += n
     },
     decode (state) {
-      const b = state.buffer.subarray(state.start, state.start += n)
-      if (b.length !== n) throw new Error('Out of bounds')
-      return b
+      if (state.end - state.start < n) throw new Error('Out of bounds')
+      return state.buffer.subarray(state.start, (state.start += n))
     }
   }
 }
@@ -314,7 +369,7 @@ exports.array = function array (enc) {
     },
     decode (state) {
       const len = uint.decode(state)
-      if (len > 1048576) throw new Error('Array is too big')
+      if (len > 0x100000) throw new Error('Array is too big')
       const arr = new Array(len)
       for (let i = 0; i < len; i++) arr[i] = enc.decode(state)
       return arr
@@ -322,6 +377,30 @@ exports.array = function array (enc) {
   }
 }
 
+exports.json = {
+  preencode (state, v) {
+    utf8.preencode(state, JSON.stringify(v))
+  },
+  encode (state, v) {
+    utf8.encode(state, JSON.stringify(v))
+  },
+  decode (state) {
+    return JSON.parse(utf8.decode(state))
+  }
+}
+
+exports.ndjson = {
+  preencode (state, v) {
+    utf8.preencode(state, JSON.stringify(v) + '\n')
+  },
+  encode (state, v) {
+    utf8.encode(state, JSON.stringify(v) + '\n')
+  },
+  decode (state) {
+    return JSON.parse(utf8.decode(state))
+  }
+}
+
 exports.from = function from (enc) {
   if (enc.preencode) return enc
   if (enc.encodingLength) return fromAbstractEncoder(enc)
@@ -336,7 +415,7 @@ function fromCodec (enc) {
     preencode (state, m) {
       tmpM = m
       tmpBuf = enc.encode(m)
-      state.end += tmpBuf.length
+      state.end += tmpBuf.byteLength
     },
     encode (state, m) {
       raw.encode(state, m === tmpM ? tmpBuf : enc.encode(m))

package/lexint.js

@@ -0,0 +1,114 @@
+module.exports = {
+  preencode,
+  encode,
+  decode
+}
+
+function preencode (state, num) {
+  if (num < 251) {
+    state.end++
+  } else if (num < 256) {
+    state.end += 2
+  } else if (num < 0x10000) {
+    state.end += 3
+  } else if (num < 0x1000000) {
+    state.end += 4
+  } else if (num < 0x100000000) {
+    state.end += 5
+  } else {
+    state.end++
+    const exp = Math.floor(Math.log(num) / Math.log(2)) - 32
+    preencode(state, exp)
+    state.end += 6
+  }
+}
+
+function encode (state, num) {
+  const max = 251
+  const x = num - max
+
+  if (num < max) {
+    state.buffer[state.start++] = num
+  } else if (num < 256) {
+    state.buffer[state.start++] = max
+    state.buffer[state.start++] = x
+  } else if (num < 0x10000) {
+    state.buffer[state.start++] = max + 1
+    state.buffer[state.start++] = x >> 8 & 0xff
+    state.buffer[state.start++] = x & 0xff
+  } else if (num < 0x1000000) {
+    state.buffer[state.start++] = max + 2
+    state.buffer[state.start++] = x >> 16
+    state.buffer[state.start++] = x >> 8 & 0xff
+    state.buffer[state.start++] = x & 0xff
+  } else if (num < 0x100000000) {
+    state.buffer[state.start++] = max + 3
+    state.buffer[state.start++] = x >> 24
+    state.buffer[state.start++] = x >> 16 & 0xff
+    state.buffer[state.start++] = x >> 8 & 0xff
+    state.buffer[state.start++] = x & 0xff
+  } else {
+    // need to use Math here as bitwise ops are 32 bit
+    const exp = Math.floor(Math.log(x) / Math.log(2)) - 32
+    state.buffer[state.start++] = 0xff
+
+    encode(state, exp)
+    const rem = x / Math.pow(2, exp - 11)
+
+    for (let i = 5; i >= 0; i--) {
+      state.buffer[state.start++] = rem / Math.pow(2, 8 * i) & 0xff
+    }
+  }
+}
+
+function decode (state) {
+  const max = 251
+
+  if (state.end - state.start < 1) throw new Error('Out of bounds')
+
+  const flag = state.buffer[state.start++]
+
+  if (flag < max) return flag
+
+  if (state.end - state.start < flag - max + 1) {
+    throw new Error('Out of bounds.')
+  }
+
+  if (flag < 252) {
+    return state.buffer[state.start++] +
+      max
+  }
+
+  if (flag < 253) {
+    return (state.buffer[state.start++] << 8) +
+      state.buffer[state.start++] +
+      max
+  }
+
+  if (flag < 254) {
+    return (state.buffer[state.start++] << 16) +
+      (state.buffer[state.start++] << 8) +
+      state.buffer[state.start++] +
+      max
+  }
+
+  // << 24 result may be interpreted as negative
+  if (flag < 255) {
+    return (state.buffer[state.start++] * 0x1000000) +
+      (state.buffer[state.start++] << 16) +
+      (state.buffer[state.start++] << 8) +
+      state.buffer[state.start++] +
+      max
+  }
+
+  const exp = decode(state)
+
+  if (state.end - state.start < 6) throw new Error('Out of bounds')
+
+  let rem = 0
+  for (let i = 5; i >= 0; i--) {
+    rem += state.buffer[state.start++] * Math.pow(2, 8 * i)
+  }
+
+  return (rem * Math.pow(2, exp - 11)) + max
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "compact-encoding",
-  "version": "2.6.0",
+  "version": "2.8.0",
   "description": "A series of compact encoding schemes for building small and fast parsers and serializers",
   "main": "index.js",
   "dependencies": {

package/test.js

@@ -170,9 +170,6 @@ tape('buffer', function (t) {
   t.is(state.start, state.end)
 
   t.exception(() => enc.buffer.decode(state))
-  state.buffer = state.buffer.subarray(0, 8)
-  state.start = 3
-  t.exception(() => enc.buffer.decode(state), 'partial throws')
 })
 
 tape('raw', function (t) {
@@ -410,3 +407,184 @@ tape('array', function (t) {
 
   t.exception(() => arr.decode(state))
 })
+
+tape('json', function (t) {
+  const state = enc.state()
+
+  enc.json.preencode(state, { a: 1, b: 2 })
+  t.alike(state, { start: 0, end: 14, buffer: null })
+
+  state.buffer = Buffer.alloc(state.end)
+  enc.json.encode(state, { a: 1, b: 2 })
+  t.alike(state, {
+    start: 14,
+    end: 14,
+    buffer: Buffer.concat([
+      Buffer.from([13]),
+      Buffer.from('{"a":1,"b":2}')
+    ])
+  })
+
+  state.start = 0
+  t.alike(enc.json.decode(state), { a: 1, b: 2 })
+
+  t.exception(() => enc.json.decode(state))
+})
+
+tape('lexint: big numbers', function (t) {
+  t.plan(1)
+
+  let prev = enc.encode(enc.lexint, 0)
+
+  let n
+  let skip = 1
+
+  for (n = 1; n < Number.MAX_VALUE; n += skip) {
+    const cur = enc.encode(enc.lexint, n)
+    if (Buffer.compare(cur, prev) < 1) break
+    prev = cur
+    skip = 1 + Math.pow(245, Math.ceil(Math.log(n) / Math.log(256)))
+  }
+  t.is(n, Infinity)
+})
+
+tape('lexint: range precision', function (t) {
+  t.plan(2)
+  const a = 1e55
+  const b = 1.0000000000001e55
+  const ha = enc.encode(enc.lexint, a).toString('hex')
+  const hb = enc.encode(enc.lexint, b).toString('hex')
+  t.not(a, b)
+  t.not(ha, hb)
+})
+
+tape('lexint: range precision', function (t) {
+  let prev = enc.encode(enc.lexint, 0)
+  const skip = 0.000000001e55
+  for (let i = 0, n = 1e55; i < 1000; n = 1e55 + skip * ++i) {
+    const cur = enc.encode(enc.lexint, n)
+    if (Buffer.compare(cur, prev) < 1) t.fail('cur <= prev')
+    prev = cur
+  }
+  t.ok(true)
+  t.end()
+})
+
+tape('lexint: small numbers', function (t) {
+  let prev = enc.encode(enc.lexint, 0)
+  for (let n = 1; n < 256 * 256 * 16; n++) {
+    const cur = enc.encode(enc.lexint, n)
+    if (Buffer.compare(cur, prev) < 1) t.fail('cur <= prev')
+    prev = cur
+  }
+  t.ok(true)
+  t.end()
+})
+
+tape('lexint: throws', function (t) {
+  t.exception(() => {
+    enc.decode(enc.lexint, Buffer.alloc(1, 251))
+  })
+
+  let num = 252
+
+  const state = {
+    start: 0,
+    end: 0,
+    buffer: null
+  }
+
+  enc.lexint.preencode(state, num)
+  state.buffer = Buffer.alloc(state.end - state.start)
+  enc.lexint.encode(state, num)
+
+  t.exception(() => {
+    enc.decode(enc.lexint, state.buffer.subarray(0, state.buffer.byteLength - 2))
+  })
+
+  num <<= 8
+
+  state.start = 0
+  state.end = 0
+  state.buffer = null
+
+  enc.lexint.preencode(state, num)
+  state.buffer = Buffer.alloc(state.end - state.start)
+  enc.lexint.encode(state, num)
+
+  t.exception(() => {
+    enc.decode(enc.lexint, state.buffer.subarray(0, state.buffer.byteLength - 2))
+  })
+
+  num <<= 8
+
+  state.start = 0
+  state.end = 0
+  state.buffer = null
+
+  enc.lexint.preencode(state, num)
+  state.buffer = Buffer.alloc(state.end - state.start)
+  enc.lexint.encode(state, num)
+
+  t.exception(() => {
+    enc.decode(enc.lexint, state.buffer.subarray(0, state.buffer.byteLength - 2))
+  })
+
+  num *= 256
+
+  state.start = 0
+  state.end = 0
+  state.buffer = null
+
+  enc.lexint.preencode(state, num)
+  state.buffer = Buffer.alloc(state.end - state.start)
+  enc.lexint.encode(state, num)
+
+  t.exception(() => {
+    enc.decode(enc.lexint, state.buffer.subarray(0, state.buffer.byteLength - 2))
+  })
+
+  num *= 256 * 256
+
+  state.start = 0
+  state.end = 0
+  state.buffer = null
+
+  enc.lexint.preencode(state, num)
+  state.buffer = Buffer.alloc(state.end - state.start)
+  enc.lexint.encode(state, num)
+
+  t.exception(() => {
+    enc.decode(enc.lexint, state.buffer.subarray(0, state.buffer.byteLength - 2))
+  })
+
+  t.end()
+})
+
+tape('lexint: unpack', function (t) {
+  let n
+  let skip = 1
+
+  for (n = 1; n < Number.MAX_VALUE; n += skip) {
+    const cur = enc.encode(enc.lexint, n)
+    compare(n, enc.decode(enc.lexint, cur))
+    skip = 1 + Math.pow(245, Math.ceil(Math.log(n) / Math.log(256)))
+  }
+  t.is(n, Infinity)
+  t.end()
+
+  function compare (a, b) {
+    const desc = a + ' !=~ ' + b
+    if (/e\+\d+$/.test(a) || /e\+\d+$/.test(b)) {
+      if (String(a).slice(0, 8) !== String(b).slice(0, 8) ||
+        /e\+(\d+)$/.exec(a)[1] !== /e\+(\d+)$/.exec(b)[1]) {
+        t.fail(desc)
+      }
+    } else {
+      if (String(a).slice(0, 8) !== String(b).slice(0, 8) ||
+       String(a).length !== String(b).length) {
+        t.fail(desc)
+      }
+    }
+  }
+})