compact-encoding 2.10.0 → 2.11.0

package/README.md

@@ -75,6 +75,7 @@ const bool = cenc.decode(cenc.bool, buf)
 The following encodings are bundled as they are primitives that can be used
 to build others on top. Feel free to PR more that are missing.
 
+* `cenc.raw` - Pass through encodes a buffer, i.e. a basic copy.
 * `cenc.uint` - Encodes a uint using [compact-uint](https://github.com/mafintosh/compact-uint).
 * `cenc.uint8` - Encodes a fixed size uint8.
 * `cenc.uint16` - Encodes a fixed size uint16. Useful for things like ports.
@@ -97,32 +98,48 @@ to build others on top. Feel free to PR more that are missing.
 * `cenc.float32` - Encodes a fixed size float32.
 * `cenc.float64` - Encodes a fixed size float64.
 * `cenc.buffer` - Encodes a buffer with its length uint prefixed. When decoding an empty buffer, `null` is returned.
-* `cenc.raw` - Pass through encodes a buffer, i.e. a basic copy.
+* `cenc.raw.buffer` - Encodes a buffer without a length prefixed.
 * `cenc.uint8array` - Encodes a uint8array with its element length uint prefixed.
+* `cenc.raw.uint8array` - Encodes a uint8array without a length prefixed.
 * `cenc.uint16array` - Encodes a uint16array with its element length uint prefixed.
+* `cenc.raw.uint16array` - Encodes a uint16array without a length prefixed.
 * `cenc.uint32array` - Encodes a uint32array with its element length uint prefixed.
+* `cenc.raw.uint32array` - Encodes a uint32array without a length prefixed.
 * `cenc.int8array` - Encodes a int8array with its element length uint prefixed.
+* `cenc.raw.int8array` - Encodes a int8array without a length prefixed.
 * `cenc.int16array` - Encodes a int16array with its element length uint prefixed.
+* `cenc.raw.int16array` - Encodes a int16array without a length prefixed.
 * `cenc.int32array` - Encodes a int32array with its element length uint prefixed.
+* `cenc.raw.int32array` - Encodes a int32array without a length prefixed.
 * `cenc.float32array` - Encodes a float32array with its element length uint prefixed.
+* `cenc.raw.float32array` - Encodes a float32array without a length prefixed.
 * `cenc.float64array` - Encodes a float64array with its element length uint prefixed.
+* `cenc.raw.float64array` - Encodes a float64array without a length prefixed.
 * `cenc.bool` - Encodes a boolean as 1 or 0.
 * `cenc.string`, `cenc.utf8` - Encodes a utf-8 string, similar to buffer.
+* `cenc.raw.string`, `cenc.raw.utf8` - Encodes a utf-8 string without a length prefixed.
 * `cenc.string.fixed(n)`, `cenc.utf8.fixed(n)` - Encodes a fixed sized utf-8 string.
 * `cenc.ascii` - Encodes an ascii string.
+* `cenc.raw.ascii` - Encodes an ascii string without a length prefixed.
 * `cenc.ascii.fixed(n)` - Encodes a fixed size ascii string.
 * `cenc.hex` - Encodes a hex string.
+* `cenc.raw.hex` - Encodes a hex string without a length prefixed.
 * `cenc.hex.fixed(n)` - Encodes a fixed size hex string.
 * `cenc.base64` - Encodes a base64 string.
+* `cenc.raw.base64` - Encodes a base64 string without a length prefixed.
 * `cenc.base64.fixed(n)` - Encodes a fixed size base64 string.
 * `cenc.utf16le`, `cenc.ucs2` - Encodes a utf16le string.
+* `cenc.raw.utf16le`, `cenc.raw.ucs2` - Encodes a utf16le string without a length prefixed.
 * `cenc.utf16le.fixed(n)`, `cenc.ucs2.fixed(n)` - Encodes a fixed size utf16le string.
 * `cenc.fixed32` - Encodes a fixed 32 byte buffer.
 * `cenc.fixed64` - Encodes a fixed 64 byte buffer.
 * `cenc.fixed(n)` - Makes a fixed sized encoder.
 * `cenc.array(enc)` - Makes an array encoder from another encoder. Arrays are uint prefixed with their length.
+* `cenc.raw.array(enc)` - Makes an array encoder from another encoder, without a length prefixed.
 * `cenc.json` - Encodes a JSON value as utf-8.
+* `cenc.raw.json` - Encodes a JSON value as utf-8 without a length prefixed.
 * `cenc.ndjson` - Encodes a JSON value as newline delimited utf-8.
+* `cenc.raw.ndjson` - Encodes a JSON value as newline delimited utf-8 without a length prefixed.
 * `cenc.from(enc)` - Makes a compact encoder from a [codec](https://github.com/mafintosh/codecs) or [abstract-encoding](https://github.com/mafintosh/abstract-encoding).
 
 ## License

package/endian.js

@@ -0,0 +1,3 @@
+const LE = exports.LE = (new Uint8Array(new Uint16Array([0xff]).buffer))[0] === 0xff
+
+exports.BE = !LE

package/index.js

@@ -1,12 +1,13 @@
 const b4a = require('b4a')
 
-const LE = (new Uint8Array(new Uint16Array([0xff]).buffer))[0] === 0xff
-const BE = !LE
+const { BE } = require('./endian')
 
 exports.state = function (start = 0, end = 0, buffer = null) {
   return { start, end, buffer, cache: null }
 }
 
+const raw = exports.raw = require('./raw')
+
 const uint = exports.uint = {
   preencode (state, n) {
     state.end += n <= 0xfc ? 1 : n <= 0xffff ? 3 : n <= 0xffffffff ? 5 : 9
@@ -211,7 +212,7 @@ exports.float64 = {
   }
 }
 
-exports.buffer = {
+const buffer = exports.buffer = {
   preencode (state, b) {
     if (b) uint8array.preencode(state, b)
     else state.end++
@@ -228,18 +229,15 @@ exports.buffer = {
   }
 }
 
-const raw = exports.raw = {
+exports.binary = {
+  ...buffer,
   preencode (state, b) {
-    state.end += b.byteLength
+    if (typeof b === 'string') utf8.preencode(state, b)
+    else buffer.preencode(state, b)
   },
   encode (state, b) {
-    state.buffer.set(b, state.start)
-    state.start += b.byteLength
-  },
-  decode (state) {
-    const b = state.buffer.subarray(state.start, state.end)
-    state.start = state.end
-    return b
+    if (typeof b === 'string') utf8.encode(state, b)
+    else buffer.encode(state, b)
   }
 }
 
@@ -417,11 +415,30 @@ exports.ndjson = {
 }
 
 exports.from = function from (enc) {
+  if (typeof enc === 'string') return fromNamed(enc)
   if (enc.preencode) return enc
   if (enc.encodingLength) return fromAbstractEncoder(enc)
   return fromCodec(enc)
 }
 
+function fromNamed (enc) {
+  switch (enc) {
+    case 'ascii': return raw.ascii
+    case 'utf-8':
+    case 'utf8': return raw.utf8
+    case 'hex': return raw.hex
+    case 'base64': return raw.base64
+    case 'utf16-le':
+    case 'utf16le':
+    case 'ucs-2':
+    case 'ucs2': return raw.ucs2
+    case 'ndjson': return raw.ndjson
+    case 'json': return raw.json
+    case 'binary':
+    default: return raw.binary
+  }
+}
+
 function fromCodec (enc) {
   let tmpM = null
   let tmpBuf = null

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "compact-encoding",
-  "version": "2.10.0",
+  "version": "2.11.0",
   "description": "A series of compact encoding schemes for building small and fast parsers and serializers",
   "main": "index.js",
   "dependencies": {

package/raw.js

@@ -0,0 +1,148 @@
+const b4a = require('b4a')
+
+const { BE } = require('./endian')
+
+exports = module.exports = {
+  preencode (state, b) {
+    state.end += b.byteLength
+  },
+  encode (state, b) {
+    state.buffer.set(b, state.start)
+    state.start += b.byteLength
+  },
+  decode (state) {
+    const b = state.buffer.subarray(state.start, state.end)
+    state.start = state.end
+    return b
+  }
+}
+
+const buffer = exports.buffer = {
+  preencode (state, b) {
+    if (b) uint8array.preencode(state, b)
+    else state.end++
+  },
+  encode (state, b) {
+    if (b) uint8array.encode(state, b)
+    else state.buffer[state.start++] = 0
+  },
+  decode (state) {
+    const b = state.buffer.subarray(state.start)
+    if (b.byteLength === 0) return null
+    state.start = state.end
+    return b
+  }
+}
+
+exports.binary = {
+  ...buffer,
+  preencode (state, b) {
+    if (typeof b === 'string') utf8.preencode(state, b)
+    else buffer.preencode(state, b)
+  },
+  encode (state, b) {
+    if (typeof b === 'string') utf8.encode(state, b)
+    else buffer.encode(state, b)
+  }
+}
+
+function typedarray (TypedArray, swap) {
+  const n = TypedArray.BYTES_PER_ELEMENT
+
+  return {
+    preencode (state, b) {
+      state.end += b.byteLength
+    },
+    encode (state, b) {
+      const view = new Uint8Array(b.buffer, b.byteOffset, b.byteLength)
+
+      if (BE && swap) swap(view)
+
+      state.buffer.set(view, state.start)
+      state.start += b.byteLength
+    },
+    decode (state) {
+      let b = state.buffer.subarray(state.start)
+      if ((b.byteOffset % n) !== 0) b = new Uint8Array(b)
+
+      if (BE && swap) swap(b)
+
+      state.start = state.end
+
+      return new TypedArray(b.buffer, b.byteOffset, b.byteLength / n)
+    }
+  }
+}
+
+const uint8array = exports.uint8array = typedarray(Uint8Array)
+exports.uint16array = typedarray(Uint16Array, b4a.swap16)
+exports.uint32array = typedarray(Uint32Array, b4a.swap32)
+
+exports.int8array = typedarray(Int8Array)
+exports.int16array = typedarray(Int16Array, b4a.swap16)
+exports.int32array = typedarray(Int32Array, b4a.swap32)
+
+exports.float32array = typedarray(Float32Array, b4a.swap32)
+exports.float64array = typedarray(Float64Array, b4a.swap64)
+
+function string (encoding) {
+  return {
+    preencode (state, s) {
+      state.end += b4a.byteLength(s, encoding)
+    },
+    encode (state, s) {
+      state.start += b4a.write(state.buffer, s, state.start, encoding)
+    },
+    decode (state) {
+      const s = b4a.toString(state.buffer, encoding, state.start)
+      state.start = state.end
+      return s
+    }
+  }
+}
+
+const utf8 = exports.string = exports.utf8 = string('utf-8')
+exports.ascii = string('ascii')
+exports.hex = string('hex')
+exports.base64 = string('base64')
+exports.ucs2 = exports.utf16le = string('utf16le')
+
+exports.array = function array (enc) {
+  return {
+    preencode (state, list) {
+      for (const value of list) enc.preencode(state, value)
+    },
+    encode (state, list) {
+      for (const value of list) enc.encode(state, value)
+    },
+    decode (state) {
+      const arr = []
+      while (state.start < state.end) arr.push(enc.decode(state))
+      return arr
+    }
+  }
+}
+
+exports.json = {
+  preencode (state, v) {
+    utf8.preencode(state, JSON.stringify(v))
+  },
+  encode (state, v) {
+    utf8.encode(state, JSON.stringify(v))
+  },
+  decode (state) {
+    return JSON.parse(utf8.decode(state))
+  }
+}
+
+exports.ndjson = {
+  preencode (state, v) {
+    utf8.preencode(state, JSON.stringify(v) + '\n')
+  },
+  encode (state, v) {
+    utf8.encode(state, JSON.stringify(v) + '\n')
+  },
+  decode (state) {
+    return JSON.parse(utf8.decode(state))
+  }
+}

package/test.js

@@ -1,8 +1,8 @@
 const enc = require('./')
-const tape = require('brittle')
+const test = require('brittle')
 const b4a = require('b4a')
 
-tape('uint', function (t) {
+test('uint', function (t) {
   const state = enc.state()
 
   enc.uint.preencode(state, 42)
@@ -29,7 +29,7 @@ tape('uint', function (t) {
   t.exception(() => enc.uint.decode(state))
 })
 
-tape('int', function (t) {
+test('int', function (t) {
   const state = enc.state()
 
   enc.int.preencode(state, 42)
@@ -51,7 +51,7 @@ tape('int', function (t) {
   t.exception(() => enc.int.decode(state))
 })
 
-tape('float64', function (t) {
+test('float64', function (t) {
   const state = enc.state()
 
   enc.float64.preencode(state, 162.2377294)
@@ -146,7 +146,7 @@ tape('float64', function (t) {
   t.is(state.start, state.end)
 })
 
-tape('buffer', function (t) {
+test('buffer', function (t) {
   const state = enc.state()
 
   enc.buffer.preencode(state, b4a.from('hi'))
@@ -173,7 +173,7 @@ tape('buffer', function (t) {
   t.exception(() => enc.buffer.decode(state))
 })
 
-tape('raw', function (t) {
+test('raw', function (t) {
   const state = enc.state()
 
   enc.raw.preencode(state, b4a.from('hi'))
@@ -188,7 +188,26 @@ tape('raw', function (t) {
   t.is(state.start, state.end)
 })
 
-tape('uint16array', function (t) {
+test('raw uint8array', function (t) {
+  const state = enc.state()
+
+  enc.raw.uint8array.preencode(state, Uint8Array.of(1, 2))
+  t.alike(state, enc.state(0, 2))
+  enc.raw.uint8array.preencode(state, Uint8Array.of(3, 4))
+  t.alike(state, enc.state(0, 4))
+
+  state.buffer = b4a.alloc(state.end)
+  enc.raw.uint8array.encode(state, Uint8Array.of(1, 2))
+  t.alike(state, enc.state(2, 4, b4a.from([1, 2, 0, 0])))
+  enc.raw.uint8array.encode(state, Uint8Array.of(3, 4))
+  t.alike(state, enc.state(4, 4, b4a.from([1, 2, 3, 4])))
+
+  state.start = 0
+  t.alike(enc.raw.uint8array.decode(state), Uint8Array.of(1, 2, 3, 4))
+  t.alike(enc.raw.uint8array.decode(state), Uint8Array.of())
+})
+
+test('uint16array', function (t) {
   const state = enc.state()
 
   enc.uint16array.preencode(state, new Uint16Array([1, 2, 3]))
@@ -205,7 +224,7 @@ tape('uint16array', function (t) {
   t.exception(() => enc.uint16array.decode(state))
 })
 
-tape('uint32array', function (t) {
+test('uint32array', function (t) {
   const state = enc.state()
 
   enc.uint32array.preencode(state, new Uint32Array([1]))
@@ -227,7 +246,7 @@ tape('uint32array', function (t) {
   t.exception(() => enc.uint32array.decode(state))
 })
 
-tape('int16array', function (t) {
+test('int16array', function (t) {
   const state = enc.state()
 
   enc.int16array.preencode(state, new Int16Array([1, -2, 3]))
@@ -244,7 +263,7 @@ tape('int16array', function (t) {
   t.exception(() => enc.int16array.decode(state))
 })
 
-tape('int32array', function (t) {
+test('int32array', function (t) {
   const state = enc.state()
 
   enc.int32array.preencode(state, new Int32Array([1, -2, 3]))
@@ -261,7 +280,7 @@ tape('int32array', function (t) {
   t.exception(() => enc.int32array.decode(state))
 })
 
-tape('float32array', function (t) {
+test('float32array', function (t) {
   const state = enc.state()
 
   enc.float32array.preencode(state, new Float32Array([1.1, -2.2, 3.3]))
@@ -278,7 +297,7 @@ tape('float32array', function (t) {
   t.exception(() => enc.float32array.decode(state))
 })
 
-tape('float64array', function (t) {
+test('float64array', function (t) {
   const state = enc.state()
 
   enc.float64array.preencode(state, new Float64Array([1.1, -2.2, 3.3]))
@@ -295,7 +314,7 @@ tape('float64array', function (t) {
   t.exception(() => enc.float64array.decode(state))
 })
 
-tape('string', function (t) {
+test('string', function (t) {
   const state = enc.state()
 
   enc.string.preencode(state, '🌾')
@@ -317,7 +336,25 @@ tape('string', function (t) {
   t.exception(() => enc.string.decode(state))
 })
 
-tape('fixed32', function (t) {
+test('raw string', function (t) {
+  const state = enc.state()
+
+  enc.raw.string.preencode(state, 'hello')
+  t.alike(state, enc.state(0, 5))
+  enc.raw.string.preencode(state, ' world')
+  t.alike(state, enc.state(0, 11))
+
+  state.buffer = b4a.alloc(state.end)
+  enc.raw.string.encode(state, 'hello')
+  enc.raw.string.encode(state, ' world')
+  t.alike(state, enc.state(11, 11, b4a.from('hello world')))
+
+  state.start = 0
+  t.is(enc.raw.string.decode(state), 'hello world')
+  t.is(enc.raw.string.decode(state), '')
+})
+
+test('fixed32', function (t) {
   const state = enc.state()
 
   enc.fixed32.preencode(state, b4a.alloc(32).fill('a'))
@@ -339,7 +376,7 @@ tape('fixed32', function (t) {
   t.exception(() => enc.fixed32.decode(state))
 })
 
-tape('fixed64', function (t) {
+test('fixed64', function (t) {
   const state = enc.state()
 
   enc.fixed64.preencode(state, b4a.alloc(64).fill('a'))
@@ -361,7 +398,7 @@ tape('fixed64', function (t) {
   t.exception(() => enc.fixed64.decode(state))
 })
 
-tape('fixed n', function (t) {
+test('fixed n', function (t) {
   const state = enc.state()
   const fixed = enc.fixed(3)
 
@@ -386,7 +423,7 @@ tape('fixed n', function (t) {
   t.exception(() => fixed.decode(state))
 })
 
-tape('array', function (t) {
+test('array', function (t) {
   const state = enc.state()
   const arr = enc.array(enc.bool)
 
@@ -409,7 +446,27 @@ tape('array', function (t) {
   t.exception(() => arr.decode(state))
 })
 
-tape('json', function (t) {
+test('raw array', function (t) {
+  const state = enc.state()
+  const arr = enc.raw.array(enc.bool)
+
+  arr.preencode(state, [true])
+  t.alike(state, enc.state(0, 1))
+  arr.preencode(state, [true])
+  t.alike(state, enc.state(0, 2))
+
+  state.buffer = b4a.alloc(state.end)
+  arr.encode(state, [true])
+  t.alike(state, enc.state(1, 2, b4a.from([1, 0])))
+  arr.encode(state, [true])
+  t.alike(state, enc.state(2, 2, b4a.from([1, 1])))
+
+  state.start = 0
+  t.alike(arr.decode(state), [true, true])
+  t.alike(arr.decode(state), [])
+})
+
+test('json', function (t) {
   const state = enc.state()
 
   enc.json.preencode(state, { a: 1, b: 2 })
@@ -428,7 +485,23 @@ tape('json', function (t) {
   t.exception(() => enc.json.decode(state))
 })
 
-tape('lexint: big numbers', function (t) {
+test('raw json', function (t) {
+  const state = enc.state()
+
+  enc.raw.json.preencode(state, { a: 1, b: 2 })
+  t.alike(state, enc.state(0, 13))
+
+  state.buffer = b4a.alloc(state.end)
+  enc.raw.json.encode(state, { a: 1, b: 2 })
+  t.alike(state, enc.state(13, 13, b4a.from('{"a":1,"b":2}')))
+
+  state.start = 0
+  t.alike(enc.raw.json.decode(state), { a: 1, b: 2 })
+
+  t.exception(() => enc.json.decode(state))
+})
+
+test('lexint: big numbers', function (t) {
   t.plan(1)
 
   let prev = enc.encode(enc.lexint, 0)
@@ -445,7 +518,7 @@ tape('lexint: big numbers', function (t) {
   t.is(n, Infinity)
 })
 
-tape('lexint: range precision', function (t) {
+test('lexint: range precision', function (t) {
   t.plan(2)
   const a = 1e55
   const b = 1.0000000000001e55
@@ -455,7 +528,7 @@ tape('lexint: range precision', function (t) {
   t.not(ha, hb)
 })
 
-tape('lexint: range precision', function (t) {
+test('lexint: range precision', function (t) {
   let prev = enc.encode(enc.lexint, 0)
   const skip = 0.000000001e55
   for (let i = 0, n = 1e55; i < 1000; n = 1e55 + skip * ++i) {
@@ -467,7 +540,7 @@ tape('lexint: range precision', function (t) {
   t.end()
 })
 
-tape('lexint: small numbers', function (t) {
+test('lexint: small numbers', function (t) {
   let prev = enc.encode(enc.lexint, 0)
   for (let n = 1; n < 256 * 256 * 16; n++) {
     const cur = enc.encode(enc.lexint, n)
@@ -478,7 +551,7 @@ tape('lexint: small numbers', function (t) {
   t.end()
 })
 
-tape('lexint: throws', function (t) {
+test('lexint: throws', function (t) {
   t.exception(() => {
     enc.decode(enc.lexint, b4a.alloc(1, 251))
   })
@@ -554,7 +627,7 @@ tape('lexint: throws', function (t) {
   t.end()
 })
 
-tape('lexint: unpack', function (t) {
+test('lexint: unpack', function (t) {
   let n
   let skip = 1