cbor-x 0.8.2 → 0.9.2

package/package.json

@@ -1,10 +1,12 @@
 {
   "name": "cbor-x",
   "author": "Kris Zyp",
-  "version": "0.8.2",
+  "version": "0.9.2",
   "description": "Ultra-fast CBOR implementation with tag extensions for records and structured cloning",
   "license": "MIT",
   "types": "./index.d.ts",
+  "main": "./dist/node.cjs",
+  "module": "./index.js",
   "keywords": [
     "CBOR",
     "performance",
@@ -16,55 +18,53 @@
     "url": "http://github.com/kriszyp/cbor-x"
   },
   "scripts": {
-    "benchmark": "node ./tests/benchmark.js",
+    "benchmark": "node ./tests/benchmark.cjs",
     "build": "rollup -c",
+    "dry-run": "npm publish --dry-run",
     "prepare": "npm run build",
-    "test": "./node_modules/.bin/mocha --experimental-json-modules tests/test**.*js -u tdd"
+    "test": "mocha tests/test**.*js -u tdd --experimental-json-modules"
   },
   "type": "module",
-  "main": "./dist/node.cjs",
   "exports": {
     ".": {
       "node": {
         "require": "./dist/node.cjs",
-        "import": "./node.js"
+        "import": "./node-index.js"
       },
-      "default": {
-        "import": "./index.js"
-      }
+      "default": "./index.js"
     },
-    "./pack": {
+    "./encode": {
       "node": {
         "import": "./index.js",
         "require": "./dist/node.cjs"
       },
       "default": {
-        "import": "./pack.js"
+        "import": "./encode.js"
       }
     },
-    "./unpack": {
+    "./decode": {
       "node": {
         "import": "./index.js",
         "require": "./dist/node.cjs"
       },
       "default": {
-        "import": "./unpack.js"
+        "import": "./decode.js"
       }
     }
   },
   "optionalDependencies": {
-    "cbor-extract": "^0.3.0"
+    "cbor-extract": "^0.3.2"
   },
   "devDependencies": {
+    "@types/node": "latest",
     "async": "^3",
     "cbor": "^5",
     "cbor-sync": "^1.0.4",
-    "@types/node": "latest",
-    "chai": "^4",
+    "chai": "^4.3.4",
     "esm": "^3.2.25",
     "mocha": "^8.1.3",
     "rollup": "^1.20.3",
-    "rollup-plugin-babel-minify": "^9.0.0",
-    "@rollup/plugin-json": "^4.1.0"
+    "@rollup/plugin-json": "^4.1.0",
+    "rollup-plugin-babel-minify": "^9.0.0"
   }
 }

package/rollup.config.js

@@ -3,7 +3,7 @@ import json  from "@rollup/plugin-json";
 
 export default [
     {
-        input: "node.js",
+        input: "node-index.js",
         output: [
             {
                 file: "dist/node.cjs",
@@ -42,4 +42,4 @@ export default [
             },
         }
     }
-];
+];

package/stream.js

@@ -1,24 +1,20 @@
 import { Transform } from 'stream'
 import { Encoder } from './encode.js'
-import { read, getPosition, Decoder, clearSource } from './decode.js'
+import { checkedRead, getPosition, Decoder, clearSource } from './decode.js'
 var DEFAULT_OPTIONS = {objectMode: true}
 
 export class EncoderStream extends Transform {
 	constructor(options) {
 		if (!options)
 			options = {}
+		options.writableObjectMode = true
 		super(options)
 		options.sequential = true
-		this.encoder = new Encoder(options)
+		this.encoder = options.encoder || new Encoder(options)
 	}
-	write(value) {
+	_transform(value, encoding, callback) {
 		this.push(this.encoder.encode(value))
-	}
-
-	end(value) {
-		if (value != null)
-			this.push(this.encoder.encode(value))
-		this.push(null)
+		callback()
 	}
 }
 
@@ -29,31 +25,33 @@ export class DecoderStream extends Transform {
 		options.objectMode = true
 		super(options)
 		options.structures = []
-		this.decoder = new Decoder(options)
+		this.decoder = options.decoder || new Decoder(options)
 	}
 	_transform(chunk, encoding, callback) {
 		if (this.incompleteBuffer) {
 			chunk = Buffer.concat([this.incompleteBuffer, chunk])
 			this.incompleteBuffer = null
 		}
-		let position = 0
-		let size = chunk.length
+		let values
 		try {
-			this.push(this.decoder.decode(chunk, size, true))
-			position = getPosition()
-			while (position < size) {
-				let value = read()
-				this.push(value)
-				position = getPosition()
-			}
+			values = this.decoder.decodeMultiple(chunk)
 		} catch(error) {
-			if (error.incomplete)
-				this.incompleteBuffer = chunk.slice(position)
+			if (error.incomplete) {
+				this.incompleteBuffer = chunk.slice(error.lastPosition)
+				values = error.values
+			}
 			else
 				throw error
 		} finally {
-			clearSource()
+			for (let value of values || []) {
+				if (value === null)
+					value = this.getNullValue()
+				this.push(value)
+			}
 		}
 		if (callback) callback()
 	}
+	getNullValue() {
+		return Symbol.for(null)
+	}
 }

package/tests/benchmark.cjs

@@ -1,4 +1,4 @@
-var cborX = tryRequire("..");
+var cborX = tryRequire("../dist/node.cjs");
 var msgpack_node = tryRequire("msgpack");
 var msgpack_msgpack = tryRequire("@msgpack/msgpack");
 var msgpack_lite = tryRequire("msgpack-lite");
@@ -12,6 +12,12 @@ var what_the_pack = tryRequire("what-the-pack");
 var avro = tryRequire('avsc')
 var cbor = tryRequire('cbor')
 var cborSync = tryRequire('cbor-sync')
+var zlib = require('zlib')
+var deflateSync = zlib.deflateSync
+var inflateSync = zlib.inflateSync
+//var deflateSync = zlib.brotliCompressSync
+//var inflateSync = zlib.brotliDecompressSync
+var constants = zlib.constants
 
 msgpack5 = msgpack5 && msgpack5();
 msgpack_codec = msgpack_codec && msgpack_codec.msgpack;
@@ -43,25 +49,59 @@ console.log(rpad("", COL1, "-"), "|", lpad(":", COL2, "-"), "|", lpad(":", COL3,
 
 var buf, obj;
 
-if (JSON) {
-  buf = bench('buf = Buffer(JSON.stringify(obj));', JSON_stringify, data);
-  obj = bench('obj = JSON.parse(buf);', JSON.parse, buf);
+if (cborX) {
+  let encoder = new cborX.Encoder({ useRecords: false, pack: true })
+  buf = bench('cbor-x packed and compress: encoder.encode(obj);', (data) => deflateSync(encoder.encode(data), { level: constants.Z_BEST_SPEED}), data);
+  console.log('size', buf.length)
+  
+  buf = bench('cbor-x packed: encoder.encode(obj);', encoder.encode.bind(encoder), data);
+
+  obj = bench('cbor-x packed: encoder.decode(buf);', encoder.decode.bind(encoder), buf);
   test(obj);
-}
+  console.log('size', buf.length)
 
-if (cborX) {
-//  let packr = new msgpackr.Packr({ objectsAsMaps: true })
+
+  encoder = new cborX.Encoder({ useRecords: true, pack: true })
+  buf = bench('cbor-x packed and useRecords and compress: encoder.encode(obj);', (data) => deflateSync(encoder.encode(data), { level: constants.Z_BEST_SPEED}), data);
+  console.log('size', buf.length)
+  
+  buf = bench('cbor-x packed and useRecords: encoder.encode(obj);', encoder.encode.bind(encoder), data);
+
+  obj = bench('cbor-x packed and useRecords: encoder.decode(buf);', encoder.decode.bind(encoder), buf);
+  test(obj);
+  console.log('size', buf.length)
+
+  buf = bench('require("cbor-x").encode(obj) and compress;', (data) => deflateSync(cborX.encode(data)), data);
+  console.log('size', buf.length)
   buf = bench('require("cbor-x").encode(obj);', cborX.encode, data);
 //    buf = bench('require("msgpack").serialize(obj);', data => {let result = packr.serialize(data); packr.resetMemory(); return result;}, data);
 
   obj = bench('require("cbor-x").decode(buf);', cborX.decode, buf);
   test(obj);
+  console.log('size', buf.length)
+
 
-  encoder = new cborX.Encoder({ structures: [] })
+  encoder = new cborX.Encoder({  })
+  buf = bench('cbor-x w/ shared structures: encoder.encode(obj);', (data) => deflateSync(encoder.encode(data), { level: constants.Z_BEST_SPEED}), data);
+  console.log('size', buf.length)
+  
   buf = bench('cbor-x w/ shared structures: encoder.encode(obj);', encoder.encode.bind(encoder), data);
 
   obj = bench('cbor-x w/ shared structures: encoder.decode(buf);', encoder.decode.bind(encoder), buf);
   test(obj);
+  console.log('size', buf.length)
+}
+if (JSON) {
+  buf = bench('buf = Buffer(JSON.stringify(obj));', JSON_stringify, data);
+  obj = bench('obj = JSON.parse(buf);', JSON.parse, buf);
+  test(obj);
+}
+
+
+if (JSON) {
+  buf = bench('buf = Buffer(JSON.stringify(obj));', JSON_stringify, data);
+  obj = bench('obj = JSON.parse(buf);', JSON.parse, buf);
+  test(obj);
 }
 
 if (msgpack_lite) {

package/tests/test-compatibility.cjs

@@ -6,6 +6,7 @@ function tryRequire(module) {
 	try {
 		return require(module)
 	} catch(error) {
+		console.log(error)
 	}
 }
 //if (typeof chai === 'undefined') { chai = require('chai') }

package/tests/test-incomplete.js

@@ -0,0 +1,41 @@
+import { encode } from '../index.js'
+import { assert } from 'chai'
+import { Encoder } from '../encode.js'
+
+const tests = {
+  string: 'interesting string',
+  number: 12345,
+  buffer: Buffer.from('hello world'),
+  bigint: 12345678910n,
+  array: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+  'many-strings': [],
+  set: new Set('abcdefghijklmnopqrstuvwxyz'.split('')),
+  object: { a: 1, b: 2, c: 3, d: 4, e: 5, f: 6 }
+}
+for (let i = 0; i < 100; i++) {
+  tests['many-strings'].push('test-data-' + i)
+}
+
+suite('encode and decode tests with partial values', function () {
+  const encoder = new Encoder({ objectMode: true, structures: [] })
+
+  for (const [label, testData] of Object.entries(tests)) {
+    test(label, () => {
+      const encoded = encoder.encode(testData)
+      assert.isTrue(Buffer.isBuffer(encoded), 'encode returns a Buffer')
+      assert.deepStrictEqual(encoder.decode(encoded, encoded.length, true), testData, 'full buffer decodes well')
+      const firstHalf = encoded.slice(0, Math.floor(encoded.length / 2))
+      let value
+      try {
+        value = encoder.decode(firstHalf, firstHalf.length, true)
+      } catch (err) {
+        if (err.incomplete !== true) {
+          assert.fail(`Should throw an error with .incomplete set to true, instead threw error <${err}>`)
+        } else {
+          return; // victory! correct outcome!
+        }
+      }
+      assert.fail(`Should throw an error with .incomplete set to true, instead returned value ${JSON.stringify(value)}`)
+    })
+  }
+})

package/tests/test-node-iterators.js

@@ -0,0 +1,72 @@
+import { encodeIter, decodeIter } from '../index.js'
+import { decode } from '../index.js'
+import { assert } from 'chai'
+
+const tests = [
+  null,
+  false,
+  true,
+  'interesting string',
+  12345,
+  123456789n,
+  123.456,
+  Buffer.from('Hello World'),
+  new Set('abcdefghijklmnopqrstuvwxyz'.split(''))
+]
+
+suite('cbor-x iterators interface tests', function () {
+  test('sync encode iterator', () => {
+    const encodings = [...encodeIter(tests)]
+    const decodings = encodings.map(x => decode(x))
+    assert.deepStrictEqual(decodings, tests)
+  })
+
+  test('async encode iterator', async () => {
+    async function * generate () {
+      for (const test of tests) {
+        await new Promise((resolve, reject) => setImmediate(resolve))
+        yield test
+      }
+    }
+
+    const chunks = []
+    for await (const chunk of encodeIter(generate())) {
+      chunks.push(chunk)
+    }
+
+    const decodings = chunks.map(x => decode(x))
+    assert.deepStrictEqual(decodings, tests)
+  })
+
+  test('sync encode and decode iterator', () => {
+    const encodings = [...encodeIter(tests)]
+    assert.isTrue(encodings.every(v => Buffer.isBuffer(v)))
+    const decodings = [...decodeIter(encodings)]
+    assert.deepStrictEqual(decodings, tests)
+
+    // also test decodings work with buffers multiple values in a buffer
+    const concatEncoding = Buffer.concat([...encodings])
+    const decodings2 = [...decodeIter([concatEncoding])]
+    assert.deepStrictEqual(decodings2, tests)
+
+    // also test decodings work with partial buffers that don't align to values perfectly
+    const half1 = concatEncoding.slice(0, Math.floor(concatEncoding.length / 2))
+    const half2 = concatEncoding.slice(Math.floor(concatEncoding.length / 2))
+    const decodings3 = [...decodeIter([half1, half2])]
+    assert.deepStrictEqual(decodings3, tests)
+  })
+
+  test('async encode and decode iterator', async () => {
+    async function * generator () {
+      for (const obj of tests) {
+        await new Promise((resolve, reject) => setImmediate(resolve))
+        yield obj
+      }
+    }
+    const yields = []
+    for await (const value of decodeIter(encodeIter(generator()))) {
+      yields.push(value)
+    }
+    assert.deepStrictEqual(yields, tests)
+  })
+})

package/tests/test-node-stream.js

@@ -1,6 +1,9 @@
-import { EncoderStream, DecoderStream } from '../node.js'
+import { EncoderStream, DecoderStream } from '../node-index.js'
 import stream from 'stream'
 import chai from 'chai'
+import util from 'util'
+import fs from 'fs'
+const finished = util.promisify(stream.finished)
 var assert = chai.assert
 
 suite('cbor-x node stream tests', function(){
@@ -47,4 +50,11 @@ suite('cbor-x node stream tests', function(){
 		bufferStream.push(new Uint8Array([1, 2]))
 		bufferStream.push(null)
 	}))
+
+	teardown(function() {
+		try {
+			fs.unlinkSync('test-output.msgpack')
+		}catch(error){}
+	})
 })
+

package/tests/test.js

@@ -1,8 +1,7 @@
 import * as CBOR from '../index.js'
 import chai from 'chai'
-//import('./test.mjs')
 import sampleData from './example4.json'
-//import inspector from 'inspector'; inspector.open(9330, null, true); debugger
+//import inspector from 'inspector'; inspector.open(9229, null, true); debugger
 function tryRequire(module) {
 	try {
 		return require(module)
@@ -85,6 +84,13 @@ suite('CBOR basic tests', function(){
 		assert.deepEqual(deserialized, data)
 	})
 
+	test('255 chars', function() {
+		const data = 'RRZG9A6I7xupPeOZhxcOcioFsuhszGOdyDUcbRf4Zef2kdPIfC9RaLO4jTM5JhuZvTsF09fbRHMGtqk7YAgu3vespeTe9l61ziZ6VrMnYu2CamK96wCkmz0VUXyqaiUoTPgzk414LS9yYrd5uh7w18ksJF5SlC2e91rukWvNqAZJjYN3jpkqHNOFchCwFrhbxq2Lrv1kSJPYCx9blRg2hGmYqTbElLTZHv20iNqwZeQbRMgSBPT6vnbCBPnOh1W'
+		var serialized = CBOR.encode(data)
+		var deserialized = CBOR.decode(serialized)
+		assert.equal(deserialized, data)
+	})
+
 	test('encode/decode sample data', function(){
 		var data = sampleData
 		var serialized = CBOR.encode(data)
@@ -102,6 +108,55 @@ suite('CBOR basic tests', function(){
 		var deserialized = encoder.decode(serialized)
 		assert.deepEqual(deserialized, data)
 	})
+	test('encode/decode sample data with packing', function(){
+		var data = sampleData
+		let encoder = new Encoder({ pack: true, useRecords: false })
+		var serialized = encoder.encode(data)
+		var deserialized = encoder.decode(serialized)
+		assert.deepEqual(deserialized, data)
+	})
+	test('encode/decode sample data with packing and records', function(){
+		var data = sampleData
+		let structures = []
+		let encoder = new Encoder({ structures, pack: true, useRecords: true })
+		var serialized = encoder.encode(data)
+		var deserialized = encoder.decode(serialized)
+		assert.deepEqual(deserialized, data)
+	})
+	test('encode/decode sample data with shared packing and records', function(){
+		let encoder = new Encoder({ useRecords: true })
+		let finishPack = encoder.findCommonStringsToPack()
+		for (let i = 0; i < 20; i++) {
+			let data = {
+				shouldShare: 'same each time',
+				shouldShare2: 'same each time 2',
+				shouldntShare: 'different each time ' + i
+			}
+			if (i == 10)
+				finishPack({})
+			var serialized = encoder.encode(data)
+			var deserialized = encoder.decode(serialized)
+			assert.deepEqual(deserialized, data)
+		}
+	})
+	test('encode/decode sample data with individual packing, shared packing and records', function(){
+		let encoder = new Encoder({ pack: true, useRecords: true })
+		let finishPack = encoder.findCommonStringsToPack()
+		for (let i = 0; i < 20; i++) {
+			let data = {
+				shouldShare: 'same each time',
+				shouldShare2: 'same each time',
+				shouldntShare: 'different each time ' + i,
+				shouldntShare2: 'different each time ' + i,
+				noPack: 'no packing ' + i,
+			}
+			if (i == 10)
+				finishPack({ threshold: 5 })
+			var serialized = encoder.encode(data)
+			var deserialized = encoder.decode(serialized)
+			assert.deepEqual(deserialized, data)
+		}
+	})
 	if (typeof Buffer != 'undefined')
 	test('replace data', function(){
 		var data1 = {
@@ -215,10 +270,13 @@ suite('CBOR basic tests', function(){
 		let fa = new Float32Array(b.buffer, 8, 2)
 		fa[0] = 2.25
 		fa[1] = 6
+		let map = new Map()
+		map.set('key', 'value')
 		let object = {
 			error: new Error('test'),
 			set: new Set(['a', 'b']),
 			regexp: /test/gi,
+			map,
 			float32Array: fa,
 			uint16Array: new Uint16Array([3,4])
 		}
@@ -228,6 +286,7 @@ suite('CBOR basic tests', function(){
 		var serialized = encoder.encode(object)
 		var deserialized = encoder.decode(serialized)
 		assert.deepEqual(Array.from(deserialized.set), Array.from(object.set))
+		assert.equal(deserialized.map.get('key'), 'value')
 		assert.equal(deserialized.error.message, object.error.message)
 		assert.equal(deserialized.regexp.test('TEST'), true)
 		assert.equal(deserialized.float32Array.constructor.name, 'Float32Array')
@@ -238,6 +297,19 @@ suite('CBOR basic tests', function(){
 		assert.equal(deserialized.uint16Array[1], 4)
 	})
 
+	test('explicit maps and sets', function () {
+		let map = new Map()
+		map.set('key', { inside: 'value'})
+		let object = {
+			set: new Set(['a', 'b']),
+			map,
+		}
+		var serialized = encode(object) // default encoder
+		var deserialized = decode(serialized)
+		assert.deepEqual(Array.from(deserialized.set), Array.from(object.set))
+		assert.equal(deserialized.map.get('key').inside, 'value')
+	})
+
 	test('object without prototype', function(){
 		var data = Object.create(null)
 		data.test = 3
@@ -253,6 +325,13 @@ suite('CBOR basic tests', function(){
 		var decoded = decode(encoded)
 		assert.equal(decoded.length, size)
 	})
+	test('little buffer', function() {
+		var data = typeof Buffer == 'undefined' ? new Uint8Array(0) : Buffer.alloc(0)
+		var encoded = encode(data)
+		assert.equal(encoded.length, 1) // make sure to use canonical form
+		var decoded = decode(encoded)
+		assert.equal(decoded.length, 0)
+	})
 
 	test('random strings', function(){
 		var data = []
@@ -300,6 +379,7 @@ suite('CBOR basic tests', function(){
 		let encoder = new Encoder({
 			mapsAsObjects: true,
 			useTimestamp32: true,
+			useTag259ForMaps: false,
 		})
 		var serialized = encoder.encode(data)
 		var deserialized = encoder.decode(serialized)

package/pack.d.ts

@@ -1,9 +0,0 @@
-import { Unpackr } from './unpack'
-export { addExtension, FLOAT32_OPTIONS } from './unpack'
-export class Packr extends Unpackr {
-	pack(value: any): Buffer
-	encode(value: any): Buffer
-}
-export class Encoder extends Packr {}
-export function pack(value: any): Buffer
-export function encode(value: any): Buffer