msgpackr 1.4.6 → 1.5.2

package/unpack.js

@@ -15,6 +15,7 @@ var currentStructures
 var srcString
 var srcStringStart = 0
 var srcStringEnd = 0
+var bundledStrings
 var referenceMap
 var currentExtensions = []
 var dataView
@@ -55,12 +56,21 @@ export class Unpackr {
 		srcStringEnd = 0
 		srcString = null
 		strings = EMPTY_ARRAY
+		bundledStrings = null
 		src = source
 		// this provides cached access to the data view for a buffer if it is getting reused, which is a recommend
 		// technique for getting data from a database where it can be copied into an existing buffer instead of creating
 		// new ones
-		dataView = source.dataView || (source.dataView = new DataView(source.buffer, source.byteOffset, source.byteLength))
-		if (this) {
+		try {
+			dataView = source.dataView || (source.dataView = new DataView(source.buffer, source.byteOffset, source.byteLength))
+		} catch(error) {
+			// if it doesn't have a buffer, maybe it is the wrong type of object
+			src = null
+			if (source instanceof Uint8Array)
+				throw error
+			throw new Error('Source must be a Uint8Array or Buffer but was a ' + ((source && typeof source == 'object') ? source.constructor.name : typeof source))
+		}
+		if (this instanceof Unpackr) {
 			currentUnpackr = this
 			if (this.structures) {
 				currentStructures = this.structures
@@ -240,7 +250,15 @@ export function read() {
 		let value
 		switch (token) {
 			case 0xc0: return null
-			case 0xc1: return C1; // "never-used", return special object to denote that
+			case 0xc1:
+				if (bundledStrings) {
+					value = read() // followed by the length of the string in characters (not bytes!)
+					if (value > 0)
+						return bundledStrings[1].slice(bundledStrings.position1, bundledStrings.position1 += value)
+					else
+						return bundledStrings[0].slice(bundledStrings.position0, bundledStrings.position0 -= value)
+				}
+				return C1; // "never-used", return special object to denote that
 			case 0xc2: return false
 			case 0xc3: return true
 			case 0xc4:
@@ -295,10 +313,11 @@ export function read() {
 				position += 4
 				return value
 			case 0xcf:
-				if (currentUnpackr.uint64AsNumber)
-					return src[position++] * 0x100000000000000 + src[position++] * 0x1000000000000 + src[position++] * 0x10000000000 + src[position++] * 0x100000000 +
-						src[position++] * 0x1000000 + (src[position++] << 16) + (src[position++] << 8) + src[position++]
-				value = dataView.getBigUint64(position)
+				if (currentUnpackr.int64AsNumber) {
+					value = dataView.getUint32(position) * 0x100000000
+					value += dataView.getUint32(position + 4)
+				} else
+					value = dataView.getBigUint64(position)
 				position += 8
 				return value
 
@@ -314,7 +333,11 @@ export function read() {
 				position += 4
 				return value
 			case 0xd3:
-				value = dataView.getBigInt64(position)
+				if (currentUnpackr.int64AsNumber) {
+					value = dataView.getInt32(position) * 0x100000000
+					value += dataView.getUint32(position + 4)
+				} else
+					value = dataView.getBigInt64(position)
 				position += 8
 				return value
 
@@ -463,8 +486,10 @@ var readFixedString = readStringJS
 var readString8 = readStringJS
 var readString16 = readStringJS
 var readString32 = readStringJS
+export let isNativeAccelerationEnabled = false
 
 export function setExtractor(extractStrings) {
+	isNativeAccelerationEnabled = true
 	readFixedString = readString(1)
 	readString8 = readString(2)
 	readString16 = readString(3)
@@ -883,6 +908,22 @@ currentExtensions[0x78] = () => {
 	return new RegExp(data[0], data[1])
 }
 
+currentExtensions[0x62] = (data) => {
+	let dataSize = (data[0] << 24) + (data[1] << 16) + (data[2] << 8) + data[3]
+	let dataPosition = position
+	position += dataSize - 4
+	bundledStrings = [read(), read()]
+	bundledStrings.position0 = 0
+	bundledStrings.position1 = 0
+	let postBundlePosition = position
+	position = dataPosition
+	try {
+		return read()
+	} finally {
+		position = postBundlePosition
+	}
+}
+
 currentExtensions[0xff] = (data) => {
 	// 32-bit date extension
 	if (data.length == 4)
@@ -896,7 +937,7 @@ currentExtensions[0xff] = (data) => {
 			((data[0] << 24) + (data[1] << 16) + (data[2] << 8) + data[3]) / 1000000 +
 			(((data[4] & 0x80) ? -0x1000000000000 : 0) + data[6] * 0x10000000000 + data[7] * 0x100000000 + data[8] * 0x1000000 + (data[9] << 16) + (data[10] << 8) + data[11]) * 1000)
 	else
-		throw new Error('Invalid timestamp length')
+		return new Date('invalid')
 } // notepack defines extension 0 to mean undefined, so use that as the default here
 // registration of bulk record definition?
 // currentExtensions[0x52] = () =>
@@ -910,6 +951,7 @@ function saveState(callback) {
 	let savedSrcString = srcString
 	let savedStrings = strings
 	let savedReferenceMap = referenceMap
+	let savedBundledStrings = bundledStrings
 
 	// TODO: We may need to revisit this if we do more external calls to user code (since it could be slow)
 	let savedSrc = new Uint8Array(src.slice(0, srcEnd)) // we copy the data in case it changes while external data is processed
@@ -926,6 +968,7 @@ function saveState(callback) {
 	srcString = savedSrcString
 	strings = savedStrings
 	referenceMap = savedReferenceMap
+	bundledStrings = savedBundledStrings
 	src = savedSrc
 	sequentialMode = savedSequentialMode
 	currentStructures = savedStructures
@@ -951,8 +994,6 @@ export const mult10 = new Array(147) // this is a table matching binary exponent
 for (let i = 0; i < 256; i++) {
 	mult10[i] = +('1e' + Math.floor(45.15 - i * 0.30103))
 }
-export const useRecords = false
-export const mapsAsObjects = true
 export const Decoder = Unpackr
 var defaultUnpackr = new Unpackr({ useRecords: false })
 export const unpack = defaultUnpackr.unpack

package/.vscode/launch.json

@@ -1,23 +0,0 @@
-{
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "type": "pwa-node",
-            "request": "launch",
-            "name": "Run msgpackr tests",
-            "runtimeArgs": [
-                "test"
-            ],
-            "runtimeExecutable": "npm",
-            "skipFiles": [
-                "<node_internals>/**"
-            ],
-            "type": "pwa-node",
-            "console": "internalConsole", 
-            "outputCapture": "std"
-        }
-    ]
-}

package/tests/benchmark-stream.cjs

@@ -1,282 +0,0 @@
-#!/usr/bin/env node
-
-var PassThrough = require("stream").PassThrough;
-var async = require("async");
-
-let { PackrStream, UnpackrStream } = require("..");
-var msgpack = require("msgpack-lite");
-var Encoder = require("msgpack-lite/lib/encoder").Encoder;
-var Decoder = require("msgpack-lite/lib/decoder").Decoder;
-var notepack = require("notepack");
-
-var pkg = require("../package.json");
-
-// a sample fluentd message
-var data = ["tag", [[1440949922, {"message": "hi there"}]]];
-var packed = msgpack.encode(data); // 30 bytes per message
-var packsize = packed.length;
-var opcount = 1000000;
-var joincount = 100;
-var packjoin = repeatbuf(packed, joincount); // 3KB per chunk
-var limit = 2;
-
-var blocksToJoin = []
-var streamForJoin = new PackrStream();
-streamForJoin.on("data", data => blocksToJoin.push(data));
-for (var j = 0; j < joincount; j++) {
-  streamForJoin.write(data);
-}
-var packjoinWithRecords = Buffer.concat(blocksToJoin)
-
-var argv = Array.prototype.slice.call(process.argv, 2);
-
-if (argv[0] === "-v") {
-  console.warn(pkg.name + " " + pkg.version);
-  process.exit(0);
-}
-
-if (argv[0] - 0) limit = argv.shift() - 0;
-
-var list = [
-  ['new PackrStream().write(obj);', encode5],
-  ['new UnpackrStream().write(buf);', decode5],
-  ['stream.write(msgpack.encode(obj));', encode1],
-  ['stream.write(msgpack.decode(buf));', decode1],
-  ['stream.write(notepack.encode(obj));', encode4],
-  ['stream.write(notepack.decode(buf));', decode4],
-  ['msgpack.Encoder().on("data",ondata).encode(obj);', encode2],
-  ['msgpack.createDecodeStream().write(buf);', decode3],
-  ['msgpack.createEncodeStream().write(obj);', encode3],
-  ['msgpack.Decoder().on("data",ondata).decode(buf);', decode2],
-//  ['stream.write(Buffer.from(JSON.stringify(obj)));', stringify],
-//  ['stream.write(JSON.parse(buf));', parse]
-];
-
-function encode5(callback) {
-  var stream = new PackrStream();
-  var cnt = counter(callback);
-  stream.on("data", cnt.inc);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount; j++) {
-    stream.write(data);
-  }
-  stream.end();
-}
-
-function encode1(callback) {
-  var stream = new PassThrough();
-  var cnt = counter(callback);
-  stream.on("data", cnt.buf);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount; j++) {
-    stream.write(msgpack.encode(data));
-  }
-  stream.end();
-}
-
-function encode2(callback) {
-  var stream = new PassThrough();
-  var cnt = counter(callback);
-  stream.on("data", cnt.buf);
-  stream.on("end", cnt.end);
-  var encoder = Encoder();
-  encoder.on("data", function(chunk) {
-    stream.write(chunk);
-  });
-  encoder.on("end", function() {
-    stream.end();
-  });
-  for (var j = 0; j < opcount; j++) {
-    encoder.encode(data);
-  }
-  encoder.end();
-}
-
-function encode3(callback) {
-  var stream = msgpack.createEncodeStream();
-  var cnt = counter(callback);
-  stream.on("data", cnt.buf);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount; j++) {
-    stream.write(data);
-  }
-  stream.end();
-}
-
-function encode4(callback) {
-  var stream = new PassThrough();
-  var cnt = counter(callback);
-  stream.on("data", cnt.buf);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount; j++) {
-    stream.write(notepack.encode(data));
-  }
-  stream.end();
-}
-
-function decode5(callback) {
-  var stream = new UnpackrStream();
-  var cnt = counter(callback);
-  stream.on("data", cnt.inc);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount / joincount; j++) {
-    stream.write(packjoinWithRecords);
-  }
-  stream.end();
-}
-
-function decode1(callback) {
-  var stream = new PassThrough({objectMode: true});
-  var cnt = counter(callback);
-  stream.on("data", cnt.inc);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount; j++) {
-    stream.write(msgpack.decode(packed));
-  }
-  stream.end();
-}
-
-function decode2(callback) {
-  var stream = new PassThrough({objectMode: true});
-  var cnt = counter(callback);
-  stream.on("data", cnt.inc);
-  stream.on("end", cnt.end);
-  var decoder = Decoder();
-  decoder.on("data", function(chunk) {
-    stream.write(chunk);
-  });
-  decoder.on("end", function() {
-    stream.end();
-  });
-  for (var j = 0; j < opcount / joincount; j++) {
-    decoder.decode(packjoin);
-  }
-  decoder.end();
-}
-
-function decode3(callback) {
-  var stream = msgpack.createDecodeStream();
-  var cnt = counter(callback);
-  stream.on("data", cnt.inc);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount / joincount; j++) {
-    stream.write(packjoin);
-  }
-  stream.end();
-}
-
-function decode4(callback) {
-  var stream = new PassThrough({objectMode: true});
-  var cnt = counter(callback);
-  stream.on("data", cnt.inc);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount; j++) {
-    stream.write(notepack.decode(packed));
-  }
-  stream.end();
-}
-
-function rpad(str, len, chr) {
-  if (!chr) chr = " ";
-  str += "";
-  while (str.length < len) str += chr;
-  return str;
-}
-
-function lpad(str, len, chr) {
-  if (!chr) chr = " ";
-  str += "";
-  while (str.length < len) str = chr + str;
-  return str;
-}
-
-function repeatbuf(buf, cnt) {
-  var array = [];
-  for (var i = 0; i < cnt; i++) {
-    array.push(buf);
-  }
-  return Buffer.concat(array);
-}
-
-function counter(callback) {
-  var cnt = 0;
-  return {buf: b, inc: i, end: e};
-
-  function b(buf) {
-    cnt += buf.length / packsize;
-  }
-
-  function i() {
-    cnt++;
-  }
-
-  function e() {
-    cnt = Math.round(cnt);
-    callback(null, cnt);
-  }
-}
-
-function run() {
-  // task filter
-  if (argv.length) {
-    list = list.filter(function(pair) {
-      var name = pair[0];
-      var match = argv.filter(function(grep) {
-        return (name.indexOf(grep) > -1);
-      });
-      return match.length;
-    });
-  }
-
-  // run tasks repeatedly
-  var tasks = [];
-  for (var i = 0; i < limit; i++) {
-    tasks.push(oneset);
-  }
-  async.series(tasks, end);
-
-  // run a series of tasks
-  function oneset(callback) {
-    async.eachSeries(list, bench, callback);
-  }
-
-  // run a single benchmark
-  function bench(pair, callback) {
-    process.stdout.write(".");
-    var func = pair[1];
-    var start = new Date() - 0;
-    func(function(err, cnt) {
-      var end = new Date() - 0;
-      var array = pair[2] || (pair[2] = []);
-      array.push(end - start);
-      pair[3] = cnt;
-      setTimeout(callback, 100);
-    });
-  }
-
-  // show result
-  function end() {
-    var title = "operation (" + opcount + " x " + limit + ")";
-    process.stdout.write("\n");
-
-    // table header
-    var COL1 = 48;
-    console.log(rpad(title, COL1), "|", "  op   ", "|", " ms  ", "|", " op/s ");
-    console.log(rpad("", COL1, "-"), "|", "------:", "|", "----:", "|", "-----:");
-
-    // table body
-    list.forEach(function(pair) {
-      var name = pair[0];
-      var op = pair[3];
-      var array = pair[2];
-      array = array.sort(function(a, b) {
-        return a > b;
-      });
-      var fastest = array[0];
-      var score = Math.floor(opcount / fastest * 1000);
-      console.log(rpad(name, COL1), "|", lpad(op, 7), "|", lpad(fastest, 5), "|", lpad(score, 6));
-    });
-  }
-}
-
-run();

package/tests/benchmark.cjs

@@ -1,199 +0,0 @@
-var msgpackr = tryRequire("../dist/node.cjs");
-var msgpack_node = tryRequire("msgpack");
-var msgpack_msgpack = tryRequire("@msgpack/msgpack");
-var msgpack_lite = tryRequire("msgpack-lite");
-var msgpack_js = tryRequire("msgpack-js");
-var msgpack_js_v5 = tryRequire("msgpack-js-v5");
-var msgpack5 = tryRequire("msgpack5");
-var msgpack_unpack = tryRequire("msgpack-unpack");
-var msgpack_codec = tryRequire("msgpack.codec");
-var notepack = tryRequire("notepack");
-var what_the_pack = tryRequire("what-the-pack");
-var avro = tryRequire('avsc')
-var cbor = tryRequire('cbor')
-
-msgpack5 = msgpack5 && msgpack5();
-msgpack_codec = msgpack_codec && msgpack_codec.msgpack;
-what_the_pack = what_the_pack && what_the_pack.initialize(2**20);
-
-var pkg = require("../package.json");
-var data = require("./example4.json");
-var packed = msgpack_lite && msgpack_lite.encode(data);
-var expected = JSON.stringify(data);
-
-var argv = Array.prototype.slice.call(process.argv, 2);
-
-if (argv[0] === "-v") {
-  console.warn(pkg.name + " " + pkg.version);
-  process.exit(0);
-}
-
-var limit = 5;
-if (argv[0] - 0) limit = argv.shift() - 0;
-limit *= 1000;
-
-var COL1 = 58;
-var COL2 = 7;
-var COL3 = 5;
-var COL4 = 6;
-
-console.log(rpad("operation", COL1), "|", "  op  ", "|", "  ms ", "|", " op/s ");
-console.log(rpad("", COL1, "-"), "|", lpad(":", COL2, "-"), "|", lpad(":", COL3, "-"), "|", lpad(":", COL4, "-"));
-
-var buf, obj;
-
-if (msgpackr) {
-  let packr = new msgpackr.Packr({ useRecords: false })
-  buf = bench('require("msgpackr").pack(obj);', msgpackr.pack, data);
-    //buf = bench('require("msgpackr").pack(obj);', data => {let result = packr.pack(data); packr.resetMemory(); return result;}, data);
-
-  obj = bench('require("msgpackr").unpack(buf);', msgpackr.unpack, buf);
-  test(obj);
-
-  packr = new msgpackr.Packr({ structures: [] })
-  buf = bench('msgpackr w/ shared structures: packr.pack(obj);', packr.pack.bind(packr), data);
-  //buf = bench('msgpackr w/ shared structures: packr.pack(obj);', data => {let result = packr.pack(data); packr.resetMemory(); return result;}, data);
-
-  obj = bench('msgpackr w/ shared structures: packr.unpack(buf);', packr.unpack.bind(packr), buf);
-  test(obj);
-}
-
-if (JSON) {
-  buf = bench('buf = Buffer(JSON.stringify(obj));', JSON_stringify, data);
-  obj = bench('obj = JSON.parse(buf);', JSON.parse, buf);
-  test(obj);
-}
-
-if (msgpack_lite) {
-  buf = bench('buf = require("msgpack-lite").encode(obj);', msgpack_lite.encode, data);
-  obj = bench('obj = require("msgpack-lite").decode(buf);', msgpack_lite.decode, packed);
-  test(obj);
-}
-
-if (msgpack_msgpack) {
-  buf = bench('buf = require("@msgpack/msgpack").encode(obj);', msgpack_msgpack.encode, data);
-  obj = bench('obj = require("@msgpack/msgpack").decode(buf);', msgpack_msgpack.decode, buf);
-  test(obj);
-}
-
-if (msgpack_node) {
-  buf = bench('buf = require("msgpack").pack(obj);', msgpack_node.pack, data);
-  obj = bench('obj = require("msgpack").unpack(buf);', msgpack_node.unpack, buf);
-  test(obj);
-}
-
-if (msgpack_codec) {
-  buf = bench('buf = Buffer(require("msgpack.codec").msgpack.pack(obj));', msgpack_codec_pack, data);
-  obj = bench('obj = require("msgpack.codec").msgpack.unpack(buf);', msgpack_codec.unpack, buf);
-  test(obj);
-}
-
-if (msgpack_js_v5) {
-  buf = bench('buf = require("msgpack-js-v5").encode(obj);', msgpack_js_v5.encode, data);
-  obj = bench('obj = require("msgpack-js-v5").decode(buf);', msgpack_js_v5.decode, buf);
-  test(obj);
-}
-
-if (msgpack_js) {
-  buf = bench('buf = require("msgpack-js").encode(obj);', msgpack_js.encode, data);
-  obj = bench('obj = require("msgpack-js").decode(buf);', msgpack_js.decode, buf);
-  test(obj);
-}
-
-if (msgpack5) {
-  buf = bench('buf = require("msgpack5")().encode(obj);', msgpack5.encode, data);
-  obj = bench('obj = require("msgpack5")().decode(buf);', msgpack5.decode, buf);
-  test(obj);
-}
-
-if (notepack) {
-  buf = bench('buf = require("notepack").encode(obj);', notepack.encode, data);
-  obj = bench('obj = require("notepack").decode(buf);', notepack.decode, buf);
-  test(obj);
-}
-if (what_the_pack) {
-  buf = bench('require("what-the-pack")... encoder.encode(obj);', what_the_pack.encode, data);
-  obj = bench('require("what-the-pack")... encoder.decode(buf);', what_the_pack.decode, buf);
-  test(obj);
-}
-
-if (msgpack_unpack) {
-  obj = bench('obj = require("msgpack-unpack").decode(buf);', msgpack_unpack, packed);
-  test(obj);
-}
-
-if (avro) {
-  const type = avro.Type.forValue(data);
-  buf = bench('require("avsc")...make schema/type...type.toBuffer(obj);', type.toBuffer.bind(type), data);
-  obj = bench('require("avsc")...make schema/type...type.fromBuffer(obj);', type.fromBuffer.bind(type), buf);
-}
-if (cbor) {
-  buf = bench('buf = require("cbor").encode(obj);', cbor.encode, data);
-  obj = bench('obj = require("cbor").decode(buf);', cbor.decode, buf);
-  test(obj);
-}
-
-function JSON_stringify(src) {
-  return Buffer(JSON.stringify(src));
-}
-
-function msgpack_codec_pack(data) {
-  return Buffer(msgpack_codec.pack(data));
-}
-
-function bench(name, func, src) {
-  if (argv.length) {
-    var match = argv.filter(function(grep) {
-      return (name.indexOf(grep) > -1);
-    });
-    if (!match.length) return SKIP;
-  }
-  var ret, duration;
-  var start = new Date() - 0;
-  var count = 0;
-  while (1) {
-    var end = new Date() - 0;
-    duration = end - start;
-    if (duration >= limit) break;
-    while ((++count) % 100) ret = func(src);
-  }
-  name = rpad(name, COL1);
-  var score = Math.floor(count / duration * 1000);
-  count = lpad(count, COL2);
-  duration = lpad(duration, COL3);
-  score = lpad(score, COL4);
-  console.log(name, "|", count, "|", duration, "|", score);
-  return ret;
-}
-
-function rpad(str, len, chr) {
-  if (!chr) chr = " ";
-  while (str.length < len) str += chr;
-  return str;
-}
-
-function lpad(str, len, chr) {
-  if (!chr) chr = " ";
-  str += "";
-  while (str.length < len) str = chr + str;
-  return str;
-}
-
-function test(actual) {
-  if (actual === SKIP) return;
-  actual = JSON.stringify(actual);
-  if (actual === expected) return;
-  console.warn("expected: " + expected);
-  console.warn("actual:   " + actual);
-}
-
-function SKIP() {
-}
-
-function tryRequire(name) {
-  try {
-    return require(name);
-  } catch (e) {
-    // ignore
-  }
-}

package/tests/example.json

@@ -1,52 +0,0 @@
-{
-  "int0": 0,
-  "int1": 1,
-  "int1-": -1,
-  "int8": 255,
-  "int8-": -255,
-  "int16": 256,
-  "int16-": -256,
-  "int32": 65536,
-  "int32-": -65536,
-  "nil": null,
-  "true": true,
-  "false": false,
-  "float": 0.5,
-  "float-": -0.5,
-  "string0": "",
-  "string1": "A",
-  "string4": "foobarbaz",
-  "string8": "Omnes viae Romam ducunt.",
-  "string16": "L’homme n’est qu’un roseau, le plus faible de la nature ; mais c’est un roseau pensant. Il ne faut pas que l’univers entier s’arme pour l’écraser : une vapeur, une goutte d’eau, suffit pour le tuer. Mais, quand l’univers l’écraserait, l’homme serait encore plus noble que ce qui le tue, puisqu’il sait qu’il meurt, et l’avantage que l’univers a sur lui, l’univers n’en sait rien. Toute notre dignité consiste donc en la pensée. C’est de là qu’il faut nous relever et non de l’espace et de la durée, que nous ne saurions remplir. Travaillons donc à bien penser : voilà le principe de la morale.",
-  "array0": [],
-  "array1": [
-    "foo"
-  ],
-  "array8": [
-    1,
-    2,
-    4,
-    8,
-    16,
-    32,
-    64,
-    128,
-    256,
-    512,
-    1024,
-    2048,
-    4096,
-    8192,
-    16384,
-    32768,
-    65536,
-    131072,
-    262144,
-    524288,
-    1048576
-  ],
-  "map0": {},
-  "map1": {
-    "foo": "bar"
-  }
-}