msgpackr 1.5.0 → 1.5.4

package/unpack.d.ts

@@ -13,6 +13,7 @@ export interface Options {
 	mapsAsObjects?: boolean
 	variableMapSize?: boolean
 	copyBuffers?: boolean
+	bundleStrings?: boolean
 	useTimestamp32?: boolean
 	largeBigIntToFloat?: boolean
 	encodeUndefinedAsNil?: boolean
@@ -21,6 +22,7 @@ export interface Options {
 	shouldShareStructure?: (keys: string[]) => boolean
 	getStructures?(): {}[]
 	saveStructures?(structures: {}[]): boolean | void
+	onInvalidDate?: () => any
 }
 interface Extension {
 	Class: Function

package/unpack.js

@@ -15,6 +15,7 @@ var currentStructures
 var srcString
 var srcStringStart = 0
 var srcStringEnd = 0
+var bundledStrings
 var referenceMap
 var currentExtensions = []
 var dataView
@@ -55,6 +56,7 @@ export class Unpackr {
 		srcStringEnd = 0
 		srcString = null
 		strings = EMPTY_ARRAY
+		bundledStrings = null
 		src = source
 		// this provides cached access to the data view for a buffer if it is getting reused, which is a recommend
 		// technique for getting data from a database where it can be copied into an existing buffer instead of creating
@@ -154,6 +156,9 @@ export function checkedRead() {
 				currentStructures.length = sharedLength
 		}
 		let result = read()
+		if (bundledStrings) // bundled strings to skip past
+			position = bundledStrings.postBundlePosition
+
 		if (position == srcEnd) {
 			// finished reading this source, cleanup references
 			if (currentStructures.restoreStructures)
@@ -248,7 +253,15 @@ export function read() {
 		let value
 		switch (token) {
 			case 0xc0: return null
-			case 0xc1: return C1; // "never-used", return special object to denote that
+			case 0xc1:
+				if (bundledStrings) {
+					value = read() // followed by the length of the string in characters (not bytes!)
+					if (value > 0)
+						return bundledStrings[1].slice(bundledStrings.position1, bundledStrings.position1 += value)
+					else
+						return bundledStrings[0].slice(bundledStrings.position0, bundledStrings.position0 -= value)
+				}
+				return C1; // "never-used", return special object to denote that
 			case 0xc2: return false
 			case 0xc3: return true
 			case 0xc4:
@@ -488,6 +501,8 @@ export function setExtractor(extractStrings) {
 		return function readString(length) {
 			let string = strings[stringPosition++]
 			if (string == null) {
+				if (bundledStrings)
+					return readStringJS(length)
 				let extraction = extractStrings(position - headerLength, srcEnd, src)
 				if (typeof extraction == 'string') {
 					string = extraction
@@ -746,6 +761,36 @@ function shortStringInJS(length) {
 	}
 }
 
+function readOnlyJSString() {
+	let token = src[position++]
+	let length
+	if (token < 0xc0) {
+		// fixstr
+		length = token - 0xa0
+	} else {
+		switch(token) {
+			case 0xd9:
+			// str 8
+				length = src[position++]
+				break
+			case 0xda:
+			// str 16
+				length = dataView.getUint16(position)
+				position += 2
+				break
+			case 0xdb:
+			// str 32
+				length = dataView.getUint32(position)
+				position += 4
+				break
+			default:
+				throw new Error('Expected string')
+		}
+	}
+	return readStringJS(length)
+}
+
+
 function readBin(length) {
 	return currentUnpackr.copyBuffers ?
 		// specifically use the copying slice (not the node one)
@@ -897,6 +942,19 @@ currentExtensions[0x78] = () => {
 	let data = read()
 	return new RegExp(data[0], data[1])
 }
+const TEMP_BUNDLE = []
+currentExtensions[0x62] = (data) => {
+	let dataSize = (data[0] << 24) + (data[1] << 16) + (data[2] << 8) + data[3]
+	let dataPosition = position
+	position += dataSize - data.length
+	bundledStrings = TEMP_BUNDLE
+	bundledStrings = [readOnlyJSString(), readOnlyJSString()]
+	bundledStrings.position0 = 0
+	bundledStrings.position1 = 0
+	bundledStrings.postBundlePosition = position
+	position = dataPosition
+	return read()
+}
 
 currentExtensions[0xff] = (data) => {
 	// 32-bit date extension
@@ -911,7 +969,7 @@ currentExtensions[0xff] = (data) => {
 			((data[0] << 24) + (data[1] << 16) + (data[2] << 8) + data[3]) / 1000000 +
 			(((data[4] & 0x80) ? -0x1000000000000 : 0) + data[6] * 0x10000000000 + data[7] * 0x100000000 + data[8] * 0x1000000 + (data[9] << 16) + (data[10] << 8) + data[11]) * 1000)
 	else
-		throw new Error('Invalid timestamp length')
+		return new Date('invalid')
 } // notepack defines extension 0 to mean undefined, so use that as the default here
 // registration of bulk record definition?
 // currentExtensions[0x52] = () =>
@@ -925,6 +983,7 @@ function saveState(callback) {
 	let savedSrcString = srcString
 	let savedStrings = strings
 	let savedReferenceMap = referenceMap
+	let savedBundledStrings = bundledStrings
 
 	// TODO: We may need to revisit this if we do more external calls to user code (since it could be slow)
 	let savedSrc = new Uint8Array(src.slice(0, srcEnd)) // we copy the data in case it changes while external data is processed
@@ -941,6 +1000,7 @@ function saveState(callback) {
 	srcString = savedSrcString
 	strings = savedStrings
 	referenceMap = savedReferenceMap
+	bundledStrings = savedBundledStrings
 	src = savedSrc
 	sequentialMode = savedSequentialMode
 	currentStructures = savedStructures

package/.vscode/launch.json

@@ -1,23 +0,0 @@
-{
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "type": "pwa-node",
-            "request": "launch",
-            "name": "Run msgpackr tests",
-            "runtimeArgs": [
-                "test"
-            ],
-            "runtimeExecutable": "npm",
-            "skipFiles": [
-                "<node_internals>/**"
-            ],
-            "type": "pwa-node",
-            "console": "internalConsole", 
-            "outputCapture": "std"
-        }
-    ]
-}

package/tests/benchmark-stream.cjs

@@ -1,282 +0,0 @@
-#!/usr/bin/env node
-
-var PassThrough = require("stream").PassThrough;
-var async = require("async");
-
-let { PackrStream, UnpackrStream } = require("..");
-var msgpack = require("msgpack-lite");
-var Encoder = require("msgpack-lite/lib/encoder").Encoder;
-var Decoder = require("msgpack-lite/lib/decoder").Decoder;
-var notepack = require("notepack");
-
-var pkg = require("../package.json");
-
-// a sample fluentd message
-var data = ["tag", [[1440949922, {"message": "hi there"}]]];
-var packed = msgpack.encode(data); // 30 bytes per message
-var packsize = packed.length;
-var opcount = 1000000;
-var joincount = 100;
-var packjoin = repeatbuf(packed, joincount); // 3KB per chunk
-var limit = 2;
-
-var blocksToJoin = []
-var streamForJoin = new PackrStream();
-streamForJoin.on("data", data => blocksToJoin.push(data));
-for (var j = 0; j < joincount; j++) {
-  streamForJoin.write(data);
-}
-var packjoinWithRecords = Buffer.concat(blocksToJoin)
-
-var argv = Array.prototype.slice.call(process.argv, 2);
-
-if (argv[0] === "-v") {
-  console.warn(pkg.name + " " + pkg.version);
-  process.exit(0);
-}
-
-if (argv[0] - 0) limit = argv.shift() - 0;
-
-var list = [
-  ['new PackrStream().write(obj);', encode5],
-  ['new UnpackrStream().write(buf);', decode5],
-  ['stream.write(msgpack.encode(obj));', encode1],
-  ['stream.write(msgpack.decode(buf));', decode1],
-  ['stream.write(notepack.encode(obj));', encode4],
-  ['stream.write(notepack.decode(buf));', decode4],
-  ['msgpack.Encoder().on("data",ondata).encode(obj);', encode2],
-  ['msgpack.createDecodeStream().write(buf);', decode3],
-  ['msgpack.createEncodeStream().write(obj);', encode3],
-  ['msgpack.Decoder().on("data",ondata).decode(buf);', decode2],
-//  ['stream.write(Buffer.from(JSON.stringify(obj)));', stringify],
-//  ['stream.write(JSON.parse(buf));', parse]
-];
-
-function encode5(callback) {
-  var stream = new PackrStream();
-  var cnt = counter(callback);
-  stream.on("data", cnt.inc);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount; j++) {
-    stream.write(data);
-  }
-  stream.end();
-}
-
-function encode1(callback) {
-  var stream = new PassThrough();
-  var cnt = counter(callback);
-  stream.on("data", cnt.buf);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount; j++) {
-    stream.write(msgpack.encode(data));
-  }
-  stream.end();
-}
-
-function encode2(callback) {
-  var stream = new PassThrough();
-  var cnt = counter(callback);
-  stream.on("data", cnt.buf);
-  stream.on("end", cnt.end);
-  var encoder = Encoder();
-  encoder.on("data", function(chunk) {
-    stream.write(chunk);
-  });
-  encoder.on("end", function() {
-    stream.end();
-  });
-  for (var j = 0; j < opcount; j++) {
-    encoder.encode(data);
-  }
-  encoder.end();
-}
-
-function encode3(callback) {
-  var stream = msgpack.createEncodeStream();
-  var cnt = counter(callback);
-  stream.on("data", cnt.buf);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount; j++) {
-    stream.write(data);
-  }
-  stream.end();
-}
-
-function encode4(callback) {
-  var stream = new PassThrough();
-  var cnt = counter(callback);
-  stream.on("data", cnt.buf);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount; j++) {
-    stream.write(notepack.encode(data));
-  }
-  stream.end();
-}
-
-function decode5(callback) {
-  var stream = new UnpackrStream();
-  var cnt = counter(callback);
-  stream.on("data", cnt.inc);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount / joincount; j++) {
-    stream.write(packjoinWithRecords);
-  }
-  stream.end();
-}
-
-function decode1(callback) {
-  var stream = new PassThrough({objectMode: true});
-  var cnt = counter(callback);
-  stream.on("data", cnt.inc);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount; j++) {
-    stream.write(msgpack.decode(packed));
-  }
-  stream.end();
-}
-
-function decode2(callback) {
-  var stream = new PassThrough({objectMode: true});
-  var cnt = counter(callback);
-  stream.on("data", cnt.inc);
-  stream.on("end", cnt.end);
-  var decoder = Decoder();
-  decoder.on("data", function(chunk) {
-    stream.write(chunk);
-  });
-  decoder.on("end", function() {
-    stream.end();
-  });
-  for (var j = 0; j < opcount / joincount; j++) {
-    decoder.decode(packjoin);
-  }
-  decoder.end();
-}
-
-function decode3(callback) {
-  var stream = msgpack.createDecodeStream();
-  var cnt = counter(callback);
-  stream.on("data", cnt.inc);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount / joincount; j++) {
-    stream.write(packjoin);
-  }
-  stream.end();
-}
-
-function decode4(callback) {
-  var stream = new PassThrough({objectMode: true});
-  var cnt = counter(callback);
-  stream.on("data", cnt.inc);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount; j++) {
-    stream.write(notepack.decode(packed));
-  }
-  stream.end();
-}
-
-function rpad(str, len, chr) {
-  if (!chr) chr = " ";
-  str += "";
-  while (str.length < len) str += chr;
-  return str;
-}
-
-function lpad(str, len, chr) {
-  if (!chr) chr = " ";
-  str += "";
-  while (str.length < len) str = chr + str;
-  return str;
-}
-
-function repeatbuf(buf, cnt) {
-  var array = [];
-  for (var i = 0; i < cnt; i++) {
-    array.push(buf);
-  }
-  return Buffer.concat(array);
-}
-
-function counter(callback) {
-  var cnt = 0;
-  return {buf: b, inc: i, end: e};
-
-  function b(buf) {
-    cnt += buf.length / packsize;
-  }
-
-  function i() {
-    cnt++;
-  }
-
-  function e() {
-    cnt = Math.round(cnt);
-    callback(null, cnt);
-  }
-}
-
-function run() {
-  // task filter
-  if (argv.length) {
-    list = list.filter(function(pair) {
-      var name = pair[0];
-      var match = argv.filter(function(grep) {
-        return (name.indexOf(grep) > -1);
-      });
-      return match.length;
-    });
-  }
-
-  // run tasks repeatedly
-  var tasks = [];
-  for (var i = 0; i < limit; i++) {
-    tasks.push(oneset);
-  }
-  async.series(tasks, end);
-
-  // run a series of tasks
-  function oneset(callback) {
-    async.eachSeries(list, bench, callback);
-  }
-
-  // run a single benchmark
-  function bench(pair, callback) {
-    process.stdout.write(".");
-    var func = pair[1];
-    var start = new Date() - 0;
-    func(function(err, cnt) {
-      var end = new Date() - 0;
-      var array = pair[2] || (pair[2] = []);
-      array.push(end - start);
-      pair[3] = cnt;
-      setTimeout(callback, 100);
-    });
-  }
-
-  // show result
-  function end() {
-    var title = "operation (" + opcount + " x " + limit + ")";
-    process.stdout.write("\n");
-
-    // table header
-    var COL1 = 48;
-    console.log(rpad(title, COL1), "|", "  op   ", "|", " ms  ", "|", " op/s ");
-    console.log(rpad("", COL1, "-"), "|", "------:", "|", "----:", "|", "-----:");
-
-    // table body
-    list.forEach(function(pair) {
-      var name = pair[0];
-      var op = pair[3];
-      var array = pair[2];
-      array = array.sort(function(a, b) {
-        return a > b;
-      });
-      var fastest = array[0];
-      var score = Math.floor(opcount / fastest * 1000);
-      console.log(rpad(name, COL1), "|", lpad(op, 7), "|", lpad(fastest, 5), "|", lpad(score, 6));
-    });
-  }
-}
-
-run();

package/tests/benchmark.cjs

@@ -1,199 +0,0 @@
-var msgpackr = tryRequire("../dist/node.cjs");
-var msgpack_node = tryRequire("msgpack");
-var msgpack_msgpack = tryRequire("@msgpack/msgpack");
-var msgpack_lite = tryRequire("msgpack-lite");
-var msgpack_js = tryRequire("msgpack-js");
-var msgpack_js_v5 = tryRequire("msgpack-js-v5");
-var msgpack5 = tryRequire("msgpack5");
-var msgpack_unpack = tryRequire("msgpack-unpack");
-var msgpack_codec = tryRequire("msgpack.codec");
-var notepack = tryRequire("notepack");
-var what_the_pack = tryRequire("what-the-pack");
-var avro = tryRequire('avsc')
-var cbor = tryRequire('cbor')
-
-msgpack5 = msgpack5 && msgpack5();
-msgpack_codec = msgpack_codec && msgpack_codec.msgpack;
-what_the_pack = what_the_pack && what_the_pack.initialize(2**20);
-
-var pkg = require("../package.json");
-var data = require("./example4.json");
-var packed = msgpack_lite && msgpack_lite.encode(data);
-var expected = JSON.stringify(data);
-
-var argv = Array.prototype.slice.call(process.argv, 2);
-
-if (argv[0] === "-v") {
-  console.warn(pkg.name + " " + pkg.version);
-  process.exit(0);
-}
-
-var limit = 5;
-if (argv[0] - 0) limit = argv.shift() - 0;
-limit *= 1000;
-
-var COL1 = 58;
-var COL2 = 7;
-var COL3 = 5;
-var COL4 = 6;
-
-console.log(rpad("operation", COL1), "|", "  op  ", "|", "  ms ", "|", " op/s ");
-console.log(rpad("", COL1, "-"), "|", lpad(":", COL2, "-"), "|", lpad(":", COL3, "-"), "|", lpad(":", COL4, "-"));
-
-var buf, obj;
-
-if (msgpackr) {
-  let packr = new msgpackr.Packr({ useRecords: false })
-  buf = bench('require("msgpackr").pack(obj);', msgpackr.pack, data);
-    //buf = bench('require("msgpackr").pack(obj);', data => {let result = packr.pack(data); packr.resetMemory(); return result;}, data);
-
-  obj = bench('require("msgpackr").unpack(buf);', msgpackr.unpack, buf);
-  test(obj);
-
-  packr = new msgpackr.Packr({ structures: [] })
-  buf = bench('msgpackr w/ shared structures: packr.pack(obj);', packr.pack.bind(packr), data);
-  //buf = bench('msgpackr w/ shared structures: packr.pack(obj);', data => {let result = packr.pack(data); packr.resetMemory(); return result;}, data);
-
-  obj = bench('msgpackr w/ shared structures: packr.unpack(buf);', packr.unpack.bind(packr), buf);
-  test(obj);
-}
-
-if (JSON) {
-  buf = bench('buf = Buffer(JSON.stringify(obj));', JSON_stringify, data);
-  obj = bench('obj = JSON.parse(buf);', JSON.parse, buf);
-  test(obj);
-}
-
-if (msgpack_lite) {
-  buf = bench('buf = require("msgpack-lite").encode(obj);', msgpack_lite.encode, data);
-  obj = bench('obj = require("msgpack-lite").decode(buf);', msgpack_lite.decode, packed);
-  test(obj);
-}
-
-if (msgpack_msgpack) {
-  buf = bench('buf = require("@msgpack/msgpack").encode(obj);', msgpack_msgpack.encode, data);
-  obj = bench('obj = require("@msgpack/msgpack").decode(buf);', msgpack_msgpack.decode, buf);
-  test(obj);
-}
-
-if (msgpack_node) {
-  buf = bench('buf = require("msgpack").pack(obj);', msgpack_node.pack, data);
-  obj = bench('obj = require("msgpack").unpack(buf);', msgpack_node.unpack, buf);
-  test(obj);
-}
-
-if (msgpack_codec) {
-  buf = bench('buf = Buffer(require("msgpack.codec").msgpack.pack(obj));', msgpack_codec_pack, data);
-  obj = bench('obj = require("msgpack.codec").msgpack.unpack(buf);', msgpack_codec.unpack, buf);
-  test(obj);
-}
-
-if (msgpack_js_v5) {
-  buf = bench('buf = require("msgpack-js-v5").encode(obj);', msgpack_js_v5.encode, data);
-  obj = bench('obj = require("msgpack-js-v5").decode(buf);', msgpack_js_v5.decode, buf);
-  test(obj);
-}
-
-if (msgpack_js) {
-  buf = bench('buf = require("msgpack-js").encode(obj);', msgpack_js.encode, data);
-  obj = bench('obj = require("msgpack-js").decode(buf);', msgpack_js.decode, buf);
-  test(obj);
-}
-
-if (msgpack5) {
-  buf = bench('buf = require("msgpack5")().encode(obj);', msgpack5.encode, data);
-  obj = bench('obj = require("msgpack5")().decode(buf);', msgpack5.decode, buf);
-  test(obj);
-}
-
-if (notepack) {
-  buf = bench('buf = require("notepack").encode(obj);', notepack.encode, data);
-  obj = bench('obj = require("notepack").decode(buf);', notepack.decode, buf);
-  test(obj);
-}
-if (what_the_pack) {
-  buf = bench('require("what-the-pack")... encoder.encode(obj);', what_the_pack.encode, data);
-  obj = bench('require("what-the-pack")... encoder.decode(buf);', what_the_pack.decode, buf);
-  test(obj);
-}
-
-if (msgpack_unpack) {
-  obj = bench('obj = require("msgpack-unpack").decode(buf);', msgpack_unpack, packed);
-  test(obj);
-}
-
-if (avro) {
-  const type = avro.Type.forValue(data);
-  buf = bench('require("avsc")...make schema/type...type.toBuffer(obj);', type.toBuffer.bind(type), data);
-  obj = bench('require("avsc")...make schema/type...type.fromBuffer(obj);', type.fromBuffer.bind(type), buf);
-}
-if (cbor) {
-  buf = bench('buf = require("cbor").encode(obj);', cbor.encode, data);
-  obj = bench('obj = require("cbor").decode(buf);', cbor.decode, buf);
-  test(obj);
-}
-
-function JSON_stringify(src) {
-  return Buffer(JSON.stringify(src));
-}
-
-function msgpack_codec_pack(data) {
-  return Buffer(msgpack_codec.pack(data));
-}
-
-function bench(name, func, src) {
-  if (argv.length) {
-    var match = argv.filter(function(grep) {
-      return (name.indexOf(grep) > -1);
-    });
-    if (!match.length) return SKIP;
-  }
-  var ret, duration;
-  var start = new Date() - 0;
-  var count = 0;
-  while (1) {
-    var end = new Date() - 0;
-    duration = end - start;
-    if (duration >= limit) break;
-    while ((++count) % 100) ret = func(src);
-  }
-  name = rpad(name, COL1);
-  var score = Math.floor(count / duration * 1000);
-  count = lpad(count, COL2);
-  duration = lpad(duration, COL3);
-  score = lpad(score, COL4);
-  console.log(name, "|", count, "|", duration, "|", score);
-  return ret;
-}
-
-function rpad(str, len, chr) {
-  if (!chr) chr = " ";
-  while (str.length < len) str += chr;
-  return str;
-}
-
-function lpad(str, len, chr) {
-  if (!chr) chr = " ";
-  str += "";
-  while (str.length < len) str = chr + str;
-  return str;
-}
-
-function test(actual) {
-  if (actual === SKIP) return;
-  actual = JSON.stringify(actual);
-  if (actual === expected) return;
-  console.warn("expected: " + expected);
-  console.warn("actual:   " + actual);
-}
-
-function SKIP() {
-}
-
-function tryRequire(name) {
-  try {
-    return require(name);
-  } catch (e) {
-    // ignore
-  }
-}

package/tests/example.json

@@ -1,52 +0,0 @@
-{
-  "int0": 0,
-  "int1": 1,
-  "int1-": -1,
-  "int8": 255,
-  "int8-": -255,
-  "int16": 256,
-  "int16-": -256,
-  "int32": 65536,
-  "int32-": -65536,
-  "nil": null,
-  "true": true,
-  "false": false,
-  "float": 0.5,
-  "float-": -0.5,
-  "string0": "",
-  "string1": "A",
-  "string4": "foobarbaz",
-  "string8": "Omnes viae Romam ducunt.",
-  "string16": "L’homme n’est qu’un roseau, le plus faible de la nature ; mais c’est un roseau pensant. Il ne faut pas que l’univers entier s’arme pour l’écraser : une vapeur, une goutte d’eau, suffit pour le tuer. Mais, quand l’univers l’écraserait, l’homme serait encore plus noble que ce qui le tue, puisqu’il sait qu’il meurt, et l’avantage que l’univers a sur lui, l’univers n’en sait rien. Toute notre dignité consiste donc en la pensée. C’est de là qu’il faut nous relever et non de l’espace et de la durée, que nous ne saurions remplir. Travaillons donc à bien penser : voilà le principe de la morale.",
-  "array0": [],
-  "array1": [
-    "foo"
-  ],
-  "array8": [
-    1,
-    2,
-    4,
-    8,
-    16,
-    32,
-    64,
-    128,
-    256,
-    512,
-    1024,
-    2048,
-    4096,
-    8192,
-    16384,
-    32768,
-    65536,
-    131072,
-    262144,
-    524288,
-    1048576
-  ],
-  "map0": {},
-  "map1": {
-    "foo": "bar"
-  }
-}