msgpackr 1.5.1 → 1.5.2

package/dist/test.js

@@ -684,6 +684,14 @@
 			var deserialized = packr.unpack(serialized);
 			assert.deepEqual(deserialized, data);
 		});
+		test('pack/unpack sample data with bundled strings', function(){
+			var data = sampleData;
+			let structures = [];
+			let packr = new Packr({ structures, bundleStrings: true });
+			var serialized = packr.pack(data);
+			var deserialized = packr.unpack(serialized);
+			assert.deepEqual(deserialized, data);
+		});
 		if (typeof Buffer != 'undefined')
 		test('replace data', function(){
 			var data1 = {
@@ -1010,16 +1018,19 @@
 			var data = {
 				map: map,
 				date: new Date(1532219539011),
+				invalidDate: new Date('invalid')
 			};
 			let packr = new Packr({
 				mapsAsObjects: true,
 				useTimestamp32: true,
+				onInvalidDate: () => 'Custom invalid date'
 			});
 			var serialized = packr.pack(data);
 			var deserialized = packr.unpack(serialized);
 			assert.equal(deserialized.map[4], 'four');
 			assert.equal(deserialized.map.three, 3);
 			assert.equal(deserialized.date.getTime(), 1532219539000);
+			assert.equal(deserialized.invalidDate, 'Custom invalid date');
 		});
 		test('key caching', function() {
 			var data = {
@@ -1199,7 +1210,7 @@
 			let structures = [];
 			var serialized = pack(data);
 			console.log('MessagePack size', serialized.length);
-			let packr = new Packr({ structures });
+			let packr = new Packr({ structures, bundleStrings: false });
 			var serialized = packr.pack(data);
 			console.log('msgpackr w/ record ext size', serialized.length);
 			for (var i = 0; i < ITERATIONS; i++) {
@@ -1210,7 +1221,7 @@
 			var data = sampleData;
 			this.timeout(10000);
 			let structures = [];
-			let packr = new Packr({ structures });
+			let packr = new Packr({ structures, bundleStrings: false });
 			let buffer = typeof Buffer != 'undefined' ? Buffer.alloc(0x10000) : new Uint8Array(0x10000);
 
 			for (var i = 0; i < ITERATIONS; i++) {

package/pack.js

@@ -13,6 +13,8 @@ let target
 let targetView
 let position = 0
 let safeEnd
+let bundledStrings = null
+const hasNonLatin = /[\u0080-\uFFFF]/
 const RECORD_SYMBOL = Symbol('record-id')
 export class Packr extends Unpackr {
 	constructor(options) {
@@ -75,6 +77,14 @@ export class Packr extends Unpackr {
 				position = (position + 7) & 0x7ffffff8 // Word align to make any future copying of this buffer faster
 			start = position
 			referenceMap = packr.structuredClone ? new Map() : null
+			if (packr.bundleStrings) {
+				bundledStrings = ['', '']
+				target[position++] = 0xd6
+				target[position++] = 0x62 // 'b'
+				bundledStrings.position = position - start
+				position += 4
+			} else
+				bundledStrings = null
 			sharedStructures = packr.structures
 			if (sharedStructures) {
 				if (sharedStructures.uninitialized)
@@ -113,6 +123,13 @@ export class Packr extends Unpackr {
 			structures = sharedStructures || []
 			try {
 				pack(value)
+				if (bundledStrings) {
+					targetView.setUint32(bundledStrings.position + start, position - bundledStrings.position - start)
+					let writeStrings = bundledStrings
+					bundledStrings = null
+					pack(writeStrings[0])
+					pack(writeStrings[1])
+				}
 				packr.offset = position // update the offset so next serialization doesn't write over our buffer, but can continue writing to same buffer sequentially
 				if (referenceMap && referenceMap.idsToInsert) {
 					position += referenceMap.idsToInsert.length * 6
@@ -123,7 +140,7 @@ export class Packr extends Unpackr {
 					referenceMap = null
 					return serialized
 				}
-				if (encodeOptions === REUSE_BUFFER_MODE) {
+				if (encodeOptions & REUSE_BUFFER_MODE) {
 					target.start = start
 					target.end = position
 					return target
@@ -162,6 +179,8 @@ export class Packr extends Unpackr {
 						return returnBuffer
 					}
 				}
+				if (encodeOptions & RESET_BUFFER_MODE)
+					position = start
 			}
 		}
 		const pack = (value) => {
@@ -172,6 +191,13 @@ export class Packr extends Unpackr {
 			var length
 			if (type === 'string') {
 				let strLength = value.length
+				if (bundledStrings && strLength >= 8 && strLength < 0x1000) {
+					let twoByte = hasNonLatin.test(value)
+					bundledStrings[twoByte ? 0 : 1] += value
+					target[position++] = 0xc1
+					pack(twoByte ? -strLength : strLength);
+					return
+				}
 				let headerSize
 				// first we estimate the header size, so we can write to the correct location
 				if (strLength < 0x20) {
@@ -638,8 +664,10 @@ extensions = [{
 			targetView.setUint32(position, date.getMilliseconds() * 4000000 + ((seconds / 1000 / 0x100000000) >> 0))
 			targetView.setUint32(position + 4, seconds)
 		} else if (isNaN(seconds)) {
-			if (this.onInvalidDate)
+			if (this.onInvalidDate) {
+				allocateForWrite(0)
 				return pack(this.onInvalidDate())
+			}
 			// Intentionally invalid timestamp
 			let { target, targetView, position} = allocateForWrite(3)
 			target[position++] = 0xd4
@@ -832,4 +860,5 @@ export const Encoder = Packr
 export { FLOAT32_OPTIONS } from './unpack.js'
 import { FLOAT32_OPTIONS } from './unpack.js'
 export const { NEVER, ALWAYS, DECIMAL_ROUND, DECIMAL_FIT } = FLOAT32_OPTIONS
-export const REUSE_BUFFER_MODE = 1000
+export const REUSE_BUFFER_MODE = 512
+export const RESET_BUFFER_MODE = 1024

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "msgpackr",
   "author": "Kris Zyp",
-  "version": "1.5.1",
+  "version": "1.5.2",
   "description": "Ultra-fast MessagePack implementation with extensions for records and structured cloning",
   "license": "MIT",
   "types": "./index.d.ts",
@@ -49,6 +49,12 @@
       "default": "./unpack.js"
     }
   },
+  "files": [
+    "/dist",
+    "*.md",
+    "/*.js",
+    "/*.ts"
+  ],
   "optionalDependencies": {
     "msgpackr-extract": "^1.0.14"
   },

package/unpack.d.ts

@@ -13,6 +13,7 @@ export interface Options {
 	mapsAsObjects?: boolean
 	variableMapSize?: boolean
 	copyBuffers?: boolean
+	bundleStrings?: boolean
 	useTimestamp32?: boolean
 	largeBigIntToFloat?: boolean
 	encodeUndefinedAsNil?: boolean
@@ -21,6 +22,7 @@ export interface Options {
 	shouldShareStructure?: (keys: string[]) => boolean
 	getStructures?(): {}[]
 	saveStructures?(structures: {}[]): boolean | void
+	onInvalidDate?: () => any
 }
 interface Extension {
 	Class: Function

package/unpack.js

@@ -15,6 +15,7 @@ var currentStructures
 var srcString
 var srcStringStart = 0
 var srcStringEnd = 0
+var bundledStrings
 var referenceMap
 var currentExtensions = []
 var dataView
@@ -55,6 +56,7 @@ export class Unpackr {
 		srcStringEnd = 0
 		srcString = null
 		strings = EMPTY_ARRAY
+		bundledStrings = null
 		src = source
 		// this provides cached access to the data view for a buffer if it is getting reused, which is a recommend
 		// technique for getting data from a database where it can be copied into an existing buffer instead of creating
@@ -248,7 +250,15 @@ export function read() {
 		let value
 		switch (token) {
 			case 0xc0: return null
-			case 0xc1: return C1; // "never-used", return special object to denote that
+			case 0xc1:
+				if (bundledStrings) {
+					value = read() // followed by the length of the string in characters (not bytes!)
+					if (value > 0)
+						return bundledStrings[1].slice(bundledStrings.position1, bundledStrings.position1 += value)
+					else
+						return bundledStrings[0].slice(bundledStrings.position0, bundledStrings.position0 -= value)
+				}
+				return C1; // "never-used", return special object to denote that
 			case 0xc2: return false
 			case 0xc3: return true
 			case 0xc4:
@@ -898,6 +908,22 @@ currentExtensions[0x78] = () => {
 	return new RegExp(data[0], data[1])
 }
 
+currentExtensions[0x62] = (data) => {
+	let dataSize = (data[0] << 24) + (data[1] << 16) + (data[2] << 8) + data[3]
+	let dataPosition = position
+	position += dataSize - 4
+	bundledStrings = [read(), read()]
+	bundledStrings.position0 = 0
+	bundledStrings.position1 = 0
+	let postBundlePosition = position
+	position = dataPosition
+	try {
+		return read()
+	} finally {
+		position = postBundlePosition
+	}
+}
+
 currentExtensions[0xff] = (data) => {
 	// 32-bit date extension
 	if (data.length == 4)
@@ -925,6 +951,7 @@ function saveState(callback) {
 	let savedSrcString = srcString
 	let savedStrings = strings
 	let savedReferenceMap = referenceMap
+	let savedBundledStrings = bundledStrings
 
 	// TODO: We may need to revisit this if we do more external calls to user code (since it could be slow)
 	let savedSrc = new Uint8Array(src.slice(0, srcEnd)) // we copy the data in case it changes while external data is processed
@@ -941,6 +968,7 @@ function saveState(callback) {
 	srcString = savedSrcString
 	strings = savedStrings
 	referenceMap = savedReferenceMap
+	bundledStrings = savedBundledStrings
 	src = savedSrc
 	sequentialMode = savedSequentialMode
 	currentStructures = savedStructures

package/.vscode/launch.json

@@ -1,23 +0,0 @@
-{
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "type": "pwa-node",
-            "request": "launch",
-            "name": "Run msgpackr tests",
-            "runtimeArgs": [
-                "test"
-            ],
-            "runtimeExecutable": "npm",
-            "skipFiles": [
-                "<node_internals>/**"
-            ],
-            "type": "pwa-node",
-            "console": "internalConsole", 
-            "outputCapture": "std"
-        }
-    ]
-}

package/tests/benchmark-stream.cjs

@@ -1,282 +0,0 @@
-#!/usr/bin/env node
-
-var PassThrough = require("stream").PassThrough;
-var async = require("async");
-
-let { PackrStream, UnpackrStream } = require("..");
-var msgpack = require("msgpack-lite");
-var Encoder = require("msgpack-lite/lib/encoder").Encoder;
-var Decoder = require("msgpack-lite/lib/decoder").Decoder;
-var notepack = require("notepack");
-
-var pkg = require("../package.json");
-
-// a sample fluentd message
-var data = ["tag", [[1440949922, {"message": "hi there"}]]];
-var packed = msgpack.encode(data); // 30 bytes per message
-var packsize = packed.length;
-var opcount = 1000000;
-var joincount = 100;
-var packjoin = repeatbuf(packed, joincount); // 3KB per chunk
-var limit = 2;
-
-var blocksToJoin = []
-var streamForJoin = new PackrStream();
-streamForJoin.on("data", data => blocksToJoin.push(data));
-for (var j = 0; j < joincount; j++) {
-  streamForJoin.write(data);
-}
-var packjoinWithRecords = Buffer.concat(blocksToJoin)
-
-var argv = Array.prototype.slice.call(process.argv, 2);
-
-if (argv[0] === "-v") {
-  console.warn(pkg.name + " " + pkg.version);
-  process.exit(0);
-}
-
-if (argv[0] - 0) limit = argv.shift() - 0;
-
-var list = [
-  ['new PackrStream().write(obj);', encode5],
-  ['new UnpackrStream().write(buf);', decode5],
-  ['stream.write(msgpack.encode(obj));', encode1],
-  ['stream.write(msgpack.decode(buf));', decode1],
-  ['stream.write(notepack.encode(obj));', encode4],
-  ['stream.write(notepack.decode(buf));', decode4],
-  ['msgpack.Encoder().on("data",ondata).encode(obj);', encode2],
-  ['msgpack.createDecodeStream().write(buf);', decode3],
-  ['msgpack.createEncodeStream().write(obj);', encode3],
-  ['msgpack.Decoder().on("data",ondata).decode(buf);', decode2],
-//  ['stream.write(Buffer.from(JSON.stringify(obj)));', stringify],
-//  ['stream.write(JSON.parse(buf));', parse]
-];
-
-function encode5(callback) {
-  var stream = new PackrStream();
-  var cnt = counter(callback);
-  stream.on("data", cnt.inc);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount; j++) {
-    stream.write(data);
-  }
-  stream.end();
-}
-
-function encode1(callback) {
-  var stream = new PassThrough();
-  var cnt = counter(callback);
-  stream.on("data", cnt.buf);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount; j++) {
-    stream.write(msgpack.encode(data));
-  }
-  stream.end();
-}
-
-function encode2(callback) {
-  var stream = new PassThrough();
-  var cnt = counter(callback);
-  stream.on("data", cnt.buf);
-  stream.on("end", cnt.end);
-  var encoder = Encoder();
-  encoder.on("data", function(chunk) {
-    stream.write(chunk);
-  });
-  encoder.on("end", function() {
-    stream.end();
-  });
-  for (var j = 0; j < opcount; j++) {
-    encoder.encode(data);
-  }
-  encoder.end();
-}
-
-function encode3(callback) {
-  var stream = msgpack.createEncodeStream();
-  var cnt = counter(callback);
-  stream.on("data", cnt.buf);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount; j++) {
-    stream.write(data);
-  }
-  stream.end();
-}
-
-function encode4(callback) {
-  var stream = new PassThrough();
-  var cnt = counter(callback);
-  stream.on("data", cnt.buf);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount; j++) {
-    stream.write(notepack.encode(data));
-  }
-  stream.end();
-}
-
-function decode5(callback) {
-  var stream = new UnpackrStream();
-  var cnt = counter(callback);
-  stream.on("data", cnt.inc);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount / joincount; j++) {
-    stream.write(packjoinWithRecords);
-  }
-  stream.end();
-}
-
-function decode1(callback) {
-  var stream = new PassThrough({objectMode: true});
-  var cnt = counter(callback);
-  stream.on("data", cnt.inc);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount; j++) {
-    stream.write(msgpack.decode(packed));
-  }
-  stream.end();
-}
-
-function decode2(callback) {
-  var stream = new PassThrough({objectMode: true});
-  var cnt = counter(callback);
-  stream.on("data", cnt.inc);
-  stream.on("end", cnt.end);
-  var decoder = Decoder();
-  decoder.on("data", function(chunk) {
-    stream.write(chunk);
-  });
-  decoder.on("end", function() {
-    stream.end();
-  });
-  for (var j = 0; j < opcount / joincount; j++) {
-    decoder.decode(packjoin);
-  }
-  decoder.end();
-}
-
-function decode3(callback) {
-  var stream = msgpack.createDecodeStream();
-  var cnt = counter(callback);
-  stream.on("data", cnt.inc);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount / joincount; j++) {
-    stream.write(packjoin);
-  }
-  stream.end();
-}
-
-function decode4(callback) {
-  var stream = new PassThrough({objectMode: true});
-  var cnt = counter(callback);
-  stream.on("data", cnt.inc);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount; j++) {
-    stream.write(notepack.decode(packed));
-  }
-  stream.end();
-}
-
-function rpad(str, len, chr) {
-  if (!chr) chr = " ";
-  str += "";
-  while (str.length < len) str += chr;
-  return str;
-}
-
-function lpad(str, len, chr) {
-  if (!chr) chr = " ";
-  str += "";
-  while (str.length < len) str = chr + str;
-  return str;
-}
-
-function repeatbuf(buf, cnt) {
-  var array = [];
-  for (var i = 0; i < cnt; i++) {
-    array.push(buf);
-  }
-  return Buffer.concat(array);
-}
-
-function counter(callback) {
-  var cnt = 0;
-  return {buf: b, inc: i, end: e};
-
-  function b(buf) {
-    cnt += buf.length / packsize;
-  }
-
-  function i() {
-    cnt++;
-  }
-
-  function e() {
-    cnt = Math.round(cnt);
-    callback(null, cnt);
-  }
-}
-
-function run() {
-  // task filter
-  if (argv.length) {
-    list = list.filter(function(pair) {
-      var name = pair[0];
-      var match = argv.filter(function(grep) {
-        return (name.indexOf(grep) > -1);
-      });
-      return match.length;
-    });
-  }
-
-  // run tasks repeatedly
-  var tasks = [];
-  for (var i = 0; i < limit; i++) {
-    tasks.push(oneset);
-  }
-  async.series(tasks, end);
-
-  // run a series of tasks
-  function oneset(callback) {
-    async.eachSeries(list, bench, callback);
-  }
-
-  // run a single benchmark
-  function bench(pair, callback) {
-    process.stdout.write(".");
-    var func = pair[1];
-    var start = new Date() - 0;
-    func(function(err, cnt) {
-      var end = new Date() - 0;
-      var array = pair[2] || (pair[2] = []);
-      array.push(end - start);
-      pair[3] = cnt;
-      setTimeout(callback, 100);
-    });
-  }
-
-  // show result
-  function end() {
-    var title = "operation (" + opcount + " x " + limit + ")";
-    process.stdout.write("\n");
-
-    // table header
-    var COL1 = 48;
-    console.log(rpad(title, COL1), "|", "  op   ", "|", " ms  ", "|", " op/s ");
-    console.log(rpad("", COL1, "-"), "|", "------:", "|", "----:", "|", "-----:");
-
-    // table body
-    list.forEach(function(pair) {
-      var name = pair[0];
-      var op = pair[3];
-      var array = pair[2];
-      array = array.sort(function(a, b) {
-        return a > b;
-      });
-      var fastest = array[0];
-      var score = Math.floor(opcount / fastest * 1000);
-      console.log(rpad(name, COL1), "|", lpad(op, 7), "|", lpad(fastest, 5), "|", lpad(score, 6));
-    });
-  }
-}
-
-run();