msgpackr 1.4.7 → 1.5.3

package/unpack.d.ts

@@ -13,6 +13,7 @@ export interface Options {
 	mapsAsObjects?: boolean
 	variableMapSize?: boolean
 	copyBuffers?: boolean
+	bundleStrings?: boolean
 	useTimestamp32?: boolean
 	largeBigIntToFloat?: boolean
 	encodeUndefinedAsNil?: boolean
@@ -21,6 +22,7 @@ export interface Options {
 	shouldShareStructure?: (keys: string[]) => boolean
 	getStructures?(): {}[]
 	saveStructures?(structures: {}[]): boolean | void
+	onInvalidDate?: () => any
 }
 interface Extension {
 	Class: Function
@@ -44,3 +46,4 @@ export function addExtension(extension: Extension): void
 export function clearSource(): void
 export function roundFloat32(float32Number: number): number
 export const C1: {}
+export let isNativeAccelerationEnabled: boolean

package/unpack.js

@@ -15,6 +15,7 @@ var currentStructures
 var srcString
 var srcStringStart = 0
 var srcStringEnd = 0
+var bundledStrings
 var referenceMap
 var currentExtensions = []
 var dataView
@@ -55,12 +56,21 @@ export class Unpackr {
 		srcStringEnd = 0
 		srcString = null
 		strings = EMPTY_ARRAY
+		bundledStrings = null
 		src = source
 		// this provides cached access to the data view for a buffer if it is getting reused, which is a recommend
 		// technique for getting data from a database where it can be copied into an existing buffer instead of creating
 		// new ones
-		dataView = source.dataView || (source.dataView = new DataView(source.buffer, source.byteOffset, source.byteLength))
-		if (this) {
+		try {
+			dataView = source.dataView || (source.dataView = new DataView(source.buffer, source.byteOffset, source.byteLength))
+		} catch(error) {
+			// if it doesn't have a buffer, maybe it is the wrong type of object
+			src = null
+			if (source instanceof Uint8Array)
+				throw error
+			throw new Error('Source must be a Uint8Array or Buffer but was a ' + ((source && typeof source == 'object') ? source.constructor.name : typeof source))
+		}
+		if (this instanceof Unpackr) {
 			currentUnpackr = this
 			if (this.structures) {
 				currentStructures = this.structures
@@ -146,6 +156,9 @@ export function checkedRead() {
 				currentStructures.length = sharedLength
 		}
 		let result = read()
+		if (bundledStrings) // bundled strings to skip past
+			position = bundledStrings.postBundlePosition
+
 		if (position == srcEnd) {
 			// finished reading this source, cleanup references
 			if (currentStructures.restoreStructures)
@@ -240,7 +253,15 @@ export function read() {
 		let value
 		switch (token) {
 			case 0xc0: return null
-			case 0xc1: return C1; // "never-used", return special object to denote that
+			case 0xc1:
+				if (bundledStrings) {
+					value = read() // followed by the length of the string in characters (not bytes!)
+					if (value > 0)
+						return bundledStrings[1].slice(bundledStrings.position1, bundledStrings.position1 += value)
+					else
+						return bundledStrings[0].slice(bundledStrings.position0, bundledStrings.position0 -= value)
+				}
+				return C1; // "never-used", return special object to denote that
 			case 0xc2: return false
 			case 0xc3: return true
 			case 0xc4:
@@ -295,10 +316,11 @@ export function read() {
 				position += 4
 				return value
 			case 0xcf:
-				if (currentUnpackr.uint64AsNumber)
-					return src[position++] * 0x100000000000000 + src[position++] * 0x1000000000000 + src[position++] * 0x10000000000 + src[position++] * 0x100000000 +
-						src[position++] * 0x1000000 + (src[position++] << 16) + (src[position++] << 8) + src[position++]
-				value = dataView.getBigUint64(position)
+				if (currentUnpackr.int64AsNumber) {
+					value = dataView.getUint32(position) * 0x100000000
+					value += dataView.getUint32(position + 4)
+				} else
+					value = dataView.getBigUint64(position)
 				position += 8
 				return value
 
@@ -314,7 +336,11 @@ export function read() {
 				position += 4
 				return value
 			case 0xd3:
-				value = dataView.getBigInt64(position)
+				if (currentUnpackr.int64AsNumber) {
+					value = dataView.getInt32(position) * 0x100000000
+					value += dataView.getUint32(position + 4)
+				} else
+					value = dataView.getBigInt64(position)
 				position += 8
 				return value
 
@@ -463,8 +489,10 @@ var readFixedString = readStringJS
 var readString8 = readStringJS
 var readString16 = readStringJS
 var readString32 = readStringJS
+export let isNativeAccelerationEnabled = false
 
 export function setExtractor(extractStrings) {
+	isNativeAccelerationEnabled = true
 	readFixedString = readString(1)
 	readString8 = readString(2)
 	readString16 = readString(3)
@@ -473,6 +501,8 @@ export function setExtractor(extractStrings) {
 		return function readString(length) {
 			let string = strings[stringPosition++]
 			if (string == null) {
+				if (bundledStrings)
+					return readStringJS(length)
 				let extraction = extractStrings(position - headerLength, srcEnd, src)
 				if (typeof extraction == 'string') {
 					string = extraction
@@ -731,6 +761,36 @@ function shortStringInJS(length) {
 	}
 }
 
+function readOnlyJSString() {
+	let token = src[position++]
+	let length
+	if (token < 0xc0) {
+		// fixstr
+		length = token - 0xa0
+	} else {
+		switch(token) {
+			case 0xd9:
+			// str 8
+				length = src[position++]
+				break
+			case 0xda:
+			// str 16
+				length = dataView.getUint16(position)
+				position += 2
+				break
+			case 0xdb:
+			// str 32
+				length = dataView.getUint32(position)
+				position += 4
+				break
+			default:
+				throw new Error('Expected string')
+		}
+	}
+	return readStringJS(length)
+}
+
+
 function readBin(length) {
 	return currentUnpackr.copyBuffers ?
 		// specifically use the copying slice (not the node one)
@@ -882,6 +942,19 @@ currentExtensions[0x78] = () => {
 	let data = read()
 	return new RegExp(data[0], data[1])
 }
+const TEMP_BUNDLE = []
+currentExtensions[0x62] = (data) => {
+	let dataSize = (data[0] << 24) + (data[1] << 16) + (data[2] << 8) + data[3]
+	let dataPosition = position
+	position += dataSize - data.length
+	bundledStrings = TEMP_BUNDLE
+	bundledStrings = [readOnlyJSString(), readOnlyJSString()]
+	bundledStrings.position0 = 0
+	bundledStrings.position1 = 0
+	bundledStrings.postBundlePosition = position
+	position = dataPosition
+	return read()
+}
 
 currentExtensions[0xff] = (data) => {
 	// 32-bit date extension
@@ -896,7 +969,7 @@ currentExtensions[0xff] = (data) => {
 			((data[0] << 24) + (data[1] << 16) + (data[2] << 8) + data[3]) / 1000000 +
 			(((data[4] & 0x80) ? -0x1000000000000 : 0) + data[6] * 0x10000000000 + data[7] * 0x100000000 + data[8] * 0x1000000 + (data[9] << 16) + (data[10] << 8) + data[11]) * 1000)
 	else
-		throw new Error('Invalid timestamp length')
+		return new Date('invalid')
 } // notepack defines extension 0 to mean undefined, so use that as the default here
 // registration of bulk record definition?
 // currentExtensions[0x52] = () =>
@@ -910,6 +983,7 @@ function saveState(callback) {
 	let savedSrcString = srcString
 	let savedStrings = strings
 	let savedReferenceMap = referenceMap
+	let savedBundledStrings = bundledStrings
 
 	// TODO: We may need to revisit this if we do more external calls to user code (since it could be slow)
 	let savedSrc = new Uint8Array(src.slice(0, srcEnd)) // we copy the data in case it changes while external data is processed
@@ -926,6 +1000,7 @@ function saveState(callback) {
 	srcString = savedSrcString
 	strings = savedStrings
 	referenceMap = savedReferenceMap
+	bundledStrings = savedBundledStrings
 	src = savedSrc
 	sequentialMode = savedSequentialMode
 	currentStructures = savedStructures
@@ -951,8 +1026,6 @@ export const mult10 = new Array(147) // this is a table matching binary exponent
 for (let i = 0; i < 256; i++) {
 	mult10[i] = +('1e' + Math.floor(45.15 - i * 0.30103))
 }
-export const useRecords = false
-export const mapsAsObjects = true
 export const Decoder = Unpackr
 var defaultUnpackr = new Unpackr({ useRecords: false })
 export const unpack = defaultUnpackr.unpack

package/.vscode/launch.json

@@ -1,23 +0,0 @@
-{
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "type": "pwa-node",
-            "request": "launch",
-            "name": "Run msgpackr tests",
-            "runtimeArgs": [
-                "test"
-            ],
-            "runtimeExecutable": "npm",
-            "skipFiles": [
-                "<node_internals>/**"
-            ],
-            "type": "pwa-node",
-            "console": "internalConsole", 
-            "outputCapture": "std"
-        }
-    ]
-}

package/tests/benchmark-stream.cjs

@@ -1,282 +0,0 @@
-#!/usr/bin/env node
-
-var PassThrough = require("stream").PassThrough;
-var async = require("async");
-
-let { PackrStream, UnpackrStream } = require("..");
-var msgpack = require("msgpack-lite");
-var Encoder = require("msgpack-lite/lib/encoder").Encoder;
-var Decoder = require("msgpack-lite/lib/decoder").Decoder;
-var notepack = require("notepack");
-
-var pkg = require("../package.json");
-
-// a sample fluentd message
-var data = ["tag", [[1440949922, {"message": "hi there"}]]];
-var packed = msgpack.encode(data); // 30 bytes per message
-var packsize = packed.length;
-var opcount = 1000000;
-var joincount = 100;
-var packjoin = repeatbuf(packed, joincount); // 3KB per chunk
-var limit = 2;
-
-var blocksToJoin = []
-var streamForJoin = new PackrStream();
-streamForJoin.on("data", data => blocksToJoin.push(data));
-for (var j = 0; j < joincount; j++) {
-  streamForJoin.write(data);
-}
-var packjoinWithRecords = Buffer.concat(blocksToJoin)
-
-var argv = Array.prototype.slice.call(process.argv, 2);
-
-if (argv[0] === "-v") {
-  console.warn(pkg.name + " " + pkg.version);
-  process.exit(0);
-}
-
-if (argv[0] - 0) limit = argv.shift() - 0;
-
-var list = [
-  ['new PackrStream().write(obj);', encode5],
-  ['new UnpackrStream().write(buf);', decode5],
-  ['stream.write(msgpack.encode(obj));', encode1],
-  ['stream.write(msgpack.decode(buf));', decode1],
-  ['stream.write(notepack.encode(obj));', encode4],
-  ['stream.write(notepack.decode(buf));', decode4],
-  ['msgpack.Encoder().on("data",ondata).encode(obj);', encode2],
-  ['msgpack.createDecodeStream().write(buf);', decode3],
-  ['msgpack.createEncodeStream().write(obj);', encode3],
-  ['msgpack.Decoder().on("data",ondata).decode(buf);', decode2],
-//  ['stream.write(Buffer.from(JSON.stringify(obj)));', stringify],
-//  ['stream.write(JSON.parse(buf));', parse]
-];
-
-function encode5(callback) {
-  var stream = new PackrStream();
-  var cnt = counter(callback);
-  stream.on("data", cnt.inc);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount; j++) {
-    stream.write(data);
-  }
-  stream.end();
-}
-
-function encode1(callback) {
-  var stream = new PassThrough();
-  var cnt = counter(callback);
-  stream.on("data", cnt.buf);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount; j++) {
-    stream.write(msgpack.encode(data));
-  }
-  stream.end();
-}
-
-function encode2(callback) {
-  var stream = new PassThrough();
-  var cnt = counter(callback);
-  stream.on("data", cnt.buf);
-  stream.on("end", cnt.end);
-  var encoder = Encoder();
-  encoder.on("data", function(chunk) {
-    stream.write(chunk);
-  });
-  encoder.on("end", function() {
-    stream.end();
-  });
-  for (var j = 0; j < opcount; j++) {
-    encoder.encode(data);
-  }
-  encoder.end();
-}
-
-function encode3(callback) {
-  var stream = msgpack.createEncodeStream();
-  var cnt = counter(callback);
-  stream.on("data", cnt.buf);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount; j++) {
-    stream.write(data);
-  }
-  stream.end();
-}
-
-function encode4(callback) {
-  var stream = new PassThrough();
-  var cnt = counter(callback);
-  stream.on("data", cnt.buf);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount; j++) {
-    stream.write(notepack.encode(data));
-  }
-  stream.end();
-}
-
-function decode5(callback) {
-  var stream = new UnpackrStream();
-  var cnt = counter(callback);
-  stream.on("data", cnt.inc);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount / joincount; j++) {
-    stream.write(packjoinWithRecords);
-  }
-  stream.end();
-}
-
-function decode1(callback) {
-  var stream = new PassThrough({objectMode: true});
-  var cnt = counter(callback);
-  stream.on("data", cnt.inc);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount; j++) {
-    stream.write(msgpack.decode(packed));
-  }
-  stream.end();
-}
-
-function decode2(callback) {
-  var stream = new PassThrough({objectMode: true});
-  var cnt = counter(callback);
-  stream.on("data", cnt.inc);
-  stream.on("end", cnt.end);
-  var decoder = Decoder();
-  decoder.on("data", function(chunk) {
-    stream.write(chunk);
-  });
-  decoder.on("end", function() {
-    stream.end();
-  });
-  for (var j = 0; j < opcount / joincount; j++) {
-    decoder.decode(packjoin);
-  }
-  decoder.end();
-}
-
-function decode3(callback) {
-  var stream = msgpack.createDecodeStream();
-  var cnt = counter(callback);
-  stream.on("data", cnt.inc);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount / joincount; j++) {
-    stream.write(packjoin);
-  }
-  stream.end();
-}
-
-function decode4(callback) {
-  var stream = new PassThrough({objectMode: true});
-  var cnt = counter(callback);
-  stream.on("data", cnt.inc);
-  stream.on("end", cnt.end);
-  for (var j = 0; j < opcount; j++) {
-    stream.write(notepack.decode(packed));
-  }
-  stream.end();
-}
-
-function rpad(str, len, chr) {
-  if (!chr) chr = " ";
-  str += "";
-  while (str.length < len) str += chr;
-  return str;
-}
-
-function lpad(str, len, chr) {
-  if (!chr) chr = " ";
-  str += "";
-  while (str.length < len) str = chr + str;
-  return str;
-}
-
-function repeatbuf(buf, cnt) {
-  var array = [];
-  for (var i = 0; i < cnt; i++) {
-    array.push(buf);
-  }
-  return Buffer.concat(array);
-}
-
-function counter(callback) {
-  var cnt = 0;
-  return {buf: b, inc: i, end: e};
-
-  function b(buf) {
-    cnt += buf.length / packsize;
-  }
-
-  function i() {
-    cnt++;
-  }
-
-  function e() {
-    cnt = Math.round(cnt);
-    callback(null, cnt);
-  }
-}
-
-function run() {
-  // task filter
-  if (argv.length) {
-    list = list.filter(function(pair) {
-      var name = pair[0];
-      var match = argv.filter(function(grep) {
-        return (name.indexOf(grep) > -1);
-      });
-      return match.length;
-    });
-  }
-
-  // run tasks repeatedly
-  var tasks = [];
-  for (var i = 0; i < limit; i++) {
-    tasks.push(oneset);
-  }
-  async.series(tasks, end);
-
-  // run a series of tasks
-  function oneset(callback) {
-    async.eachSeries(list, bench, callback);
-  }
-
-  // run a single benchmark
-  function bench(pair, callback) {
-    process.stdout.write(".");
-    var func = pair[1];
-    var start = new Date() - 0;
-    func(function(err, cnt) {
-      var end = new Date() - 0;
-      var array = pair[2] || (pair[2] = []);
-      array.push(end - start);
-      pair[3] = cnt;
-      setTimeout(callback, 100);
-    });
-  }
-
-  // show result
-  function end() {
-    var title = "operation (" + opcount + " x " + limit + ")";
-    process.stdout.write("\n");
-
-    // table header
-    var COL1 = 48;
-    console.log(rpad(title, COL1), "|", "  op   ", "|", " ms  ", "|", " op/s ");
-    console.log(rpad("", COL1, "-"), "|", "------:", "|", "----:", "|", "-----:");
-
-    // table body
-    list.forEach(function(pair) {
-      var name = pair[0];
-      var op = pair[3];
-      var array = pair[2];
-      array = array.sort(function(a, b) {
-        return a > b;
-      });
-      var fastest = array[0];
-      var score = Math.floor(opcount / fastest * 1000);
-      console.log(rpad(name, COL1), "|", lpad(op, 7), "|", lpad(fastest, 5), "|", lpad(score, 6));
-    });
-  }
-}
-
-run();