msgpackr 1.7.0-alpha2 → 1.7.0-alpha3

package/index.js

@@ -1,5 +1,5 @@
-export { Packr, Encoder, addExtension, pack, encode, NEVER, ALWAYS, DECIMAL_ROUND, DECIMAL_FIT, REUSE_BUFFER_MODE } from './pack.js'
-export { Unpackr, Decoder, C1, unpack, unpackMultiple, decode, FLOAT32_OPTIONS, clearSource, roundFloat32, isNativeAccelerationEnabled } from './unpack.js'
-export { decodeIter, encodeIter } from './iterators.js'
-export const useRecords = false
-export const mapsAsObjects = true
+export { Packr, Encoder, addExtension, pack, encode, NEVER, ALWAYS, DECIMAL_ROUND, DECIMAL_FIT, REUSE_BUFFER_MODE } from './pack.js'
+export { Unpackr, Decoder, C1, unpack, unpackMultiple, decode, FLOAT32_OPTIONS, clearSource, roundFloat32, isNativeAccelerationEnabled } from './unpack.js'
+export { decodeIter, encodeIter } from './iterators.js'
+export const useRecords = false
+export const mapsAsObjects = true

package/iterators.js

@@ -1,87 +1,87 @@
-import { Packr } from './pack.js'
-import { Unpackr } from './unpack.js'
-
-/**
- * Given an Iterable first argument, returns an Iterable where each value is packed as a Buffer
- * If the argument is only Async Iterable, the return value will be an Async Iterable.
- * @param {Iterable|Iterator|AsyncIterable|AsyncIterator} objectIterator - iterable source, like a Readable object stream, an array, Set, or custom object
- * @param {options} [options] - msgpackr pack options
- * @returns {IterableIterator|Promise.<AsyncIterableIterator>}
- */
-export function packIter (objectIterator, options = {}) {
-  if (!objectIterator || typeof objectIterator !== 'object') {
-    throw new Error('first argument must be an Iterable, Async Iterable, or a Promise for an Async Iterable')
-  } else if (typeof objectIterator[Symbol.iterator] === 'function') {
-    return packIterSync(objectIterator, options)
-  } else if (typeof objectIterator.then === 'function' || typeof objectIterator[Symbol.asyncIterator] === 'function') {
-    return packIterAsync(objectIterator, options)
-  } else {
-    throw new Error('first argument must be an Iterable, Async Iterable, Iterator, Async Iterator, or a Promise')
-  }
-}
-
-function * packIterSync (objectIterator, options) {
-  const packr = new Packr(options)
-  for (const value of objectIterator) {
-    yield packr.pack(value)
-  }
-}
-
-async function * packIterAsync (objectIterator, options) {
-  const packr = new Packr(options)
-  for await (const value of objectIterator) {
-    yield packr.pack(value)
-  }
-}
-
-/**
- * Given an Iterable/Iterator input which yields buffers, returns an IterableIterator which yields sync decoded objects
- * Or, given an Async Iterable/Iterator which yields promises resolving in buffers, returns an AsyncIterableIterator.
- * @param {Iterable|Iterator|AsyncIterable|AsyncIterableIterator} bufferIterator
- * @param {object} [options] - unpackr options
- * @returns {IterableIterator|Promise.<AsyncIterableIterator}
- */
-export function unpackIter (bufferIterator, options = {}) {
-  if (!bufferIterator || typeof bufferIterator !== 'object') {
-    throw new Error('first argument must be an Iterable, Async Iterable, Iterator, Async Iterator, or a promise')
-  }
-
-  const unpackr = new Unpackr(options)
-  let incomplete
-  const parser = (chunk) => {
-    let yields
-    // if there's incomplete data from previous chunk, concatinate and try again
-    if (incomplete) {
-      chunk = Buffer.concat([incomplete, chunk])
-      incomplete = undefined
-    }
-
-    try {
-      yields = unpackr.unpackMultiple(chunk)
-    } catch (err) {
-      if (err.incomplete) {
-        incomplete = chunk.slice(err.lastPosition)
-        yields = err.values
-      } else {
-        throw err
-      }
-    }
-    return yields
-  }
-
-  if (typeof bufferIterator[Symbol.iterator] === 'function') {
-    return (function * iter () {
-      for (const value of bufferIterator) {
-        yield * parser(value)
-      }
-    })()
-  } else if (typeof bufferIterator[Symbol.asyncIterator] === 'function') {
-    return (async function * iter () {
-      for await (const value of bufferIterator) {
-        yield * parser(value)
-      }
-    })()
-  }
-}
-export const decodeIter = unpackIter
+import { Packr } from './pack.js'
+import { Unpackr } from './unpack.js'
+
+/**
+ * Given an Iterable first argument, returns an Iterable where each value is packed as a Buffer
+ * If the argument is only Async Iterable, the return value will be an Async Iterable.
+ * @param {Iterable|Iterator|AsyncIterable|AsyncIterator} objectIterator - iterable source, like a Readable object stream, an array, Set, or custom object
+ * @param {options} [options] - msgpackr pack options
+ * @returns {IterableIterator|Promise.<AsyncIterableIterator>}
+ */
+export function packIter (objectIterator, options = {}) {
+  if (!objectIterator || typeof objectIterator !== 'object') {
+    throw new Error('first argument must be an Iterable, Async Iterable, or a Promise for an Async Iterable')
+  } else if (typeof objectIterator[Symbol.iterator] === 'function') {
+    return packIterSync(objectIterator, options)
+  } else if (typeof objectIterator.then === 'function' || typeof objectIterator[Symbol.asyncIterator] === 'function') {
+    return packIterAsync(objectIterator, options)
+  } else {
+    throw new Error('first argument must be an Iterable, Async Iterable, Iterator, Async Iterator, or a Promise')
+  }
+}
+
+function * packIterSync (objectIterator, options) {
+  const packr = new Packr(options)
+  for (const value of objectIterator) {
+    yield packr.pack(value)
+  }
+}
+
+async function * packIterAsync (objectIterator, options) {
+  const packr = new Packr(options)
+  for await (const value of objectIterator) {
+    yield packr.pack(value)
+  }
+}
+
+/**
+ * Given an Iterable/Iterator input which yields buffers, returns an IterableIterator which yields sync decoded objects
+ * Or, given an Async Iterable/Iterator which yields promises resolving in buffers, returns an AsyncIterableIterator.
+ * @param {Iterable|Iterator|AsyncIterable|AsyncIterableIterator} bufferIterator
+ * @param {object} [options] - unpackr options
+ * @returns {IterableIterator|Promise.<AsyncIterableIterator}
+ */
+export function unpackIter (bufferIterator, options = {}) {
+  if (!bufferIterator || typeof bufferIterator !== 'object') {
+    throw new Error('first argument must be an Iterable, Async Iterable, Iterator, Async Iterator, or a promise')
+  }
+
+  const unpackr = new Unpackr(options)
+  let incomplete
+  const parser = (chunk) => {
+    let yields
+    // if there's incomplete data from previous chunk, concatinate and try again
+    if (incomplete) {
+      chunk = Buffer.concat([incomplete, chunk])
+      incomplete = undefined
+    }
+
+    try {
+      yields = unpackr.unpackMultiple(chunk)
+    } catch (err) {
+      if (err.incomplete) {
+        incomplete = chunk.slice(err.lastPosition)
+        yields = err.values
+      } else {
+        throw err
+      }
+    }
+    return yields
+  }
+
+  if (typeof bufferIterator[Symbol.iterator] === 'function') {
+    return (function * iter () {
+      for (const value of bufferIterator) {
+        yield * parser(value)
+      }
+    })()
+  } else if (typeof bufferIterator[Symbol.asyncIterator] === 'function') {
+    return (async function * iter () {
+      for await (const value of bufferIterator) {
+        yield * parser(value)
+      }
+    })()
+  }
+}
+export const decodeIter = unpackIter
 export const encodeIter = packIter

package/pack.d.ts

@@ -1,9 +1,9 @@
-import { Unpackr } from './unpack'
-export { addExtension, FLOAT32_OPTIONS } from './unpack'
-export class Packr extends Unpackr {
-	pack(value: any): Buffer
-	encode(value: any): Buffer
-}
-export class Encoder extends Packr {}
-export function pack(value: any): Buffer
-export function encode(value: any): Buffer
+import { Unpackr } from './unpack'
+export { addExtension, FLOAT32_OPTIONS } from './unpack'
+export class Packr extends Unpackr {
+	pack(value: any): Buffer
+	encode(value: any): Buffer
+}
+export class Encoder extends Packr {}
+export function pack(value: any): Buffer
+export function encode(value: any): Buffer

package/pack.js

@@ -15,7 +15,7 @@ let targetView
 let position = 0
 let safeEnd
 let bundledStrings = null
-let writeStructSlots
+let writeStructSlots, prepareStructures
 const MAX_BUNDLE_SIZE = 0xf000
 const hasNonLatin = /[\u0080-\uFFFF]/
 export const RECORD_SYMBOL = Symbol('record-id')
@@ -28,7 +28,6 @@ export class Packr extends Unpackr {
 		let hasSharedUpdate
 		let structures
 		let referenceMap
-		let lastSharedStructuresLength = 0
 		let encodeUtf8 = ByteArray.prototype.utf8Write ? function(string, position) {
 			return target.utf8Write(string, position, 0xffffffff)
 		} : (textEncoder && textEncoder.encodeInto) ?
@@ -114,7 +113,7 @@ export class Packr extends Unpackr {
 						}
 						transition[RECORD_SYMBOL] = i + 0x40
 					}
-					lastSharedStructuresLength = sharedLength
+					this.lastNamedStructuresLength = sharedLength
 				}
 				if (!isSequential) {
 					structures.nextId = sharedLength + 0x40
@@ -169,12 +168,13 @@ export class Packr extends Unpackr {
 					if (hasSharedUpdate && packr.saveStructures) {
 						// we can't rely on start/end with REUSE_BUFFER_MODE since they will (probably) change when we save
 						let returnBuffer = target.subarray(start, position)
-						if (packr.saveStructures(structures, lastSharedStructuresLength) === false) {
+						let newSharedData = prepareStructures ? prepareStructures(structures, packr) : structures;
+						if (packr.saveStructures(newSharedData, newSharedData.isCompatible || packr.lastNamedStructuresLength || 0) === false) {
 							// get updated structures and try again if the update failed
 							packr._mergeStructures(packr.getStructures())
 							return packr.pack(value)
 						}
-						lastSharedStructuresLength = sharedLength
+						packr.lastNamedStructuresLength = sharedLength
 						return returnBuffer
 					}
 				}
@@ -292,7 +292,7 @@ export class Packr extends Unpackr {
 			} else if (type === 'number') {
 				if (value >>> 0 === value) {// positive integer, 32-bit or less
 					// positive uint
-					if (value < 0x40 || (value < 0x80 && this.useRecords === false)) {
+					if (value < 0x20 || (value < 0x80 && this.useRecords === false) || (value < 0x40 && !this.randomAccessStructure)) {
 						target[position++] = value
 					} else if (value < 0x100) {
 						target[position++] = 0xcc
@@ -692,16 +692,18 @@ export class Packr extends Unpackr {
 			}
 		}
 		const writeStruct = (object, safePrototype) => {
-			let newPosition = writeStructSlots(object, target, position, structures, makeRoom, (value, newPosition) => {
+			let newPosition = writeStructSlots(object, target, position, structures, makeRoom, (value, newPosition, notifySharedUpdate) => {
+				if (notifySharedUpdate)
+					hasSharedUpdate = true;
 				position = newPosition;
 				if (start > 0) {
 					pack(value);
 					if (start == 0)
-						return { position, targetView }; // indicate the buffer was re-allocated
+						return { position, targetView, target }; // indicate the buffer was re-allocated
 				} else
 					pack(value);
 				return position;
-			})
+			}, this);
 			if (newPosition === 0) // bail and go to a msgpack object
 				return writeObject(object, true);
 			position = newPosition;
@@ -716,6 +718,8 @@ export class Packr extends Unpackr {
 	clearSharedData() {
 		if (this.structures)
 			this.structures = []
+		if (this.typedStructs)
+			this.typedStructs = []
 	}
 }
 
@@ -942,8 +946,9 @@ export function addExtension(extension) {
 	}
 	unpackAddExtension(extension)
 }
-export function setWriteStructSlots(func) {
-	writeStructSlots = func;
+export function setWriteStructSlots(writeSlots, makeStructures) {
+	writeStructSlots = writeSlots;
+	prepareStructures = makeStructures;
 }
 
 let defaultPackr = new Packr({ useRecords: false })

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "msgpackr",
   "author": "Kris Zyp",
-  "version": "1.7.0-alpha2",
+  "version": "1.7.0-alpha3",
   "description": "Ultra-fast MessagePack implementation with extensions for records and structured cloning",
   "license": "MIT",
   "types": "./index.d.ts",

package/rollup.config.js

@@ -1,49 +1,49 @@
-import minify from "rollup-plugin-babel-minify";
-import json  from "@rollup/plugin-json";
-
-export default [
-    {
-        input: "node-index.js",
-        output: [
-            {
-                file: "dist/node.cjs",
-                format: "cjs",
-                sourcemap: true
-            }
-        ]
-    },
-    {
-        input: "index.js",
-        output: {
-            file: "dist/index.js",
-            format: "umd",
-            name: "msgpackr",
-            sourcemap: true
-        }
-    },    
-    {
-        input: "index.js",
-        plugins: [minify({
-        })],
-        output: {
-            file: "dist/index.min.js",
-            format: "umd",
-            name: "msgpackr",
-            sourcemap: true
-        }
-    },
-    {
-        input: "tests/test.js",
-        plugins: [json()],
-        external: ['chai', '../index.js'],
-        output: {
-            file: "dist/test.js",
-            format: "iife",
-            sourcemap: true,
-            globals: {
-                chai: 'chai',
-                './index.js': 'msgpackr',
-            },
-        }
-    }
-];
+import minify from "rollup-plugin-babel-minify";
+import json  from "@rollup/plugin-json";
+
+export default [
+    {
+        input: "node-index.js",
+        output: [
+            {
+                file: "dist/node.cjs",
+                format: "cjs",
+                sourcemap: true
+            }
+        ]
+    },
+    {
+        input: "index.js",
+        output: {
+            file: "dist/index.js",
+            format: "umd",
+            name: "msgpackr",
+            sourcemap: true
+        }
+    },    
+    {
+        input: "index.js",
+        plugins: [minify({
+        })],
+        output: {
+            file: "dist/index.min.js",
+            format: "umd",
+            name: "msgpackr",
+            sourcemap: true
+        }
+    },
+    {
+        input: "tests/test.js",
+        plugins: [json()],
+        external: ['chai', '../index.js'],
+        output: {
+            file: "dist/test.js",
+            format: "iife",
+            sourcemap: true,
+            globals: {
+                chai: 'chai',
+                './index.js': 'msgpackr',
+            },
+        }
+    }
+];

package/stream.js

@@ -1,57 +1,57 @@
-import { Transform } from 'stream'
-import { Packr } from './pack.js'
-import { Unpackr } from './unpack.js'
-var DEFAULT_OPTIONS = {objectMode: true}
-
-export class PackrStream extends Transform {
-	constructor(options) {
-		if (!options)
-			options = {}
-		options.writableObjectMode = true
-		super(options)
-		options.sequential = true
-		this.packr = options.packr || new Packr(options)
-	}
-	_transform(value, encoding, callback) {
-		this.push(this.packr.pack(value))
-		callback()
-	}
-}
-
-export class UnpackrStream extends Transform {
-	constructor(options) {
-		if (!options)
-			options = {}
-		options.objectMode = true
-		super(options)
-		options.structures = []
-		this.unpackr = options.unpackr || new Unpackr(options)
-	}
-	_transform(chunk, encoding, callback) {
-		if (this.incompleteBuffer) {
-			chunk = Buffer.concat([this.incompleteBuffer, chunk])
-			this.incompleteBuffer = null
-		}
-		let values
-		try {
-			values = this.unpackr.unpackMultiple(chunk)
-		} catch(error) {
-			if (error.incomplete) {
-				this.incompleteBuffer = chunk.slice(error.lastPosition)
-				values = error.values
-			}
-			else
-				throw error
-		} finally {
-			for (let value of values || []) {
-				if (value === null)
-					value = this.getNullValue()
-				this.push(value)
-			}
-		}
-		if (callback) callback()
-	}
-	getNullValue() {
-		return Symbol.for(null)
-	}
-}
+import { Transform } from 'stream'
+import { Packr } from './pack.js'
+import { Unpackr } from './unpack.js'
+var DEFAULT_OPTIONS = {objectMode: true}
+
+export class PackrStream extends Transform {
+	constructor(options) {
+		if (!options)
+			options = {}
+		options.writableObjectMode = true
+		super(options)
+		options.sequential = true
+		this.packr = options.packr || new Packr(options)
+	}
+	_transform(value, encoding, callback) {
+		this.push(this.packr.pack(value))
+		callback()
+	}
+}
+
+export class UnpackrStream extends Transform {
+	constructor(options) {
+		if (!options)
+			options = {}
+		options.objectMode = true
+		super(options)
+		options.structures = []
+		this.unpackr = options.unpackr || new Unpackr(options)
+	}
+	_transform(chunk, encoding, callback) {
+		if (this.incompleteBuffer) {
+			chunk = Buffer.concat([this.incompleteBuffer, chunk])
+			this.incompleteBuffer = null
+		}
+		let values
+		try {
+			values = this.unpackr.unpackMultiple(chunk)
+		} catch(error) {
+			if (error.incomplete) {
+				this.incompleteBuffer = chunk.slice(error.lastPosition)
+				values = error.values
+			}
+			else
+				throw error
+		} finally {
+			for (let value of values || []) {
+				if (value === null)
+					value = this.getNullValue()
+				this.push(value)
+			}
+		}
+		if (callback) callback()
+	}
+	getNullValue() {
+		return Symbol.for(null)
+	}
+}