cbor-x 1.4.0 → 1.5.0

package/encode.js

@@ -7,10 +7,11 @@ let extensions, extensionClasses
 const hasNodeBuffer = typeof Buffer !== 'undefined'
 const ByteArrayAllocate = hasNodeBuffer ? Buffer.allocUnsafeSlow : Uint8Array
 const ByteArray = hasNodeBuffer ? Buffer : Uint8Array
-const RECORD_INLINE_ID = 0xdfff // temporary first-come first-serve tag // proposed tag: 0x7265 // 're'
+const BlobConstructor = typeof Blob === 'undefined' ? {} : Blob;
 const MAX_STRUCTURES = 0x100
 const MAX_BUFFER_SIZE = hasNodeBuffer ? 0x100000000 : 0x7fd00000
 let serializationId = 1
+let throwOnIterable
 let target
 let targetView
 let position = 0
@@ -175,7 +176,10 @@ export class Encoder extends Decoder {
 					}
 				}
 			}
+			throwOnIterable = encodeOptions & THROW_ON_ITERABLE;
 			try {
+				if (throwOnIterable)
+					return;
 				encode(value)
 				if (bundledStrings) {
 					writeBundles(start, encode)
@@ -536,6 +540,11 @@ export class Encoder extends Decoder {
 							}
 						}
 						if (value[Symbol.iterator]) {
+							if (throwOnIterable) {
+								let error = new Error('Iterable should be serialized as iterator')
+								error.iteratorNotHandled = true;
+								throw error;
+							}
 							target[position++] = 0x9f // indefinite length array
 							for (let entry of value) {
 								encode(entry)
@@ -543,6 +552,11 @@ export class Encoder extends Decoder {
 							target[position++] = 0xff // stop-code
 							return
 						}
+						if (value[Symbol.asyncIterator] || constructor === BlobConstructor) {
+							let error = new Error('Iterable/blob should be serialized as iterator')
+							error.iteratorNotHandled = true;
+							throw error;
+						}
 						// no extension found, write as object
 						writeObject(value, !value.hasOwnProperty) // if it doesn't have hasOwnProperty, don't do hasOwnProperty checks
 					}
@@ -707,7 +721,10 @@ export class Encoder extends Decoder {
 					writeArrayHeader(length + 2)
 					encode(0xe000 + recordId)
 					encode(keys)
-					for (let v of Object.values(object)) encode(v)
+					if (safePrototype === null) return; // special exit for iterator
+					for (let key in object)
+						if (safePrototype || object.hasOwnProperty(key))
+							encode(object[key])
 					return
 				}
 			}
@@ -716,6 +733,7 @@ export class Encoder extends Decoder {
 			} else {
 				writeArrayHeader(length)
 			}
+			if (safePrototype === null) return; // special exit for iterator
 			for (let key in object)
 				if (safePrototype || object.hasOwnProperty(key))
 					encode(object[key])
@@ -741,6 +759,130 @@ export class Encoder extends Decoder {
 			safeEnd = newBuffer.length - 10
 			return target = newBuffer
 		}
+		let chunkThreshold = 100;
+		let continuedChunkThreshold = 1000;
+		this.encodeAsIterable = function(value, options) {
+			return startEncoding(value, options, encodeObjectAsIterable);
+		}
+		this.encodeAsAsyncIterable = function(value, options) {
+			return startEncoding(value, options, encodeObjectAsAsyncIterable);
+		}
+
+		function* encodeObjectAsIterable(object, iterateProperties, finalIterable) {
+			let constructor = object.constructor;
+			if (constructor === Object) {
+				let useRecords = encoder.useRecords !== false;
+				if (useRecords)
+					writeObject(object, null); // write the record identifier
+				else
+					writeEntityLength(Object.keys(object).length, 0xa0);
+				for (let key in object) {
+					let value = object[key];
+					if (!useRecords) encode(key);
+					if (value && typeof value === 'object') {
+						if (iterateProperties[key])
+							yield* encodeObjectAsIterable(value, iterateProperties[key]);
+						else
+							yield* tryEncode(value, iterateProperties, key);
+					} else encode(value);
+				}
+			} else if (constructor === Array) {
+				writeArrayHeader(object.length);
+				for (let i = 0; i < length; i++) {
+					let value = object[i];
+					if (value && (typeof value === 'object' || position - start > chunkThreshold)) {
+						if (iterateProperties.element)
+							yield* encodeObjectAsIterable(value, iterateProperties.element);
+						else
+							yield* tryEncode(value, iterateProperties, 'element');
+					} else encode(value);
+				}
+			} else if (object[Symbol.iterator]) {
+				target[position++] = 0x9f; // start indefinite array
+				for (let value of object) {
+					if (value && (typeof value === 'object' || position - start > chunkThreshold)) {
+						if (iterateProperties.element)
+							yield* encodeObjectAsIterable(value, iterateProperties.element);
+						else
+							yield* tryEncode(value, iterateProperties, 'element');
+					} else encode(value);
+				}
+				target[position++] = 0xff; // stop byte
+			} else if (constructor === BlobConstructor){
+				writeEntityLength(object.size, 0x40); // encode as binary data
+				yield target.subarray(start, position);
+				yield object; // directly return blobs, they have to be encoded asynchronously
+				restartEncoding();
+			} else if (object[Symbol.asyncIterator]) {
+				target[position++] = 0x9f; // start indefinite array
+				yield target.subarray(start, position);
+				yield object; // directly return async iterators, they have to be encoded asynchronously
+				restartEncoding();
+				target[position++] = 0xff; // stop byte
+			} else {
+				encode(object);
+			}
+			if (finalIterable && position > start) yield target.subarray(start, position);
+			else if (position - start > chunkThreshold) {
+				yield target.subarray(start, position);
+				restartEncoding();
+			}
+		}
+		function* tryEncode(value, iterateProperties, key) {
+			let restart = position - start;
+			try {
+				encode(value);
+				if (position - start > chunkThreshold) {
+					yield target.subarray(start, position);
+					restartEncoding();
+				}
+			} catch (error) {
+				if (error.iteratorNotHandled) {
+					iterateProperties[key] = {};
+					position = start + restart; // restart our position so we don't have partial data from last encode
+					yield* encodeObjectAsIterable.call(this, value, iterateProperties[key]);
+				} else throw error;
+			}
+		}
+		function restartEncoding() {
+			chunkThreshold = continuedChunkThreshold;
+			encoder.encode(null, THROW_ON_ITERABLE); // restart encoding
+		}
+		function startEncoding(value, options, encodeIterable) {
+			if (options && options.chunkThreshold) // explicitly specified chunk sizes
+				chunkThreshold = continuedChunkThreshold = options.chunkThreshold;
+			else // we start with a smaller threshold to get initial bytes sent quickly
+				chunkThreshold = 100;
+			if (value && typeof value === 'object') {
+				encoder.encode(null, THROW_ON_ITERABLE); // start encoding
+				return encodeIterable(value, encoder.iterateProperties || (encoder.iterateProperties = {}), true);
+			}
+			return [encoder.encode(value)];
+		}
+
+		async function* encodeObjectAsAsyncIterable(value, iterateProperties) {
+			for (let encodedValue of encodeObjectAsIterable(value, iterateProperties, true)) {
+				let constructor = encodedValue.constructor;
+				if (constructor === ByteArray || constructor === Uint8Array)
+					yield encodedValue;
+				else if (constructor === BlobConstructor) {
+					let reader = encodedValue.stream().getReader();
+					let next;
+					while (!(next = await reader.read()).done) {
+						yield next.value;
+					}
+				} else if (encodedValue[Symbol.asyncIterator]) {
+					for await (let asyncValue of encodedValue) {
+						restartEncoding();
+						if (asyncValue)
+							yield* encodeObjectAsAsyncIterable(asyncValue, iterateProperties.async || (iterateProperties.async = {}));
+						else yield encoder.encode(asyncValue);
+					}
+				} else {
+					yield encodedValue;
+				}
+			}
+		}
 	}
 	useBuffer(buffer) {
 		// this means we are finished using our own buffer and we can write over it safely
@@ -776,6 +918,23 @@ export class Encoder extends Decoder {
 		return saveResults
 	}
 }
+function writeEntityLength(length, majorValue) {
+	if (length < 0x18)
+		target[position++] = majorValue | length
+	else if (length < 0x100) {
+		target[position++] = majorValue | 0x18
+		target[position++] = length
+	} else if (length < 0x10000) {
+		target[position++] = majorValue | 0x19
+		target[position++] = length >> 8
+		target[position++] = length & 0xff
+	} else {
+		target[position++] = majorValue | 0x1a
+		targetView.setUint32(position, length)
+		position += 4
+	}
+
+}
 class SharedData {
 	constructor(structures, values, version) {
 		this.structures = structures
@@ -1033,9 +1192,13 @@ export function addExtension(extension) {
 }
 let defaultEncoder = new Encoder({ useRecords: false })
 export const encode = defaultEncoder.encode
+export const encodeAsIterable = defaultEncoder.encodeAsIterable
+export const encodeAsAsyncIterable = defaultEncoder.encodeAsAsyncIterable
 export { FLOAT32_OPTIONS } from './decode.js'
 import { FLOAT32_OPTIONS } from './decode.js'
 export const { NEVER, ALWAYS, DECIMAL_ROUND, DECIMAL_FIT } = FLOAT32_OPTIONS
 export const REUSE_BUFFER_MODE = 512
 export const RESET_BUFFER_MODE = 1024
+export const THROW_ON_ITERABLE = 2048
+
 

package/index.d.ts

@@ -26,12 +26,14 @@ export interface Options {
 	saveStructures?(structures: {}[]): boolean | void
 	onInvalidDate?: () => any
 	tagUint8Array?: boolean
+	pack?: boolean
 }
-interface Extension {
-	Class: Function
+type ClassOf<T> = new (...args: any[]) => T;
+interface Extension<T, R> {
+	Class: ClassOf<T>
 	tag: number
-	encode(value: any): Buffer | Uint8Array
-	decode(messagePack: Buffer | Uint8Array): any
+	encode(value: T, encodeFn: (data: R) => Uint8Array): Buffer | Uint8Array
+	decode(item: R): T
 }
 export class Decoder {
 	constructor(options?: Options)
@@ -40,7 +42,7 @@ export class Decoder {
 }
 export function decode(messagePack: Buffer | Uint8Array): any
 export function decodeMultiple(messagePack: Buffer | Uint8Array, forEach?: (value: any) => any): [] | void
-export function addExtension(extension: Extension): void
+export function addExtension<T, R>(extension: Extension<T, R>): void
 export function clearSource(): void
 export function roundFloat32(float32Number: number): number
 export let isNativeAccelerationEnabled: boolean
@@ -49,6 +51,8 @@ export class Encoder extends Decoder {
 	encode(value: any): Buffer
 }
 export function encode(value: any): Buffer
+export function encodeAsIterable(value: any): Iterable<Buffer | Blob | AsyncIterable<Buffer>>
+export function encodeAsAsyncIterable(value: any): AsyncIterable<Buffer>
 
 import { Transform, Readable } from 'stream'
 

package/index.js

@@ -1,3 +1,3 @@
-export { Encoder, addExtension, encode, NEVER, ALWAYS, DECIMAL_ROUND, DECIMAL_FIT, REUSE_BUFFER_MODE } from './encode.js'
+export { Encoder, addExtension, encode, encodeAsIterable, encodeAsAsyncIterable, NEVER, ALWAYS, DECIMAL_ROUND, DECIMAL_FIT, REUSE_BUFFER_MODE } from './encode.js'
 export { Tag, Decoder, decodeMultiple, decode, FLOAT32_OPTIONS, clearSource, roundFloat32, isNativeAccelerationEnabled } from './decode.js'
 export { decodeIter, encodeIter } from './iterators.js'

package/node-index.js

@@ -1,4 +1,4 @@
-export { Encoder, addExtension, encode, NEVER, ALWAYS, DECIMAL_ROUND, DECIMAL_FIT, REUSE_BUFFER_MODE } from './encode.js'
+export { Encoder, addExtension, encode, encodeAsIterable, encodeAsAsyncIterable, NEVER, ALWAYS, DECIMAL_ROUND, DECIMAL_FIT, REUSE_BUFFER_MODE } from './encode.js'
 export { Tag, Decoder, decodeMultiple, decode, FLOAT32_OPTIONS, clearSource, roundFloat32, isNativeAccelerationEnabled } from './decode.js'
 export { EncoderStream, DecoderStream } from './stream.js'
 export { decodeIter, encodeIter } from './iterators.js'

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "cbor-x",
   "author": "Kris Zyp",
-  "version": "1.4.0",
+  "version": "1.5.0",
   "description": "Ultra-fast and conformant CBOR (RFC 8949) implementation with support for numerous tag extensions including records and structured cloning",
   "license": "MIT",
   "types": "./index.d.ts",
@@ -64,10 +64,12 @@
     "/*.ts"
   ],
   "optionalDependencies": {
-    "cbor-extract": "^2.0.2"
+    "cbor-extract": "^2.1.1"
   },
   "devDependencies": {
-    "@rollup/plugin-json": "^4.1.0",
+    "@rollup/plugin-json": "^5.0.1",
+    "@rollup/plugin-replace": "^5.0.1",
+    "@rollup/plugin-terser": "^0.1.0",
     "@types/node": "latest",
     "async": "^3",
     "cbor": "^5",
@@ -75,8 +77,7 @@
     "chai": "^4.3.4",
     "cpy-cli": "^4.1.0",
     "esm": "^3.2.25",
-    "mocha": "^8.1.3",
-    "rollup": "^1.20.3",
-    "rollup-plugin-babel-minify": "^9.0.0"
+    "mocha": "^10.1.0",
+    "rollup": "^3.2.5"
   }
 }

package/rollup.config.js

@@ -1,5 +1,6 @@
-import minify from "rollup-plugin-babel-minify";
-import json  from "@rollup/plugin-json";
+import terser from '@rollup/plugin-terser';
+import json from "@rollup/plugin-json";
+import replace from "@rollup/plugin-replace";
 
 export default [
     {
@@ -7,7 +8,8 @@ export default [
         output: [
             {
                 file: "dist/node.cjs",
-                format: "cjs"
+                format: "cjs",
+                sourcemap: true
             }
         ]
     },
@@ -16,17 +18,57 @@ export default [
         output: {
             file: "dist/index.js",
             format: "umd",
-            name: "CBOR"
+            name: "CBOR",
+            sourcemap: true
         }
-    },    
+    },
+    {
+        input: "index.js",
+        plugins: [
+            replace({ Function: 'BlockedFunction '})
+        ],
+        output: {
+            file: "dist/index-no-eval.cjs",
+            format: "umd",
+            name: "CBOR",
+            sourcemap: true
+        },
+    },
+    {
+        input: "decode.js",
+        plugins: [
+            replace({ Function: 'BlockedFunction '})
+        ],
+        output: {
+            file: "dist/decode-no-eval.cjs",
+            format: "umd",
+            name: "CBOR",
+            sourcemap: true
+        },
+    },
     {
         input: "index.js",
-        plugins: [minify({
-        })],
+        plugins: [
+            terser({})
+        ],
         output: {
             file: "dist/index.min.js",
             format: "umd",
-            name: "CBOR"
+            name: "CBOR",
+            sourcemap: true
+        }
+    },
+    {
+        input: "index.js",
+        plugins: [
+            replace({ Function: 'BlockedFunction '}),
+            terser({})
+        ],
+        output: {
+            file: "dist/index-no-eval.min.js",
+            format: "umd",
+            name: "CBOR",
+            sourcemap: true
         }
     },
     {
@@ -36,6 +78,7 @@ export default [
         output: {
             file: "dist/test.js",
             format: "iife",
+            sourcemap: true,
             globals: {
                 chai: 'chai',
                 './index.js': 'CBOR',

package/stream.js

@@ -12,8 +12,10 @@ export class EncoderStream extends Transform {
 		options.sequential = true
 		this.encoder = options.encoder || new Encoder(options)
 	}
-	_transform(value, encoding, callback) {
-		this.push(this.encoder.encode(value))
+	async _transform(value, encoding, callback) {
+		for await (let chunk of this.encoder.encodeAsAsyncIterable(value)) {
+			this.push(chunk)
+		}
 		callback()
 	}
 }