ordered-binary 1.1.0 → 1.2.0

package/README.md

@@ -1,8 +1,34 @@
+[![npm version](https://img.shields.io/npm/dw/ordered-binary)](https://www.npmjs.org/package/ordered-binary)
+[![npm version](https://img.shields.io/npm/v/ordered-binary.svg?style=flat-square)](https://www.npmjs.org/package/ordered-binary)
+[![license](https://img.shields.io/badge/license-MIT-brightgreen)](LICENSE)
 <a href="https://dev.doctorevidence.com/"><img src="./assets/powers-dre.png" width="203" /></a>
 
-The ordered-binary provides a representation of JavaScript primitives as NodeJS Buffers, such that there binary values are naturally ordered such that it matches primitive ordering. For example, since -2.0321 > -2.04, then `toBufferKey(-2.0321)` will be greater than `toBufferKey(-2.04)` as a binary representation, in left-to-right evaluation. This is particular useful for storing keys as binaries with something like LevelDB, to avoid any custom sorting.
+The ordered-binary package provides a representation of JavaScript primitives, serialized into binary format (NodeJS Buffers or Uint8Arrays), such that the binary values are naturally ordered such that it matches the natural ordering or values. For example, since -2.0321 > -2.04, then `toBufferKey(-2.0321)` will be greater than `toBufferKey(-2.04)` as a binary representation, in left-to-right evaluation. This is particular useful for storing keys as binaries with something like LMDB or LevelDB, to avoid any custom sorting.
 
-The main module exports two functions:
+The ordered-binary package supports strings, numbers, booleans, symbols, null, as well as an array of primitives. Here is an example of ordering of primitive values:
+```
+Buffer.from([0]) // buffers are left unchanged, and this is the minimum value
+Symbol.for('even symbols')
+-10 // negative supported
+-1.1 // decimals supported
+400
+3E10
+'Hello'
+['Hello', 'World']
+'World'
+'hello'
+['hello', 1, 'world']
+['hello', 'world']
+Buffer.from([0xff])
+```
+
+
+The main module exports these functions:
+
+`writeKey(key: string | number | boolean | null | Array, target: Buffer, position: integer, inSequence?: boolean)` - Writes the provide key to the target buffer
+
+`readKey(buffer, start, end, inSequence)` - Reads the key from the buffer, given the provided start and end, as a primitive value
 
 `toBufferKey(jsPrimitive)` - This accepts a string, number, or boolean as the argument, and returns a `Buffer`.
+
 `fromBufferKey(bufferKey, multiple)` - This accepts a Buffer and returns a JavaScript primitive value. This can also parse buffers that hold multiple values delimited by a byte `30`, by setting the second argument to true (in which case it will return an array).

package/dist/index.cjs

@@ -0,0 +1,343 @@
+'use strict';
+
+Object.defineProperty(exports, '__esModule', { value: true });
+
+/*
+control character types:
+1 - metadata
+2 - symbols
+6 - false
+7 - true
+8- 16 - negative doubles
+16-24 positive doubles
+27 - String starts with a character 27 or less or is an empty string
+0 - multipart separator
+> 27 normal string characters
+*/
+/*
+* Convert arbitrary scalar values to buffer bytes with type preservation and type-appropriate ordering
+*/
+
+const float64Array = new Float64Array(2);
+const int32Array = new Int32Array(float64Array.buffer, 0, 4);
+new Uint8Array(float64Array.buffer, 2, 6);
+new Uint8Array(float64Array.buffer, 0, 8);
+let nullTerminate = false;
+let textEncoder;
+try {
+	textEncoder = new TextEncoder();
+} catch (error) {}
+
+/*
+* Convert arbitrary scalar values to buffer bytes with type preservation and type-appropriate ordering
+*/
+function writeKey(key, target, position, inSequence) {
+	let targetView = target.dataView;
+	if (!targetView)
+		targetView  = target.dataView = new DataView(target.buffer, target.byteOffset, ((target.byteLength + 3) >> 2) << 2);
+	switch (typeof key) {
+	case 'string':
+		let strLength = key.length;
+		let c1 = key.charCodeAt(0);
+		if (!(c1 >= 28)) // escape character
+			target[position++] = 27;
+		if (strLength < 0x40) {
+			let i, c2;
+			for (i = 0; i < strLength; i++) {
+				c1 = key.charCodeAt(i);
+				if (c1 < 0x80) {
+					target[position++] = c1;
+				} else if (c1 < 0x800) {
+					target[position++] = c1 >> 6 | 0xc0;
+					target[position++] = c1 & 0x3f | 0x80;
+				} else if (
+					(c1 & 0xfc00) === 0xd800 &&
+					((c2 = key.charCodeAt(i + 1)) & 0xfc00) === 0xdc00
+				) {
+					c1 = 0x10000 + ((c1 & 0x03ff) << 10) + (c2 & 0x03ff);
+					i++;
+					target[position++] = c1 >> 18 | 0xf0;
+					target[position++] = c1 >> 12 & 0x3f | 0x80;
+					target[position++] = c1 >> 6 & 0x3f | 0x80;
+					target[position++] = c1 & 0x3f | 0x80;
+				} else {
+					target[position++] = c1 >> 12 | 0xe0;
+					target[position++] = c1 >> 6 & 0x3f | 0x80;
+					target[position++] = c1 & 0x3f | 0x80;
+				}
+			}
+		} else if (target.utf8Write) {
+			position += target.utf8Write(key, position, 2000);
+		} else {
+			position += textEncoder.encodeInto(key, target.subarray(position)).written;
+		}
+		break
+	case 'number':
+		float64Array[0] = key;
+		let lowInt = int32Array[0];
+		let highInt = int32Array[1];
+		let length;
+		if (key < 0) {
+			targetView.setInt32(position + 4, ~((lowInt >>> 4) | (highInt << 28)));
+			targetView.setInt32(position + 0, (highInt ^ 0x7fffffff) >>> 4);
+			targetView.setInt32(position + 8, ((lowInt & 0xf) ^ 0xf) << 4, true); // just always do the null termination here
+			return position + 9
+		} else if ((lowInt & 0xf) || inSequence) {
+			length = 9;
+		} else if (lowInt & 0xfffff)
+			length = 8;
+		else if (lowInt || (highInt & 0xf))
+			length = 6;
+		else
+			length = 4;
+		// switching order to go to little endian
+		targetView.setInt32(position + 0, (highInt >>> 4) | 0x10000000);
+		targetView.setInt32(position + 4, (lowInt >>> 4) | (highInt << 28));
+		// if (length == 9 || nullTerminate)
+		targetView.setInt32(position + 8, (lowInt & 0xf) << 4, true);
+		return position + length;
+	case 'object':
+		if (key) {
+			if (key instanceof Array) {
+				for (let i = 0, l = key.length; i < l; i++) {
+					if (i > 0)
+						target[position++] = 0;
+					position = writeKey(key[i], target, position, true);
+				}
+				break
+			} else if (key instanceof Uint8Array) {
+				target.set(key, position);
+				position += key.length;
+				break
+			} else {
+				throw new Error('Unable to serialize object as a key')
+			}
+		} else // null
+			target[position++] = 0;
+			break
+	case 'boolean':
+		targetView.setUint32(position++, key ? 7 : 6, true);
+		return position
+	case 'bigint':
+		return writeKey(Number(key), target, position, inSequence)
+	case 'undefined':
+		return position
+	// undefined is interpreted as the absence of a key, signified by zero length
+	case 'symbol':
+		target[position++] = 2;
+		return writeKey(key.description, target, position, inSequence)
+	default:
+		throw new Error('Can not serialize key of type ' + typeof key)
+	}
+	if (nullTerminate && !inSequence)
+		targetView.setUint32(position, 0);
+	return position
+}
+
+let position;
+function readKey(buffer, start, end, inSequence) {
+	buffer[end] = 0; // make sure it is null terminated
+	position = start;
+	let controlByte = buffer[position];
+	let value;
+	if (controlByte < 24) {
+	if (controlByte < 8) {
+		position++;
+		if (controlByte == 6) {
+			value = false;
+		} else if (controlByte == 7) {
+			value = true;
+		} else if (controlByte == 0) {
+			value = null;
+		} else if (controlByte == 2) {
+			value = Symbol.for(readString(buffer));
+		} else
+			return Uint8Array.prototype.slice.call(buffer, start, end)
+	} else {
+		let dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, ((buffer.byteLength + 3) >> 2) << 2));
+		let highInt = dataView.getInt32(position) << 4;
+		let size = end - position;
+		let lowInt;
+		if (size > 4) {
+			lowInt = dataView.getInt32(position + 4);
+			highInt |= lowInt >>> 28;
+			if (size <= 6) { // clear the last bits
+				lowInt &= -0x1000;
+			}
+			lowInt = lowInt << 4;
+			if (size > 8) {
+				lowInt = lowInt | buffer[position + 8] >> 4;
+			}
+		} else
+			lowInt = 0;
+		if (controlByte < 16) {
+			// negative gets negated
+			highInt = highInt ^ 0x7fffffff;
+			lowInt = ~lowInt;
+		}
+		int32Array[1] = highInt;
+		int32Array[0] = lowInt;
+		value = float64Array[0];
+		position += 9;
+	}
+	} else {
+	if (controlByte == 27) {
+		position++;
+	}
+	value = readString(buffer);
+	/*let strStart = position
+	let strEnd = end
+	for (; position < end; position++) {
+		if (buffer[position] == 0) {
+		break
+		}
+	}
+	value = buffer.toString('utf8', strStart, position++)*/
+	}
+	while (position < end) {
+		if (buffer[position] === 0)
+			position++;
+		if (inSequence) {
+			encoder.position = position;
+			return value
+		}
+		let nextValue = readKey(buffer, position, end, true);
+		if (value instanceof Array) {
+			value.push(nextValue);
+		} else
+			value = [ value, nextValue ];
+	}
+	return value
+}
+const encoder = {
+	writeKey,
+	readKey,
+};
+const toBufferKey = (key) => {
+	let buffer = Buffer.alloc(2048);
+	return buffer.slice(0, writeKey(key, buffer, 0, 2048) + 1)
+};
+const fromBufferKey = (sourceBuffer) => {
+	return readKey(sourceBuffer, 0, sourceBuffer.length - 1)
+};
+const fromCharCode = String.fromCharCode;
+function makeStringBuilder() {
+	let stringBuildCode = '(source) => {';
+	let previous = [];
+	for (let i = 0; i < 0x30; i++) {
+		let v = fromCharCode((i & 0xf) + 97) + fromCharCode((i >> 4) + 97);
+		stringBuildCode += `
+		let ${v} = source[position++]
+		if (${v} === 0)
+		return fromCharCode(${previous})
+		else if (${v} >= 0x80)
+		${v} = finishUtf8(${v}, source)
+		`;
+		previous.push(v);
+		if (i == 1000000) // this just exists to prevent rollup from doing dead code elimination on finishUtf8
+			finishUtf8();
+	}
+	stringBuildCode += `return fromCharCode(${previous}) + readString(source)}`;
+	return stringBuildCode
+}
+
+let pendingSurrogate;
+function finishUtf8(byte1, src) {
+	if ((byte1 & 0xe0) === 0xc0) {
+		// 2 bytes
+		const byte2 = src[position++] & 0x3f;
+		return ((byte1 & 0x1f) << 6) | byte2
+	} else if ((byte1 & 0xf0) === 0xe0) {
+		// 3 bytes
+		const byte2 = src[position++] & 0x3f;
+		const byte3 = src[position++] & 0x3f;
+		return ((byte1 & 0x1f) << 12) | (byte2 << 6) | byte3
+	} else if ((byte1 & 0xf8) === 0xf0) {
+		// 4 bytes
+		if (pendingSurrogate) {
+		byte1 = pendingSurrogate;
+		pendingSurrogate = null;
+		position += 3;
+		return byte1
+		}
+		const byte2 = src[position++] & 0x3f;
+		const byte3 = src[position++] & 0x3f;
+		const byte4 = src[position++] & 0x3f;
+		let unit = ((byte1 & 0x07) << 0x12) | (byte2 << 0x0c) | (byte3 << 0x06) | byte4;
+		if (unit > 0xffff) {
+		unit -= 0x10000;
+		unit = 0xdc00 | (unit & 0x3ff);
+		pendingSurrogate = ((unit >>> 10) & 0x3ff) | 0xd800;
+		position -= 4; // reset so we can return the next part of the surrogate pair
+		}
+		return unit
+	} else {
+		return byte1
+	}
+}
+
+const enableNullTermination = () => nullTerminate = true;
+
+const readString = eval(makeStringBuilder());
+
+function compareKeys(a, b) {
+	// compare with type consistency that matches binary comparison
+	if (typeof a == 'object') {
+		if (!a) {
+			return b == null ? 0 : -1
+		}
+		if (a.compare) {
+			if (b == null) {
+				return 1
+			} else if (b.compare) {
+				return a.compare(b)
+			} else {
+				return -1
+			}
+		}
+		let arrayComparison;
+		if (b instanceof Array) {
+			let i = 0;
+			while((arrayComparison = compareKeys(a[i], b[i])) == 0 && i <= a.length)  {
+				i++;
+			}
+			return arrayComparison
+		}
+		arrayComparison = compareKeys(a[0], b);
+		if (arrayComparison == 0 && a.length > 1)
+			return 1
+		return arrayComparison
+	} else if (typeof a == typeof b) {
+		if (typeof a === 'symbol') {
+			a = Symbol.keyFor(a);
+			b = Symbol.keyFor(b);
+		}
+		return a < b ? -1 : a === b ? 0 : 1
+	}
+	else if (typeof b == 'object') {
+		if (b instanceof Array)
+			return -compareKeys(b, a)
+		return 1
+	} else {
+		return typeOrder[typeof a] < typeOrder[typeof b] ? -1 : 1
+	}
+}
+const typeOrder = {
+	symbol: 0,
+	undefined: 1,
+	boolean: 2,
+	number: 3,
+	string: 4
+};
+const MINIMUM_KEY = null;
+const MAXIMUM_KEY = Buffer.from([0xff]);
+
+exports.MAXIMUM_KEY = MAXIMUM_KEY;
+exports.MINIMUM_KEY = MINIMUM_KEY;
+exports.compareKeys = compareKeys;
+exports.enableNullTermination = enableNullTermination;
+exports.encoder = encoder;
+exports.fromBufferKey = fromBufferKey;
+exports.readKey = readKey;
+exports.toBufferKey = toBufferKey;
+exports.writeKey = writeKey;

package/{main.mjs → index.js}

@@ -19,6 +19,11 @@ const int32Array = new Int32Array(float64Array.buffer, 0, 4)
 const uint8Array6 = new Uint8Array(float64Array.buffer, 2, 6)
 const uint8Array8 = new Uint8Array(float64Array.buffer, 0, 8)
 let nullTerminate = false
+let textEncoder
+try {
+	textEncoder = new TextEncoder()
+} catch (error) {}
+
 /*
 * Convert arbitrary scalar values to buffer bytes with type preservation and type-appropriate ordering
 */
@@ -30,9 +35,9 @@ export function writeKey(key, target, position, inSequence) {
 	case 'string':
 		let strLength = key.length
 		let c1 = key.charCodeAt(0)
-		if (c1 < 28) // escape character
+		if (!(c1 >= 28)) // escape character
 			target[position++] = 27
-		if (strLength < 0x20) {
+		if (strLength < 0x40) {
 			let i, c2
 			for (i = 0; i < strLength; i++) {
 				c1 = key.charCodeAt(i)
@@ -57,8 +62,10 @@ export function writeKey(key, target, position, inSequence) {
 					target[position++] = c1 & 0x3f | 0x80
 				}
 			}
-		} else {
+		} else if (target.utf8Write) {
 			position += target.utf8Write(key, position, 2000)
+		} else {
+			position += textEncoder.encodeInto(key, target.subarray(position)).written
 		}
 		break
 	case 'number':
@@ -214,15 +221,17 @@ function makeStringBuilder() {
 	let stringBuildCode = '(source) => {'
 	let previous = []
 	for (let i = 0; i < 0x30; i++) {
-	let v = fromCharCode((i & 0xf) + 97) + fromCharCode((i >> 4) + 97)
-	stringBuildCode += `
-	let ${v} = source[position++]
-	if (${v} === 0)
-	return fromCharCode(${previous})
-	else if (${v} >= 0x80)
-	${v} = finishUtf8(${v}, source)
-`
-	previous.push(v)
+		let v = fromCharCode((i & 0xf) + 97) + fromCharCode((i >> 4) + 97)
+		stringBuildCode += `
+		let ${v} = source[position++]
+		if (${v} === 0)
+		return fromCharCode(${previous})
+		else if (${v} >= 0x80)
+		${v} = finishUtf8(${v}, source)
+		`
+		previous.push(v)
+		if (i == 1000000) // this just exists to prevent rollup from doing dead code elimination on finishUtf8
+			finishUtf8()
 	}
 	stringBuildCode += `return fromCharCode(${previous}) + readString(source)}`
 	return stringBuildCode
@@ -268,7 +277,7 @@ export const enableNullTermination = () => nullTerminate = true
 const readString = eval(makeStringBuilder())
 
 export function compareKeys(a, b) {
-	// compare with type consistency that matches ordered-binary
+	// compare with type consistency that matches binary comparison
 	if (typeof a == 'object') {
 		if (!a) {
 			return b == null ? 0 : -1
@@ -316,3 +325,5 @@ const typeOrder = {
 	number: 3,
 	string: 4
 }
+export const MINIMUM_KEY = null
+export const MAXIMUM_KEY = Buffer.from([0xff])

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "ordered-binary",
   "author": "Kris Zyp",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "Conversion of JavaScript primitives to and from Buffer with binary order matching natural primitive order",
   "license": "MIT",
   "repository": {
@@ -9,19 +9,28 @@
     "url": "http://github.com/kriszyp/ordered-binary"
   },
   "scripts": {
+    "build": "rollup -c",
+    "prepare": "rollup -c",
     "test": "mocha tests -u tdd"
   },
   "type": "module",
-  "main": "index.cjs",
-  "module": "main.mjs",
-  "typings": "./index.d.ts",
-  "dependencies": {
-    "esm": "^3.2.25"
+  "module": "index.js",
+  "exports": {
+    ".": {
+      "require": "./dist/index.cjs",
+      "import": "./index.js"
+    },
+    "./index.js": {
+      "require": "./dist/index.cjs",
+      "import": "./index.js"
+    }
   },
+  "typings": "./index.d.ts",
   "optionalDependencies": {},
   "devDependencies": {
     "@types/node": "latest",
     "chai": "^4",
-    "mocha": "^4"
+    "mocha": "^8.1.3",
+    "rollup": "^1.20.3"
   }
 }

package/rollup.config.js

@@ -0,0 +1,11 @@
+export default [
+    {
+        input: "index.js",
+        output: [
+            {
+                file: "dist/index.cjs",
+                format: "cjs"
+            }
+        ]
+    }
+];

package/tests/test.js

@@ -1,6 +1,6 @@
-const { assert } = require('chai')
+import { assert } from 'chai'
 
-const { toBufferKey, fromBufferKey, readKey, writeKey } = require('../index')
+import { toBufferKey, fromBufferKey, readKey, writeKey } from '../index.js'
 
 function assertBufferComparison(lesser, greater) {
   for (let i = 0; i < lesser.length; i++) {
@@ -43,6 +43,7 @@ suite('key buffers', () => {
   test('string equivalence', () => {
     assert.strictEqual(fromBufferKey(toBufferKey('4')), '4')
     assert.strictEqual(fromBufferKey(toBufferKey('hello')), 'hello')
+    assert.strictEqual(fromBufferKey(toBufferKey('')), '')
   })
   test('string comparison', () => {
     assertBufferComparison(toBufferKey('4'), toBufferKey('5'))

package/index.cjs

@@ -1,2 +0,0 @@
-require = require('esm')(module)
-Object.assign(exports, require('./main.mjs'))