ordered-binary 1.5.0 → 1.5.2

package/dist/index.cjs

@@ -2,8 +2,6 @@
 
 Object.defineProperty(exports, '__esModule', { value: true });
 
-require('constants');
-
 /*
 control character types:
 1 - metadata
@@ -16,6 +14,9 @@ control character types:
 0 - multipart separator
 > 27 normal string characters
 */
+/*
+* Convert arbitrary scalar values to buffer bytes with type preservation and type-appropriate ordering
+*/
 
 const float64Array = new Float64Array(2);
 const int32Array = new Int32Array(float64Array.buffer, 0, 4);
@@ -125,11 +126,20 @@ function writeKey(key, target, position, inSequence) {
 		let asFloat = Number(key);
 		if (BigInt(asFloat) > key) {
 			float64Array[0] = asFloat;
-			if (int32Array[0])
-				int32Array[0]--;
-			else {
-				int32Array[1]--;
-				int32Array[0] = 0xffffffff;
+			if (asFloat > 0) {
+				if (int32Array[0])
+					int32Array[0]--;
+				else {
+					int32Array[1]--;
+					int32Array[0] = 0xffffffff;
+				}
+			} else {
+				if (int32Array[0] < 0xffffffff)
+					int32Array[0]++;
+				else {
+					int32Array[1]++;
+					int32Array[0] = 0;
+				}
 			}
 			asFloat = float64Array[0];
 		}
@@ -138,7 +148,7 @@ function writeKey(key, target, position, inSequence) {
 			return writeKey(asFloat, target, position, inSequence)
 		writeKey(asFloat, target, position, inSequence);
 		position += 9; // always increment by 9 if we are adding fractional bits
-		let exponent = BigInt((int32Array[1] >> 20) - 1079);
+		let exponent = BigInt((int32Array[1] >> 20 & 0x7ff) - 1079);
 		let nextByte = difference >> exponent;
 		target[position - 1] |= Number(nextByte);
 		difference -= nextByte << exponent;
@@ -167,7 +177,6 @@ function writeKey(key, target, position, inSequence) {
 
 let position;
 function readKey(buffer, start, end, inSequence) {
-	buffer[end] = 0; // make sure it is null terminated
 	position = start;
 	let controlByte = buffer[position];
 	let value;
@@ -181,11 +190,18 @@ function readKey(buffer, start, end, inSequence) {
 			} else if (controlByte == 0) {
 				value = null;
 			} else if (controlByte == 2) {
-				value = Symbol.for(readString(buffer));
+				value = Symbol.for(readStringSafely(buffer, end));
 			} else
 				return Uint8Array.prototype.slice.call(buffer, start, end)
 		} else {
-			let dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, ((buffer.byteLength + 3) >> 2) << 2));
+			let dataView;
+			try {
+				dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, ((buffer.byteLength + 3) >> 2) << 2));
+			} catch(error) {
+				// if it is write at the end of the ArrayBuffer, we may need to retry with the exact remaining bytes
+				dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, buffer.buffer.byteLength - buffer.byteOffset));
+			}
+
 			let highInt = dataView.getInt32(position) << 4;
 			let size = end - position;
 			let lowInt;
@@ -193,7 +209,7 @@ function readKey(buffer, start, end, inSequence) {
 				lowInt = dataView.getInt32(position + 4);
 				highInt |= lowInt >>> 28;
 				if (size <= 6) { // clear the last bits
-					lowInt &= -0x1000;
+					lowInt &= -0x10000;
 				}
 				lowInt = lowInt << 4;
 				if (size > 8) {
@@ -214,8 +230,7 @@ function readKey(buffer, start, end, inSequence) {
 				// convert the float to bigint, and then we will add precision as we enumerate through the
 				// extra bytes
 				value = BigInt(value);
-				let exponent = highInt >> 20;
-				// TODO: negatives?
+				let exponent = highInt >> 20 & 0x7ff;
 				let next_byte = buffer[position - 1] & 0xf;
 				value += BigInt(next_byte) << BigInt(exponent - 1079);
 				while ((next_byte = buffer[position]) > 0 && position++ < end) {
@@ -227,15 +242,7 @@ function readKey(buffer, start, end, inSequence) {
 		if (controlByte == 27) {
 			position++;
 		}
-		value = readString(buffer);
-		/*let strStart = position
-		let strEnd = end
-		for (; position < end; position++) {
-			if (buffer[position] == 0) {
-			break
-			}
-		}
-		value = buffer.toString('utf8', strStart, position++)*/
+		value = readStringSafely(buffer, end);
 	}
 	while (position < end) {
 		if (buffer[position] === 0)
@@ -364,7 +371,19 @@ const readString =
 			'getPosition() { return position },' +
 			'readString }'))(fromCharCode)) :
 		eval(makeStringBuilder());
-
+function readStringSafely(source, end) {
+	if (source[end] > 0) {
+		let previous = source[end];
+		try {
+			// read string expects a null terminator, that is a 0 or undefined from reading past the end of the buffer, so we
+			// have to ensure that, but do so safely, restoring the buffer to its original state
+			source[end] = 0;
+			return readString(source)
+		} finally {
+			source[end] = previous;
+		}
+	} else return readString(source);
+}
 function compareKeys(a, b) {
 	// compare with type consistency that matches binary comparison
 	if (typeof a == 'object') {

package/index.js

@@ -14,8 +14,6 @@ control character types:
 * Convert arbitrary scalar values to buffer bytes with type preservation and type-appropriate ordering
 */
 
-import exp from "constants"
-
 const float64Array = new Float64Array(2)
 const int32Array = new Int32Array(float64Array.buffer, 0, 4)
 let nullTerminate = false
@@ -124,11 +122,20 @@ export function writeKey(key, target, position, inSequence) {
 		let asFloat = Number(key)
 		if (BigInt(asFloat) > key) {
 			float64Array[0] = asFloat;
-			if (int32Array[0])
-				int32Array[0]--;
-			else {
-				int32Array[1]--;
-				int32Array[0] = 0xffffffff;
+			if (asFloat > 0) {
+				if (int32Array[0])
+					int32Array[0]--;
+				else {
+					int32Array[1]--;
+					int32Array[0] = 0xffffffff;
+				}
+			} else {
+				if (int32Array[0] < 0xffffffff)
+					int32Array[0]++;
+				else {
+					int32Array[1]++;
+					int32Array[0] = 0;
+				}
 			}
 			asFloat = float64Array[0];
 		}
@@ -137,7 +144,7 @@ export function writeKey(key, target, position, inSequence) {
 			return writeKey(asFloat, target, position, inSequence)
 		writeKey(asFloat, target, position, inSequence)
 		position += 9; // always increment by 9 if we are adding fractional bits
-		let exponent = BigInt((int32Array[1] >> 20) - 1079);
+		let exponent = BigInt((int32Array[1] >> 20 & 0x7ff) - 1079);
 		let nextByte = difference >> exponent;
 		target[position - 1] |= Number(nextByte);
 		difference -= nextByte << exponent;
@@ -166,7 +173,6 @@ export function writeKey(key, target, position, inSequence) {
 
 let position
 export function readKey(buffer, start, end, inSequence) {
-	buffer[end] = 0 // make sure it is null terminated
 	position = start
 	let controlByte = buffer[position]
 	let value
@@ -180,11 +186,18 @@ export function readKey(buffer, start, end, inSequence) {
 			} else if (controlByte == 0) {
 				value = null
 			} else if (controlByte == 2) {
-				value = Symbol.for(readString(buffer))
+				value = Symbol.for(readStringSafely(buffer, end))
 			} else
 				return Uint8Array.prototype.slice.call(buffer, start, end)
 		} else {
-			let dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, ((buffer.byteLength + 3) >> 2) << 2))
+			let dataView;
+			try {
+				dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, ((buffer.byteLength + 3) >> 2) << 2))
+			} catch(error) {
+				// if it is write at the end of the ArrayBuffer, we may need to retry with the exact remaining bytes
+				dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, buffer.buffer.byteLength - buffer.byteOffset))
+			}
+
 			let highInt = dataView.getInt32(position) << 4
 			let size = end - position
 			let lowInt
@@ -192,7 +205,7 @@ export function readKey(buffer, start, end, inSequence) {
 				lowInt = dataView.getInt32(position + 4)
 				highInt |= lowInt >>> 28
 				if (size <= 6) { // clear the last bits
-					lowInt &= -0x1000
+					lowInt &= -0x10000
 				}
 				lowInt = lowInt << 4
 				if (size > 8) {
@@ -213,8 +226,7 @@ export function readKey(buffer, start, end, inSequence) {
 				// convert the float to bigint, and then we will add precision as we enumerate through the
 				// extra bytes
 				value = BigInt(value);
-				let exponent = highInt >> 20;
-				// TODO: negatives?
+				let exponent = highInt >> 20 & 0x7ff;
 				let next_byte = buffer[position - 1] & 0xf;
 				value += BigInt(next_byte) << BigInt(exponent - 1079);
 				while ((next_byte = buffer[position]) > 0 && position++ < end) {
@@ -226,15 +238,7 @@ export function readKey(buffer, start, end, inSequence) {
 		if (controlByte == 27) {
 			position++
 		}
-		value = readString(buffer)
-		/*let strStart = position
-		let strEnd = end
-		for (; position < end; position++) {
-			if (buffer[position] == 0) {
-			break
-			}
-		}
-		value = buffer.toString('utf8', strStart, position++)*/
+		value = readStringSafely(buffer, end)
 	}
 	while (position < end) {
 		if (buffer[position] === 0)
@@ -363,7 +367,19 @@ const readString =
 			'getPosition() { return position },' +
 			'readString }'))(fromCharCode)) :
 		eval(makeStringBuilder())
-
+function readStringSafely(source, end) {
+	if (source[end] > 0) {
+		let previous = source[end]
+		try {
+			// read string expects a null terminator, that is a 0 or undefined from reading past the end of the buffer, so we
+			// have to ensure that, but do so safely, restoring the buffer to its original state
+			source[end] = 0
+			return readString(source)
+		} finally {
+			source[end] = previous
+		}
+	} else return readString(source);
+}
 export function compareKeys(a, b) {
 	// compare with type consistency that matches binary comparison
 	if (typeof a == 'object') {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "ordered-binary",
   "author": "Kris Zyp",
-  "version": "1.5.0",
+  "version": "1.5.2",
   "description": "Conversion of JavaScript primitives to and from Buffer with binary order matching natural primitive order",
   "license": "MIT",
   "repository": {

package/tests/test.js

@@ -27,6 +27,7 @@ suite('key buffers', () => {
     assert.strictEqual(fromBufferKey(toBufferKey(-4)), -4)
     assert.strictEqual(fromBufferKey(toBufferKey(3.4)), 3.4)
     assert.strictEqual(fromBufferKey(toBufferKey(Math.PI)), Math.PI)
+    assert.strictEqual(fromBufferKey(toBufferKey(2002225)), 2002225)
     assert.strictEqual(fromBufferKey(toBufferKey(9377288)), 9377288)
     assert.strictEqual(fromBufferKey(toBufferKey(1503579323825)), 1503579323825)
     assert.strictEqual(fromBufferKey(toBufferKey(1503579323825.3523532)), 1503579323825.3523532)
@@ -35,6 +36,18 @@ suite('key buffers', () => {
     assert.strictEqual(fromBufferKey(toBufferKey(-0.00005032)), -0.00005032)
     assert.strictEqual(fromBufferKey(toBufferKey(0.00000000000000000000000005431)), 0.00000000000000000000000005431)
   })
+  test('within buffer equivalence', () => {
+    let buffer = Buffer.alloc(1024, 0xff);
+    let length = writeKey(2002225, buffer, 0);
+    let position = length;
+    buffer[position++] = 0xff;
+    buffer[position++] = 0xff;
+    assert.strictEqual(readKey(buffer, 0, length), 2002225);
+    length = writeKey('hello, world', buffer, 0);
+    assert.strictEqual(readKey(buffer, 0, length), 'hello, world');
+    // ensure that reading the string didn't modify the buffer after the string (or is restored at least)
+    assert.strictEqual(buffer[length], 0xff);
+  });
   test('number comparison', () => {
     assertBufferComparison(toBufferKey(4), toBufferKey(5))
     assertBufferComparison(toBufferKey(1503579323824), toBufferKey(1503579323825))
@@ -47,6 +60,7 @@ suite('key buffers', () => {
     assertBufferComparison(toBufferKey(-5236532532532), toBufferKey(-5236532532531))
   })
   test('bigint equivalence', () => {
+    assert.strictEqual(fromBufferKey(toBufferKey(-35913040084491349n)), -35913040084491349n)
     assert.strictEqual(fromBufferKey(toBufferKey(6135421331404949076605986n)), 6135421331404949076605986n)
     assert.strictEqual(fromBufferKey(toBufferKey(0xfffffffffffffffffffffn)), 0xfffffffffffffffffffffn)
     assert.strictEqual(fromBufferKey(toBufferKey(12345678901234567890n)), 12345678901234567890n)
@@ -60,12 +74,16 @@ suite('key buffers', () => {
       num *= BigInt(Math.floor(random() * 3 + 1));
       num -= BigInt(Math.floor(random() * 1000));
       assert.strictEqual(BigInt(fromBufferKey(toBufferKey(num))), num)
+      assert.strictEqual(BigInt(fromBufferKey(toBufferKey(-num))), -num)
     }
     assert.strictEqual(fromBufferKey(toBufferKey(-352n)), -352)
   })
   test('bigint comparison', () => {
     assertBufferComparison(toBufferKey(0xfffffffffffffffffffffn), toBufferKey(0x100fffffffffffffffffffn))
     assertBufferComparison(toBufferKey(12345678901234567890), toBufferKey(12345678901234567890n))
+    assertBufferComparison(toBufferKey(6135421331404949076605986n), toBufferKey(6135421331404949076605987n))
+    assertBufferComparison(toBufferKey(-6135421331404949076605986n), toBufferKey(-6135421331404949076605985n))
+    assertBufferComparison(toBufferKey(-35913040084491349n), toBufferKey(-35913040084491348n))
   })
 
   test('string equivalence', () => {