@andrew_l/tl-pack 0.1.6 → 0.1.8

package/README.md

@@ -120,9 +120,13 @@ const extensions = [
         this.writeBytes(value.id);
       }
     },
-    // in the browser you possibly need to decode as a plain string
     decode() {
       const bytes = this.readBytes();
+
+      if (IS_BROWSER) {
+        return hex(bytes);
+      }
+
       return new ObjectId(bytes);
     },
   }),
@@ -146,6 +150,24 @@ console.log(reader.readObject());
   lastName: 'L.'
 }
  */
+
+const byteToHex: string[] = [];
+
+for (let n = 0; n <= 0xff; ++n) {
+  const hexOctet = n.toString(16).padStart(2, '0');
+  byteToHex.push(hexOctet);
+}
+
+function hex(arrayBuffer: Uint8Array) {
+  const buff = new Uint8Array(arrayBuffer);
+  const hexOctets = new Array(buff.length);
+
+  for (let i = 0; i < buff.length; ++i) {
+    hexOctets[i] = byteToHex[buff[i]];
+  }
+
+  return hexOctets.join('');
+}
 ```
 
 ## Dictionary

package/dist/BinaryReader.d.ts

@@ -77,7 +77,7 @@ export declare class BinaryReader {
     readGzip(): Uint8Array;
     private readCore;
     getDictionaryValue(index: number): string | undefined;
-    readDictionary(): string | null;
+    readDictionary(): null | string;
     readMap(checkConstructor?: boolean): Record<string, any>;
     decode(value: Buffer | Uint8Array): any;
     /**

package/dist/BinaryReader.js

@@ -1,7 +1,7 @@
 import pako from 'pako';
 import { CORE_TYPES } from './constants.js';
 import { Dictionary } from './dictionary.js';
-import { bytesToUtf8, float32, float64, int32 } from './helpers.js';
+import { float32, float64, int32, utf8Read } from './helpers.js';
 export class BinaryReader {
     target;
     _last;
@@ -160,10 +160,8 @@ export class BinaryReader {
     readString() {
         const length = this.readLength();
         this.assertRead(length);
-        const bytes = this.target.subarray(this.offset, this.offset + length);
-        this.offset += bytes.length;
-        // return pako.inflateRaw(bytes, { to: 'string' });
-        const result = bytesToUtf8(bytes);
+        const result = utf8Read(this.target, length, this.offset);
+        this.offset += length;
         this._last = result;
         return result;
     }
@@ -190,7 +188,7 @@ export class BinaryReader {
      */
     readDate() {
         const value = this.readDouble();
-        return new Date(value * 1000);
+        return new Date(value);
     }
     /**
      * Reads a object.
@@ -267,6 +265,15 @@ export class BinaryReader {
                 return this.readDouble();
             case CORE_TYPES.Map:
                 return this.readMap(false);
+            case CORE_TYPES.DictIndex: {
+                const idx = this.readLength();
+                return this.getDictionaryValue(idx);
+            }
+            case CORE_TYPES.DictValue: {
+                const value = this.readString();
+                this.dictionaryExtended.maybeInsert(value);
+                return value;
+            }
             case CORE_TYPES.Repeat: {
                 const size = this.readLength();
                 this._repeat = { pool: size - 1, value: this._lastObject };

package/dist/BinaryWriter.js

@@ -1,25 +1,8 @@
 import pako from 'pako';
-import { CORE_TYPES } from './constants.js';
+import { CORE_TYPES, MAX_BUFFER_SIZE } from './constants.js';
 import { Dictionary } from './dictionary.js';
-import { coreType, float32, float64, int32 } from './helpers.js';
-const hasNodeBuffer = typeof Buffer !== 'undefined';
-const MAX_BUFFER_SIZE = hasNodeBuffer ? 0x100000000 : 0x7fd00000;
-const textEncoder = new TextEncoder();
+import { byteArrayAllocate, coreType, float32, float64, int32, utf8Write, } from './helpers.js';
 const noop = Symbol();
-const writeUtf8 = hasNodeBuffer
-    ? function (target, value, offset) {
-        const length = target.utf8Write(value, offset, 0xffffffff);
-        return length;
-    }
-    : function (target, value, offset) {
-        return textEncoder.encodeInto(value, target.subarray(offset)).written;
-    };
-function byteArrayAllocate(length) {
-    if (hasNodeBuffer) {
-        return Buffer.allocUnsafeSlow(length);
-    }
-    return new Uint8Array(length);
-}
 const NO_CONSTRUCTOR = new Set([CORE_TYPES.BoolFalse, CORE_TYPES.BoolTrue, CORE_TYPES.Null]);
 const SUPPORT_COMPRESSION = new Set([CORE_TYPES.String]);
 export class BinaryWriter {
@@ -77,7 +60,6 @@ export class BinaryWriter {
             newSize = ((Math.max((end - start) << 2, target.length - 1) >> 12) + 1) << 12;
         }
         const newBuffer = byteArrayAllocate(newSize);
-        const newView = new DataView(newBuffer.buffer, 0, newSize);
         end = Math.min(end, target.length);
         if ('copy' in target) {
             target.copy(newBuffer, 0, start, end);
@@ -152,7 +134,7 @@ export class BinaryWriter {
     writeDate(value) {
         let timestamp = 0;
         if (value instanceof Date) {
-            timestamp = Math.floor(value.getTime() / 1000);
+            timestamp = value.getTime();
         }
         else if (typeof value === 'number') {
             timestamp = value;
@@ -162,8 +144,9 @@ export class BinaryWriter {
     writeString(value) {
         // const compressed = pako.deflateRaw(value, { level: 9 });
         // this.writeBytes(compressed);
+        const strLength = value.length;
         let start = this.offset;
-        let require = value.length << 2;
+        let require = strLength << 2;
         if (require < 254) {
             require += 1;
             this.offset += 1;
@@ -173,7 +156,7 @@ export class BinaryWriter {
             this.offset += 4;
         }
         this.allocate(require);
-        const bytes = writeUtf8(this.target, value, this.offset);
+        const bytes = utf8Write(this.target, value, this.offset);
         if (require < 254) {
             this.target[start++] = bytes;
         }
@@ -265,7 +248,12 @@ export class BinaryWriter {
         const start = this.offset;
         this.allocate(1);
         this.offset++;
+        let edgeExt;
         for (const ext of this.extensions.values()) {
+            if (ext.token === -1) {
+                edgeExt = ext;
+                continue;
+            }
             ext.encode.call(this, value);
             const processed = start < this.offset;
             if (processed) {
@@ -277,6 +265,10 @@ export class BinaryWriter {
             }
         }
         this.offset = start;
+        if (edgeExt) {
+            edgeExt.encode.call(this, value);
+            return start < this.offset;
+        }
         return false;
     }
     writeObject(value) {
@@ -366,6 +358,11 @@ export class BinaryWriter {
                 return this.writeNull();
             }
             case CORE_TYPES.String: {
+                // write short strings into dictionary
+                if (value.length <= 0x10) {
+                    this.offset--;
+                    return this.wireDictionary(value);
+                }
                 return this.writeString(value);
             }
             case CORE_TYPES.Vector: {

package/dist/constants.d.ts

@@ -22,3 +22,5 @@ export declare enum CORE_TYPES {
     Repeat = 20,
     GZIP = 25
 }
+export declare const HAS_NODE_BUFFER: boolean;
+export declare const MAX_BUFFER_SIZE: number;

package/dist/constants.js

@@ -23,3 +23,5 @@ export var CORE_TYPES;
     CORE_TYPES[CORE_TYPES["Repeat"] = 20] = "Repeat";
     CORE_TYPES[CORE_TYPES["GZIP"] = 25] = "GZIP";
 })(CORE_TYPES || (CORE_TYPES = {}));
+export const HAS_NODE_BUFFER = typeof Buffer !== 'undefined';
+export const MAX_BUFFER_SIZE = HAS_NODE_BUFFER ? 0x100000000 : 0x7fd00000;

package/dist/helpers.d.ts

@@ -3,6 +3,9 @@ import { CORE_TYPES } from './constants.js';
 export declare const int32: Int32Array;
 export declare const float32: Float32Array;
 export declare const float64: Float64Array;
-export declare function bytesToUtf8(bytes: Buffer | Uint8Array): string;
-export declare function utf8ToBytes(string: string): Uint8Array;
+export declare function byteArrayAllocate(length: number): Buffer | Uint8Array;
 export declare function coreType(value: any): CORE_TYPES;
+export declare function utf8Read(target: Buffer | Uint8Array, length: number, offset: number): string;
+export declare function utf8ReadShort(target: Buffer | Uint8Array, length: number, offset: number): string | undefined;
+export declare const utf8Write: (target: any, value: string, offset: number) => number;
+export declare const utf8WriteShort: (target: any, value: string, offset: number) => number;

package/dist/helpers.js

@@ -1,16 +1,15 @@
-import { CORE_TYPES } from './constants.js';
-const textEncoder = new TextEncoder();
-const textDecoder = new TextDecoder();
+import { CORE_TYPES, HAS_NODE_BUFFER } from './constants.js';
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
+const fromCharCode = String.fromCharCode;
 export const int32 = new Int32Array(2);
 export const float32 = new Float32Array(int32.buffer);
 export const float64 = new Float64Array(int32.buffer);
-export function bytesToUtf8(bytes) {
-    return textDecoder.decode(bytes);
-}
-export function utf8ToBytes(string) {
-    const buff = new Uint8Array(string.length << 2);
-    const length = textEncoder.encodeInto(string, buff).written;
-    return buff.subarray(0, length);
+export function byteArrayAllocate(length) {
+    if (HAS_NODE_BUFFER) {
+        return Buffer.allocUnsafeSlow(length);
+    }
+    return new Uint8Array(length);
 }
 export function coreType(value) {
     switch (typeof value) {
@@ -59,6 +58,245 @@ export function coreType(value) {
     }
     return CORE_TYPES.None;
 }
+export function utf8Read(target, length, offset) {
+    let result;
+    if (length < 16) {
+        if ((result = utf8ReadShort(target, length, offset)))
+            return result;
+    }
+    if (length > 64 && decoder)
+        return decoder.decode(target.subarray(offset, (offset += length)));
+    const end = offset + length;
+    const units = [];
+    result = '';
+    while (offset < end) {
+        const byte1 = target[offset++];
+        if ((byte1 & 0x80) === 0) {
+            // 1 byte
+            units.push(byte1);
+        }
+        else if ((byte1 & 0xe0) === 0xc0) {
+            // 2 bytes
+            const byte2 = target[offset++] & 0x3f;
+            units.push(((byte1 & 0x1f) << 6) | byte2);
+        }
+        else if ((byte1 & 0xf0) === 0xe0) {
+            // 3 bytes
+            const byte2 = target[offset++] & 0x3f;
+            const byte3 = target[offset++] & 0x3f;
+            units.push(((byte1 & 0x1f) << 12) | (byte2 << 6) | byte3);
+        }
+        else if ((byte1 & 0xf8) === 0xf0) {
+            // 4 bytes
+            const byte2 = target[offset++] & 0x3f;
+            const byte3 = target[offset++] & 0x3f;
+            const byte4 = target[offset++] & 0x3f;
+            let unit = ((byte1 & 0x07) << 0x12) | (byte2 << 0x0c) | (byte3 << 0x06) | byte4;
+            if (unit > 0xffff) {
+                unit -= 0x10000;
+                units.push(((unit >>> 10) & 0x3ff) | 0xd800);
+                unit = 0xdc00 | (unit & 0x3ff);
+            }
+            units.push(unit);
+        }
+        else {
+            units.push(byte1);
+        }
+        if (units.length >= 0x1000) {
+            result += fromCharCode.apply(String, units);
+            units.length = 0;
+        }
+    }
+    if (units.length > 0) {
+        result += fromCharCode.apply(String, units);
+    }
+    return result;
+}
+export function utf8ReadShort(target, length, offset) {
+    if (length < 4) {
+        if (length < 2) {
+            if (length === 0)
+                return '';
+            else {
+                let a = target[offset++];
+                if ((a & 0x80) > 1) {
+                    offset -= 1;
+                    return;
+                }
+                return fromCharCode(a);
+            }
+        }
+        else {
+            let a = target[offset++];
+            let b = target[offset++];
+            if ((a & 0x80) > 0 || (b & 0x80) > 0) {
+                offset -= 2;
+                return;
+            }
+            if (length < 3)
+                return fromCharCode(a, b);
+            let c = target[offset++];
+            if ((c & 0x80) > 0) {
+                offset -= 3;
+                return;
+            }
+            return fromCharCode(a, b, c);
+        }
+    }
+    else {
+        let a = target[offset++];
+        let b = target[offset++];
+        let c = target[offset++];
+        let d = target[offset++];
+        if ((a & 0x80) > 0 || (b & 0x80) > 0 || (c & 0x80) > 0 || (d & 0x80) > 0) {
+            offset -= 4;
+            return;
+        }
+        if (length < 6) {
+            if (length === 4)
+                return fromCharCode(a, b, c, d);
+            else {
+                let e = target[offset++];
+                if ((e & 0x80) > 0) {
+                    offset -= 5;
+                    return;
+                }
+                return fromCharCode(a, b, c, d, e);
+            }
+        }
+        else if (length < 8) {
+            let e = target[offset++];
+            let f = target[offset++];
+            if ((e & 0x80) > 0 || (f & 0x80) > 0) {
+                offset -= 6;
+                return;
+            }
+            if (length < 7)
+                return fromCharCode(a, b, c, d, e, f);
+            let g = target[offset++];
+            if ((g & 0x80) > 0) {
+                offset -= 7;
+                return;
+            }
+            return fromCharCode(a, b, c, d, e, f, g);
+        }
+        else {
+            let e = target[offset++];
+            let f = target[offset++];
+            let g = target[offset++];
+            let h = target[offset++];
+            if ((e & 0x80) > 0 || (f & 0x80) > 0 || (g & 0x80) > 0 || (h & 0x80) > 0) {
+                offset -= 8;
+                return;
+            }
+            if (length < 10) {
+                if (length === 8)
+                    return fromCharCode(a, b, c, d, e, f, g, h);
+                else {
+                    let i = target[offset++];
+                    if ((i & 0x80) > 0) {
+                        offset -= 9;
+                        return;
+                    }
+                    return fromCharCode(a, b, c, d, e, f, g, h, i);
+                }
+            }
+            else if (length < 12) {
+                let i = target[offset++];
+                let j = target[offset++];
+                if ((i & 0x80) > 0 || (j & 0x80) > 0) {
+                    offset -= 10;
+                    return;
+                }
+                if (length < 11)
+                    return fromCharCode(a, b, c, d, e, f, g, h, i, j);
+                let k = target[offset++];
+                if ((k & 0x80) > 0) {
+                    offset -= 11;
+                    return;
+                }
+                return fromCharCode(a, b, c, d, e, f, g, h, i, j, k);
+            }
+            else {
+                let i = target[offset++];
+                let j = target[offset++];
+                let k = target[offset++];
+                let l = target[offset++];
+                if ((i & 0x80) > 0 || (j & 0x80) > 0 || (k & 0x80) > 0 || (l & 0x80) > 0) {
+                    offset -= 12;
+                    return;
+                }
+                if (length < 14) {
+                    if (length === 12)
+                        return fromCharCode(a, b, c, d, e, f, g, h, i, j, k, l);
+                    else {
+                        let m = target[offset++];
+                        if ((m & 0x80) > 0) {
+                            offset -= 13;
+                            return;
+                        }
+                        return fromCharCode(a, b, c, d, e, f, g, h, i, j, k, l, m);
+                    }
+                }
+                else {
+                    let m = target[offset++];
+                    let n = target[offset++];
+                    if ((m & 0x80) > 0 || (n & 0x80) > 0) {
+                        offset -= 14;
+                        return;
+                    }
+                    if (length < 15)
+                        return fromCharCode(a, b, c, d, e, f, g, h, i, j, k, l, m, n);
+                    let o = target[offset++];
+                    if ((o & 0x80) > 0) {
+                        offset -= 15;
+                        return;
+                    }
+                    return fromCharCode(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o);
+                }
+            }
+        }
+    }
+}
+export const utf8Write = HAS_NODE_BUFFER
+    ? function (target, value, offset) {
+        return value.length < 0x40
+            ? utf8WriteShort(target, value, offset)
+            : target.utf8Write(value, offset, 0xffffffff);
+    }
+    : function (target, value, offset) {
+        return value.length < 0x40
+            ? utf8WriteShort(target, value, offset)
+            : encoder.encodeInto(value, target.subarray(offset)).written;
+    };
+export const utf8WriteShort = (target, value, offset) => {
+    let i, c1, c2, strPosition = offset;
+    const strLength = value.length;
+    for (i = 0; i < strLength; i++) {
+        c1 = value.charCodeAt(i);
+        if (c1 < 0x80) {
+            target[strPosition++] = c1;
+        }
+        else if (c1 < 0x800) {
+            target[strPosition++] = (c1 >> 6) | 0xc0;
+            target[strPosition++] = (c1 & 0x3f) | 0x80;
+        }
+        else if ((c1 & 0xfc00) === 0xd800 && ((c2 = value.charCodeAt(i + 1)) & 0xfc00) === 0xdc00) {
+            c1 = 0x10000 + ((c1 & 0x03ff) << 10) + (c2 & 0x03ff);
+            i++;
+            target[strPosition++] = (c1 >> 18) | 0xf0;
+            target[strPosition++] = ((c1 >> 12) & 0x3f) | 0x80;
+            target[strPosition++] = ((c1 >> 6) & 0x3f) | 0x80;
+            target[strPosition++] = (c1 & 0x3f) | 0x80;
+        }
+        else {
+            target[strPosition++] = (c1 >> 12) | 0xe0;
+            target[strPosition++] = ((c1 >> 6) & 0x3f) | 0x80;
+            target[strPosition++] = (c1 & 0x3f) | 0x80;
+        }
+    }
+    return strPosition - offset;
+};
 /** @ts-ignore */
 const isObject = (val) => toString.call(val) === '[object Object]';
 function isPlainObject(value) {

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@andrew_l/tl-pack",
-	"version": "0.1.6",
+	"version": "0.1.8",
   "description": "Binary serialization library",
 	"type": "module",
 	"scripts": {