@andrew_l/tl-pack 0.0.4 → 0.0.6

package/README.md

@@ -0,0 +1,138 @@
+# TLPack for JavaScript/NodeJs
+
+![license](https://img.shields.io/npm/l/%40andrew_l%2Ftl-pack)![npm version](https://img.shields.io/npm/v/%40andrew_l%2Ftl-pack)![npm bundle size](https://img.shields.io/bundlephobia/minzip/%40andrew_l%2Ftl-pack)
+
+This library is another implementation of binary serialization like **MessagePack** inspired by TL (Type Language) developed by old VK team.
+
+You don't needed in schema for data serialization/deserialization unlike official TL implementation at least in this version.
+
+## Benchmark
+
+A bit faster and a bit more compact than **@msgpack/msgpack**
+
+## Synopsis
+
+```javascript
+import { BinaryWriter, BinaryReader } from '@andrew_l/tl-pack';
+
+const writer = new BinaryWriter();
+
+// use to compress this part of data
+// writer.writeObjectGzip
+writer.writeObject({
+	null: null,
+	uint8: 255,
+	uint16: 256,
+	uint32: 65536,
+	int8: -128,
+	int16: -32768,
+	int32: -2147483648,
+	double: 3.14,
+	string: 'Hello world',
+	vector: [1, 2, 3, 4, 5, { text: 'hi' }],
+	map: { foo: 'bar' },
+	date: new Date(),
+});
+
+const reader = new BinaryReader(writer.getBuffer());
+
+console.log(reader.readObject());
+```
+
+## Supported Types
+
+| Constructor ID | Byte Size          |
+| -------------- | ------------------ |
+| Binary         | 5 + sizeof(object) |
+| BoolFalse      | 1                  |
+| BoolTrue       | 1                  |
+| Null           | 1                  |
+| Date           | 4                  |
+| Vector         | 5 + n \* sizeof(n) |
+| VectorDynamic  | 2 + n \* sizeof(n) |
+| Int8           | 1                  |
+| Int16          | 2                  |
+| Int32          | 4                  |
+| UInt8          | 1                  |
+| UInt16         | 2                  |
+| UInt32         | 4                  |
+| Float          | 4                  |
+| Double         | 8                  |
+| Map            | 2 + sizeof(object) |
+| String         | 5 + sizeof(object) |
+| GZIP           | 5 + sizeof(object) |
+
+## Stream Example (NodeJs Only)
+
+```javascript
+import { Readable } from 'node:stream';
+import { TLEncode, TLDecode } from '@andrew_l/tl-pack/stream';
+
+const values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
+
+const dataStream = new Readable({ objectMode: true });
+
+dataStream._read = () => {
+	const chunk = values.shift();
+
+	dataStream.push(chunk || null);
+};
+
+const encode = new TLEncode();
+const decode = new TLDecode();
+
+dataStream.pipe(encode).pipe(decode);
+
+decode.on('data', (data) => console.log('stream', data));
+decode.on('error', console.error);
+```
+
+## Custom Types
+
+```javascript
+import mongoose from 'mongoose';
+import { BinaryWriter, BinaryReader, createExtension } from '@andrew_l/tl-pack';
+
+const ObjectId = mongoose.Types.ObjectId;
+
+const extensions = [
+	// Reserving token - 100 for ObjectId types
+	createExtension(100, {
+		encode(value) {
+			if (value?._bsontype === 'ObjectID') {
+				return value.toJSON(); // (string)
+			}
+		},
+		// on client side, you possibly need a decoding as a plain string.
+		decode(value) {
+			return new ObjectId(value);
+		},
+	}),
+];
+
+const writer = new BinaryWriter({ extensions });
+
+writer.writeObject({
+	_id: new Object(),
+	firstName: 'Andrew',
+	lastName: 'L.',
+});
+
+const reader = new BinaryReader(writer.getBuffer(), { extensions });
+
+console.log(reader.readObject());
+```
+
+## Dictionary
+
+A dictionary is used to replace strings with numeric indexes, which saves the resulting buffer size. In stream mode, the dictionary is grown while the stream is alive.
+
+**Static**
+Dictionary with you initialize a BinaryWriter/BinaryReader.
+
+**Dynamic**
+Dictionary which appends while encoding and decoding keys of an object (Map).
+
+## Production
+
+No way!

package/dist/BinaryReader.d.ts

@@ -1,13 +1,16 @@
 /// <reference types="node" resolution-mode="require"/>
+import { Dictionary } from './dictionary.js';
 import { TLExtension } from './extension.js';
 export interface BinaryReaderOptions {
+    dictionary?: string[] | Dictionary;
     extensions?: TLExtension[];
 }
 export declare class BinaryReader {
     private target;
     private targetView;
     private _last?;
-    private _dict;
+    private dictionary?;
+    private dictionaryExtended;
     private extensions;
     offset: number;
     length: number;
@@ -42,6 +45,7 @@ export declare class BinaryReader {
     getBuffer(): Uint8Array | Buffer;
     readNull(): null;
     readLength(): number;
+    readAll(): any[];
     /**
      * @returns {Uint8Array | Buffer}
      */
@@ -66,7 +70,10 @@ export declare class BinaryReader {
      * Reads a object.
      */
     readObject(): any;
+    readObjectGzip(): any;
+    readGzip(): Uint8Array;
     private readCore;
+    getDictionaryValue(index: number): string | undefined;
     readDictionary(): string | null;
     readMap(checkConstructor?: boolean): Record<string, any>;
     decode(value: Buffer | Uint8Array): any;

package/dist/BinaryReader.js

@@ -1,10 +1,13 @@
+import pako from 'pako';
 import { CORE_TYPES } from './constants.js';
+import { Dictionary } from './dictionary.js';
 import { bytesToUtf8 } from './helpers.js';
 export class BinaryReader {
     target;
     targetView;
     _last;
-    _dict;
+    dictionary;
+    dictionaryExtended;
     extensions;
     offset;
     length;
@@ -16,7 +19,6 @@ export class BinaryReader {
         this.target = data;
         this.targetView = new DataView(data.buffer, 0, data.length);
         this._last = undefined;
-        this._dict = new Map();
         this.offset = 0;
         this.length = data.length;
         this.extensions = new Map();
@@ -25,6 +27,19 @@ export class BinaryReader {
                 this.extensions.set(ext.token, ext);
             });
         }
+        if (!options) {
+            this.dictionary = new Dictionary();
+        }
+        else if (options.dictionary instanceof Dictionary) {
+            this.dictionary = options.dictionary;
+        }
+        else if (Array.isArray(options.dictionary)) {
+            this.dictionary = new Dictionary(options.dictionary);
+        }
+        else {
+            this.dictionary = new Dictionary();
+        }
+        this.dictionaryExtended = new Dictionary(undefined, this.dictionary.size);
     }
     readByte() {
         this.assertRead(1);
@@ -119,6 +134,13 @@ export class BinaryReader {
         }
         return firstByte;
     }
+    readAll() {
+        const result = [];
+        while (this.length > this.offset) {
+            result.push(this.readObject());
+        }
+        return result;
+    }
     /**
      * @returns {Uint8Array | Buffer}
      */
@@ -138,6 +160,7 @@ export class BinaryReader {
         this.assertRead(length);
         const bytes = this.target.subarray(this.offset, this.offset + length);
         this.offset += bytes.length;
+        // return pako.inflateRaw(bytes, { to: 'string' });
         return bytesToUtf8(bytes);
     }
     /**
@@ -169,7 +192,6 @@ export class BinaryReader {
      * Reads a object.
      */
     readObject() {
-        const offset = this.offset;
         const constructorId = this.readByte();
         const ext = this.extensions.get(constructorId);
         let value;
@@ -182,10 +204,23 @@ export class BinaryReader {
         }
         return value;
     }
+    readObjectGzip() {
+        const bytes = this.readGzip();
+        const reader = new BinaryReader(bytes);
+        reader.extensions = this.extensions;
+        reader.dictionary = this.dictionary;
+        reader.dictionaryExtended = this.dictionaryExtended;
+        return reader.readObject();
+    }
+    readGzip() {
+        return pako.inflateRaw(this.readBytes());
+    }
     readCore(constructorId) {
         switch (constructorId) {
             case CORE_TYPES.None:
                 return this.readObject();
+            case CORE_TYPES.GZIP:
+                return this.readObjectGzip();
             case CORE_TYPES.BoolTrue:
                 return true;
             case CORE_TYPES.BoolFalse:
@@ -221,18 +256,28 @@ export class BinaryReader {
         }
         throw new Error(`Invalid constructor = ${CORE_TYPES[constructorId] || constructorId}, offset = ${this.offset - 1}`);
     }
+    getDictionaryValue(index) {
+        let value;
+        if (this.dictionary) {
+            value = this.dictionary.getValue(index);
+        }
+        if (value === undefined) {
+            value = this.dictionaryExtended.getValue(index);
+        }
+        return value;
+    }
     readDictionary() {
         const constructorId = this.readByte();
         let key = null;
         switch (constructorId) {
             case CORE_TYPES.DictIndex: {
                 const idx = this.readLength();
-                key = this._dict.get(idx);
+                key = this.getDictionaryValue(idx);
                 break;
             }
             case CORE_TYPES.DictValue: {
                 key = this.readString();
-                this._dict.set(this._dict.size + 1, key);
+                this.dictionaryExtended.maybeInsert(key);
                 break;
             }
             case CORE_TYPES.None: {

package/dist/BinaryWriter.d.ts

@@ -1,12 +1,17 @@
 /// <reference types="node" resolution-mode="require"/>
+import { Dictionary } from './dictionary.js';
 import { TLExtension } from './extension.js';
 export interface BinaryWriterOptions {
+    gzip?: boolean;
+    dictionary?: string[] | Dictionary;
     extensions?: TLExtension[];
 }
 export declare class BinaryWriter {
+    private withGzip;
     private target;
     private targetView;
-    private dict;
+    private dictionary?;
+    private dictionaryExtended;
     private extensions;
     offset: number;
     constructor(options?: BinaryWriterOptions);
@@ -29,8 +34,10 @@ export declare class BinaryWriter {
     writeVector(value: Array<any>): void;
     writeMap(object: Record<string, any>): void;
     wireDictionary(value: string): void;
+    writeGzip(value: any): void;
     encode(value: any): Uint8Array | Buffer;
     private _writeCustom;
     writeObject(value: any): void;
+    writeObjectGzip(value: any): void;
     private writeCore;
 }

package/dist/BinaryWriter.js

@@ -1,4 +1,6 @@
+import pako from 'pako';
 import { CORE_TYPES } from './constants.js';
+import { Dictionary } from './dictionary.js';
 import { coreType } from './helpers.js';
 const hasNodeBuffer = typeof Buffer !== 'undefined';
 const MAX_BUFFER_SIZE = hasNodeBuffer ? 0x100000000 : 0x7fd00000;
@@ -17,23 +19,40 @@ function byteArrayAllocate(length) {
     }
     return new Uint8Array(length);
 }
+const NO_CONSTRUCTOR = new Set([CORE_TYPES.BoolFalse, CORE_TYPES.BoolTrue, CORE_TYPES.Null]);
+const SUPPORT_COMPRESSION = new Set([CORE_TYPES.String]);
 export class BinaryWriter {
+    withGzip;
     target;
     targetView;
-    dict;
+    dictionary;
+    dictionaryExtended;
     extensions;
     offset;
     constructor(options) {
-        this.dict = new Map();
         this.offset = 0;
         this.extensions = new Map();
+        this.withGzip = !!options && !!options.gzip;
+        this.target = byteArrayAllocate(8192);
+        this.targetView = new DataView(this.target.buffer, 0, this.target.length);
         if (options && options.extensions) {
             options.extensions.forEach((ext) => {
                 this.extensions.set(ext.token, ext);
             });
         }
-        this.target = byteArrayAllocate(8192);
-        this.targetView = new DataView(this.target.buffer, 0, this.target.length);
+        if (!options) {
+            this.dictionary = new Dictionary();
+        }
+        else if (options.dictionary instanceof Dictionary) {
+            this.dictionary = options.dictionary;
+        }
+        else if (Array.isArray(options.dictionary)) {
+            this.dictionary = new Dictionary(options.dictionary);
+        }
+        else {
+            this.dictionary = new Dictionary();
+        }
+        this.dictionaryExtended = new Dictionary(undefined, this.dictionary.size);
     }
     allocate(size) {
         const position = this.offset + size;
@@ -139,6 +158,8 @@ export class BinaryWriter {
         this.writeDouble(timestamp);
     }
     writeString(value) {
+        // const compressed = pako.deflateRaw(value, { level: 9 });
+        // this.writeBytes(compressed);
         let start = this.offset;
         let require = value.length << 2;
         if (require < 254) {
@@ -204,22 +225,31 @@ export class BinaryWriter {
         this.writeByte(CORE_TYPES.None);
     }
     wireDictionary(value) {
-        if (this.dict.has(value)) {
-            const idx = this.dict.get(value);
-            this.writeCore(CORE_TYPES.DictIndex, idx);
+        let idx;
+        if (this.dictionary) {
+            idx = this.dictionary.getIndex(value);
         }
-        else {
-            const newIndex = this.dict.size + 1;
-            this.dict.set(value, newIndex);
+        if (idx === undefined) {
+            idx = this.dictionaryExtended.getIndex(value);
+        }
+        if (idx === undefined) {
+            this.dictionaryExtended.maybeInsert(value);
             this.writeCore(CORE_TYPES.DictValue, value);
         }
+        else {
+            this.writeCore(CORE_TYPES.DictIndex, idx);
+        }
+    }
+    writeGzip(value) {
+        const compressed = pako.deflateRaw(value, { level: 9 });
+        this.writeBytes(compressed);
     }
     encode(value) {
-        const start = this.offset;
+        this.offset = 0;
+        this.target = byteArrayAllocate(256);
+        this.targetView = new DataView(this.target.buffer, 0, this.target.length);
         this.writeObject(value);
-        const end = this.offset;
-        this.offset = start;
-        return this.target.subarray(start, end);
+        return this.getBuffer();
     }
     _writeCustom(value) {
         for (const ext of this.extensions.values()) {
@@ -255,11 +285,27 @@ export class BinaryWriter {
         }
         this.writeCore(constructorId, value);
     }
+    writeObjectGzip(value) {
+        const writer = new BinaryWriter();
+        writer.extensions = this.extensions;
+        writer.dictionary = this.dictionary;
+        writer.dictionaryExtended = this.dictionaryExtended;
+        writer.writeObject(value);
+        this.writeCore(CORE_TYPES.GZIP, writer.getBuffer());
+    }
     writeCore(constructorId, value) {
-        if (![CORE_TYPES.BoolFalse, CORE_TYPES.BoolTrue, CORE_TYPES.Null].includes(constructorId)) {
+        // console.log('write', { constructorId: CORE_TYPES[constructorId] || constructorId, value });
+        if (this.withGzip && SUPPORT_COMPRESSION.has(constructorId)) {
+            this.writeObjectGzip(value);
+            return;
+        }
+        else if (!NO_CONSTRUCTOR.has(constructorId)) {
             this.writeByte(constructorId);
         }
         switch (constructorId) {
+            case CORE_TYPES.GZIP: {
+                return this.writeGzip(value);
+            }
             case CORE_TYPES.DictIndex: {
                 return this.writeLength(value);
             }

package/dist/constants.d.ts

@@ -17,5 +17,6 @@ export declare enum CORE_TYPES {
     Map = 15,
     DictValue = 16,
     DictIndex = 17,
-    String = 18
+    String = 18,
+    GZIP = 19
 }

package/dist/constants.js

@@ -19,5 +19,5 @@ export var CORE_TYPES;
     CORE_TYPES[CORE_TYPES["DictValue"] = 16] = "DictValue";
     CORE_TYPES[CORE_TYPES["DictIndex"] = 17] = "DictIndex";
     CORE_TYPES[CORE_TYPES["String"] = 18] = "String";
+    CORE_TYPES[CORE_TYPES["GZIP"] = 19] = "GZIP";
 })(CORE_TYPES || (CORE_TYPES = {}));
-;

package/dist/dictionary.d.ts

@@ -0,0 +1,17 @@
+export declare function createDictionary(values?: string[]): Dictionary;
+export declare class Dictionary {
+    private _count;
+    private _map;
+    private _index;
+    private _offset;
+    constructor(values?: string[], offset?: number);
+    get size(): number;
+    /**
+     * Returns inserted index or nothing
+     */
+    maybeInsert(word: string): number | undefined;
+    getValue(index: number): string | undefined;
+    getIndex(value: string): number | undefined;
+    hasValue(value: string): boolean;
+    hasIndex(index: number): boolean;
+}

package/dist/dictionary.js

@@ -0,0 +1,51 @@
+export function createDictionary(values) {
+    return new Dictionary(values);
+}
+export class Dictionary {
+    _count = 0;
+    _map;
+    _index;
+    _offset;
+    constructor(values, offset = 0) {
+        this._index = [];
+        this._map = new Map();
+        this._offset = offset;
+        if (Array.isArray(values) && values.length) {
+            values.forEach((word) => {
+                if (this._map.has(word))
+                    return;
+                this._map.set(word, this._count++);
+                this._index.push(word);
+            });
+        }
+    }
+    get size() {
+        return this._count;
+    }
+    /**
+     * Returns inserted index or nothing
+     */
+    maybeInsert(word) {
+        if (this._map.has(word))
+            return;
+        this._map.set(word, this._count++);
+        this._index.push(word);
+        return this._count + this._offset;
+    }
+    getValue(index) {
+        return this._index[index - this._offset];
+    }
+    getIndex(value) {
+        const idx = this._map.get(value);
+        if (idx === undefined) {
+            return idx;
+        }
+        return idx + this._offset;
+    }
+    hasValue(value) {
+        return this._map.has(value);
+    }
+    hasIndex(index) {
+        return this._index[index - this._offset] !== undefined;
+    }
+}

package/dist/index.d.ts

@@ -2,3 +2,4 @@ export * from './BinaryWriter.js';
 export * from './BinaryReader.js';
 export * from './constants.js';
 export * from './extension.js';
+export * from './dictionary.js';

package/dist/index.js

@@ -2,3 +2,4 @@ export * from './BinaryWriter.js';
 export * from './BinaryReader.js';
 export * from './constants.js';
 export * from './extension.js';
+export * from './dictionary.js';

package/dist/stream.d.ts

@@ -1,13 +1,20 @@
 /// <reference types="node" resolution-mode="require"/>
 /// <reference types="node" resolution-mode="require"/>
 import { Transform, type TransformCallback, type TransformOptions } from 'stream';
+import { BinaryWriter, BinaryWriterOptions } from './BinaryWriter.js';
+import { BinaryReader } from './BinaryReader.js';
+export interface TLEncodeOptions extends BinaryWriterOptions {
+    streamOptions?: TransformOptions;
+    writeVectorWhenEmpty?: boolean;
+}
 export declare class TLEncode extends Transform {
-    private writer;
-    constructor(options?: TransformOptions);
+    writer: BinaryWriter;
+    count: number;
+    constructor(options?: TLEncodeOptions);
     _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void;
 }
 export declare class TLDecode extends Transform {
-    private reader;
+    reader: BinaryReader;
     private incompleteBuffer;
     constructor(options?: TransformOptions);
     _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void;

package/dist/stream.js

@@ -3,16 +3,31 @@ import { BinaryWriter } from './BinaryWriter.js';
 import { BinaryReader } from './BinaryReader.js';
 export class TLEncode extends Transform {
     writer;
+    count;
     constructor(options) {
-        if (!options)
-            options = {};
-        options.writableObjectMode = true;
-        super(options);
-        this.writer = new BinaryWriter();
+        const opts = options || {};
+        opts.streamOptions = { writableObjectMode: true, ...(opts.streamOptions || {}) };
+        super(opts.streamOptions);
+        const writer = new BinaryWriter(options);
+        const customFlush = opts.streamOptions.flush;
+        this._flush = (callback) => {
+            if (this.count === 0 && opts.writeVectorWhenEmpty) {
+                this.push(writer.encode([]));
+            }
+            if (customFlush) {
+                customFlush.call(this, callback);
+            }
+            else {
+                callback();
+            }
+        };
+        this.writer = writer;
+        this.count = 0;
     }
     _transform(chunk, encoding, callback) {
         const buff = this.writer.encode(chunk);
         this.push(buff);
+        this.count++;
         callback();
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@andrew_l/tl-pack",
-	"version": "0.0.4",
+	"version": "0.0.6",
 	"description": "",
 	"type": "module",
 	"scripts": {
@@ -10,10 +10,18 @@
 	"keywords": [],
 	"author": "Andrew L.",
 	"license": "ISC",
-	"types": "./dist",
 	"files": [
 		"dist/**"
 	],
+	"repository": {
+		"type": "git",
+		"url": "https://github.com/men232/tl-pack.git"
+	},
+	"bugs": {
+		"url": "https://github.com/men232/tl-pack/issues"
+	},
+	"main": "./dist/index.js",
+	"types": "./dist/index.d.ts",
 	"exports": {
 		".": {
 			"import": "./dist/index.js",
@@ -26,10 +34,14 @@
 	},
 	"devDependencies": {
 		"@types/node": "^20.3.2",
+		"@types/pako": "^2.0.0",
 		"eslint-config-prettier": "^8.8.0",
 		"nodemon": "^2.0.22",
 		"prettier": "^2.8.8",
 		"ts-node": "^10.9.1",
 		"typescript": "^5.1.5"
+	},
+	"dependencies": {
+		"pako": "^2.1.0"
 	}
 }