@andrew_l/tl-pack 0.0.4 → 0.0.5

package/dist/BinaryReader.d.ts

@@ -1,13 +1,16 @@
 /// <reference types="node" resolution-mode="require"/>
+import { Dictionary } from './dictionary.js';
 import { TLExtension } from './extension.js';
 export interface BinaryReaderOptions {
+    dictionary?: string[] | Dictionary;
     extensions?: TLExtension[];
 }
 export declare class BinaryReader {
     private target;
     private targetView;
     private _last?;
-    private _dict;
+    private dictionary?;
+    private dictionaryExtended;
     private extensions;
     offset: number;
     length: number;
@@ -42,6 +45,7 @@ export declare class BinaryReader {
     getBuffer(): Uint8Array | Buffer;
     readNull(): null;
     readLength(): number;
+    readAll(): any[];
     /**
      * @returns {Uint8Array | Buffer}
      */
@@ -66,7 +70,10 @@ export declare class BinaryReader {
      * Reads a object.
      */
     readObject(): any;
+    readObjectGzip(): any;
+    readGzip(): Uint8Array;
     private readCore;
+    getDictionaryValue(index: number): string | undefined;
     readDictionary(): string | null;
     readMap(checkConstructor?: boolean): Record<string, any>;
     decode(value: Buffer | Uint8Array): any;

package/dist/BinaryReader.js

@@ -1,10 +1,13 @@
+import pako from 'pako';
 import { CORE_TYPES } from './constants.js';
+import { Dictionary } from './dictionary.js';
 import { bytesToUtf8 } from './helpers.js';
 export class BinaryReader {
     target;
     targetView;
     _last;
-    _dict;
+    dictionary;
+    dictionaryExtended;
     extensions;
     offset;
     length;
@@ -16,7 +19,6 @@ export class BinaryReader {
         this.target = data;
         this.targetView = new DataView(data.buffer, 0, data.length);
         this._last = undefined;
-        this._dict = new Map();
         this.offset = 0;
         this.length = data.length;
         this.extensions = new Map();
@@ -25,6 +27,19 @@ export class BinaryReader {
                 this.extensions.set(ext.token, ext);
             });
         }
+        if (!options) {
+            this.dictionary = new Dictionary();
+        }
+        else if (options.dictionary instanceof Dictionary) {
+            this.dictionary = options.dictionary;
+        }
+        else if (Array.isArray(options.dictionary)) {
+            this.dictionary = new Dictionary(options.dictionary);
+        }
+        else {
+            this.dictionary = new Dictionary();
+        }
+        this.dictionaryExtended = new Dictionary(undefined, this.dictionary.size);
     }
     readByte() {
         this.assertRead(1);
@@ -119,6 +134,13 @@ export class BinaryReader {
         }
         return firstByte;
     }
+    readAll() {
+        const result = [];
+        while (this.length > this.offset) {
+            result.push(this.readObject());
+        }
+        return result;
+    }
     /**
      * @returns {Uint8Array | Buffer}
      */
@@ -138,6 +160,7 @@ export class BinaryReader {
         this.assertRead(length);
         const bytes = this.target.subarray(this.offset, this.offset + length);
         this.offset += bytes.length;
+        // return pako.inflateRaw(bytes, { to: 'string' });
         return bytesToUtf8(bytes);
     }
     /**
@@ -169,7 +192,6 @@ export class BinaryReader {
      * Reads a object.
      */
     readObject() {
-        const offset = this.offset;
         const constructorId = this.readByte();
         const ext = this.extensions.get(constructorId);
         let value;
@@ -182,10 +204,23 @@ export class BinaryReader {
         }
         return value;
     }
+    readObjectGzip() {
+        const bytes = this.readGzip();
+        const reader = new BinaryReader(bytes);
+        reader.extensions = this.extensions;
+        reader.dictionary = this.dictionary;
+        reader.dictionaryExtended = this.dictionaryExtended;
+        return reader.readObject();
+    }
+    readGzip() {
+        return pako.inflateRaw(this.readBytes());
+    }
     readCore(constructorId) {
         switch (constructorId) {
             case CORE_TYPES.None:
                 return this.readObject();
+            case CORE_TYPES.GZIP:
+                return this.readObjectGzip();
             case CORE_TYPES.BoolTrue:
                 return true;
             case CORE_TYPES.BoolFalse:
@@ -221,18 +256,28 @@ export class BinaryReader {
         }
         throw new Error(`Invalid constructor = ${CORE_TYPES[constructorId] || constructorId}, offset = ${this.offset - 1}`);
     }
+    getDictionaryValue(index) {
+        let value;
+        if (this.dictionary) {
+            value = this.dictionary.getValue(index);
+        }
+        if (value === undefined) {
+            value = this.dictionaryExtended.getValue(index);
+        }
+        return value;
+    }
     readDictionary() {
         const constructorId = this.readByte();
         let key = null;
         switch (constructorId) {
             case CORE_TYPES.DictIndex: {
                 const idx = this.readLength();
-                key = this._dict.get(idx);
+                key = this.getDictionaryValue(idx);
                 break;
             }
             case CORE_TYPES.DictValue: {
                 key = this.readString();
-                this._dict.set(this._dict.size + 1, key);
+                this.dictionaryExtended.maybeInsert(key);
                 break;
             }
             case CORE_TYPES.None: {

package/dist/BinaryWriter.d.ts

@@ -1,12 +1,17 @@
 /// <reference types="node" resolution-mode="require"/>
+import { Dictionary } from './dictionary.js';
 import { TLExtension } from './extension.js';
 export interface BinaryWriterOptions {
+    gzip?: boolean;
+    dictionary?: string[] | Dictionary;
     extensions?: TLExtension[];
 }
 export declare class BinaryWriter {
+    private withGzip;
     private target;
     private targetView;
-    private dict;
+    private dictionary?;
+    private dictionaryExtended;
     private extensions;
     offset: number;
     constructor(options?: BinaryWriterOptions);
@@ -29,8 +34,10 @@ export declare class BinaryWriter {
     writeVector(value: Array<any>): void;
     writeMap(object: Record<string, any>): void;
     wireDictionary(value: string): void;
+    writeGzip(value: any): void;
     encode(value: any): Uint8Array | Buffer;
     private _writeCustom;
     writeObject(value: any): void;
+    writeObjectGzip(value: any): void;
     private writeCore;
 }

package/dist/BinaryWriter.js

@@ -1,4 +1,6 @@
+import pako from 'pako';
 import { CORE_TYPES } from './constants.js';
+import { Dictionary } from './dictionary.js';
 import { coreType } from './helpers.js';
 const hasNodeBuffer = typeof Buffer !== 'undefined';
 const MAX_BUFFER_SIZE = hasNodeBuffer ? 0x100000000 : 0x7fd00000;
@@ -17,23 +19,40 @@ function byteArrayAllocate(length) {
     }
     return new Uint8Array(length);
 }
+const NO_CONSTRUCTOR = new Set([CORE_TYPES.BoolFalse, CORE_TYPES.BoolTrue, CORE_TYPES.Null]);
+const SUPPORT_COMPRESSION = new Set([CORE_TYPES.String]);
 export class BinaryWriter {
+    withGzip;
     target;
     targetView;
-    dict;
+    dictionary;
+    dictionaryExtended;
     extensions;
     offset;
     constructor(options) {
-        this.dict = new Map();
         this.offset = 0;
         this.extensions = new Map();
+        this.withGzip = !!options && !!options.gzip;
+        this.target = byteArrayAllocate(8192);
+        this.targetView = new DataView(this.target.buffer, 0, this.target.length);
         if (options && options.extensions) {
             options.extensions.forEach((ext) => {
                 this.extensions.set(ext.token, ext);
             });
         }
-        this.target = byteArrayAllocate(8192);
-        this.targetView = new DataView(this.target.buffer, 0, this.target.length);
+        if (!options) {
+            this.dictionary = new Dictionary();
+        }
+        else if (options.dictionary instanceof Dictionary) {
+            this.dictionary = options.dictionary;
+        }
+        else if (Array.isArray(options.dictionary)) {
+            this.dictionary = new Dictionary(options.dictionary);
+        }
+        else {
+            this.dictionary = new Dictionary();
+        }
+        this.dictionaryExtended = new Dictionary(undefined, this.dictionary.size);
     }
     allocate(size) {
         const position = this.offset + size;
@@ -139,6 +158,8 @@ export class BinaryWriter {
         this.writeDouble(timestamp);
     }
     writeString(value) {
+        // const compressed = pako.deflateRaw(value, { level: 9 });
+        // this.writeBytes(compressed);
         let start = this.offset;
         let require = value.length << 2;
         if (require < 254) {
@@ -204,22 +225,31 @@ export class BinaryWriter {
         this.writeByte(CORE_TYPES.None);
     }
     wireDictionary(value) {
-        if (this.dict.has(value)) {
-            const idx = this.dict.get(value);
-            this.writeCore(CORE_TYPES.DictIndex, idx);
+        let idx;
+        if (this.dictionary) {
+            idx = this.dictionary.getIndex(value);
         }
-        else {
-            const newIndex = this.dict.size + 1;
-            this.dict.set(value, newIndex);
+        if (idx === undefined) {
+            idx = this.dictionaryExtended.getIndex(value);
+        }
+        if (idx === undefined) {
+            this.dictionaryExtended.maybeInsert(value);
             this.writeCore(CORE_TYPES.DictValue, value);
         }
+        else {
+            this.writeCore(CORE_TYPES.DictIndex, idx);
+        }
+    }
+    writeGzip(value) {
+        const compressed = pako.deflateRaw(value, { level: 9 });
+        this.writeBytes(compressed);
     }
     encode(value) {
-        const start = this.offset;
+        this.offset = 0;
+        this.target = byteArrayAllocate(256);
+        this.targetView = new DataView(this.target.buffer, 0, this.target.length);
         this.writeObject(value);
-        const end = this.offset;
-        this.offset = start;
-        return this.target.subarray(start, end);
+        return this.getBuffer();
     }
     _writeCustom(value) {
         for (const ext of this.extensions.values()) {
@@ -255,11 +285,27 @@ export class BinaryWriter {
         }
         this.writeCore(constructorId, value);
     }
+    writeObjectGzip(value) {
+        const writer = new BinaryWriter();
+        writer.extensions = this.extensions;
+        writer.dictionary = this.dictionary;
+        writer.dictionaryExtended = this.dictionaryExtended;
+        writer.writeObject(value);
+        this.writeCore(CORE_TYPES.GZIP, writer.getBuffer());
+    }
     writeCore(constructorId, value) {
-        if (![CORE_TYPES.BoolFalse, CORE_TYPES.BoolTrue, CORE_TYPES.Null].includes(constructorId)) {
+        // console.log('write', { constructorId: CORE_TYPES[constructorId] || constructorId, value });
+        if (this.withGzip && SUPPORT_COMPRESSION.has(constructorId)) {
+            this.writeObjectGzip(value);
+            return;
+        }
+        else if (!NO_CONSTRUCTOR.has(constructorId)) {
             this.writeByte(constructorId);
         }
         switch (constructorId) {
+            case CORE_TYPES.GZIP: {
+                return this.writeGzip(value);
+            }
             case CORE_TYPES.DictIndex: {
                 return this.writeLength(value);
             }

package/dist/constants.d.ts

@@ -17,5 +17,6 @@ export declare enum CORE_TYPES {
     Map = 15,
     DictValue = 16,
     DictIndex = 17,
-    String = 18
+    String = 18,
+    GZIP = 19
 }

package/dist/constants.js

@@ -19,5 +19,5 @@ export var CORE_TYPES;
     CORE_TYPES[CORE_TYPES["DictValue"] = 16] = "DictValue";
     CORE_TYPES[CORE_TYPES["DictIndex"] = 17] = "DictIndex";
     CORE_TYPES[CORE_TYPES["String"] = 18] = "String";
+    CORE_TYPES[CORE_TYPES["GZIP"] = 19] = "GZIP";
 })(CORE_TYPES || (CORE_TYPES = {}));
-;

package/dist/dictionary.d.ts

@@ -0,0 +1,17 @@
+export declare function createDictionary(values?: string[]): Dictionary;
+export declare class Dictionary {
+    private _count;
+    private _map;
+    private _index;
+    private _offset;
+    constructor(values?: string[], offset?: number);
+    get size(): number;
+    /**
+     * Returns inserted index or nothing
+     */
+    maybeInsert(word: string): number | undefined;
+    getValue(index: number): string | undefined;
+    getIndex(value: string): number | undefined;
+    hasValue(value: string): boolean;
+    hasIndex(index: number): boolean;
+}

package/dist/dictionary.js

@@ -0,0 +1,51 @@
+export function createDictionary(values) {
+    return new Dictionary(values);
+}
+export class Dictionary {
+    _count = 0;
+    _map;
+    _index;
+    _offset;
+    constructor(values, offset = 0) {
+        this._index = [];
+        this._map = new Map();
+        this._offset = offset;
+        if (Array.isArray(values) && values.length) {
+            values.forEach((word) => {
+                if (this._map.has(word))
+                    return;
+                this._map.set(word, this._count++);
+                this._index.push(word);
+            });
+        }
+    }
+    get size() {
+        return this._count;
+    }
+    /**
+     * Returns inserted index or nothing
+     */
+    maybeInsert(word) {
+        if (this._map.has(word))
+            return;
+        this._map.set(word, this._count++);
+        this._index.push(word);
+        return this._count + this._offset;
+    }
+    getValue(index) {
+        return this._index[index - this._offset];
+    }
+    getIndex(value) {
+        const idx = this._map.get(value);
+        if (idx === undefined) {
+            return idx;
+        }
+        return idx + this._offset;
+    }
+    hasValue(value) {
+        return this._map.has(value);
+    }
+    hasIndex(index) {
+        return this._index[index - this._offset] !== undefined;
+    }
+}

package/dist/index.d.ts

@@ -2,3 +2,4 @@ export * from './BinaryWriter.js';
 export * from './BinaryReader.js';
 export * from './constants.js';
 export * from './extension.js';
+export * from './dictionary.js';

package/dist/index.js

@@ -2,3 +2,4 @@ export * from './BinaryWriter.js';
 export * from './BinaryReader.js';
 export * from './constants.js';
 export * from './extension.js';
+export * from './dictionary.js';

package/dist/stream.d.ts

@@ -1,13 +1,20 @@
 /// <reference types="node" resolution-mode="require"/>
 /// <reference types="node" resolution-mode="require"/>
 import { Transform, type TransformCallback, type TransformOptions } from 'stream';
+import { BinaryWriter, BinaryWriterOptions } from './BinaryWriter.js';
+import { BinaryReader } from './BinaryReader.js';
+export interface TLEncodeOptions extends BinaryWriterOptions {
+    streamOptions?: TransformOptions;
+    writeVectorWhenEmpty?: boolean;
+}
 export declare class TLEncode extends Transform {
-    private writer;
-    constructor(options?: TransformOptions);
+    writer: BinaryWriter;
+    count: number;
+    constructor(options?: TLEncodeOptions);
     _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void;
 }
 export declare class TLDecode extends Transform {
-    private reader;
+    reader: BinaryReader;
     private incompleteBuffer;
     constructor(options?: TransformOptions);
     _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void;

package/dist/stream.js

@@ -3,16 +3,31 @@ import { BinaryWriter } from './BinaryWriter.js';
 import { BinaryReader } from './BinaryReader.js';
 export class TLEncode extends Transform {
     writer;
+    count;
     constructor(options) {
-        if (!options)
-            options = {};
-        options.writableObjectMode = true;
-        super(options);
-        this.writer = new BinaryWriter();
+        const opts = options || {};
+        opts.streamOptions = { writableObjectMode: true, ...(opts.streamOptions || {}) };
+        super(opts.streamOptions);
+        const writer = new BinaryWriter(options);
+        const customFlush = opts.streamOptions.flush;
+        this._flush = (callback) => {
+            if (this.count === 0 && opts.writeVectorWhenEmpty) {
+                this.push(writer.encode([]));
+            }
+            if (customFlush) {
+                customFlush.call(this, callback);
+            }
+            else {
+                callback();
+            }
+        };
+        this.writer = writer;
+        this.count = 0;
     }
     _transform(chunk, encoding, callback) {
         const buff = this.writer.encode(chunk);
         this.push(buff);
+        this.count++;
         callback();
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@andrew_l/tl-pack",
-	"version": "0.0.4",
+	"version": "0.0.5",
 	"description": "",
 	"type": "module",
 	"scripts": {
@@ -26,10 +26,14 @@
 	},
 	"devDependencies": {
 		"@types/node": "^20.3.2",
+		"@types/pako": "^2.0.0",
 		"eslint-config-prettier": "^8.8.0",
 		"nodemon": "^2.0.22",
 		"prettier": "^2.8.8",
 		"ts-node": "^10.9.1",
 		"typescript": "^5.1.5"
+	},
+	"dependencies": {
+		"pako": "^2.1.0"
 	}
 }