@peers-app/peers-sdk 0.7.18 → 0.7.20

package/dist/device/connection.test.js

@@ -230,16 +230,17 @@ describe(connection_1.Connection, () => {
         const result = await clientConnection.doHandshake(serverAddress).catch((err) => String(err));
         expect(result).toMatch(/Inconsistent public keys/);
     });
-    it("should handle RPC calls with large arguments through chunking", async () => {
+    it("should handle RPC calls with large arguments and responses through chunking", async () => {
         const { clientSocket, serverSocket } = createTestSocketPair();
         const clientDevice = new device_1.Device();
         const serverDevice = new device_1.Device();
         const clientConnection = new connection_1.Connection(clientSocket, clientDevice);
         const serverConnection = new connection_1.Connection(serverSocket, serverDevice, ['localhost']);
-        clientConnection.maxChunkSize = 50;
-        serverConnection.maxChunkSize = 50;
-        clientConnection.socket.maxChunkSize = 50;
-        serverConnection.socket.maxChunkSize = 50;
+        const chunkSize = 256; // 256 minimum chunk size due to encoding overheads
+        clientConnection.maxChunkSize = chunkSize;
+        serverConnection.maxChunkSize = chunkSize;
+        clientConnection.socket.maxChunkSize = chunkSize;
+        serverConnection.socket.maxChunkSize = chunkSize;
         await clientConnection.doHandshake('localhost');
         const clientEmitSpy = jest.spyOn(clientConnection.socket.socket, 'emit');
         const serverEmitSpy = jest.spyOn(serverConnection.socket.socket, 'emit');
@@ -259,15 +260,21 @@ describe(connection_1.Connection, () => {
         expect(result.received.message).toBe("A".repeat(500));
         expect(result.received.metadata).toBe("B".repeat(300));
         expect(result.received.payload).toBe("C".repeat(400));
-        expect(result.processedSize).toBeGreaterThan(1000); // Should be large
+        expect(JSON.stringify(largeData).length).toEqual(1241);
         // Count chunk emissions
         const clientChunkCalls = clientEmitSpy.mock.calls.filter((call) => call[0] === clientConnection.socket.safeSocketChunkEventName);
         const serverChunkCalls = serverEmitSpy.mock.calls.filter((call) => call[0] === serverConnection.socket.safeSocketChunkEventName);
-        // The data gets wrapped by Connection layer (encryption/signing), making it larger
-        // 41 chunks at 50 bytes each = ~2050 bytes (41 * 50)
-        // 43 chunks at 50 bytes each = ~2150 bytes (43 * 50)
-        expect(clientChunkCalls.length).toBe(41); // ~2050 bytes wrapped data / 50 bytes per chunk
-        expect(serverChunkCalls.length).toBe(43); // ~2150 bytes wrapped data / 50 bytes per chunk
+        expect(clientChunkCalls.length).toBe(4);
+        expect(serverChunkCalls.length).toBe(5);
+        // expect calls to never have chunks greater than chunk size
+        for (const call of clientChunkCalls) {
+            const chunkData = call[1];
+            expect(chunkData.length).toBeLessThanOrEqual(chunkSize);
+        }
+        for (const call of serverChunkCalls) {
+            const chunkData = call[1];
+            expect(chunkData.length).toBeLessThanOrEqual(chunkSize);
+        }
     });
     it("should reject handshake if device timestamps are too far apart", async () => {
         const { clientSocket, serverSocket } = createTestSocketPair();

package/dist/device/get-trust-level.js

@@ -9,9 +9,21 @@ function getTrustLevelFn(me, serverUrl) {
         const userContext = await (0, context_1.getUserContext)();
         const userDataContext = userContext.userDataContext;
         if (deviceInfo.userId === me.userId && deviceInfo.publicKey === me.publicKey && deviceInfo.publicBoxKey === me.publicBoxKey) {
-            if (deviceInfo.deviceId === (0, data_1.thisDeviceId)()) {
-                // return TrustLevel.Untrusted;
-            }
+            // if (deviceInfo.deviceId === thisDeviceId()) {
+            //   return TrustLevel.Untrusted;
+            // }
+            const device = await (0, data_1.Devices)(userDataContext).get(deviceInfo.deviceId) || {
+                deviceId: deviceInfo.deviceId,
+                userId: deviceInfo.userId,
+                firstSeen: new Date(),
+                lastSeen: new Date(),
+                trustLevel: socket_type_1.TrustLevel.Trusted,
+                serverUrl,
+            };
+            device.lastSeen = new Date();
+            device.trustLevel = socket_type_1.TrustLevel.Trusted;
+            console.log(`Updating my own device: ${deviceInfo.deviceId}`);
+            await (0, data_1.Devices)(userDataContext).save(device);
             return socket_type_1.TrustLevel.Trusted;
         }
         // await Devices().delete(deviceInfo.deviceId);

package/dist/device/streamed-socket.d.ts

@@ -1,11 +1,4 @@
 import { ISocket, RPCCallback } from "./socket.type";
-export interface IMessageChunk {
-    messageId: string;
-    chunkIndex: number;
-    totalChunks?: number;
-    eventName?: string;
-    data: string;
-}
 export declare class StreamedSocket implements ISocket {
     readonly socket: ISocket;
     maxChunkSize: number;

package/dist/device/streamed-socket.js

@@ -2,9 +2,50 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.StreamedSocket = void 0;
 const lodash_1 = require("lodash");
-const serial_json_1 = require("../serial-json");
 const utils_1 = require("../utils");
 const socket_type_1 = require("./socket.type");
+const tx_encoding_1 = require("./tx-encoding");
+const msgpack_1 = require("@msgpack/msgpack");
+// Encode chunk metadata + data into a single Uint8Array
+function encodeChunk(chunk) {
+    // Encode metadata using msgpack
+    const metadata = {
+        messageId: chunk.messageId,
+        chunkIndex: chunk.chunkIndex,
+        ...(chunk.totalChunks !== undefined && { totalChunks: chunk.totalChunks }),
+        ...(chunk.eventName !== undefined && { eventName: chunk.eventName }),
+    };
+    const metadataBytes = (0, msgpack_1.encode)(metadata);
+    const metadataLength = metadataBytes.length;
+    // Create combined buffer: [4 bytes length][metadata][data]
+    const combined = new Uint8Array(4 + metadataLength + chunk.data.length);
+    // Write metadata length as 32-bit unsigned integer (big-endian)
+    const view = new DataView(combined.buffer, combined.byteOffset, combined.byteLength);
+    view.setUint32(0, metadataLength, false); // big-endian
+    // Write metadata
+    combined.set(metadataBytes, 4);
+    // Write data
+    combined.set(chunk.data, 4 + metadataLength);
+    return combined;
+}
+// Decode binary chunk back into IMessageChunk
+function decodeChunk(bytes) {
+    // Read metadata length
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    const metadataLength = view.getUint32(0, false); // big-endian
+    // Extract metadata
+    const metadataBytes = bytes.subarray(4, 4 + metadataLength);
+    const metadata = (0, msgpack_1.decode)(metadataBytes);
+    // Extract data
+    const data = bytes.subarray(4 + metadataLength);
+    return {
+        messageId: metadata.messageId,
+        chunkIndex: metadata.chunkIndex,
+        totalChunks: metadata.totalChunks,
+        eventName: metadata.eventName,
+        data: data,
+    };
+}
 class StreamedSocket {
     socket;
     maxChunkSize;
@@ -13,7 +54,10 @@ class StreamedSocket {
     constructor(socket, maxChunkSize = socket_type_1.DEFAULT_MAX_CHUNK_SIZE) {
         this.socket = socket;
         this.maxChunkSize = maxChunkSize;
-        socket.on(this.safeSocketChunkEventName, (chunk) => this.handleChunk(chunk));
+        socket.on(this.safeSocketChunkEventName, (encodedChunk) => {
+            const chunk = decodeChunk(encodedChunk);
+            this.handleChunk(chunk);
+        });
     }
     get id() {
         return this.socket.id;
@@ -26,8 +70,8 @@ class StreamedSocket {
     }
     callbacks = {};
     emit(eventName, args, callback) {
-        const strArgs = (0, serial_json_1.toJSONString)(args);
-        // if (strArgs.length < this.maxChunkSize) {
+        const encoded = (0, tx_encoding_1.txEncode)(args);
+        // if (encoded.length < this.maxChunkSize) {
         //   // If the data is small enough, send it directly
         //   this.socket.emit(eventName, args, callback);
         //   return;
@@ -37,25 +81,34 @@ class StreamedSocket {
             this.callbacks[messageId] = callback;
         }
         const chunks = [];
-        for (let i = 0; i < strArgs.length; i += this.maxChunkSize) {
-            const chunkData = strArgs.slice(i, i + this.maxChunkSize);
+        // Reserve space for metadata overhead (typically ~50-100 bytes)
+        const metadataOverhead = 128;
+        if (this.maxChunkSize < (metadataOverhead * 2)) {
+            throw new Error(`maxChunkSize ${this.maxChunkSize} is too small to accommodate metadata overhead, must be greater than ${metadataOverhead * 2}`);
+        }
+        const effectiveChunkSize = this.maxChunkSize - metadataOverhead;
+        for (let i = 0; i < encoded.length; i += effectiveChunkSize) {
+            const chunkData = encoded.subarray(i, i + effectiveChunkSize);
             chunks.push({
                 messageId,
-                chunkIndex: Math.floor(i / this.maxChunkSize),
-                data: chunkData
+                chunkIndex: Math.floor(i / effectiveChunkSize),
+                data: chunkData,
             });
             if (i === 0) {
                 // The first chunk contains metadata about the chunks
                 chunks[0].eventName = eventName;
-                chunks[0].totalChunks = Math.ceil(strArgs.length / this.maxChunkSize);
+                chunks[0].totalChunks = Math.ceil(encoded.length / effectiveChunkSize);
             }
         }
         if (chunks.length > 1) {
             console.debug(`Sending ${chunks.length} chunks for event ${eventName} with messageId ${messageId}`);
         }
         for (const chunk of chunks) {
+            // Encode chunk with metadata into single Uint8Array
+            const encodedChunk = encodeChunk(chunk);
             // TODO on error or timeout retry
-            this.socket.emit(this.safeSocketChunkEventName, chunk, lodash_1.noop);
+            // TODO respond to backpressure from socket
+            this.socket.emit(this.safeSocketChunkEventName, encodedChunk, lodash_1.noop);
         }
     }
     handlers = {};
@@ -98,10 +151,17 @@ class StreamedSocket {
     async processChunks(chunks) {
         try {
             const chunkZero = chunks[0];
-            const strData = chunks.map(c => c.data).join('');
+            // Reassemble Uint8Array chunks (already decoded from binary format)
+            const totalSize = chunks.reduce((sum, chunk) => sum + chunk.data.length, 0);
+            const reassembled = new Uint8Array(totalSize);
+            let offset = 0;
+            for (const chunk of chunks) {
+                reassembled.set(chunk.data, offset);
+                offset += chunk.data.length;
+            }
             delete this.chunkBuffers[chunkZero.messageId];
             const eventName = chunkZero.eventName;
-            const args = (0, serial_json_1.fromJSONString)(strData);
+            const args = (0, tx_encoding_1.txDecode)(reassembled);
             if (eventName === this.safeSocketResponseEventName) {
                 this.handleResponse(args.messageId, args.error, args.result);
                 return;

package/dist/device/streamed-socket.test.js

@@ -2,6 +2,17 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 const streamed_socket_1 = require("./streamed-socket");
 const utils_1 = require("../utils");
+const tx_encoding_1 = require("./tx-encoding");
+const msgpack_1 = require("@msgpack/msgpack");
+// Helper to decode the binary chunk format
+function decodeChunk(bytes) {
+    const view = new DataView(bytes.buffer, bytes.byteOffset);
+    const metadataLength = view.getUint32(0, false);
+    const metadataBytes = bytes.subarray(4, 4 + metadataLength);
+    const metadata = (0, msgpack_1.decode)(metadataBytes);
+    const data = bytes.subarray(4 + metadataLength);
+    return { ...metadata, data };
+}
 function createMockSocket() {
     const emittedEvents = [];
     const socket = {
@@ -35,10 +46,171 @@ describe('StreamedSocket', () => {
         expect(emittedEvents).toHaveLength(1);
         const chunkEvent = emittedEvents[0];
         expect(chunkEvent.eventName).toBe('__streamed-socket-chunk');
-        expect(chunkEvent.args.messageId).toBeTruthy();
-        expect(chunkEvent.args.chunkIndex).toBe(0);
-        expect(chunkEvent.args.totalChunks).toBe(1);
-        expect(chunkEvent.args.eventName).toBe('test-event');
-        expect(JSON.parse(chunkEvent.args.data)).toEqual(testData);
+        // Args should be a Uint8Array (binary encoded chunk)
+        expect(chunkEvent.args).toBeInstanceOf(Uint8Array);
+        // Decode the binary chunk
+        const decodedChunk = decodeChunk(chunkEvent.args);
+        expect(decodedChunk.messageId).toBeTruthy();
+        expect(decodedChunk.chunkIndex).toBe(0);
+        expect(decodedChunk.totalChunks).toBe(1);
+        expect(decodedChunk.eventName).toBe('test-event');
+        // Decode the actual data
+        const decoded = (0, tx_encoding_1.txDecode)(decodedChunk.data);
+        expect(decoded).toEqual(testData);
+    });
+    it('should emit multiple chunks for large messages', () => {
+        const mock = createMockSocket();
+        const streamedSocket = new streamed_socket_1.StreamedSocket(mock.socket, 256); // Small chunk size to force chunking
+        // Create a large test object
+        const testData = {
+            message: 'x'.repeat(200),
+            array: [1, 2, 3, 4, 5],
+            nested: { deep: 'value' }
+        };
+        streamedSocket.emit('test-event', testData, () => { });
+        const emittedEvents = mock.getEmittedEvents();
+        expect(emittedEvents.length).toBeGreaterThan(1);
+        // Decode all chunks
+        const decodedChunks = emittedEvents.map(event => {
+            expect(event.args).toBeInstanceOf(Uint8Array);
+            return decodeChunk(event.args);
+        });
+        // Verify first chunk has metadata
+        const firstChunk = decodedChunks[0];
+        expect(firstChunk.eventName).toBe('test-event');
+        expect(firstChunk.totalChunks).toBe(emittedEvents.length);
+        expect(firstChunk.chunkIndex).toBe(0);
+        // Verify all chunks have the same messageId
+        const messageId = firstChunk.messageId;
+        decodedChunks.forEach(chunk => {
+            expect(chunk.messageId).toBe(messageId);
+        });
+        // Verify chunks are in order
+        decodedChunks.forEach((chunk, index) => {
+            expect(chunk.chunkIndex).toBe(index);
+        });
+        // Verify all chunk data are Uint8Arrays
+        decodedChunks.forEach(chunk => {
+            expect(chunk.data).toBeInstanceOf(Uint8Array);
+        });
+        // Reassemble chunks and verify data integrity
+        const reassembled = new Uint8Array(decodedChunks.reduce((sum, chunk) => sum + chunk.data.length, 0));
+        let offset = 0;
+        for (const chunk of decodedChunks) {
+            reassembled.set(chunk.data, offset);
+            offset += chunk.data.length;
+        }
+        const decoded = (0, tx_encoding_1.txDecode)(reassembled);
+        expect(decoded).toEqual(testData);
+    });
+    it('should handle binary data in messages', () => {
+        const mock = createMockSocket();
+        const streamedSocket = new streamed_socket_1.StreamedSocket(mock.socket, 1000);
+        const testData = {
+            binaryData: new Uint8Array([1, 2, 3, 4, 5]),
+            metadata: { name: 'test' }
+        };
+        streamedSocket.emit('test-event', testData, () => { });
+        const emittedEvents = mock.getEmittedEvents();
+        expect(emittedEvents).toHaveLength(1);
+        const decodedChunk = decodeChunk(emittedEvents[0].args);
+        const decoded = (0, tx_encoding_1.txDecode)(decodedChunk.data);
+        expect(decoded.binaryData).toBeInstanceOf(Uint8Array);
+        expect(decoded.binaryData).toEqual(testData.binaryData);
+        expect(decoded.metadata).toEqual(testData.metadata);
+    });
+    it('should handle special types in messages', () => {
+        const mock = createMockSocket();
+        const streamedSocket = new streamed_socket_1.StreamedSocket(mock.socket, 1000);
+        const testData = {
+            date: new Date('2023-01-01T00:00:00.000Z'),
+            nan: NaN,
+            infinity: Infinity,
+            undef: undefined,
+            nullVal: null
+        };
+        streamedSocket.emit('test-event', testData, () => { });
+        const emittedEvents = mock.getEmittedEvents();
+        expect(emittedEvents).toHaveLength(1);
+        const decodedChunk = decodeChunk(emittedEvents[0].args);
+        const decoded = (0, tx_encoding_1.txDecode)(decodedChunk.data);
+        expect(decoded.date).toBeInstanceOf(Date);
+        expect(decoded.date.toISOString()).toBe('2023-01-01T00:00:00.000Z');
+        expect(Number.isNaN(decoded.nan)).toBe(true);
+        expect(decoded.infinity).toBe(Infinity);
+        expect(decoded.undef).toBeUndefined();
+        expect(decoded.nullVal).toBeNull();
+    });
+    it('should demonstrate msgpack efficiency: structured data vs JSON string (uncompressed)', () => {
+        const mock1 = createMockSocket();
+        const mock2 = createMockSocket();
+        const streamedSocket1 = new streamed_socket_1.StreamedSocket(mock1.socket, 1000000); // Large chunk to avoid splitting
+        const streamedSocket2 = new streamed_socket_1.StreamedSocket(mock2.socket, 1000000);
+        // Create a small structured object that won't trigger compression (< 1024 bytes)
+        const smallObject = {
+            users: Array.from({ length: 5 }, (_, i) => ({
+                id: i + 1000000, // Use larger numbers to make JSON less compressible
+                name: `User${i}_${Math.random().toString(36).substring(7)}`, // Add randomness
+                email: `user${i}@example.com`,
+                score: Math.random() * 1000,
+                active: i % 2 === 0,
+            }))
+        };
+        // Convert to JSON string
+        const jsonString = JSON.stringify(smallObject);
+        console.log(`Original JSON string size: ${jsonString.length} bytes`);
+        // Send as structured object (msgpack can optimize)
+        streamedSocket1.emit('test-object', smallObject, () => { });
+        // Send as JSON string (msgpack treats it as opaque string)
+        streamedSocket2.emit('test-string', { jsonPayload: jsonString }, () => { });
+        const objectEvents = mock1.getEmittedEvents();
+        const stringEvents = mock2.getEmittedEvents();
+        expect(objectEvents).toHaveLength(1);
+        expect(stringEvents).toHaveLength(1);
+        // Decode and measure sizes
+        const objectChunk = decodeChunk(objectEvents[0].args);
+        const stringChunk = decodeChunk(stringEvents[0].args);
+        const objectSize = objectChunk.data.length;
+        const stringSize = stringChunk.data.length;
+        console.log(`Msgpack with structured data: ${objectSize} bytes (${((1 - objectSize / jsonString.length) * 100).toFixed(1)}% reduction from JSON)`);
+        console.log(`Msgpack with JSON string: ${stringSize} bytes (${((1 - stringSize / jsonString.length) * 100).toFixed(1)}% reduction from JSON)`);
+        console.log(`Efficiency gain: structured data is ${((1 - objectSize / stringSize) * 100).toFixed(1)}% smaller than JSON string`);
+        // Msgpack should be more efficient with structured data than with a JSON string
+        expect(objectSize).toBeLessThan(stringSize);
+        // Structured data should be at least 10% more efficient
+        const efficiency = (1 - objectSize / stringSize) * 100;
+        expect(efficiency).toBeGreaterThan(10);
+        // Verify data integrity
+        const decodedObject = (0, tx_encoding_1.txDecode)(objectChunk.data);
+        const decodedString = (0, tx_encoding_1.txDecode)(stringChunk.data);
+        expect(decodedObject.users).toHaveLength(5);
+        expect(decodedString.jsonPayload).toBe(jsonString);
+    });
+    it('should demonstrate compression effectiveness for large repetitive data', () => {
+        const mock = createMockSocket();
+        const streamedSocket = new streamed_socket_1.StreamedSocket(mock.socket, 1000000);
+        // Create large repetitive object that will compress well
+        const largeObject = {
+            data: Array.from({ length: 200 }, () => ({
+                type: 'event',
+                status: 'active',
+                category: 'user_action',
+                timestamp: new Date('2024-01-01'),
+                metadata: { app: 'test', version: '1.0.0' }
+            }))
+        };
+        const jsonString = JSON.stringify(largeObject);
+        console.log(`Large repetitive data - JSON size: ${jsonString.length} bytes`);
+        streamedSocket.emit('test-compressed', largeObject, () => { });
+        const events = mock.getEmittedEvents();
+        const chunk = decodeChunk(events[0].args);
+        console.log(`After msgpack + compression: ${chunk.data.length} bytes`);
+        console.log(`Compression ratio: ${((1 - chunk.data.length / jsonString.length) * 100).toFixed(1)}% reduction`);
+        // Should achieve significant compression (>80% reduction)
+        expect(chunk.data.length).toBeLessThan(jsonString.length * 0.2);
+        // Verify data integrity
+        const decoded = (0, tx_encoding_1.txDecode)(chunk.data);
+        expect(decoded.data).toHaveLength(200);
+        expect(decoded.data[0].type).toBe('event');
     });
 });

package/dist/device/tx-encoding.d.ts

@@ -0,0 +1,2 @@
+export declare function txEncode(data: unknown): Uint8Array;
+export declare function txDecode(data: Uint8Array): unknown;

package/dist/device/tx-encoding.js

@@ -0,0 +1,44 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.txEncode = txEncode;
+exports.txDecode = txDecode;
+const msgpack_1 = require("@msgpack/msgpack");
+const fflate_1 = require("fflate");
+const serial_json_1 = require("../serial-json");
+function txEncode(data) {
+    const noCycles = (0, serial_json_1.toJSON)(data); // remove cycles and encode dates, etc.
+    // First encode to msgpack
+    const encoded = (0, msgpack_1.encode)(noCycles);
+    let body;
+    let flag;
+    if (encoded.length > 1024) {
+        // Compress big payloads
+        body = (0, fflate_1.compressSync)(encoded);
+        flag = 1; // compressed
+    }
+    else {
+        body = encoded;
+        flag = 0; // not compressed
+    }
+    const out = new Uint8Array(1 + body.length);
+    out[0] = flag;
+    out.set(body, 1);
+    return out;
+}
+function txDecode(data) {
+    if (data.length === 0) {
+        throw new Error("Empty payload");
+    }
+    const flag = data[0];
+    let body = data.subarray(1);
+    if (flag === 1) {
+        body = (0, fflate_1.decompressSync)(body);
+    }
+    else if (flag !== 0) {
+        // Optional: future versioning or legacy handling
+        throw new Error(`Unknown tx flag: ${flag}`);
+    }
+    const unpacked = (0, msgpack_1.decode)(body);
+    const restored = (0, serial_json_1.fromJSON)(unpacked); // restore cycles, dates, etc.
+    return restored;
+}

package/dist/device/tx-encoding.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/device/tx-encoding.test.js

@@ -0,0 +1,267 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const tx_encoding_1 = require("./tx-encoding");
+describe('tx-encode-decode', () => {
+    describe('txEncode and txDecode', () => {
+        it('should encode and decode simple objects', () => {
+            const testObj = {
+                name: 'test',
+                number: 42,
+                bool: true,
+                str: 'hello world'
+            };
+            const encoded = (0, tx_encoding_1.txEncode)(testObj);
+            expect(encoded).toBeInstanceOf(Uint8Array);
+            expect(encoded.length).toBeGreaterThan(0);
+            const decoded = (0, tx_encoding_1.txDecode)(encoded);
+            expect(decoded).toEqual(testObj);
+        });
+        it('should encode and decode arrays', () => {
+            const testArray = [1, 2, 3, 'four', { five: 5 }];
+            const encoded = (0, tx_encoding_1.txEncode)(testArray);
+            const decoded = (0, tx_encoding_1.txDecode)(encoded);
+            expect(decoded).toEqual(testArray);
+        });
+        it('should encode and decode nested objects', () => {
+            const testObj = {
+                level1: {
+                    level2: {
+                        level3: {
+                            value: 'deep nested value',
+                            number: 123
+                        },
+                        array: [1, 2, 3]
+                    }
+                }
+            };
+            const encoded = (0, tx_encoding_1.txEncode)(testObj);
+            const decoded = (0, tx_encoding_1.txDecode)(encoded);
+            expect(decoded).toEqual(testObj);
+        });
+        it('should encode and decode special types (dates, etc.)', () => {
+            const testObj = {
+                date: new Date('2023-01-01T00:00:00.000Z'),
+                nan: NaN,
+                infinity: Infinity,
+                negInfinity: -Infinity,
+                undef: undefined,
+                nullVal: null
+            };
+            const encoded = (0, tx_encoding_1.txEncode)(testObj);
+            const decoded = (0, tx_encoding_1.txDecode)(encoded);
+            expect(decoded.date).toBeInstanceOf(Date);
+            expect(decoded.date.toISOString()).toBe('2023-01-01T00:00:00.000Z');
+            expect(Number.isNaN(decoded.nan)).toBe(true);
+            expect(decoded.infinity).toBe(Infinity);
+            expect(decoded.negInfinity).toBe(-Infinity);
+            expect(decoded.undef).toBeUndefined();
+            expect(decoded.nullVal).toBeNull();
+        });
+        it('should encode and decode Uint8Array', () => {
+            const testObj = {
+                data: new Uint8Array([72, 101, 108, 108, 111]),
+                metadata: { size: 5 }
+            };
+            const encoded = (0, tx_encoding_1.txEncode)(testObj);
+            const decoded = (0, tx_encoding_1.txDecode)(encoded);
+            expect(decoded.data).toBeInstanceOf(Uint8Array);
+            expect(decoded.data).toEqual(testObj.data);
+            expect(decoded.metadata).toEqual(testObj.metadata);
+        });
+        it('should encode and decode Buffer', () => {
+            const testObj = {
+                buffer: Buffer.from('Hello Buffer'),
+                name: 'test'
+            };
+            const encoded = (0, tx_encoding_1.txEncode)(testObj);
+            const decoded = (0, tx_encoding_1.txDecode)(encoded);
+            expect(Buffer.isBuffer(decoded.buffer)).toBe(true);
+            expect(decoded.buffer).toEqual(testObj.buffer);
+            expect(decoded.name).toBe('test');
+        });
+    });
+    describe('compression threshold', () => {
+        it('should not compress small payloads (< 1024 bytes)', () => {
+            const smallObj = { message: 'small' };
+            const encoded = (0, tx_encoding_1.txEncode)(smallObj);
+            // First byte is the flag
+            expect(encoded[0]).toBe(0); // flag = 0 means not compressed
+        });
+        it('should compress large payloads (> 1024 bytes)', () => {
+            // Create an object that will exceed 1024 bytes when encoded
+            const largeObj = {
+                data: 'x'.repeat(2000),
+                array: Array.from({ length: 100 }, (_, i) => ({ id: i, value: `item ${i}` }))
+            };
+            const jsonStr = JSON.stringify(largeObj);
+            expect(jsonStr.length).toBeGreaterThan(1024);
+            const encoded = (0, tx_encoding_1.txEncode)(largeObj);
+            // First byte is the flag
+            expect(encoded[0]).toBe(1); // flag = 1 means compressed
+            // Verify it can still be decoded correctly
+            const decoded = (0, tx_encoding_1.txDecode)(encoded);
+            expect(decoded).toEqual(largeObj);
+        });
+        it('should have smaller encoded size for large compressible data', () => {
+            const largeRepetitiveObj = {
+                data: 'repeat '.repeat(500),
+                values: Array.from({ length: 200 }, () => 'same string')
+            };
+            const encoded = (0, tx_encoding_1.txEncode)(largeRepetitiveObj);
+            // Compressed size should be significantly smaller than the raw data
+            const rawDataSize = JSON.stringify(largeRepetitiveObj).length;
+            expect(encoded.length).toBeLessThan(rawDataSize / 2);
+            // Verify correctness
+            const decoded = (0, tx_encoding_1.txDecode)(encoded);
+            expect(decoded).toEqual(largeRepetitiveObj);
+        });
+    });
+    describe('error handling', () => {
+        it('should throw error on empty payload', () => {
+            const emptyPayload = new Uint8Array(0);
+            expect(() => (0, tx_encoding_1.txDecode)(emptyPayload)).toThrow('Empty payload');
+        });
+        it('should throw error on unknown flag', () => {
+            // Create a payload with an invalid flag (2)
+            const invalidPayload = new Uint8Array([2, 0, 0, 0]);
+            expect(() => (0, tx_encoding_1.txDecode)(invalidPayload)).toThrow('Unknown tx flag: 2');
+        });
+        it('should throw error on unknown flag (255)', () => {
+            const invalidPayload = new Uint8Array([255, 0, 0, 0]);
+            expect(() => (0, tx_encoding_1.txDecode)(invalidPayload)).toThrow('Unknown tx flag: 255');
+        });
+    });
+    describe('round-trip consistency', () => {
+        it('should handle empty objects', () => {
+            const emptyObj = {};
+            const encoded = (0, tx_encoding_1.txEncode)(emptyObj);
+            const decoded = (0, tx_encoding_1.txDecode)(encoded);
+            expect(decoded).toEqual(emptyObj);
+        });
+        it('should handle empty arrays', () => {
+            const emptyArray = [];
+            const encoded = (0, tx_encoding_1.txEncode)(emptyArray);
+            const decoded = (0, tx_encoding_1.txDecode)(encoded);
+            expect(decoded).toEqual(emptyArray);
+        });
+        it('should handle strings with special characters', () => {
+            const testObj = {
+                emoji: '🚀🎉',
+                unicode: 'Hello 世界',
+                special: 'line1\nline2\ttab',
+                quotes: 'He said "hello"'
+            };
+            const encoded = (0, tx_encoding_1.txEncode)(testObj);
+            const decoded = (0, tx_encoding_1.txDecode)(encoded);
+            expect(decoded).toEqual(testObj);
+        });
+        it('should handle very large numbers', () => {
+            const testObj = {
+                maxSafeInt: Number.MAX_SAFE_INTEGER,
+                minSafeInt: Number.MIN_SAFE_INTEGER,
+                large: 9007199254740991,
+                small: -9007199254740991
+            };
+            const encoded = (0, tx_encoding_1.txEncode)(testObj);
+            const decoded = (0, tx_encoding_1.txDecode)(encoded);
+            expect(decoded).toEqual(testObj);
+        });
+        it('should handle mixed type arrays', () => {
+            const testArray = [
+                1,
+                'string',
+                true,
+                null,
+                undefined,
+                { obj: 'value' },
+                [1, 2, 3],
+                new Date('2023-01-01'),
+                new Uint8Array([1, 2, 3])
+            ];
+            const encoded = (0, tx_encoding_1.txEncode)(testArray);
+            const decoded = (0, tx_encoding_1.txDecode)(encoded);
+            expect(decoded[0]).toBe(1);
+            expect(decoded[1]).toBe('string');
+            expect(decoded[2]).toBe(true);
+            expect(decoded[3]).toBeNull();
+            expect(decoded[4]).toBeUndefined();
+            expect(decoded[5]).toEqual({ obj: 'value' });
+            expect(decoded[6]).toEqual([1, 2, 3]);
+            expect(decoded[7]).toBeInstanceOf(Date);
+            expect(decoded[8]).toBeInstanceOf(Uint8Array);
+            expect(decoded[8]).toEqual(new Uint8Array([1, 2, 3]));
+        });
+        it('should handle objects with circular references removed by toJSON', () => {
+            // toJSON handles circular references, so this should work
+            const obj = { name: 'test' };
+            obj.self = obj; // circular reference
+            const encoded = (0, tx_encoding_1.txEncode)(obj);
+            const decoded = (0, tx_encoding_1.txDecode)(encoded);
+            // The circular reference should be handled by toJSON/fromJSON
+            expect(decoded).toHaveProperty('name', 'test');
+            expect(decoded).toHaveProperty('self');
+            expect(decoded.self).toBe(decoded); // self-reference restored
+        });
+        it('should preserve data integrity across multiple encode/decode cycles', () => {
+            const original = {
+                id: 123,
+                name: 'multi-cycle test',
+                data: new Uint8Array([1, 2, 3, 4, 5]),
+                timestamp: new Date('2023-06-15T12:00:00.000Z')
+            };
+            // Encode and decode multiple times
+            let current = original;
+            for (let i = 0; i < 5; i++) {
+                const encoded = (0, tx_encoding_1.txEncode)(current);
+                current = (0, tx_encoding_1.txDecode)(encoded);
+            }
+            expect(current.id).toBe(original.id);
+            expect(current.name).toBe(original.name);
+            expect(current.data).toEqual(original.data);
+            expect(current.timestamp).toEqual(original.timestamp);
+        });
+    });
+    describe('payload structure', () => {
+        it('should have flag byte as first byte', () => {
+            const testObj = { test: 'value' };
+            const encoded = (0, tx_encoding_1.txEncode)(testObj);
+            // First byte should be 0 or 1
+            expect(encoded[0]).toBeGreaterThanOrEqual(0);
+            expect(encoded[0]).toBeLessThanOrEqual(1);
+        });
+        it('should have body after flag byte', () => {
+            const testObj = { test: 'value' };
+            const encoded = (0, tx_encoding_1.txEncode)(testObj);
+            // Should have at least 2 bytes (flag + body)
+            expect(encoded.length).toBeGreaterThan(1);
+        });
+    });
+    describe('edge cases', () => {
+        it('should handle objects at exactly 1024 bytes threshold', () => {
+            // Create an object that when msgpack-encoded is very close to 1024 bytes
+            // This is tricky to get exact, so we just verify it works
+            const testObj = {
+                data: 'x'.repeat(1000)
+            };
+            const encoded = (0, tx_encoding_1.txEncode)(testObj);
+            const decoded = (0, tx_encoding_1.txDecode)(encoded);
+            expect(decoded).toEqual(testObj);
+        });
+        it('should handle primitive values', () => {
+            expect((0, tx_encoding_1.txDecode)((0, tx_encoding_1.txEncode)(42))).toBe(42);
+            expect((0, tx_encoding_1.txDecode)((0, tx_encoding_1.txEncode)('string'))).toBe('string');
+            expect((0, tx_encoding_1.txDecode)((0, tx_encoding_1.txEncode)(true))).toBe(true);
+            expect((0, tx_encoding_1.txDecode)((0, tx_encoding_1.txEncode)(false))).toBe(false);
+            expect((0, tx_encoding_1.txDecode)((0, tx_encoding_1.txEncode)(null))).toBe(null);
+        });
+        it('should handle objects with only undefined values', () => {
+            const testObj = {
+                a: undefined,
+                b: undefined
+            };
+            const encoded = (0, tx_encoding_1.txEncode)(testObj);
+            const decoded = (0, tx_encoding_1.txDecode)(encoded);
+            expect(decoded).toEqual(testObj);
+        });
+    });
+});

package/dist/index.d.ts

@@ -17,6 +17,7 @@ export * from "./device/device";
 export * from "./device/device-election";
 export * from "./device/get-trust-level";
 export * from "./device/socket.type";
+export * from "./device/tx-encoding";
 export * from "./package-loader";
 export * from "./events";
 export * from "./data";

package/dist/index.js

@@ -34,6 +34,7 @@ __exportStar(require("./device/device"), exports);
 __exportStar(require("./device/device-election"), exports);
 __exportStar(require("./device/get-trust-level"), exports);
 __exportStar(require("./device/socket.type"), exports);
+__exportStar(require("./device/tx-encoding"), exports);
 __exportStar(require("./package-loader"), exports);
 __exportStar(require("./events"), exports);
 __exportStar(require("./data"), exports);

package/dist/keys.js

@@ -22,12 +22,13 @@ exports.addSignatureToObject = addSignatureToObject;
 exports.verifyObjectSignature = verifyObjectSignature;
 exports.isObjectSignatureValid = isObjectSignatureValid;
 exports.getPublicKeyFromObjectSignature = getPublicKeyFromObjectSignature;
+const sha2_1 = require("@noble/hashes/sha2");
+const buffer_1 = require("buffer");
 const nacl = require("tweetnacl");
 const utils = require("tweetnacl-util");
 const tweetnacl_util_1 = require("tweetnacl-util");
+const tx_encoding_1 = require("./device/tx-encoding");
 const serial_json_1 = require("./serial-json");
-const sha2_1 = require("@noble/hashes/sha2");
-const buffer_1 = require("buffer");
 globalThis.Buffer = buffer_1.Buffer; // shim for browsers/RN
 const _stableStringify = require('fast-json-stable-stringify');
 function setPRNG(fn) {
@@ -114,10 +115,10 @@ function signObjectWithSecretKey(obj, secretKey) {
     }
     const objStr = (0, serial_json_1.toJSONString)(obj);
     const objDecoded = (0, tweetnacl_util_1.decodeUTF8)(objStr);
-    const objSigned = nacl.sign.detached(objDecoded, _secretKey);
+    const signatureByteArray = nacl.sign.detached(objDecoded, _secretKey);
     return {
         contents: obj,
-        signature: encodeBase64(objSigned),
+        signature: encodeBase64(signatureByteArray),
         publicKey: encodeBase64(_secretKey.slice(32)),
     };
 }
@@ -146,9 +147,8 @@ function boxDataWithKeys(data, toPublicBoxKey, mySecretKey) {
     const boxKeyPair = nacl.box.keyPair.fromSecretKey(_secretKey.slice(0, 32));
     const _toPublicBoxKey = decodeBase64(toPublicBoxKey);
     const nonce = nacl.randomBytes(24);
-    data = (0, serial_json_1.toJSONString)(data);
-    const dataDecoded = (0, tweetnacl_util_1.decodeUTF8)(data);
-    const dataBoxed = nacl.box(dataDecoded, nonce, _toPublicBoxKey, boxKeyPair.secretKey);
+    const dataByteArray = (0, tx_encoding_1.txEncode)(data);
+    const dataBoxed = nacl.box(dataByteArray, nonce, _toPublicBoxKey, boxKeyPair.secretKey);
     return {
         contents: encodeBase64(dataBoxed),
         nonce: encodeBase64(nonce),
@@ -161,12 +161,11 @@ function openBoxWithSecretKey(box, mySecretKey) {
     const boxedData = decodeBase64(box.contents);
     const nonce = decodeBase64(box.nonce);
     const _fromPublicBoxKey = decodeBase64(box.fromPublicKey);
-    const dataAry = nacl.box.open(boxedData, nonce, _fromPublicBoxKey, boxKeyPair.secretKey);
-    if (dataAry === null) {
+    const dataByteArray = nacl.box.open(boxedData, nonce, _fromPublicBoxKey, boxKeyPair.secretKey);
+    if (dataByteArray === null) {
         throw new Error('Message was null or verification failed');
     }
-    const dataStr = (0, tweetnacl_util_1.encodeUTF8)(dataAry);
-    return (0, serial_json_1.fromJSONString)(dataStr);
+    return (0, tx_encoding_1.txDecode)(dataByteArray);
 }
 function encryptString(data, secretKey) {
     let _secretKey = decodeBase64(secretKey);

package/dist/logging/console-logs.table.d.ts

@@ -13,9 +13,9 @@ export declare const consoleLogSchema: z.ZodObject<{
     stackTrace: z.ZodOptional<z.ZodString>;
 }, "strip", z.ZodTypeAny, {
     message: string;
+    level: "error" | "debug" | "info" | "log" | "warn";
     timestamp: number;
     logId: string;
-    level: "error" | "debug" | "info" | "log" | "warn";
     process: string;
     processInstanceId: string;
     source?: string | undefined;
@@ -23,8 +23,8 @@ export declare const consoleLogSchema: z.ZodObject<{
     stackTrace?: string | undefined;
 }, {
     message: string;
-    logId: string;
     level: "error" | "debug" | "info" | "log" | "warn";
+    logId: string;
     process: string;
     processInstanceId: string;
     source?: string | undefined;

package/dist/types/peer-device.d.ts

@@ -9,13 +9,13 @@ export interface IPeerDevice {
     notifyOfChanges(deviceId: string, timestampLastApplied: number): Promise<void>;
     sendDeviceMessage(message: IDeviceMessage): Promise<any>;
 }
-export interface IDeviceMessage {
+export interface IDeviceMessage<T = any> {
     deviceMessageId: string;
     fromDeviceId: string;
     toDeviceId: string;
     dataContextId: string;
     ttl: number;
-    payload: any;
+    payload: T;
     hops: string[];
     suggestedPath?: string[];
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peers-app/peers-sdk",
-  "version": "0.7.18",
+  "version": "0.7.20",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/peers-app/peers-sdk.git"
@@ -30,8 +30,10 @@
     "release": "yarn release:patch"
   },
   "dependencies": {
+    "@msgpack/msgpack": "^3.1.2",
     "@noble/hashes": "^1.8.0",
     "fast-json-stable-stringify": "^2.1.0",
+    "fflate": "^0.8.2",
     "lodash": "^4.17.21",
     "moment": "^2.30.1",
     "moment-timezone": "^0.5.46",