@peers-app/peers-sdk 0.7.17 → 0.7.19

package/dist/device/connection.test.js

@@ -230,16 +230,17 @@ describe(connection_1.Connection, () => {
         const result = await clientConnection.doHandshake(serverAddress).catch((err) => String(err));
         expect(result).toMatch(/Inconsistent public keys/);
     });
-    it("should handle RPC calls with large arguments through chunking", async () => {
+    it("should handle RPC calls with large arguments and responses through chunking", async () => {
         const { clientSocket, serverSocket } = createTestSocketPair();
         const clientDevice = new device_1.Device();
         const serverDevice = new device_1.Device();
         const clientConnection = new connection_1.Connection(clientSocket, clientDevice);
         const serverConnection = new connection_1.Connection(serverSocket, serverDevice, ['localhost']);
-        clientConnection.maxChunkSize = 50;
-        serverConnection.maxChunkSize = 50;
-        clientConnection.socket.maxChunkSize = 50;
-        serverConnection.socket.maxChunkSize = 50;
+        const chunkSize = 256; // 256 minimum chunk size due to encoding overheads
+        clientConnection.maxChunkSize = chunkSize;
+        serverConnection.maxChunkSize = chunkSize;
+        clientConnection.socket.maxChunkSize = chunkSize;
+        serverConnection.socket.maxChunkSize = chunkSize;
         await clientConnection.doHandshake('localhost');
         const clientEmitSpy = jest.spyOn(clientConnection.socket.socket, 'emit');
         const serverEmitSpy = jest.spyOn(serverConnection.socket.socket, 'emit');
@@ -259,15 +260,21 @@ describe(connection_1.Connection, () => {
         expect(result.received.message).toBe("A".repeat(500));
         expect(result.received.metadata).toBe("B".repeat(300));
         expect(result.received.payload).toBe("C".repeat(400));
-        expect(result.processedSize).toBeGreaterThan(1000); // Should be large
+        expect(JSON.stringify(largeData).length).toEqual(1241);
         // Count chunk emissions
         const clientChunkCalls = clientEmitSpy.mock.calls.filter((call) => call[0] === clientConnection.socket.safeSocketChunkEventName);
         const serverChunkCalls = serverEmitSpy.mock.calls.filter((call) => call[0] === serverConnection.socket.safeSocketChunkEventName);
-        // The data gets wrapped by Connection layer (encryption/signing), making it larger
-        // 41 chunks at 50 bytes each = ~2050 bytes (41 * 50)
-        // 43 chunks at 50 bytes each = ~2150 bytes (43 * 50)
-        expect(clientChunkCalls.length).toBe(41); // ~2050 bytes wrapped data / 50 bytes per chunk
-        expect(serverChunkCalls.length).toBe(43); // ~2150 bytes wrapped data / 50 bytes per chunk
+        expect(clientChunkCalls.length).toBe(4);
+        expect(serverChunkCalls.length).toBe(5);
+        // expect calls to never have chunks greater than chunk size
+        for (const call of clientChunkCalls) {
+            const chunkData = call[1];
+            expect(chunkData.length).toBeLessThanOrEqual(chunkSize);
+        }
+        for (const call of serverChunkCalls) {
+            const chunkData = call[1];
+            expect(chunkData.length).toBeLessThanOrEqual(chunkSize);
+        }
     });
     it("should reject handshake if device timestamps are too far apart", async () => {
         const { clientSocket, serverSocket } = createTestSocketPair();

package/dist/device/get-trust-level.js

@@ -9,9 +9,21 @@ function getTrustLevelFn(me, serverUrl) {
         const userContext = await (0, context_1.getUserContext)();
         const userDataContext = userContext.userDataContext;
         if (deviceInfo.userId === me.userId && deviceInfo.publicKey === me.publicKey && deviceInfo.publicBoxKey === me.publicBoxKey) {
-            if (deviceInfo.deviceId === (0, data_1.thisDeviceId)()) {
-                // return TrustLevel.Untrusted;
-            }
+            // if (deviceInfo.deviceId === thisDeviceId()) {
+            //   return TrustLevel.Untrusted;
+            // }
+            const device = await (0, data_1.Devices)(userDataContext).get(deviceInfo.deviceId) || {
+                deviceId: deviceInfo.deviceId,
+                userId: deviceInfo.userId,
+                firstSeen: new Date(),
+                lastSeen: new Date(),
+                trustLevel: socket_type_1.TrustLevel.Trusted,
+                serverUrl,
+            };
+            device.lastSeen = new Date();
+            device.trustLevel = socket_type_1.TrustLevel.Trusted;
+            console.log(`Updating my own device: ${deviceInfo.deviceId}`);
+            await (0, data_1.Devices)(userDataContext).save(device);
             return socket_type_1.TrustLevel.Trusted;
         }
         // await Devices().delete(deviceInfo.deviceId);

package/dist/device/streamed-socket.d.ts

@@ -1,11 +1,4 @@
 import { ISocket, RPCCallback } from "./socket.type";
-export interface IMessageChunk {
-    messageId: string;
-    chunkIndex: number;
-    totalChunks?: number;
-    eventName?: string;
-    data: string;
-}
 export declare class StreamedSocket implements ISocket {
     readonly socket: ISocket;
     maxChunkSize: number;

package/dist/device/streamed-socket.js

@@ -2,9 +2,50 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.StreamedSocket = void 0;
 const lodash_1 = require("lodash");
-const serial_json_1 = require("../serial-json");
 const utils_1 = require("../utils");
 const socket_type_1 = require("./socket.type");
+const tx_encoding_1 = require("./tx-encoding");
+const msgpack_1 = require("@msgpack/msgpack");
+// Encode chunk metadata + data into a single Uint8Array
+function encodeChunk(chunk) {
+    // Encode metadata using msgpack
+    const metadata = {
+        messageId: chunk.messageId,
+        chunkIndex: chunk.chunkIndex,
+        ...(chunk.totalChunks !== undefined && { totalChunks: chunk.totalChunks }),
+        ...(chunk.eventName !== undefined && { eventName: chunk.eventName }),
+    };
+    const metadataBytes = (0, msgpack_1.encode)(metadata);
+    const metadataLength = metadataBytes.length;
+    // Create combined buffer: [4 bytes length][metadata][data]
+    const combined = new Uint8Array(4 + metadataLength + chunk.data.length);
+    // Write metadata length as 32-bit unsigned integer (big-endian)
+    const view = new DataView(combined.buffer, combined.byteOffset, combined.byteLength);
+    view.setUint32(0, metadataLength, false); // big-endian
+    // Write metadata
+    combined.set(metadataBytes, 4);
+    // Write data
+    combined.set(chunk.data, 4 + metadataLength);
+    return combined;
+}
+// Decode binary chunk back into IMessageChunk
+function decodeChunk(bytes) {
+    // Read metadata length
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    const metadataLength = view.getUint32(0, false); // big-endian
+    // Extract metadata
+    const metadataBytes = bytes.subarray(4, 4 + metadataLength);
+    const metadata = (0, msgpack_1.decode)(metadataBytes);
+    // Extract data
+    const data = bytes.subarray(4 + metadataLength);
+    return {
+        messageId: metadata.messageId,
+        chunkIndex: metadata.chunkIndex,
+        totalChunks: metadata.totalChunks,
+        eventName: metadata.eventName,
+        data: data,
+    };
+}
 class StreamedSocket {
     socket;
     maxChunkSize;
@@ -13,7 +54,10 @@ class StreamedSocket {
     constructor(socket, maxChunkSize = socket_type_1.DEFAULT_MAX_CHUNK_SIZE) {
         this.socket = socket;
         this.maxChunkSize = maxChunkSize;
-        socket.on(this.safeSocketChunkEventName, (chunk) => this.handleChunk(chunk));
+        socket.on(this.safeSocketChunkEventName, (encodedChunk) => {
+            const chunk = decodeChunk(encodedChunk);
+            this.handleChunk(chunk);
+        });
     }
     get id() {
         return this.socket.id;
@@ -26,8 +70,8 @@ class StreamedSocket {
     }
     callbacks = {};
     emit(eventName, args, callback) {
-        const strArgs = (0, serial_json_1.toJSONString)(args);
-        // if (strArgs.length < this.maxChunkSize) {
+        const encoded = (0, tx_encoding_1.txEncode)(args);
+        // if (encoded.length < this.maxChunkSize) {
         //   // If the data is small enough, send it directly
         //   this.socket.emit(eventName, args, callback);
         //   return;
@@ -37,25 +81,34 @@ class StreamedSocket {
             this.callbacks[messageId] = callback;
         }
         const chunks = [];
-        for (let i = 0; i < strArgs.length; i += this.maxChunkSize) {
-            const chunkData = strArgs.slice(i, i + this.maxChunkSize);
+        // Reserve space for metadata overhead (typically ~50-100 bytes)
+        const metadataOverhead = 128;
+        if (this.maxChunkSize < (metadataOverhead * 2)) {
+            throw new Error(`maxChunkSize ${this.maxChunkSize} is too small to accommodate metadata overhead, must be greater than ${metadataOverhead * 2}`);
+        }
+        const effectiveChunkSize = this.maxChunkSize - metadataOverhead;
+        for (let i = 0; i < encoded.length; i += effectiveChunkSize) {
+            const chunkData = encoded.subarray(i, i + effectiveChunkSize);
             chunks.push({
                 messageId,
-                chunkIndex: Math.floor(i / this.maxChunkSize),
-                data: chunkData
+                chunkIndex: Math.floor(i / effectiveChunkSize),
+                data: chunkData,
             });
             if (i === 0) {
                 // The first chunk contains metadata about the chunks
                 chunks[0].eventName = eventName;
-                chunks[0].totalChunks = Math.ceil(strArgs.length / this.maxChunkSize);
+                chunks[0].totalChunks = Math.ceil(encoded.length / effectiveChunkSize);
             }
         }
         if (chunks.length > 1) {
             console.debug(`Sending ${chunks.length} chunks for event ${eventName} with messageId ${messageId}`);
         }
         for (const chunk of chunks) {
+            // Encode chunk with metadata into single Uint8Array
+            const encodedChunk = encodeChunk(chunk);
             // TODO on error or timeout retry
-            this.socket.emit(this.safeSocketChunkEventName, chunk, lodash_1.noop);
+            // TODO respond to backpressure from socket
+            this.socket.emit(this.safeSocketChunkEventName, encodedChunk, lodash_1.noop);
         }
     }
     handlers = {};
@@ -98,10 +151,17 @@ class StreamedSocket {
     async processChunks(chunks) {
         try {
             const chunkZero = chunks[0];
-            const strData = chunks.map(c => c.data).join('');
+            // Reassemble Uint8Array chunks (already decoded from binary format)
+            const totalSize = chunks.reduce((sum, chunk) => sum + chunk.data.length, 0);
+            const reassembled = new Uint8Array(totalSize);
+            let offset = 0;
+            for (const chunk of chunks) {
+                reassembled.set(chunk.data, offset);
+                offset += chunk.data.length;
+            }
             delete this.chunkBuffers[chunkZero.messageId];
             const eventName = chunkZero.eventName;
-            const args = (0, serial_json_1.fromJSONString)(strData);
+            const args = (0, tx_encoding_1.txDecode)(reassembled);
             if (eventName === this.safeSocketResponseEventName) {
                 this.handleResponse(args.messageId, args.error, args.result);
                 return;

package/dist/device/streamed-socket.test.js

@@ -2,6 +2,17 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 const streamed_socket_1 = require("./streamed-socket");
 const utils_1 = require("../utils");
+const tx_encoding_1 = require("./tx-encoding");
+const msgpack_1 = require("@msgpack/msgpack");
+// Helper to decode the binary chunk format
+function decodeChunk(bytes) {
+    const view = new DataView(bytes.buffer, bytes.byteOffset);
+    const metadataLength = view.getUint32(0, false);
+    const metadataBytes = bytes.subarray(4, 4 + metadataLength);
+    const metadata = (0, msgpack_1.decode)(metadataBytes);
+    const data = bytes.subarray(4 + metadataLength);
+    return { ...metadata, data };
+}
 function createMockSocket() {
     const emittedEvents = [];
     const socket = {
@@ -35,10 +46,171 @@ describe('StreamedSocket', () => {
         expect(emittedEvents).toHaveLength(1);
         const chunkEvent = emittedEvents[0];
         expect(chunkEvent.eventName).toBe('__streamed-socket-chunk');
-        expect(chunkEvent.args.messageId).toBeTruthy();
-        expect(chunkEvent.args.chunkIndex).toBe(0);
-        expect(chunkEvent.args.totalChunks).toBe(1);
-        expect(chunkEvent.args.eventName).toBe('test-event');
-        expect(JSON.parse(chunkEvent.args.data)).toEqual(testData);
+        // Args should be a Uint8Array (binary encoded chunk)
+        expect(chunkEvent.args).toBeInstanceOf(Uint8Array);
+        // Decode the binary chunk
+        const decodedChunk = decodeChunk(chunkEvent.args);
+        expect(decodedChunk.messageId).toBeTruthy();
+        expect(decodedChunk.chunkIndex).toBe(0);
+        expect(decodedChunk.totalChunks).toBe(1);
+        expect(decodedChunk.eventName).toBe('test-event');
+        // Decode the actual data
+        const decoded = (0, tx_encoding_1.txDecode)(decodedChunk.data);
+        expect(decoded).toEqual(testData);
+    });
+    it('should emit multiple chunks for large messages', () => {
+        const mock = createMockSocket();
+        const streamedSocket = new streamed_socket_1.StreamedSocket(mock.socket, 256); // Small chunk size to force chunking
+        // Create a large test object
+        const testData = {
+            message: 'x'.repeat(200),
+            array: [1, 2, 3, 4, 5],
+            nested: { deep: 'value' }
+        };
+        streamedSocket.emit('test-event', testData, () => { });
+        const emittedEvents = mock.getEmittedEvents();
+        expect(emittedEvents.length).toBeGreaterThan(1);
+        // Decode all chunks
+        const decodedChunks = emittedEvents.map(event => {
+            expect(event.args).toBeInstanceOf(Uint8Array);
+            return decodeChunk(event.args);
+        });
+        // Verify first chunk has metadata
+        const firstChunk = decodedChunks[0];
+        expect(firstChunk.eventName).toBe('test-event');
+        expect(firstChunk.totalChunks).toBe(emittedEvents.length);
+        expect(firstChunk.chunkIndex).toBe(0);
+        // Verify all chunks have the same messageId
+        const messageId = firstChunk.messageId;
+        decodedChunks.forEach(chunk => {
+            expect(chunk.messageId).toBe(messageId);
+        });
+        // Verify chunks are in order
+        decodedChunks.forEach((chunk, index) => {
+            expect(chunk.chunkIndex).toBe(index);
+        });
+        // Verify all chunk data are Uint8Arrays
+        decodedChunks.forEach(chunk => {
+            expect(chunk.data).toBeInstanceOf(Uint8Array);
+        });
+        // Reassemble chunks and verify data integrity
+        const reassembled = new Uint8Array(decodedChunks.reduce((sum, chunk) => sum + chunk.data.length, 0));
+        let offset = 0;
+        for (const chunk of decodedChunks) {
+            reassembled.set(chunk.data, offset);
+            offset += chunk.data.length;
+        }
+        const decoded = (0, tx_encoding_1.txDecode)(reassembled);
+        expect(decoded).toEqual(testData);
+    });
+    it('should handle binary data in messages', () => {
+        const mock = createMockSocket();
+        const streamedSocket = new streamed_socket_1.StreamedSocket(mock.socket, 1000);
+        const testData = {
+            binaryData: new Uint8Array([1, 2, 3, 4, 5]),
+            metadata: { name: 'test' }
+        };
+        streamedSocket.emit('test-event', testData, () => { });
+        const emittedEvents = mock.getEmittedEvents();
+        expect(emittedEvents).toHaveLength(1);
+        const decodedChunk = decodeChunk(emittedEvents[0].args);
+        const decoded = (0, tx_encoding_1.txDecode)(decodedChunk.data);
+        expect(decoded.binaryData).toBeInstanceOf(Uint8Array);
+        expect(decoded.binaryData).toEqual(testData.binaryData);
+        expect(decoded.metadata).toEqual(testData.metadata);
+    });
+    it('should handle special types in messages', () => {
+        const mock = createMockSocket();
+        const streamedSocket = new streamed_socket_1.StreamedSocket(mock.socket, 1000);
+        const testData = {
+            date: new Date('2023-01-01T00:00:00.000Z'),
+            nan: NaN,
+            infinity: Infinity,
+            undef: undefined,
+            nullVal: null
+        };
+        streamedSocket.emit('test-event', testData, () => { });
+        const emittedEvents = mock.getEmittedEvents();
+        expect(emittedEvents).toHaveLength(1);
+        const decodedChunk = decodeChunk(emittedEvents[0].args);
+        const decoded = (0, tx_encoding_1.txDecode)(decodedChunk.data);
+        expect(decoded.date).toBeInstanceOf(Date);
+        expect(decoded.date.toISOString()).toBe('2023-01-01T00:00:00.000Z');
+        expect(Number.isNaN(decoded.nan)).toBe(true);
+        expect(decoded.infinity).toBe(Infinity);
+        expect(decoded.undef).toBeUndefined();
+        expect(decoded.nullVal).toBeNull();
+    });
+    it('should demonstrate msgpack efficiency: structured data vs JSON string (uncompressed)', () => {
+        const mock1 = createMockSocket();
+        const mock2 = createMockSocket();
+        const streamedSocket1 = new streamed_socket_1.StreamedSocket(mock1.socket, 1000000); // Large chunk to avoid splitting
+        const streamedSocket2 = new streamed_socket_1.StreamedSocket(mock2.socket, 1000000);
+        // Create a small structured object that won't trigger compression (< 1024 bytes)
+        const smallObject = {
+            users: Array.from({ length: 5 }, (_, i) => ({
+                id: i + 1000000, // Use larger numbers to make JSON less compressible
+                name: `User${i}_${Math.random().toString(36).substring(7)}`, // Add randomness
+                email: `user${i}@example.com`,
+                score: Math.random() * 1000,
+                active: i % 2 === 0,
+            }))
+        };
+        // Convert to JSON string
+        const jsonString = JSON.stringify(smallObject);
+        console.log(`Original JSON string size: ${jsonString.length} bytes`);
+        // Send as structured object (msgpack can optimize)
+        streamedSocket1.emit('test-object', smallObject, () => { });
+        // Send as JSON string (msgpack treats it as opaque string)
+        streamedSocket2.emit('test-string', { jsonPayload: jsonString }, () => { });
+        const objectEvents = mock1.getEmittedEvents();
+        const stringEvents = mock2.getEmittedEvents();
+        expect(objectEvents).toHaveLength(1);
+        expect(stringEvents).toHaveLength(1);
+        // Decode and measure sizes
+        const objectChunk = decodeChunk(objectEvents[0].args);
+        const stringChunk = decodeChunk(stringEvents[0].args);
+        const objectSize = objectChunk.data.length;
+        const stringSize = stringChunk.data.length;
+        console.log(`Msgpack with structured data: ${objectSize} bytes (${((1 - objectSize / jsonString.length) * 100).toFixed(1)}% reduction from JSON)`);
+        console.log(`Msgpack with JSON string: ${stringSize} bytes (${((1 - stringSize / jsonString.length) * 100).toFixed(1)}% reduction from JSON)`);
+        console.log(`Efficiency gain: structured data is ${((1 - objectSize / stringSize) * 100).toFixed(1)}% smaller than JSON string`);
+        // Msgpack should be more efficient with structured data than with a JSON string
+        expect(objectSize).toBeLessThan(stringSize);
+        // Structured data should be at least 10% more efficient
+        const efficiency = (1 - objectSize / stringSize) * 100;
+        expect(efficiency).toBeGreaterThan(10);
+        // Verify data integrity
+        const decodedObject = (0, tx_encoding_1.txDecode)(objectChunk.data);
+        const decodedString = (0, tx_encoding_1.txDecode)(stringChunk.data);
+        expect(decodedObject.users).toHaveLength(5);
+        expect(decodedString.jsonPayload).toBe(jsonString);
+    });
+    it('should demonstrate compression effectiveness for large repetitive data', () => {
+        const mock = createMockSocket();
+        const streamedSocket = new streamed_socket_1.StreamedSocket(mock.socket, 1000000);
+        // Create large repetitive object that will compress well
+        const largeObject = {
+            data: Array.from({ length: 200 }, () => ({
+                type: 'event',
+                status: 'active',
+                category: 'user_action',
+                timestamp: new Date('2024-01-01'),
+                metadata: { app: 'test', version: '1.0.0' }
+            }))
+        };
+        const jsonString = JSON.stringify(largeObject);
+        console.log(`Large repetitive data - JSON size: ${jsonString.length} bytes`);
+        streamedSocket.emit('test-compressed', largeObject, () => { });
+        const events = mock.getEmittedEvents();
+        const chunk = decodeChunk(events[0].args);
+        console.log(`After msgpack + compression: ${chunk.data.length} bytes`);
+        console.log(`Compression ratio: ${((1 - chunk.data.length / jsonString.length) * 100).toFixed(1)}% reduction`);
+        // Should achieve significant compression (>80% reduction)
+        expect(chunk.data.length).toBeLessThan(jsonString.length * 0.2);
+        // Verify data integrity
+        const decoded = (0, tx_encoding_1.txDecode)(chunk.data);
+        expect(decoded.data).toHaveLength(200);
+        expect(decoded.data[0].type).toBe('event');
     });
 });

package/dist/device/tx-encoding.d.ts

@@ -0,0 +1,2 @@
+export declare function txEncode(data: unknown): Uint8Array;
+export declare function txDecode(data: Uint8Array): unknown;

package/dist/device/tx-encoding.js

@@ -0,0 +1,44 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.txEncode = txEncode;
+exports.txDecode = txDecode;
+const msgpack_1 = require("@msgpack/msgpack");
+const fflate_1 = require("fflate");
+const serial_json_1 = require("../serial-json");
+function txEncode(data) {
+    const noCycles = (0, serial_json_1.toJSON)(data); // remove cycles and encode dates, etc.
+    // First encode to msgpack
+    const encoded = (0, msgpack_1.encode)(noCycles);
+    let body;
+    let flag;
+    if (encoded.length > 1024) {
+        // Compress big payloads
+        body = (0, fflate_1.compressSync)(encoded);
+        flag = 1; // compressed
+    }
+    else {
+        body = encoded;
+        flag = 0; // not compressed
+    }
+    const out = new Uint8Array(1 + body.length);
+    out[0] = flag;
+    out.set(body, 1);
+    return out;
+}
+function txDecode(data) {
+    if (data.length === 0) {
+        throw new Error("Empty payload");
+    }
+    const flag = data[0];
+    let body = data.subarray(1);
+    if (flag === 1) {
+        body = (0, fflate_1.decompressSync)(body);
+    }
+    else if (flag !== 0) {
+        // Optional: future versioning or legacy handling
+        throw new Error(`Unknown tx flag: ${flag}`);
+    }
+    const unpacked = (0, msgpack_1.decode)(body);
+    const restored = (0, serial_json_1.fromJSON)(unpacked); // restore cycles, dates, etc.
+    return restored;
+}

package/dist/device/tx-encoding.test.d.ts

@@ -0,0 +1 @@
+export {};