@sqlitecloud/drivers 1.0.193 → 1.0.245

package/lib/drivers/connection-tls.d.ts

@@ -19,6 +19,7 @@ export declare class SQLiteCloudTlsConnection extends SQLiteCloudConnection {
     private startedOn;
     private executingCommands?;
     private processCallback?;
+    private pendingChunks;
     /** Handles data received in response to an outbound command sent by processCommands */
     private processCommandsData;
     /** Completes a transaction initiated by processCommands */

package/lib/drivers/connection-tls.js

@@ -45,6 +45,7 @@ class SQLiteCloudTlsConnection extends connection_1.SQLiteCloudConnection {
         // buffer to accumulate incoming data until an whole command is received and can be parsed
         this.buffer = Buffer.alloc(0);
         this.startedOn = new Date();
+        this.pendingChunks = [];
     }
     /** True if connection is open */
     get connected() {
@@ -137,10 +138,11 @@ class SQLiteCloudTlsConnection extends connection_1.SQLiteCloudConnection {
     }
     /** Handles data received in response to an outbound command sent by processCommands */
     processCommandsData(data) {
-        var _a, _b, _c, _d, _e, _f;
+        var _a, _b, _c, _d, _e, _f, _g;
         try {
             // append data to buffer as it arrives
             if (data.length && data.length > 0) {
+                // console.debug(`processCommandsData - received ${data.length} bytes`)
                 this.buffer = Buffer.concat([this.buffer, data]);
             }
             let dataType = (_a = this.buffer) === null || _a === void 0 ? void 0 : _a.subarray(0, 1).toString();
@@ -154,22 +156,33 @@ class SQLiteCloudTlsConnection extends connection_1.SQLiteCloudConnection {
                             bufferString = bufferString.substring(0, 100) + '...' + bufferString.substring(bufferString.length - 40);
                         }
                         const elapsedMs = new Date().getTime() - this.startedOn.getTime();
-                        console.debug(`<- ${bufferString} (${elapsedMs}ms)`);
+                        console.debug(`<- ${bufferString} (${bufferString.length} bytes, ${elapsedMs}ms)`);
                     }
                     // need to decompress this buffer before decoding?
                     if (dataType === protocol_1.CMD_COMPRESSED) {
-                        ;
-                        ({ buffer: this.buffer, dataType } = (0, protocol_1.decompressBuffer)(this.buffer));
-                    }
-                    if (dataType !== protocol_1.CMD_ROWSET_CHUNK) {
-                        const { data } = (0, protocol_1.popData)(this.buffer);
-                        (_c = this.processCommandsFinish) === null || _c === void 0 ? void 0 : _c.call(this, null, data);
+                        const decompressResults = (0, protocol_1.decompressBuffer)(this.buffer);
+                        if (decompressResults.dataType === protocol_1.CMD_ROWSET_CHUNK) {
+                            this.pendingChunks.push(decompressResults.buffer);
+                            this.buffer = decompressResults.remainingBuffer;
+                            this.processCommandsData(Buffer.alloc(0));
+                            return;
+                        }
+                        else {
+                            const { data } = (0, protocol_1.popData)(decompressResults.buffer);
+                            (_c = this.processCommandsFinish) === null || _c === void 0 ? void 0 : _c.call(this, null, data);
+                        }
                     }
                     else {
-                        // check if rowset received the ending chunk in which case it can be unpacked
-                        if ((0, protocol_1.bufferEndsWith)(this.buffer, protocol_1.ROWSET_CHUNKS_END)) {
-                            const parsedData = (0, protocol_1.parseRowsetChunks)([this.buffer]);
-                            (_d = this.processCommandsFinish) === null || _d === void 0 ? void 0 : _d.call(this, null, parsedData);
+                        if (dataType !== protocol_1.CMD_ROWSET_CHUNK) {
+                            const { data } = (0, protocol_1.popData)(this.buffer);
+                            (_d = this.processCommandsFinish) === null || _d === void 0 ? void 0 : _d.call(this, null, data);
+                        }
+                        else {
+                            const completeChunk = (0, protocol_1.bufferEndsWith)(this.buffer, protocol_1.ROWSET_CHUNKS_END);
+                            if (completeChunk) {
+                                const parsedData = (0, protocol_1.parseRowsetChunks)([...this.pendingChunks, this.buffer]);
+                                (_e = this.processCommandsFinish) === null || _e === void 0 ? void 0 : _e.call(this, null, parsedData);
+                            }
                         }
                     }
                 }
@@ -179,14 +192,15 @@ class SQLiteCloudTlsConnection extends connection_1.SQLiteCloudConnection {
                 const lastChar = this.buffer.subarray(this.buffer.length - 1, this.buffer.length).toString('utf8');
                 if (lastChar == ' ') {
                     const { data } = (0, protocol_1.popData)(this.buffer);
-                    (_e = this.processCommandsFinish) === null || _e === void 0 ? void 0 : _e.call(this, null, data);
+                    (_f = this.processCommandsFinish) === null || _f === void 0 ? void 0 : _f.call(this, null, data);
                 }
             }
         }
         catch (error) {
-            console.assert(error instanceof Error);
+            console.error(`processCommandsData - error: ${error}`);
+            console.assert(error instanceof Error, 'An error occoured while processing data');
             if (error instanceof Error) {
-                (_f = this.processCommandsFinish) === null || _f === void 0 ? void 0 : _f.call(this, error);
+                (_g = this.processCommandsFinish) === null || _g === void 0 ? void 0 : _g.call(this, error);
             }
         }
     }
@@ -204,6 +218,7 @@ class SQLiteCloudTlsConnection extends connection_1.SQLiteCloudConnection {
             this.processCallback(error, result);
         }
         this.buffer = Buffer.alloc(0);
+        this.pendingChunks = [];
     }
     /** Disconnect immediately, release connection, no events. */
     close() {

package/lib/drivers/protocol.d.ts

@@ -24,6 +24,7 @@ export declare function parseCommandLength(data: Buffer): number;
 export declare function decompressBuffer(buffer: Buffer): {
     buffer: Buffer;
     dataType: string;
+    remainingBuffer: Buffer;
 };
 /** Parse error message or extended error message */
 export declare function parseError(buffer: Buffer, spaceIndex: number): never;

package/lib/drivers/protocol.js

@@ -6,6 +6,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.formatCommand = exports.popData = exports.parseRowsetChunks = exports.bufferEndsWith = exports.bufferStartsWith = exports.parseRowsetHeader = exports.parseArray = exports.parseError = exports.decompressBuffer = exports.parseCommandLength = exports.hasCommandLength = exports.ROWSET_CHUNKS_END = exports.CMD_PUBSUB = exports.CMD_ARRAY = exports.CMD_COMMAND = exports.CMD_COMPRESSED = exports.CMD_BLOB = exports.CMD_NULL = exports.CMD_JSON = exports.CMD_ROWSET_CHUNK = exports.CMD_ROWSET = exports.CMD_FLOAT = exports.CMD_INT = exports.CMD_ERROR = exports.CMD_ZEROSTRING = exports.CMD_STRING = void 0;
 const types_1 = require("./types");
 const rowset_1 = require("./rowset");
+// https://www.npmjs.com/package/lz4js
 const lz4 = require('lz4js');
 // The server communicates with clients via commands defined in
 // SQLiteCloud Server Protocol (SCSP), see more at:
@@ -44,26 +45,31 @@ function parseCommandLength(data) {
 exports.parseCommandLength = parseCommandLength;
 /** Receive a compressed buffer, decompress with lz4, return buffer and datatype */
 function decompressBuffer(buffer) {
+    // https://github.com/sqlitecloud/sdk/blob/master/PROTOCOL.md#scsp-compression
+    // jest test/database.test.ts -t "select large result set"
+    // starts with %<commandLength> <compressed> <uncompressed>
     const spaceIndex = buffer.indexOf(' ');
-    buffer = buffer.subarray(spaceIndex + 1);
-    // extract compressed size
-    const compressedSize = parseInt(buffer.subarray(0, buffer.indexOf(' ') + 1).toString('utf8'));
-    buffer = buffer.subarray(buffer.indexOf(' ') + 1);
-    // extract decompressed size
-    const decompressedSize = parseInt(buffer.subarray(0, buffer.indexOf(' ') + 1).toString('utf8'));
-    buffer = buffer.subarray(buffer.indexOf(' ') + 1);
+    const commandLength = parseInt(buffer.subarray(1, spaceIndex).toString('utf8'));
+    let commandBuffer = buffer.subarray(spaceIndex + 1, spaceIndex + 1 + commandLength);
+    const remainingBuffer = buffer.subarray(spaceIndex + 1 + commandLength);
+    // extract compressed + decompressed  size
+    const compressedSize = parseInt(commandBuffer.subarray(0, commandBuffer.indexOf(' ') + 1).toString('utf8'));
+    commandBuffer = commandBuffer.subarray(commandBuffer.indexOf(' ') + 1);
+    const decompressedSize = parseInt(commandBuffer.subarray(0, commandBuffer.indexOf(' ') + 1).toString('utf8'));
+    commandBuffer = commandBuffer.subarray(commandBuffer.indexOf(' ') + 1);
     // extract compressed dataType
-    const dataType = buffer.subarray(0, 1).toString('utf8');
-    const decompressedBuffer = Buffer.alloc(decompressedSize);
-    const compressedBuffer = buffer.subarray(buffer.length - compressedSize);
+    const dataType = commandBuffer.subarray(0, 1).toString('utf8');
+    let decompressedBuffer = Buffer.alloc(decompressedSize);
+    const compressedBuffer = commandBuffer.subarray(commandBuffer.length - compressedSize);
     // lz4js library is javascript and doesn't have types so we silence the type check
     // eslint-disable-next-line
     const decompressionResult = lz4.decompressBlock(compressedBuffer, decompressedBuffer, 0, compressedSize, 0);
-    buffer = Buffer.concat([buffer.subarray(0, buffer.length - compressedSize), decompressedBuffer]);
+    // the entire command is composed of the header (which is not compressed) + the decompressed block
+    decompressedBuffer = Buffer.concat([commandBuffer.subarray(0, commandBuffer.length - compressedSize), decompressedBuffer]);
     if (decompressionResult <= 0 || decompressionResult !== decompressedSize) {
         throw new Error(`lz4 decompression error at offset ${decompressionResult}`);
     }
-    return { buffer, dataType };
+    return { buffer: decompressedBuffer, dataType, remainingBuffer };
 }
 exports.decompressBuffer = decompressBuffer;
 /** Parse error message or extended error message */
@@ -117,7 +123,7 @@ function parseRowsetHeader(buffer) {
     buffer = buffer.subarray(buffer.indexOf(':') + 1);
     // extract rowset header
     const { data, fwdBuffer } = popIntegers(buffer, 3);
-    return {
+    const result = {
         index,
         metadata: {
             version: data[0],
@@ -127,6 +133,8 @@ function parseRowsetHeader(buffer) {
         },
         fwdBuffer
     };
+    // console.debug(`parseRowsetHeader`, result)
+    return result;
 }
 exports.parseRowsetHeader = parseRowsetHeader;
 /** Extract column names and, optionally, more metadata out of a rowset's header */
@@ -195,7 +203,8 @@ function parseRowsetChunks(buffers) {
     const data = [];
     // validate and skip data type
     const dataType = buffer.subarray(0, 1).toString();
-    console.assert(dataType === exports.CMD_ROWSET_CHUNK);
+    if (dataType !== exports.CMD_ROWSET_CHUNK)
+        throw new Error(`parseRowsetChunks - dataType: ${dataType} should be CMD_ROWSET_CHUNK`);
     buffer = buffer.subarray(buffer.indexOf(' ') + 1);
     while (buffer.length > 0 && !bufferStartsWith(buffer, exports.ROWSET_CHUNKS_END)) {
         // chunk header, eg: 0:VERS NROWS NCOLS
@@ -240,21 +249,24 @@ function popData(buffer) {
     }
     // first character is the data type
     console.assert(buffer && buffer instanceof Buffer);
-    const dataType = buffer.subarray(0, 1).toString('utf8');
-    console.assert(dataType !== exports.CMD_COMPRESSED, "Compressed data shouldn't be decompressed before parsing");
-    console.assert(dataType !== exports.CMD_ROWSET_CHUNK, 'Chunked data should be parsed by parseRowsetChunks');
+    let dataType = buffer.subarray(0, 1).toString('utf8');
+    if (dataType == exports.CMD_COMPRESSED)
+        throw new Error('Compressed data should be decompressed before parsing');
+    if (dataType == exports.CMD_ROWSET_CHUNK)
+        throw new Error('Chunked data should be parsed by parseRowsetChunks');
     let spaceIndex = buffer.indexOf(' ');
     if (spaceIndex === -1) {
         spaceIndex = buffer.length - 1;
     }
-    let commandEnd = -1;
+    let commandEnd = -1, commandLength = -1;
     if (dataType === exports.CMD_INT || dataType === exports.CMD_FLOAT || dataType === exports.CMD_NULL) {
         commandEnd = spaceIndex + 1;
     }
     else {
-        const commandLength = parseInt(buffer.subarray(1, spaceIndex).toString());
+        commandLength = parseInt(buffer.subarray(1, spaceIndex).toString());
         commandEnd = spaceIndex + 1 + commandLength;
     }
+    // console.debug(`popData - dataType: ${dataType}, spaceIndex: ${spaceIndex}, commandLength: ${commandLength}, commandEnd: ${commandEnd}`)
     switch (dataType) {
         case exports.CMD_INT:
             return popResults(parseInt(buffer.subarray(1, spaceIndex).toString()));
@@ -282,7 +294,9 @@ function popData(buffer) {
             parseError(buffer, spaceIndex); // throws custom error
             break;
     }
-    throw new TypeError(`Data type: ${dataType} is not defined in SCSP`);
+    const msg = `popData - Data type: ${Number(dataType)} '${dataType}'  is not defined in SCSP, spaceIndex: ${spaceIndex}, commandLength: ${commandLength}, commandEnd: ${commandEnd}`;
+    console.error(msg);
+    throw new TypeError(msg);
 }
 exports.popData = popData;
 /** Format a command to be sent via SCSP protocol */

package/lib/drivers/types.d.ts

@@ -7,7 +7,7 @@ import tls from 'tls';
 /** Default timeout value for queries */
 export declare const DEFAULT_TIMEOUT: number;
 /** Default tls connection port */
-export declare const DEFAULT_PORT = 9960;
+export declare const DEFAULT_PORT = 8860;
 /**
  * Configuration for SQLite cloud connection
  * @note Options are all lowecase so they 1:1 compatible with C SDK

package/lib/drivers/types.js

@@ -7,7 +7,7 @@ exports.SQLiteCloudArrayType = exports.SQLiteCloudError = exports.DEFAULT_PORT =
 /** Default timeout value for queries */
 exports.DEFAULT_TIMEOUT = 300 * 1000;
 /** Default tls connection port */
-exports.DEFAULT_PORT = 9960;
+exports.DEFAULT_PORT = 8860;
 /** Custom error reported by SQLiteCloud drivers */
 class SQLiteCloudError extends Error {
     constructor(message, args) {

package/lib/drivers/utilities.js

@@ -6,6 +6,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.parseBooleanToZeroOne = exports.parseBoolean = exports.parseconnectionstring = exports.validateConfiguration = exports.popCallback = exports.getUpdateResults = exports.prepareSql = exports.escapeSqlParameter = exports.getInitializationCommands = exports.anonimizeError = exports.anonimizeCommand = exports.isNode = exports.isBrowser = void 0;
 const types_1 = require("./types");
 const types_2 = require("./types");
+const whatwg_url_1 = require("whatwg-url");
 //
 // determining running environment, thanks to browser-or-node
 // https://www.npmjs.com/package/browser-or-node
@@ -200,7 +201,7 @@ function validateConfiguration(config) {
     config.clientid || (config.clientid = 'SQLiteCloud');
     config.verbose = parseBoolean(config.verbose);
     config.noblob = parseBoolean(config.noblob);
-    config.compression = parseBoolean(config.compression);
+    config.compression = config.compression != undefined && config.compression != null ? parseBoolean(config.compression) : true; // default: true
     config.create = parseBoolean(config.create);
     config.non_linearizable = parseBoolean(config.non_linearizable);
     config.insecure = parseBoolean(config.insecure);
@@ -235,7 +236,7 @@ function parseconnectionstring(connectionstring) {
         // the sqlitecloud: protocol is not recognized by the URL constructor in browsers
         // so we need to replace it with https: to make it work
         const knownProtocolUrl = connectionstring.replace('sqlitecloud:', 'https:');
-        const url = new URL(knownProtocolUrl);
+        const url = new whatwg_url_1.URL(knownProtocolUrl);
         // all lowecase options
         const options = {};
         url.searchParams.forEach((value, key) => {