@sqlitecloud/drivers 1.0.193 → 1.0.255

package/README.md

@@ -12,6 +12,38 @@
 npm install @sqlitecloud/drivers
 ```
 
+## React Native / Expo Install
+
+You also have to install Peer Dependencies
+
+```bash
+npm install @sqlitecloud/drivers react-native-tcp-socket react-native-fast-base64
+```
+
+React Native run IOS
+
+```bash
+cd ios && pod install && cd .. && npm run ios
+```
+
+React Native run Android (without ./ in Windows)
+
+```bash
+cd android && ./gradlew clean build && cd .. && npm run android
+```
+
+Expo run IOS
+
+```bash
+npx expo prebuild && npx expo run:ios
+```
+
+Expo run Android
+
+```bash
+npx expo prebuild && npx expo run:android
+```
+
 ## Usage
 
 ```ts
@@ -60,6 +92,10 @@ Pub/Sub is a messaging pattern that allows multiple applications to communicate
 
 Pub/Sub Documentation: [https://docs.sqlitecloud.io/docs/pub-sub](https://docs.sqlitecloud.io/docs/pub-sub)
 
+## Examples
+
+Check out all the supported platforms with related examples [here](https://github.com/sqlitecloud/sqlitecloud-js/tree/main/examples)!
+
 ## More
 
 How do I deploy SQLite in the cloud?  

package/lib/drivers/connection-tls.d.ts

@@ -19,6 +19,7 @@ export declare class SQLiteCloudTlsConnection extends SQLiteCloudConnection {
     private startedOn;
     private executingCommands?;
     private processCallback?;
+    private pendingChunks;
     /** Handles data received in response to an outbound command sent by processCommands */
     private processCommandsData;
     /** Completes a transaction initiated by processCommands */

package/lib/drivers/connection-tls.js

@@ -31,6 +31,8 @@ const types_1 = require("./types");
 const connection_1 = require("./connection");
 const utilities_1 = require("./utilities");
 const protocol_1 = require("./protocol");
+// explicitly importing buffer library to allow cross-platform support by replacing it
+const buffer_1 = require("buffer");
 const tls = __importStar(require("tls"));
 /**
  * Implementation of SQLiteCloudConnection that connects to the database using specific tls APIs
@@ -43,8 +45,9 @@ class SQLiteCloudTlsConnection extends connection_1.SQLiteCloudConnection {
         // onData is called when data is received, it will process the data until all data is retrieved for a response
         // when response is complete or there's an error, finish is called to call the results callback set by processCommands...
         // buffer to accumulate incoming data until an whole command is received and can be parsed
-        this.buffer = Buffer.alloc(0);
+        this.buffer = buffer_1.Buffer.alloc(0);
         this.startedOn = new Date();
+        this.pendingChunks = [];
     }
     /** True if connection is open */
     get connected() {
@@ -70,7 +73,14 @@ class SQLiteCloudTlsConnection extends connection_1.SQLiteCloudConnection {
             // https://r2.nodejs.org/docs/v6.11.4/api/tls.html#tls_class_tls_tlssocket
             servername: config.host
         };
-        this.socket = tls.connect(connectionOptions, () => {
+        // tls.connect in the react-native-tcp-socket library is tls.connectTLS
+        let connector = tls.connect;
+        // @ts-ignore
+        if (typeof tls.connectTLS !== 'undefined') {
+            // @ts-ignore
+            connector = tls.connectTLS;
+        }
+        this.socket = connector(connectionOptions, () => {
             var _a;
             if (this.config.verbose) {
                 console.debug(`SQLiteCloudTlsConnection - connected to ${this.config.host}, authorized: ${(_a = this.socket) === null || _a === void 0 ? void 0 : _a.authorized}`);
@@ -109,7 +119,7 @@ class SQLiteCloudTlsConnection extends connection_1.SQLiteCloudConnection {
             return this;
         }
         // reset buffer and rowset chunks, define response callback
-        this.buffer = Buffer.alloc(0);
+        this.buffer = buffer_1.Buffer.alloc(0);
         this.startedOn = new Date();
         this.processCallback = callback;
         this.executingCommands = commands;
@@ -137,11 +147,12 @@ class SQLiteCloudTlsConnection extends connection_1.SQLiteCloudConnection {
     }
     /** Handles data received in response to an outbound command sent by processCommands */
     processCommandsData(data) {
-        var _a, _b, _c, _d, _e, _f;
+        var _a, _b, _c, _d, _e, _f, _g;
         try {
             // append data to buffer as it arrives
             if (data.length && data.length > 0) {
-                this.buffer = Buffer.concat([this.buffer, data]);
+                // console.debug(`processCommandsData - received ${data.length} bytes`)
+                this.buffer = buffer_1.Buffer.concat([this.buffer, data]);
             }
             let dataType = (_a = this.buffer) === null || _a === void 0 ? void 0 : _a.subarray(0, 1).toString();
             if ((0, protocol_1.hasCommandLength)(dataType)) {
@@ -154,22 +165,33 @@ class SQLiteCloudTlsConnection extends connection_1.SQLiteCloudConnection {
                             bufferString = bufferString.substring(0, 100) + '...' + bufferString.substring(bufferString.length - 40);
                         }
                         const elapsedMs = new Date().getTime() - this.startedOn.getTime();
-                        console.debug(`<- ${bufferString} (${elapsedMs}ms)`);
+                        console.debug(`<- ${bufferString} (${bufferString.length} bytes, ${elapsedMs}ms)`);
                     }
                     // need to decompress this buffer before decoding?
                     if (dataType === protocol_1.CMD_COMPRESSED) {
-                        ;
-                        ({ buffer: this.buffer, dataType } = (0, protocol_1.decompressBuffer)(this.buffer));
-                    }
-                    if (dataType !== protocol_1.CMD_ROWSET_CHUNK) {
-                        const { data } = (0, protocol_1.popData)(this.buffer);
-                        (_c = this.processCommandsFinish) === null || _c === void 0 ? void 0 : _c.call(this, null, data);
+                        const decompressResults = (0, protocol_1.decompressBuffer)(this.buffer);
+                        if (decompressResults.dataType === protocol_1.CMD_ROWSET_CHUNK) {
+                            this.pendingChunks.push(decompressResults.buffer);
+                            this.buffer = decompressResults.remainingBuffer;
+                            this.processCommandsData(buffer_1.Buffer.alloc(0));
+                            return;
+                        }
+                        else {
+                            const { data } = (0, protocol_1.popData)(decompressResults.buffer);
+                            (_c = this.processCommandsFinish) === null || _c === void 0 ? void 0 : _c.call(this, null, data);
+                        }
                     }
                     else {
-                        // check if rowset received the ending chunk in which case it can be unpacked
-                        if ((0, protocol_1.bufferEndsWith)(this.buffer, protocol_1.ROWSET_CHUNKS_END)) {
-                            const parsedData = (0, protocol_1.parseRowsetChunks)([this.buffer]);
-                            (_d = this.processCommandsFinish) === null || _d === void 0 ? void 0 : _d.call(this, null, parsedData);
+                        if (dataType !== protocol_1.CMD_ROWSET_CHUNK) {
+                            const { data } = (0, protocol_1.popData)(this.buffer);
+                            (_d = this.processCommandsFinish) === null || _d === void 0 ? void 0 : _d.call(this, null, data);
+                        }
+                        else {
+                            const completeChunk = (0, protocol_1.bufferEndsWith)(this.buffer, protocol_1.ROWSET_CHUNKS_END);
+                            if (completeChunk) {
+                                const parsedData = (0, protocol_1.parseRowsetChunks)([...this.pendingChunks, this.buffer]);
+                                (_e = this.processCommandsFinish) === null || _e === void 0 ? void 0 : _e.call(this, null, parsedData);
+                            }
                         }
                     }
                 }
@@ -179,14 +201,15 @@ class SQLiteCloudTlsConnection extends connection_1.SQLiteCloudConnection {
                 const lastChar = this.buffer.subarray(this.buffer.length - 1, this.buffer.length).toString('utf8');
                 if (lastChar == ' ') {
                     const { data } = (0, protocol_1.popData)(this.buffer);
-                    (_e = this.processCommandsFinish) === null || _e === void 0 ? void 0 : _e.call(this, null, data);
+                    (_f = this.processCommandsFinish) === null || _f === void 0 ? void 0 : _f.call(this, null, data);
                 }
             }
         }
         catch (error) {
-            console.assert(error instanceof Error);
+            console.error(`processCommandsData - error: ${error}`);
+            console.assert(error instanceof Error, 'An error occoured while processing data');
             if (error instanceof Error) {
-                (_f = this.processCommandsFinish) === null || _f === void 0 ? void 0 : _f.call(this, error);
+                (_g = this.processCommandsFinish) === null || _g === void 0 ? void 0 : _g.call(this, error);
             }
         }
     }
@@ -203,7 +226,8 @@ class SQLiteCloudTlsConnection extends connection_1.SQLiteCloudConnection {
         if (this.processCallback) {
             this.processCallback(error, result);
         }
-        this.buffer = Buffer.alloc(0);
+        this.buffer = buffer_1.Buffer.alloc(0);
+        this.pendingChunks = [];
     }
     /** Disconnect immediately, release connection, no events. */
     close() {

package/lib/drivers/protocol.d.ts

@@ -24,6 +24,7 @@ export declare function parseCommandLength(data: Buffer): number;
 export declare function decompressBuffer(buffer: Buffer): {
     buffer: Buffer;
     dataType: string;
+    remainingBuffer: Buffer;
 };
 /** Parse error message or extended error message */
 export declare function parseError(buffer: Buffer, spaceIndex: number): never;

package/lib/drivers/protocol.js

@@ -6,6 +6,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.formatCommand = exports.popData = exports.parseRowsetChunks = exports.bufferEndsWith = exports.bufferStartsWith = exports.parseRowsetHeader = exports.parseArray = exports.parseError = exports.decompressBuffer = exports.parseCommandLength = exports.hasCommandLength = exports.ROWSET_CHUNKS_END = exports.CMD_PUBSUB = exports.CMD_ARRAY = exports.CMD_COMMAND = exports.CMD_COMPRESSED = exports.CMD_BLOB = exports.CMD_NULL = exports.CMD_JSON = exports.CMD_ROWSET_CHUNK = exports.CMD_ROWSET = exports.CMD_FLOAT = exports.CMD_INT = exports.CMD_ERROR = exports.CMD_ZEROSTRING = exports.CMD_STRING = void 0;
 const types_1 = require("./types");
 const rowset_1 = require("./rowset");
+// explicitly importing buffer library to allow cross-platform support by replacing it
+const buffer_1 = require("buffer");
+// https://www.npmjs.com/package/lz4js
 const lz4 = require('lz4js');
 // The server communicates with clients via commands defined in
 // SQLiteCloud Server Protocol (SCSP), see more at:
@@ -44,26 +47,31 @@ function parseCommandLength(data) {
 exports.parseCommandLength = parseCommandLength;
 /** Receive a compressed buffer, decompress with lz4, return buffer and datatype */
 function decompressBuffer(buffer) {
+    // https://github.com/sqlitecloud/sdk/blob/master/PROTOCOL.md#scsp-compression
+    // jest test/database.test.ts -t "select large result set"
+    // starts with %<commandLength> <compressed> <uncompressed>
     const spaceIndex = buffer.indexOf(' ');
-    buffer = buffer.subarray(spaceIndex + 1);
-    // extract compressed size
-    const compressedSize = parseInt(buffer.subarray(0, buffer.indexOf(' ') + 1).toString('utf8'));
-    buffer = buffer.subarray(buffer.indexOf(' ') + 1);
-    // extract decompressed size
-    const decompressedSize = parseInt(buffer.subarray(0, buffer.indexOf(' ') + 1).toString('utf8'));
-    buffer = buffer.subarray(buffer.indexOf(' ') + 1);
+    const commandLength = parseInt(buffer.subarray(1, spaceIndex).toString('utf8'));
+    let commandBuffer = buffer.subarray(spaceIndex + 1, spaceIndex + 1 + commandLength);
+    const remainingBuffer = buffer.subarray(spaceIndex + 1 + commandLength);
+    // extract compressed + decompressed  size
+    const compressedSize = parseInt(commandBuffer.subarray(0, commandBuffer.indexOf(' ') + 1).toString('utf8'));
+    commandBuffer = commandBuffer.subarray(commandBuffer.indexOf(' ') + 1);
+    const decompressedSize = parseInt(commandBuffer.subarray(0, commandBuffer.indexOf(' ') + 1).toString('utf8'));
+    commandBuffer = commandBuffer.subarray(commandBuffer.indexOf(' ') + 1);
     // extract compressed dataType
-    const dataType = buffer.subarray(0, 1).toString('utf8');
-    const decompressedBuffer = Buffer.alloc(decompressedSize);
-    const compressedBuffer = buffer.subarray(buffer.length - compressedSize);
+    const dataType = commandBuffer.subarray(0, 1).toString('utf8');
+    let decompressedBuffer = buffer_1.Buffer.alloc(decompressedSize);
+    const compressedBuffer = commandBuffer.subarray(commandBuffer.length - compressedSize);
     // lz4js library is javascript and doesn't have types so we silence the type check
     // eslint-disable-next-line
     const decompressionResult = lz4.decompressBlock(compressedBuffer, decompressedBuffer, 0, compressedSize, 0);
-    buffer = Buffer.concat([buffer.subarray(0, buffer.length - compressedSize), decompressedBuffer]);
+    // the entire command is composed of the header (which is not compressed) + the decompressed block
+    decompressedBuffer = buffer_1.Buffer.concat([commandBuffer.subarray(0, commandBuffer.length - compressedSize), decompressedBuffer]);
     if (decompressionResult <= 0 || decompressionResult !== decompressedSize) {
         throw new Error(`lz4 decompression error at offset ${decompressionResult}`);
     }
-    return { buffer, dataType };
+    return { buffer: decompressedBuffer, dataType, remainingBuffer };
 }
 exports.decompressBuffer = decompressBuffer;
 /** Parse error message or extended error message */
@@ -117,7 +125,7 @@ function parseRowsetHeader(buffer) {
     buffer = buffer.subarray(buffer.indexOf(':') + 1);
     // extract rowset header
     const { data, fwdBuffer } = popIntegers(buffer, 3);
-    return {
+    const result = {
         index,
         metadata: {
             version: data[0],
@@ -127,6 +135,8 @@ function parseRowsetHeader(buffer) {
         },
         fwdBuffer
     };
+    // console.debug(`parseRowsetHeader`, result)
+    return result;
 }
 exports.parseRowsetHeader = parseRowsetHeader;
 /** Extract column names and, optionally, more metadata out of a rowset's header */
@@ -187,7 +197,7 @@ exports.bufferEndsWith = bufferEndsWith;
  * @see https://github.com/sqlitecloud/sdk/blob/master/PROTOCOL.md#scsp-rowset-chunk
  */
 function parseRowsetChunks(buffers) {
-    let buffer = Buffer.concat(buffers);
+    let buffer = buffer_1.Buffer.concat(buffers);
     if (!bufferStartsWith(buffer, exports.CMD_ROWSET_CHUNK) || !bufferEndsWith(buffer, exports.ROWSET_CHUNKS_END)) {
         throw new Error('SQLiteCloudConnection.parseRowsetChunks - invalid chunks buffer');
     }
@@ -195,7 +205,8 @@ function parseRowsetChunks(buffers) {
     const data = [];
     // validate and skip data type
     const dataType = buffer.subarray(0, 1).toString();
-    console.assert(dataType === exports.CMD_ROWSET_CHUNK);
+    if (dataType !== exports.CMD_ROWSET_CHUNK)
+        throw new Error(`parseRowsetChunks - dataType: ${dataType} should be CMD_ROWSET_CHUNK`);
     buffer = buffer.subarray(buffer.indexOf(' ') + 1);
     while (buffer.length > 0 && !bufferStartsWith(buffer, exports.ROWSET_CHUNKS_END)) {
         // chunk header, eg: 0:VERS NROWS NCOLS
@@ -239,22 +250,25 @@ function popData(buffer) {
         return { data, fwdBuffer };
     }
     // first character is the data type
-    console.assert(buffer && buffer instanceof Buffer);
-    const dataType = buffer.subarray(0, 1).toString('utf8');
-    console.assert(dataType !== exports.CMD_COMPRESSED, "Compressed data shouldn't be decompressed before parsing");
-    console.assert(dataType !== exports.CMD_ROWSET_CHUNK, 'Chunked data should be parsed by parseRowsetChunks');
+    console.assert(buffer && buffer instanceof buffer_1.Buffer);
+    let dataType = buffer.subarray(0, 1).toString('utf8');
+    if (dataType == exports.CMD_COMPRESSED)
+        throw new Error('Compressed data should be decompressed before parsing');
+    if (dataType == exports.CMD_ROWSET_CHUNK)
+        throw new Error('Chunked data should be parsed by parseRowsetChunks');
     let spaceIndex = buffer.indexOf(' ');
     if (spaceIndex === -1) {
         spaceIndex = buffer.length - 1;
     }
-    let commandEnd = -1;
+    let commandEnd = -1, commandLength = -1;
     if (dataType === exports.CMD_INT || dataType === exports.CMD_FLOAT || dataType === exports.CMD_NULL) {
         commandEnd = spaceIndex + 1;
     }
     else {
-        const commandLength = parseInt(buffer.subarray(1, spaceIndex).toString());
+        commandLength = parseInt(buffer.subarray(1, spaceIndex).toString());
         commandEnd = spaceIndex + 1 + commandLength;
     }
+    // console.debug(`popData - dataType: ${dataType}, spaceIndex: ${spaceIndex}, commandLength: ${commandLength}, commandEnd: ${commandEnd}`)
     switch (dataType) {
         case exports.CMD_INT:
             return popResults(parseInt(buffer.subarray(1, spaceIndex).toString()));
@@ -282,12 +296,14 @@ function popData(buffer) {
             parseError(buffer, spaceIndex); // throws custom error
             break;
     }
-    throw new TypeError(`Data type: ${dataType} is not defined in SCSP`);
+    const msg = `popData - Data type: ${Number(dataType)} '${dataType}'  is not defined in SCSP, spaceIndex: ${spaceIndex}, commandLength: ${commandLength}, commandEnd: ${commandEnd}`;
+    console.error(msg);
+    throw new TypeError(msg);
 }
 exports.popData = popData;
 /** Format a command to be sent via SCSP protocol */
 function formatCommand(command) {
-    const commandLength = Buffer.byteLength(command, 'utf-8');
+    const commandLength = buffer_1.Buffer.byteLength(command, 'utf-8');
     return `+${commandLength} ${command}`;
 }
 exports.formatCommand = formatCommand;

package/lib/drivers/types.d.ts

@@ -7,7 +7,7 @@ import tls from 'tls';
 /** Default timeout value for queries */
 export declare const DEFAULT_TIMEOUT: number;
 /** Default tls connection port */
-export declare const DEFAULT_PORT = 9960;
+export declare const DEFAULT_PORT = 8860;
 /**
  * Configuration for SQLite cloud connection
  * @note Options are all lowecase so they 1:1 compatible with C SDK

package/lib/drivers/types.js

@@ -7,7 +7,7 @@ exports.SQLiteCloudArrayType = exports.SQLiteCloudError = exports.DEFAULT_PORT =
 /** Default timeout value for queries */
 exports.DEFAULT_TIMEOUT = 300 * 1000;
 /** Default tls connection port */
-exports.DEFAULT_PORT = 9960;
+exports.DEFAULT_PORT = 8860;
 /** Custom error reported by SQLiteCloud drivers */
 class SQLiteCloudError extends Error {
     constructor(message, args) {

package/lib/drivers/utilities.js

@@ -6,6 +6,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.parseBooleanToZeroOne = exports.parseBoolean = exports.parseconnectionstring = exports.validateConfiguration = exports.popCallback = exports.getUpdateResults = exports.prepareSql = exports.escapeSqlParameter = exports.getInitializationCommands = exports.anonimizeError = exports.anonimizeCommand = exports.isNode = exports.isBrowser = void 0;
 const types_1 = require("./types");
 const types_2 = require("./types");
+// explicitly importing these libraries to allow cross-platform support by replacing them
+const whatwg_url_1 = require("whatwg-url");
+const buffer_1 = require("buffer");
 //
 // determining running environment, thanks to browser-or-node
 // https://www.npmjs.com/package/browser-or-node
@@ -93,7 +96,7 @@ function escapeSqlParameter(param) {
         return param ? '1' : '0';
     }
     // serialize buffer as X'...' hex encoded string
-    if (Buffer.isBuffer(param)) {
+    if (buffer_1.Buffer.isBuffer(param)) {
         return `X'${param.toString('hex')}'`;
     }
     if (typeof param === 'object') {
@@ -117,7 +120,7 @@ function prepareSql(sql, ...params) {
         const index = matchIndex ? parseInt(matchIndex) : parameterIndex;
         parameterIndex++;
         let sqlParameter;
-        if (params[0] && typeof params[0] === 'object' && !(params[0] instanceof Buffer)) {
+        if (params[0] && typeof params[0] === 'object' && !(params[0] instanceof buffer_1.Buffer)) {
             sqlParameter = params[0][index];
         }
         if (!sqlParameter) {
@@ -200,7 +203,7 @@ function validateConfiguration(config) {
     config.clientid || (config.clientid = 'SQLiteCloud');
     config.verbose = parseBoolean(config.verbose);
     config.noblob = parseBoolean(config.noblob);
-    config.compression = parseBoolean(config.compression);
+    config.compression = config.compression != undefined && config.compression != null ? parseBoolean(config.compression) : true; // default: true
     config.create = parseBoolean(config.create);
     config.non_linearizable = parseBoolean(config.non_linearizable);
     config.insecure = parseBoolean(config.insecure);
@@ -235,7 +238,7 @@ function parseconnectionstring(connectionstring) {
         // the sqlitecloud: protocol is not recognized by the URL constructor in browsers
         // so we need to replace it with https: to make it work
         const knownProtocolUrl = connectionstring.replace('sqlitecloud:', 'https:');
-        const url = new URL(knownProtocolUrl);
+        const url = new whatwg_url_1.URL(knownProtocolUrl);
         // all lowecase options
         const options = {};
         url.searchParams.forEach((value, key) => {