@op-engineering/op-sqlite 1.0.2 → 1.0.3

package/README.md

@@ -37,9 +37,9 @@ You can find the [benchmarking code in the example app](https://github.com/OP-En
 | ------------ | ------------- | ---------- |
 | quick-sqlite | 2719          | 8851       |
 | expo-sqlite  | 2293          | 10626      |
-| op-sqlite    | 488           | 981        |
+| op-sqlite    | 507           | 1125       |
 
-Memory consumption is also 50% to 80% reduced compared to `react-native-quick-sqlite`. `expo-sqlite` OOMS itself even when trying to insert 300k elements in a row. Queries that before might have OOM now can run just fine.
+Memory consumption is also is also 1/4 compared to `react-native-quick-sqlite`. This query used to take 1.2gb of peak memory usage, now runs in 250mbs.
 
 # DB Paths
 

package/android/CMakeLists.txt

@@ -31,6 +31,8 @@ add_library(
   ../cpp/sqlbatchexecutor.cpp
   ../cpp/DynamicHostObject.cpp
   ../cpp/DynamicHostObject.h
+  ../cpp/DumbHostObject.cpp
+  ../cpp/DumbHostObject.h
   ../cpp/macros.h
   ../cpp/types.h
   cpp-adapter.cpp

package/cpp/DumbHostObject.cpp

@@ -0,0 +1,37 @@
+#include "DumbHostObject.h"
+#include "utils.h"
+#include <iostream>
+
+namespace osp {
+
+    namespace jsi = facebook::jsi;
+
+    DumbHostObject::DumbHostObject(std::shared_ptr<std::vector<DynamicHostObject>> metadata) {
+        this->metadata = metadata;
+    };
+
+    std::vector<jsi::PropNameID> DumbHostObject::getPropertyNames(jsi::Runtime &rt) {
+        std::vector<jsi::PropNameID> keys;
+        
+        for (auto field : *metadata) {
+            // TODO improve this by generating the propName once on metadata creation
+            keys.push_back(jsi::PropNameID::forAscii(rt, std::get<std::string>(field.fields[0].second)));
+        }
+        
+        return keys;
+    }
+
+    jsi::Value DumbHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &propNameID) {
+        auto name = propNameID.utf8(rt);
+        auto fields = metadata.get();
+        for(int i = 0; i < fields->size(); i++) {
+            auto fieldName = std::get<std::string>(fields->at(i).fields[0].second);
+            if(fieldName == name) {
+                return toJSI(rt, values.at(i));
+            }
+        }
+
+        return {};
+    }
+
+}

package/cpp/DumbHostObject.h

@@ -0,0 +1,33 @@
+#ifndef DumbHostObject_h
+#define DumbHostObject_h
+
+#include <stdio.h>
+
+#include <jsi/jsi.h>
+#include <any>
+#include <vector>
+#include "types.h"
+#include "DynamicHostObject.h"
+
+namespace osp {
+
+    namespace jsi = facebook::jsi;
+
+    class JSI_EXPORT DumbHostObject: public jsi::HostObject {
+    public:
+        DumbHostObject() {};
+        
+        DumbHostObject(std::shared_ptr<std::vector<DynamicHostObject>> metadata);
+
+        std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime &rt);
+
+        jsi::Value get(jsi::Runtime &rt, const jsi::PropNameID &propNameID);
+
+        std::vector<jsVal> values;
+        
+        std::shared_ptr<std::vector<DynamicHostObject>> metadata;
+    };
+
+}
+
+#endif /* DumbHostObject_h */

package/cpp/DynamicHostObject.h

@@ -14,13 +14,10 @@ namespace osp {
     public:
         DynamicHostObject() {};
 
-        virtual ~DynamicHostObject() {};
-
         std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime &rt);
 
         jsi::Value get(jsi::Runtime &rt, const jsi::PropNameID &propNameID);
 
-
         std::vector<std::pair<std::string, jsVal>> fields;
     };
 

package/cpp/bindings.cpp

@@ -8,6 +8,7 @@
 #include <string>
 #include "macros.h"
 #include <iostream>
+#include "DumbHostObject.h"
 
 namespace osp {
 
@@ -188,17 +189,17 @@ void install(jsi::Runtime &rt, std::shared_ptr<react::CallInvoker> jsCallInvoker
             params = toAnyVec(rt, originalParams);
         }
         
-        std::vector<std::shared_ptr<DynamicHostObject>> results;
-        std::vector<std::shared_ptr<DynamicHostObject>> metadata;
+        std::vector<DumbHostObject> results;
+        std::shared_ptr<std::vector<DynamicHostObject>> metadata = std::make_shared<std::vector<DynamicHostObject>>();
         
         try {
-            auto status = sqliteExecute(dbName, query, &params, &results, &metadata);
+            auto status = sqliteExecute(dbName, query, &params, &results, metadata);
             
             if(status.type == SQLiteError) {
                 throw jsi::JSError(rt, status.message);
             }
             
-            auto jsiResult = createResult(rt, status, &results, &metadata);
+            auto jsiResult = createResult(rt, status, &results, metadata);
             return jsiResult;
         } catch(std::exception &e) {
             throw jsi::JSError(rt, e.what());
@@ -228,19 +229,19 @@ void install(jsi::Runtime &rt, std::shared_ptr<react::CallInvoker> jsCallInvoker
             {
                 try
                 {
-                    std::vector<std::shared_ptr<DynamicHostObject>> results;
-                    std::vector<std::shared_ptr<DynamicHostObject>> metadata;
-                    
-                    auto status = sqliteExecute(dbName, query, params.get(), &results, &metadata);
+                    std::vector<DumbHostObject> results;
+                    std::shared_ptr<std::vector<DynamicHostObject>> metadata = std::make_shared<std::vector<DynamicHostObject>>();;
                     
+                    auto status = sqliteExecute(dbName, query, params.get(), &results, metadata);
+                
                     invoker->invokeAsync([&rt,
-                                           results = std::make_shared<std::vector<std::shared_ptr<DynamicHostObject>>>(results),
-                                           metadata = std::make_shared<std::vector<std::shared_ptr<DynamicHostObject>>>(metadata),
+                                           results = std::make_shared<std::vector<DumbHostObject>>(results),
+                                           metadata,
                                            status = std::move(status),
                                            resolve,
                                            reject] {
                         if(status.type == SQLiteOk) {
-                            auto jsiResult = createResult(rt, status, results.get(), metadata.get());
+                            auto jsiResult = createResult(rt, status, results.get(), metadata);
                             resolve->asObject(rt).asFunction(rt).call(rt, std::move(jsiResult));
                         } else {
                             auto errorCtr = rt.global().getPropertyAsFunction(rt, "Error");

package/cpp/bridge.cpp

@@ -8,6 +8,7 @@
 #include <unordered_map>
 #include "logs.h"
 #include "DynamicHostObject.h"
+#include "DumbHostObject.h"
 #include <variant>
 
 namespace osp {
@@ -208,7 +209,7 @@ inline void bindStatement(sqlite3_stmt *statement, std::vector<jsVal> *values)
     {
         int sqIndex = ii + 1;
         jsVal value = values->at(ii);
-
+        
         if (std::holds_alternative<bool>(value))
         {
             sqlite3_bind_int(statement, sqIndex, std::get<bool>(value));
@@ -237,22 +238,22 @@ inline void bindStatement(sqlite3_stmt *statement, std::vector<jsVal> *values)
         } else {
             sqlite3_bind_null(statement, sqIndex);
         }
-//        else if (value.type() == typeid(const char*))
-//        {
-//            sqlite3_bind_text(statement, sqIndex, str.c_str(), str.length(), SQLITE_TRANSIENT);
-//            return jsi::String::createFromAscii(rt, std::any_cast<const char*>(value));
-//        }
+        //        else if (value.type() == typeid(const char*))
+        //        {
+        //            sqlite3_bind_text(statement, sqIndex, str.c_str(), str.length(), SQLITE_TRANSIENT);
+        //            return jsi::String::createFromAscii(rt, std::any_cast<const char*>(value));
+        //        }
     }
 }
 
 BridgeResult sqliteExecute(std::string const dbName,
-                             std::string const &query,
-                             std::vector<jsVal> *params,
-                             std::vector<std::shared_ptr<DynamicHostObject>> *results,
-                             std::vector<std::shared_ptr<DynamicHostObject>> *metadatas)
+                           std::string const &query,
+                           std::vector<jsVal> *params,
+                           std::vector<DumbHostObject> *results,
+                           std::shared_ptr<std::vector<DynamicHostObject>> metadatas)
 {
     
-    if (dbMap.count(dbName) == 0)
+    if (dbMap.find(dbName) == dbMap.end())
     {
         return {
             .type = SQLiteError,
@@ -283,7 +284,6 @@ BridgeResult sqliteExecute(std::string const dbName,
     
     int result, i, count, column_type;
     std::string column_name, column_declared_type;
-    std::shared_ptr<DynamicHostObject> row;
     
     while (isConsuming)
     {
@@ -291,24 +291,23 @@ BridgeResult sqliteExecute(std::string const dbName,
         
         switch (result)
         {
-            case SQLITE_ROW:
+            case SQLITE_ROW: {
                 if(results == NULL)
                 {
                     break;
                 }
                 
                 i = 0;
-                row = std::make_shared<DynamicHostObject>();
+                DumbHostObject row = DumbHostObject(metadatas);
+                
                 count = sqlite3_column_count(statement);
                 
                 while (i < count)
                 {
                     column_type = sqlite3_column_type(statement, i);
-                    column_name = sqlite3_column_name(statement, i);
                     
                     switch (column_type)
                     {
-                            
                         case SQLITE_INTEGER:
                         {
                             /**
@@ -316,14 +315,14 @@ BridgeResult sqliteExecute(std::string const dbName,
                              * only represent Integers up to 53 bits
                              */
                             double column_value = sqlite3_column_double(statement, i);
-                            row->fields.push_back(std::make_pair(column_name, jsVal(column_value)));
+                            row.values.push_back(jsVal(column_value));
                             break;
                         }
                             
                         case SQLITE_FLOAT:
                         {
                             double column_value = sqlite3_column_double(statement, i);
-                            row->fields.push_back(std::make_pair(column_name, jsVal(column_value)));
+                            row.values.push_back(jsVal(column_value));
                             break;
                         }
                             
@@ -332,7 +331,7 @@ BridgeResult sqliteExecute(std::string const dbName,
                             const char *column_value = reinterpret_cast<const char *>(sqlite3_column_text(statement, i));
                             int byteLen = sqlite3_column_bytes(statement, i);
                             // Specify length too; in case string contains NULL in the middle (which SQLite supports!)
-                            row->fields.push_back(std::make_pair(column_name, jsVal(std::string(column_value, byteLen))));
+                            row.values.push_back(jsVal(std::string(column_value, byteLen)));
                             break;
                         }
                             
@@ -342,42 +341,43 @@ BridgeResult sqliteExecute(std::string const dbName,
                             const void *blob = sqlite3_column_blob(statement, i);
                             uint8_t *data = new uint8_t[blob_size];
                             memcpy(data, blob, blob_size);
-                            row->fields.push_back(std::make_pair(column_name, jsVal(JSBuffer {
+                            row.values.push_back(jsVal(JSBuffer {
                                 .data =  std::shared_ptr<uint8_t>{data},
                                 .size =  static_cast<size_t>(blob_size)
-                            })));
+                            }));
                             break;
                         }
                             
                         case SQLITE_NULL:
                             // Intentionally left blank
-
+                            
                         default:
-                            row->fields.push_back(std::make_pair(column_name, jsVal(NULL)));
+                            row.values.push_back(jsVal(NULL));
                             break;
                     }
                     i++;
                 }
                 results->push_back(row);
                 break;
-
+            }
+                
             case SQLITE_DONE:
-                if(metadatas != nullptr)
+                i = 0;
+                count = sqlite3_column_count(statement);
+                
+                while (i < count)
                 {
-                    i = 0;
-                    count = sqlite3_column_count(statement);
-                    while (i < count)
-                    {
-                        column_name = sqlite3_column_name(statement, i);
-                        const char *type = sqlite3_column_decltype(statement, i);
-                        auto metadata = std::make_shared<DynamicHostObject>();
-                        metadata->fields.push_back(std::make_pair("name", column_name));
-                        metadata->fields.push_back(std::make_pair("index", i));
-                        metadata->fields.push_back(std::make_pair("type", type));
-                        
-                        i++;
-                    }
+                    column_name = sqlite3_column_name(statement, i);
+                    const char *type = sqlite3_column_decltype(statement, i);
+                    auto metadata = DynamicHostObject();
+                    metadata.fields.push_back(std::make_pair("name", column_name));
+                    metadata.fields.push_back(std::make_pair("index", i));
+                    metadata.fields.push_back(std::make_pair("type", type));
+                    
+                    metadatas->push_back(metadata);
+                    i++;
                 }
+                
                 isConsuming = false;
                 break;
                 

package/cpp/bridge.h

@@ -4,6 +4,7 @@
 #include "utils.h"
 #include <vector>
 #include "DynamicHostObject.h"
+#include "DumbHostObject.h"
 #include "types.h"
 
 namespace osp {
@@ -23,8 +24,8 @@ BridgeResult sqliteDetachDb(std::string const mainDBName, std::string const alia
 BridgeResult sqliteExecute(std::string const dbName,
                            std::string const &query,
                            std::vector<jsVal> *values,
-                           std::vector<std::shared_ptr<DynamicHostObject>> *result,
-                           std::vector<std::shared_ptr<DynamicHostObject>> *metadata);
+                           std::vector<DumbHostObject> *results,
+                           std::shared_ptr<std::vector<DynamicHostObject>> metadatas);
 
 BridgeResult sqliteExecuteLiteral(std::string const dbName, std::string const &query);
 

package/cpp/utils.cpp

@@ -118,8 +118,8 @@ std::vector<jsVal> toAnyVec(jsi::Runtime &rt, jsi::Value const &params)
 
 jsi::Value createResult(jsi::Runtime &rt,
                         BridgeResult status,
-                        std::vector<std::shared_ptr<DynamicHostObject>> *results,
-                        std::vector<std::shared_ptr<DynamicHostObject>> *metadata)
+                        std::vector<DumbHostObject> *results,
+                        std::shared_ptr<std::vector<DynamicHostObject>> metadata)
 {
     if(status.type == SQLiteError) {
         throw std::invalid_argument(status.message);
@@ -142,7 +142,8 @@ jsi::Value createResult(jsi::Runtime &rt,
         auto array = jsi::Array(rt, rowCount);
         for (int i = 0; i < rowCount; i++)
         {
-            array.setValueAtIndex(rt, i, jsi::Object::createFromHostObject(rt, results->at(i)));
+            auto obj = results->at(i);
+            array.setValueAtIndex(rt, i, jsi::Object::createFromHostObject(rt, std::make_shared<DumbHostObject>(obj)));
         }
         rows.setProperty(rt, "_array", std::move(array));
         res.setProperty(rt, "rows", std::move(rows));
@@ -154,7 +155,7 @@ jsi::Value createResult(jsi::Runtime &rt,
         auto column_array = jsi::Array(rt, column_count);
         for (int i = 0; i < column_count; i++) {
             auto column = metadata->at(i);
-            column_array.setValueAtIndex(rt, i, jsi::Object::createFromHostObject(rt, column));
+            column_array.setValueAtIndex(rt, i, jsi::Object::createFromHostObject(rt, std::make_shared<DynamicHostObject>(column)));
         }
         res.setProperty(rt, "metadata", std::move(column_array));
     }

package/cpp/utils.h

@@ -9,6 +9,7 @@
 #include <any>
 #include "types.h"
 #include "DynamicHostObject.h"
+#include "DumbHostObject.h"
 
 namespace osp {
 
@@ -46,8 +47,8 @@ std::vector<jsVal> toAnyVec(jsi::Runtime &rt, jsi::Value const &args);
 
 jsi::Value createResult(jsi::Runtime &rt,
                         BridgeResult status,
-                        std::vector<std::shared_ptr<DynamicHostObject>> *results,
-                        std::vector<std::shared_ptr<DynamicHostObject>> *metadata);
+                        std::vector<DumbHostObject> *results,
+                        std::shared_ptr<std::vector<DynamicHostObject>> metadata);
 
 BatchResult importSQLFile(std::string dbName, std::string fileLocation);
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@op-engineering/op-sqlite",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "description": "Next generation SQLite for React Native",
   "main": "lib/commonjs/index",
   "module": "lib/module/index",