lbug 0.12.3-dev.17 → 0.12.3-dev.18

package/lbug-source/CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.15)
 
-project(Lbug VERSION 0.12.3.17 LANGUAGES CXX C)
+project(Lbug VERSION 0.12.3.18 LANGUAGES CXX C)
 
 option(SINGLE_THREADED "Single-threaded mode" FALSE)
 if(SINGLE_THREADED)

package/lbug-source/src/include/storage/table/parquet_rel_table.h

@@ -21,7 +21,6 @@ struct ParquetRelTableScanState final : RelTableScanState {
     uint64_t currentRowGroup = 0;
 
     // Per-scan-state readers for thread safety
-    std::unique_ptr<processor::ParquetReader> nodeMappingReader;
     std::unique_ptr<processor::ParquetReader> indicesReader;
     std::unique_ptr<processor::ParquetReader> indptrReader;
 
@@ -42,7 +41,7 @@ class ParquetRelTable final : public RelTable {
 public:
     ParquetRelTable(catalog::RelGroupCatalogEntry* relGroupEntry, common::table_id_t fromTableID,
         common::table_id_t toTableID, const StorageManager* storageManager,
-        MemoryManager* memoryManager, std::string fromNodeTableName);
+        MemoryManager* memoryManager);
 
     void initScanState(transaction::Transaction* transaction, TableScanState& scanState,
         bool resetCachedBoundNodeSelVec = true) const override;
@@ -68,24 +67,17 @@ public:
 
 private:
     catalog::RelGroupCatalogEntry* relGroupEntry; // Store reference to table schema
-    std::string nodeMappingFilePath;
     std::string indicesFilePath;
     std::string indptrFilePath;
-    mutable std::unique_ptr<processor::ParquetReader> nodeMappingReader;
     mutable std::unique_ptr<processor::ParquetReader> indicesReader;
     mutable std::unique_ptr<processor::ParquetReader> indptrReader;
     mutable std::mutex parquetReaderMutex;
     mutable std::mutex indptrDataMutex;
     mutable std::vector<common::offset_t> indptrData; // Cached indptr data for CSR format
-    mutable common::internal_id_map_t<common::offset_t>
-        nodeMapping; // Maps node IDs to CSR node IDs
-    mutable std::unordered_map<common::offset_t, common::offset_t>
-        csrToNodeTableIdMap; // Reverse mapping: CSR node ID to node table ID
 
     void initializeParquetReaders(transaction::Transaction* transaction) const;
     void initializeIndptrReader(transaction::Transaction* transaction) const;
     void loadIndptrData(transaction::Transaction* transaction) const;
-    void loadNodeMappingData(transaction::Transaction* transaction) const;
     bool scanInternalByRowGroups(transaction::Transaction* transaction,
         ParquetRelTableScanState& parquetRelScanState);
     bool scanRowGroupForBoundNodes(transaction::Transaction* transaction,

package/lbug-source/src/storage/storage_manager.cpp

@@ -96,9 +96,8 @@ void StorageManager::createNodeTable(NodeTableCatalogEntry* entry) {
 void StorageManager::addRelTable(RelGroupCatalogEntry* entry, const RelTableCatalogInfo& info) {
     if (!entry->getStorage().empty()) {
         // Create parquet-backed rel table
-        std::string fromNodeTableName = tableNameCache.at(info.nodePair.srcTableID);
         tables[info.oid] = std::make_unique<ParquetRelTable>(entry, info.nodePair.srcTableID,
-            info.nodePair.dstTableID, this, &memoryManager, fromNodeTableName);
+            info.nodePair.dstTableID, this, &memoryManager);
     } else {
         // Create regular rel table
         tables[info.oid] = std::make_unique<RelTable>(entry, info.nodePair.srcTableID,
@@ -306,10 +305,8 @@ void StorageManager::deserialize(main::ClientContext* context, const Catalog* ca
             KU_ASSERT(!tables.contains(info.oid));
             if (!relGroupEntry->getStorage().empty()) {
                 // Create parquet-backed rel table
-                std::string fromNodeTableName = tableNameCache.at(info.nodePair.srcTableID);
-                tables[info.oid] =
-                    std::make_unique<ParquetRelTable>(relGroupEntry, info.nodePair.srcTableID,
-                        info.nodePair.dstTableID, this, &memoryManager, fromNodeTableName);
+                tables[info.oid] = std::make_unique<ParquetRelTable>(relGroupEntry,
+                    info.nodePair.srcTableID, info.nodePair.dstTableID, this, &memoryManager);
             } else {
                 // Create regular rel table
                 tables[info.oid] = std::make_unique<RelTable>(relGroupEntry,

package/lbug-source/src/storage/table/parquet_rel_table.cpp

@@ -42,8 +42,7 @@ void ParquetRelTableScanState::setToTable(const Transaction* transaction, Table*
 }
 
 ParquetRelTable::ParquetRelTable(RelGroupCatalogEntry* relGroupEntry, table_id_t fromTableID,
-    table_id_t toTableID, const StorageManager* storageManager, MemoryManager* memoryManager,
-    std::string fromNodeTableName)
+    table_id_t toTableID, const StorageManager* storageManager, MemoryManager* memoryManager)
     : RelTable{relGroupEntry, fromTableID, toTableID, storageManager, memoryManager},
       relGroupEntry{relGroupEntry} {
     std::string storage = relGroupEntry->getStorage();
@@ -58,7 +57,6 @@ ParquetRelTable::ParquetRelTable(RelGroupCatalogEntry* relGroupEntry, table_id_t
     // prefix_indices_{relName}.parquet, prefix_indptr_{relName}.parquet,
     // prefix_metadata_{relName}.parquet
     std::string prefix = storage;
-    nodeMappingFilePath = prefix + "_mapping_" + fromNodeTableName + ".parquet";
     indicesFilePath = prefix + "_indices_" + relName + ".parquet";
     indptrFilePath = prefix + "_indptr_" + relName + ".parquet";
 }
@@ -75,12 +73,6 @@ void ParquetRelTable::initScanState(Transaction* transaction, TableScanState& sc
     auto& parquetRelScanState = static_cast<ParquetRelTableScanState&>(relScanState);
 
     // Initialize readers if not already done for this scan state
-    if (!parquetRelScanState.nodeMappingReader) {
-        std::vector<bool> columnSkips; // Read all columns
-        auto context = transaction->getClientContext();
-        parquetRelScanState.nodeMappingReader =
-            std::make_unique<ParquetReader>(nodeMappingFilePath, columnSkips, context);
-    }
     if (!parquetRelScanState.indicesReader) {
         std::vector<bool> columnSkips; // Read all columns
         auto context = transaction->getClientContext();
@@ -94,8 +86,7 @@ void ParquetRelTable::initScanState(Transaction* transaction, TableScanState& sc
             std::make_unique<ParquetReader>(indptrFilePath, columnSkips, context);
     }
 
-    // Load shared data (node mapping and indptr) - these are thread-safe to read
-    loadNodeMappingData(transaction);
+    // Load shared indptr data - thread-safe to read
     if (!indptrFilePath.empty()) {
         loadIndptrData(transaction);
     }
@@ -127,14 +118,8 @@ void ParquetRelTable::initScanState(Transaction* transaction, TableScanState& sc
 }
 
 void ParquetRelTable::initializeParquetReaders(Transaction* transaction) const {
-    if (!nodeMappingReader || !indicesReader) {
+    if (!indicesReader) {
         std::lock_guard lock(parquetReaderMutex);
-        if (!nodeMappingReader) {
-            std::vector<bool> columnSkips; // Read all columns
-            auto context = transaction->getClientContext();
-            nodeMappingReader =
-                std::make_unique<ParquetReader>(nodeMappingFilePath, columnSkips, context);
-        }
         if (!indicesReader) {
             std::vector<bool> columnSkips; // Read all columns
             auto context = transaction->getClientContext();
@@ -154,71 +139,6 @@ void ParquetRelTable::initializeIndptrReader(Transaction* transaction) const {
     }
 }
 
-void ParquetRelTable::loadNodeMappingData(Transaction* transaction) const {
-    if (nodeMapping.empty() && !nodeMappingFilePath.empty()) {
-        std::lock_guard lock(parquetReaderMutex);
-        if (nodeMapping.empty()) {
-            // Initialize node mapping reader if not already done
-            if (!nodeMappingReader) {
-                std::vector<bool> columnSkips; // Read all columns
-                auto context = transaction->getClientContext();
-                nodeMappingReader =
-                    std::make_unique<ParquetReader>(nodeMappingFilePath, columnSkips, context);
-            }
-
-            // Initialize scan to populate column types
-            auto context = transaction->getClientContext();
-            auto vfs = VirtualFileSystem::GetUnsafe(*context);
-            std::vector<uint64_t> groupsToRead;
-            for (uint64_t i = 0; i < nodeMappingReader->getNumRowsGroups(); ++i) {
-                groupsToRead.push_back(i);
-            }
-
-            ParquetReaderScanState scanState;
-            nodeMappingReader->initializeScan(scanState, groupsToRead, vfs);
-
-            // Check if the node mapping file has columns
-            auto numColumns = nodeMappingReader->getNumColumns();
-            if (numColumns < 2) {
-                throw RuntimeException("Node mapping parquet file must have at least 2 columns");
-            }
-
-            // Validate column types for node mapping
-            const auto& csrNodeIdType = nodeMappingReader->getColumnType(0);
-            const auto& nodeTableIdType = nodeMappingReader->getColumnType(1);
-            if (!LogicalTypeUtils::isIntegral(csrNodeIdType.getLogicalTypeID()) ||
-                !LogicalTypeUtils::isIntegral(nodeTableIdType.getLogicalTypeID())) {
-                throw RuntimeException(
-                    "Node mapping parquet file columns must be integer types (columns 0 and 1)");
-            }
-
-            // Read the node mapping data
-            DataChunk dataChunk(2);
-
-            // Get column types
-            for (uint32_t i = 0; i < 2 && i < numColumns; ++i) {
-                const auto& columnTypeRef = nodeMappingReader->getColumnType(i);
-                auto columnType = columnTypeRef.copy();
-                auto vector = std::make_shared<ValueVector>(std::move(columnType));
-                dataChunk.insert(i, vector);
-            }
-
-            // Read all node mapping values
-            while (nodeMappingReader->scanInternal(scanState, dataChunk)) {
-                auto selSize = dataChunk.state->getSelVector().getSelSize();
-                for (size_t i = 0; i < selSize; ++i) {
-                    auto csrNodeId = dataChunk.getValueVector(0).getValue<common::offset_t>(i);
-                    auto nodeTableId = dataChunk.getValueVector(1).getValue<common::offset_t>(i);
-                    nodeMapping[common::internalID_t(nodeTableId, getFromNodeTableID())] =
-                        csrNodeId;
-                    // Also create reverse mapping for destination node lookups
-                    csrToNodeTableIdMap[csrNodeId] = nodeTableId;
-                }
-            }
-        }
-    }
-}
-
 void ParquetRelTable::loadIndptrData(Transaction* transaction) const {
     if (indptrData.empty() && !indptrFilePath.empty()) {
         std::lock_guard lock(indptrDataMutex);
@@ -278,9 +198,7 @@ bool ParquetRelTable::scanInternal(Transaction* transaction, TableScanState& sca
     // Get the ParquetRelTableScanState
     auto& parquetRelScanState = static_cast<ParquetRelTableScanState&>(relScanState);
 
-    // Readers are now initialized per scan state in initScanState
-    // Load shared data (node mapping and indptr) - these are thread-safe to read
-    loadNodeMappingData(transaction);
+    // Load shared indptr data - thread-safe to read
     if (!indptrFilePath.empty()) {
         loadIndptrData(transaction);
     }

package/lbug-source/test/test_files/demo_db/demo_db_graph_std.test

@@ -6,14 +6,11 @@
 
 -LOG MatchUserLivesInCity
 -STATEMENT MATCH (u:user)-[l:livesin]->(c:city) RETURN u.name, u.age, c.name;
----- 7
-Adam|30|Guelph
-Adam|30|Guelph
+---- 4
+Adam|30|Waterloo
 Karissa|40|Waterloo
-Noura|25|Kitchener
-Noura|25|Waterloo
-Noura|25|Waterloo
 Zhang|50|Kitchener
+Noura|25|Guelph
 
 -LOG MatchSingleNodeLabel
 -STATEMENT MATCH (a:user) RETURN a.name, a.age;
@@ -32,12 +29,49 @@ Guelph|75000
 
 -LOG MatchFollowsRel
 -STATEMENT MATCH (a:user)-[e:follows]->(b:user) RETURN a.name, b.name, e.since;
----- 8
-Adam|Zhang|2022
-Karissa|Noura|2020
-Karissa|Zhang|2020
-Noura|Karissa|2020
-Noura|Zhang|2021
-Zhang|Adam|2022
-Zhang|Karissa|2020
-Zhang|Noura|2021
+---- 4
+Adam|Karissa|2020
+Adam|Zhang|2020
+Karissa|Zhang|2021
+Zhang|Noura|2022
+
+-LOG MatchLivesInWithCityPopulation
+-STATEMENT MATCH (u:user)-[l:livesin]->(c:city) RETURN u.name, c.name, c.population ORDER BY c.population DESC;
+---- 4
+Zhang|Kitchener|200000
+Adam|Waterloo|150000
+Karissa|Waterloo|150000
+Noura|Guelph|75000
+
+-LOG MatchLivesInFilterByCity
+-STATEMENT MATCH (u:user)-[l:livesin]->(c:city) WHERE c.name = 'Waterloo' RETURN u.name, u.age;
+---- 2
+Adam|30
+Karissa|40
+
+-LOG MatchLivesInFilterByCityPopulation
+-STATEMENT MATCH (u:user)-[l:livesin]->(c:city) WHERE c.population > 100000 RETURN u.name, c.name ORDER BY u.name;
+---- 3
+Adam|Waterloo
+Karissa|Waterloo
+Zhang|Kitchener
+
+-LOG CountUsersPerCity
+-STATEMENT MATCH (u:user)-[l:livesin]->(c:city) RETURN c.name, COUNT(*) AS num_users ORDER BY num_users DESC;
+---- 3
+Waterloo|2
+Guelph|1
+Kitchener|1
+
+-LOG MatchFollowsWithDestinationAge
+-STATEMENT MATCH (a:user)-[e:follows]->(b:user) WHERE b.age > 30 RETURN a.name, b.name, b.age ORDER BY b.age DESC;
+---- 3
+Adam|Zhang|50
+Karissa|Zhang|50
+Adam|Karissa|40
+
+-LOG MatchFollowsFilterBySourceAndDest
+-STATEMENT MATCH (a:user)-[e:follows]->(b:user) WHERE a.age < 40 AND b.age >= 40 RETURN a.name, b.name;
+---- 2
+Adam|Karissa
+Adam|Zhang

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lbug",
-  "version": "0.12.3-dev.17",
+  "version": "0.12.3-dev.18",
   "description": "An in-process property graph database management system built for query speed and scalability.",
   "main": "index.js",
   "module": "./index.mjs",