lbug 0.12.3-dev.17 → 0.12.3-dev.19

package/lbug-source/CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.15)
 
-project(Lbug VERSION 0.12.3.17 LANGUAGES CXX C)
+project(Lbug VERSION 0.12.3.19 LANGUAGES CXX C)
 
 option(SINGLE_THREADED "Single-threaded mode" FALSE)
 if(SINGLE_THREADED)

package/lbug-source/benchmark/serializer.py

@@ -39,7 +39,17 @@ def serialize(lbug_exec_path, dataset_name, dataset_path, serialized_graph_path,
         with open(os.path.join(dataset_path, 'schema.cypher'), 'r') as f:
             serialize_queries += f.readlines()
         with open(os.path.join(dataset_path, 'copy.cypher'), 'r') as f:
-            serialize_queries += f.readlines()
+            copy_lines = f.readlines()
+        # Fix relative paths in copy.cypher
+        for line in copy_lines:
+            # Replace quoted paths with absolute paths
+            def replace_path(match):
+                path = match.group(1)
+                if not os.path.isabs(path):
+                    return '"' + os.path.join(dataset_path, path) + '"'
+                return match.group(0)
+            fixed_line = re.sub(r'"([^"]*)"', replace_path, line)
+            serialize_queries.append(fixed_line.strip())
     else:
         with open(os.path.join(base_dir, 'serialize.cypher'), 'r') as f:
             serialize_queries += f.readlines()

package/lbug-source/dataset/demo-db/csv/copy.cypher

@@ -1,4 +1,4 @@
-COPY User From "dataset/demo-db/csv/user.csv"
-COPY City FROM "dataset/demo-db/csv/city.csv"
-COPY Follows FROM "dataset/demo-db/csv/follows.csv"
-COPY LivesIn FROM "dataset/demo-db/csv/lives-in.csv"
+COPY User From "user.csv"
+COPY City FROM "city.csv"
+COPY Follows FROM "follows.csv"
+COPY LivesIn FROM "lives-in.csv"

package/lbug-source/dataset/demo-db/parquet/copy.cypher

@@ -1,4 +1,4 @@
-COPY User From "dataset/demo-db/parquet/user.parquet";
-COPY City FROM "dataset/demo-db/parquet/city.parquet";
-COPY Follows FROM "dataset/demo-db/parquet/follows.parquet";
-COPY LivesIn FROM "dataset/demo-db/parquet/lives-in.parquet";
+COPY User From "user.parquet";
+COPY City FROM "city.parquet";
+COPY Follows FROM "follows.parquet";
+COPY LivesIn FROM "lives-in.parquet";

package/lbug-source/src/include/storage/table/parquet_rel_table.h

@@ -21,7 +21,6 @@ struct ParquetRelTableScanState final : RelTableScanState {
     uint64_t currentRowGroup = 0;
 
     // Per-scan-state readers for thread safety
-    std::unique_ptr<processor::ParquetReader> nodeMappingReader;
     std::unique_ptr<processor::ParquetReader> indicesReader;
     std::unique_ptr<processor::ParquetReader> indptrReader;
 
@@ -42,7 +41,7 @@ class ParquetRelTable final : public RelTable {
 public:
     ParquetRelTable(catalog::RelGroupCatalogEntry* relGroupEntry, common::table_id_t fromTableID,
         common::table_id_t toTableID, const StorageManager* storageManager,
-        MemoryManager* memoryManager, std::string fromNodeTableName);
+        MemoryManager* memoryManager);
 
     void initScanState(transaction::Transaction* transaction, TableScanState& scanState,
         bool resetCachedBoundNodeSelVec = true) const override;
@@ -68,24 +67,17 @@ public:
 
 private:
     catalog::RelGroupCatalogEntry* relGroupEntry; // Store reference to table schema
-    std::string nodeMappingFilePath;
     std::string indicesFilePath;
     std::string indptrFilePath;
-    mutable std::unique_ptr<processor::ParquetReader> nodeMappingReader;
     mutable std::unique_ptr<processor::ParquetReader> indicesReader;
     mutable std::unique_ptr<processor::ParquetReader> indptrReader;
     mutable std::mutex parquetReaderMutex;
     mutable std::mutex indptrDataMutex;
     mutable std::vector<common::offset_t> indptrData; // Cached indptr data for CSR format
-    mutable common::internal_id_map_t<common::offset_t>
-        nodeMapping; // Maps node IDs to CSR node IDs
-    mutable std::unordered_map<common::offset_t, common::offset_t>
-        csrToNodeTableIdMap; // Reverse mapping: CSR node ID to node table ID
 
     void initializeParquetReaders(transaction::Transaction* transaction) const;
     void initializeIndptrReader(transaction::Transaction* transaction) const;
     void loadIndptrData(transaction::Transaction* transaction) const;
-    void loadNodeMappingData(transaction::Transaction* transaction) const;
     bool scanInternalByRowGroups(transaction::Transaction* transaction,
         ParquetRelTableScanState& parquetRelScanState);
     bool scanRowGroupForBoundNodes(transaction::Transaction* transaction,

package/lbug-source/src/storage/storage_manager.cpp

@@ -96,9 +96,8 @@ void StorageManager::createNodeTable(NodeTableCatalogEntry* entry) {
 void StorageManager::addRelTable(RelGroupCatalogEntry* entry, const RelTableCatalogInfo& info) {
     if (!entry->getStorage().empty()) {
         // Create parquet-backed rel table
-        std::string fromNodeTableName = tableNameCache.at(info.nodePair.srcTableID);
         tables[info.oid] = std::make_unique<ParquetRelTable>(entry, info.nodePair.srcTableID,
-            info.nodePair.dstTableID, this, &memoryManager, fromNodeTableName);
+            info.nodePair.dstTableID, this, &memoryManager);
     } else {
         // Create regular rel table
         tables[info.oid] = std::make_unique<RelTable>(entry, info.nodePair.srcTableID,
@@ -306,10 +305,8 @@ void StorageManager::deserialize(main::ClientContext* context, const Catalog* ca
             KU_ASSERT(!tables.contains(info.oid));
             if (!relGroupEntry->getStorage().empty()) {
                 // Create parquet-backed rel table
-                std::string fromNodeTableName = tableNameCache.at(info.nodePair.srcTableID);
-                tables[info.oid] =
-                    std::make_unique<ParquetRelTable>(relGroupEntry, info.nodePair.srcTableID,
-                        info.nodePair.dstTableID, this, &memoryManager, fromNodeTableName);
+                tables[info.oid] = std::make_unique<ParquetRelTable>(relGroupEntry,
+                    info.nodePair.srcTableID, info.nodePair.dstTableID, this, &memoryManager);
             } else {
                 // Create regular rel table
                 tables[info.oid] = std::make_unique<RelTable>(relGroupEntry,

package/lbug-source/src/storage/table/parquet_rel_table.cpp

@@ -42,8 +42,7 @@ void ParquetRelTableScanState::setToTable(const Transaction* transaction, Table*
 }
 
 ParquetRelTable::ParquetRelTable(RelGroupCatalogEntry* relGroupEntry, table_id_t fromTableID,
-    table_id_t toTableID, const StorageManager* storageManager, MemoryManager* memoryManager,
-    std::string fromNodeTableName)
+    table_id_t toTableID, const StorageManager* storageManager, MemoryManager* memoryManager)
     : RelTable{relGroupEntry, fromTableID, toTableID, storageManager, memoryManager},
       relGroupEntry{relGroupEntry} {
     std::string storage = relGroupEntry->getStorage();
@@ -58,7 +57,6 @@ ParquetRelTable::ParquetRelTable(RelGroupCatalogEntry* relGroupEntry, table_id_t
     // prefix_indices_{relName}.parquet, prefix_indptr_{relName}.parquet,
     // prefix_metadata_{relName}.parquet
     std::string prefix = storage;
-    nodeMappingFilePath = prefix + "_mapping_" + fromNodeTableName + ".parquet";
     indicesFilePath = prefix + "_indices_" + relName + ".parquet";
     indptrFilePath = prefix + "_indptr_" + relName + ".parquet";
 }
@@ -75,12 +73,6 @@ void ParquetRelTable::initScanState(Transaction* transaction, TableScanState& sc
     auto& parquetRelScanState = static_cast<ParquetRelTableScanState&>(relScanState);
 
     // Initialize readers if not already done for this scan state
-    if (!parquetRelScanState.nodeMappingReader) {
-        std::vector<bool> columnSkips; // Read all columns
-        auto context = transaction->getClientContext();
-        parquetRelScanState.nodeMappingReader =
-            std::make_unique<ParquetReader>(nodeMappingFilePath, columnSkips, context);
-    }
     if (!parquetRelScanState.indicesReader) {
         std::vector<bool> columnSkips; // Read all columns
         auto context = transaction->getClientContext();
@@ -94,8 +86,7 @@ void ParquetRelTable::initScanState(Transaction* transaction, TableScanState& sc
             std::make_unique<ParquetReader>(indptrFilePath, columnSkips, context);
     }
 
-    // Load shared data (node mapping and indptr) - these are thread-safe to read
-    loadNodeMappingData(transaction);
+    // Load shared indptr data - thread-safe to read
     if (!indptrFilePath.empty()) {
         loadIndptrData(transaction);
     }
@@ -127,14 +118,8 @@ void ParquetRelTable::initScanState(Transaction* transaction, TableScanState& sc
 }
 
 void ParquetRelTable::initializeParquetReaders(Transaction* transaction) const {
-    if (!nodeMappingReader || !indicesReader) {
+    if (!indicesReader) {
         std::lock_guard lock(parquetReaderMutex);
-        if (!nodeMappingReader) {
-            std::vector<bool> columnSkips; // Read all columns
-            auto context = transaction->getClientContext();
-            nodeMappingReader =
-                std::make_unique<ParquetReader>(nodeMappingFilePath, columnSkips, context);
-        }
         if (!indicesReader) {
             std::vector<bool> columnSkips; // Read all columns
             auto context = transaction->getClientContext();
@@ -154,71 +139,6 @@ void ParquetRelTable::initializeIndptrReader(Transaction* transaction) const {
     }
 }
 
-void ParquetRelTable::loadNodeMappingData(Transaction* transaction) const {
-    if (nodeMapping.empty() && !nodeMappingFilePath.empty()) {
-        std::lock_guard lock(parquetReaderMutex);
-        if (nodeMapping.empty()) {
-            // Initialize node mapping reader if not already done
-            if (!nodeMappingReader) {
-                std::vector<bool> columnSkips; // Read all columns
-                auto context = transaction->getClientContext();
-                nodeMappingReader =
-                    std::make_unique<ParquetReader>(nodeMappingFilePath, columnSkips, context);
-            }
-
-            // Initialize scan to populate column types
-            auto context = transaction->getClientContext();
-            auto vfs = VirtualFileSystem::GetUnsafe(*context);
-            std::vector<uint64_t> groupsToRead;
-            for (uint64_t i = 0; i < nodeMappingReader->getNumRowsGroups(); ++i) {
-                groupsToRead.push_back(i);
-            }
-
-            ParquetReaderScanState scanState;
-            nodeMappingReader->initializeScan(scanState, groupsToRead, vfs);
-
-            // Check if the node mapping file has columns
-            auto numColumns = nodeMappingReader->getNumColumns();
-            if (numColumns < 2) {
-                throw RuntimeException("Node mapping parquet file must have at least 2 columns");
-            }
-
-            // Validate column types for node mapping
-            const auto& csrNodeIdType = nodeMappingReader->getColumnType(0);
-            const auto& nodeTableIdType = nodeMappingReader->getColumnType(1);
-            if (!LogicalTypeUtils::isIntegral(csrNodeIdType.getLogicalTypeID()) ||
-                !LogicalTypeUtils::isIntegral(nodeTableIdType.getLogicalTypeID())) {
-                throw RuntimeException(
-                    "Node mapping parquet file columns must be integer types (columns 0 and 1)");
-            }
-
-            // Read the node mapping data
-            DataChunk dataChunk(2);
-
-            // Get column types
-            for (uint32_t i = 0; i < 2 && i < numColumns; ++i) {
-                const auto& columnTypeRef = nodeMappingReader->getColumnType(i);
-                auto columnType = columnTypeRef.copy();
-                auto vector = std::make_shared<ValueVector>(std::move(columnType));
-                dataChunk.insert(i, vector);
-            }
-
-            // Read all node mapping values
-            while (nodeMappingReader->scanInternal(scanState, dataChunk)) {
-                auto selSize = dataChunk.state->getSelVector().getSelSize();
-                for (size_t i = 0; i < selSize; ++i) {
-                    auto csrNodeId = dataChunk.getValueVector(0).getValue<common::offset_t>(i);
-                    auto nodeTableId = dataChunk.getValueVector(1).getValue<common::offset_t>(i);
-                    nodeMapping[common::internalID_t(nodeTableId, getFromNodeTableID())] =
-                        csrNodeId;
-                    // Also create reverse mapping for destination node lookups
-                    csrToNodeTableIdMap[csrNodeId] = nodeTableId;
-                }
-            }
-        }
-    }
-}
-
 void ParquetRelTable::loadIndptrData(Transaction* transaction) const {
     if (indptrData.empty() && !indptrFilePath.empty()) {
         std::lock_guard lock(indptrDataMutex);
@@ -278,9 +198,7 @@ bool ParquetRelTable::scanInternal(Transaction* transaction, TableScanState& sca
     // Get the ParquetRelTableScanState
     auto& parquetRelScanState = static_cast<ParquetRelTableScanState&>(relScanState);
 
-    // Readers are now initialized per scan state in initScanState
-    // Load shared data (node mapping and indptr) - these are thread-safe to read
-    loadNodeMappingData(transaction);
+    // Load shared indptr data - thread-safe to read
     if (!indptrFilePath.empty()) {
         loadIndptrData(transaction);
     }

package/lbug-source/test/test_files/demo_db/demo_db_graph_std.test

@@ -6,14 +6,11 @@
 
 -LOG MatchUserLivesInCity
 -STATEMENT MATCH (u:user)-[l:livesin]->(c:city) RETURN u.name, u.age, c.name;
----- 7
-Adam|30|Guelph
-Adam|30|Guelph
+---- 4
+Adam|30|Waterloo
 Karissa|40|Waterloo
-Noura|25|Kitchener
-Noura|25|Waterloo
-Noura|25|Waterloo
 Zhang|50|Kitchener
+Noura|25|Guelph
 
 -LOG MatchSingleNodeLabel
 -STATEMENT MATCH (a:user) RETURN a.name, a.age;
@@ -32,12 +29,49 @@ Guelph|75000
 
 -LOG MatchFollowsRel
 -STATEMENT MATCH (a:user)-[e:follows]->(b:user) RETURN a.name, b.name, e.since;
----- 8
-Adam|Zhang|2022
-Karissa|Noura|2020
-Karissa|Zhang|2020
-Noura|Karissa|2020
-Noura|Zhang|2021
-Zhang|Adam|2022
-Zhang|Karissa|2020
-Zhang|Noura|2021
+---- 4
+Adam|Karissa|2020
+Adam|Zhang|2020
+Karissa|Zhang|2021
+Zhang|Noura|2022
+
+-LOG MatchLivesInWithCityPopulation
+-STATEMENT MATCH (u:user)-[l:livesin]->(c:city) RETURN u.name, c.name, c.population ORDER BY c.population DESC;
+---- 4
+Zhang|Kitchener|200000
+Adam|Waterloo|150000
+Karissa|Waterloo|150000
+Noura|Guelph|75000
+
+-LOG MatchLivesInFilterByCity
+-STATEMENT MATCH (u:user)-[l:livesin]->(c:city) WHERE c.name = 'Waterloo' RETURN u.name, u.age;
+---- 2
+Adam|30
+Karissa|40
+
+-LOG MatchLivesInFilterByCityPopulation
+-STATEMENT MATCH (u:user)-[l:livesin]->(c:city) WHERE c.population > 100000 RETURN u.name, c.name ORDER BY u.name;
+---- 3
+Adam|Waterloo
+Karissa|Waterloo
+Zhang|Kitchener
+
+-LOG CountUsersPerCity
+-STATEMENT MATCH (u:user)-[l:livesin]->(c:city) RETURN c.name, COUNT(*) AS num_users ORDER BY num_users DESC;
+---- 3
+Waterloo|2
+Guelph|1
+Kitchener|1
+
+-LOG MatchFollowsWithDestinationAge
+-STATEMENT MATCH (a:user)-[e:follows]->(b:user) WHERE b.age > 30 RETURN a.name, b.name, b.age ORDER BY b.age DESC;
+---- 3
+Adam|Zhang|50
+Karissa|Zhang|50
+Adam|Karissa|40
+
+-LOG MatchFollowsFilterBySourceAndDest
+-STATEMENT MATCH (a:user)-[e:follows]->(b:user) WHERE a.age < 40 AND b.age >= 40 RETURN a.name, b.name;
+---- 2
+Adam|Karissa
+Adam|Zhang

package/lbug-source/test/test_helper/test_helper.cpp

@@ -32,6 +32,7 @@ void TestHelper::executeScript(const std::string& cypherScript, Connection& conn
         std::cout << "cypherScript: " << cypherScript << " doesn't exist. Skipping..." << std::endl;
         return;
     }
+    auto cypherDir = std::filesystem::path(cypherScript).parent_path();
     std::ifstream file(cypherScript);
     if (!file.is_open()) {
         throw Exception(stringFormat("Error opening file: {}, errno: {}.", cypherScript, errno));
@@ -68,7 +69,14 @@ void TestHelper::executeScript(const std::string& cypherScript, Connection& conn
             index = end + 1;
         }
         for (auto& csvFilePath : csvFilePaths) {
-            auto fullPath = appendLbugRootPath(csvFilePath);
+            std::string fullPath = csvFilePath;
+            if (std::filesystem::path(csvFilePath).is_relative()) {
+                if (std::filesystem::path(csvFilePath).parent_path().empty()) {
+                    fullPath = (cypherDir / csvFilePath).string();
+                } else {
+                    fullPath = appendLbugRootPath(csvFilePath);
+                }
+            }
             line.replace(line.find(csvFilePath), csvFilePath.length(), fullPath);
         }
         // Also handle storage = 'path' for parquet tables

package/lbug-source/tools/shell/embedded_shell.cpp

@@ -271,6 +271,22 @@ void completion(const char* buffer, linenoiseCompletions* lc) {
         return;
     }
 
+    // RETURN *; completion for MATCH and CALL queries.
+    // Trigger when buffer ends with ')' or ') ' after a MATCH pattern or CALL function.
+    if (regex_search(buf, std::regex(R"(\)\s*$)"))) {
+        // Check for MATCH pattern: MATCH(var:Table) or MATCH (var:Table)
+        bool isMatchQuery =
+            regex_search(buf, std::regex(R"(^\s*MATCH\s*\()", std::regex_constants::icase));
+        // Check for CALL function: CALL func_name(...) or CALL func_name (...)
+        bool isCallFunction =
+            regex_search(buf, std::regex(R"(^\s*CALL\s+\w+\s*\()", std::regex_constants::icase));
+        if (isMatchQuery || isCallFunction) {
+            std::string suffix = buf.back() == ')' ? " RETURN *;" : "RETURN *;";
+            linenoiseAddCompletion(lc, (buf + suffix).c_str());
+            return;
+        }
+    }
+
     // Node table name completion. Match patterns that include an open bracket `(` with no closing
     // bracket `)`, and a colon `:` sometime after the open bracket.
     if (regex_search(buf, std::regex("^[^]*\\([^\\)]*:[^\\)]*$"))) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lbug",
-  "version": "0.12.3-dev.17",
+  "version": "0.12.3-dev.19",
   "description": "An in-process property graph database management system built for query speed and scalability.",
   "main": "index.js",
   "module": "./index.mjs",