lbug 0.12.3-dev.15 → 0.12.3-dev.17

package/lbug-source/src/include/catalog/catalog_entry/rel_group_catalog_entry.h

@@ -34,10 +34,10 @@ public:
     RelGroupCatalogEntry() = default;
     RelGroupCatalogEntry(std::string tableName, common::RelMultiplicity srcMultiplicity,
         common::RelMultiplicity dstMultiplicity, common::ExtendDirection storageDirection,
-        std::vector<RelTableCatalogInfo> relTableInfos)
+        std::vector<RelTableCatalogInfo> relTableInfos, std::string storage = "")
         : TableCatalogEntry{type_, std::move(tableName)}, srcMultiplicity{srcMultiplicity},
           dstMultiplicity{dstMultiplicity}, storageDirection{storageDirection},
-          relTableInfos{std::move(relTableInfos)} {
+          relTableInfos{std::move(relTableInfos)}, storage{std::move(storage)} {
         propertyCollection =
             PropertyDefinitionCollection{1}; // Skip NBR_NODE_ID column as the first one.
     }
@@ -53,6 +53,7 @@ public:
     }
 
     common::ExtendDirection getStorageDirection() const { return storageDirection; }
+    const std::string& getStorage() const { return storage; }
 
     common::idx_t getNumRelTables() const { return relTableInfos.size(); }
     const std::vector<RelTableCatalogInfo>& getRelEntryInfos() const { return relTableInfos; }
@@ -97,6 +98,7 @@ private:
     // TODO(Guodong): Avoid using extend direction for storage direction
     common::ExtendDirection storageDirection = common::ExtendDirection::BOTH;
     std::vector<RelTableCatalogInfo> relTableInfos;
+    std::string storage;
 };
 
 } // namespace catalog

package/lbug-source/src/include/common/constants.h

@@ -83,6 +83,7 @@ struct StorageConstants {
 
 struct TableOptionConstants {
     static constexpr char REL_STORAGE_DIRECTION_OPTION[] = "STORAGE_DIRECTION";
+    static constexpr char REL_STORAGE_OPTION[] = "STORAGE";
 };
 
 // Hash Index Configurations

package/lbug-source/src/include/function/table/simple_table_function.h

@@ -134,6 +134,12 @@ struct FileInfoFunction final {
     static function_set getFunctionSet();
 };
 
+struct DiskSizeInfoFunction final {
+    static constexpr const char* name = "DISK_SIZE_INFO";
+
+    static function_set getFunctionSet();
+};
+
 struct ShowAttachedDatabasesFunction final {
     static constexpr const char* name = "SHOW_ATTACHED_DATABASES";
 

package/lbug-source/src/include/parser/ddl/create_table_info.h

@@ -36,8 +36,10 @@ struct CreateTableInfo {
 
 struct ExtraCreateNodeTableInfo final : ExtraCreateTableInfo {
     std::string pKName;
+    options_t options;
 
-    explicit ExtraCreateNodeTableInfo(std::string pKName) : pKName{std::move(pKName)} {}
+    explicit ExtraCreateNodeTableInfo(std::string pKName, options_t options = {})
+        : pKName{std::move(pKName)}, options{std::move(options)} {}
 };
 
 struct ExtraCreateRelTableGroupInfo final : ExtraCreateTableInfo {

package/lbug-source/src/include/processor/operator/scan/scan_node_table.h

@@ -24,7 +24,7 @@ public:
     void initialize(const transaction::Transaction* transaction, storage::NodeTable* table,
         ScanNodeTableProgressSharedState& progressSharedState);
 
-    void nextMorsel(storage::NodeTableScanState& scanState,
+    void nextMorsel(storage::TableScanState& scanState,
         ScanNodeTableProgressSharedState& progressSharedState);
 
     common::SemiMask* getSemiMask() const { return semiMask.get(); }
@@ -116,7 +116,7 @@ private:
 
 private:
     common::idx_t currentTableIdx;
-    std::unique_ptr<storage::NodeTableScanState> scanState;
+    std::unique_ptr<storage::TableScanState> scanState;
     std::vector<ScanNodeTableInfo> tableInfos;
     std::vector<std::shared_ptr<ScanNodeTableSharedState>> sharedStates;
     std::shared_ptr<ScanNodeTableProgressSharedState> progressSharedState;

package/lbug-source/src/include/storage/storage_manager.h

@@ -97,6 +97,7 @@ private:
     bool enableCompression;
     bool inMemory;
     std::vector<IndexType> registeredIndexTypes;
+    std::unordered_map<common::table_id_t, std::string> tableNameCache;
 };
 
 } // namespace storage

package/lbug-source/src/include/storage/table/node_table.h

@@ -107,7 +107,7 @@ private:
 
 class StorageManager;
 
-class LBUG_API NodeTable final : public Table {
+class LBUG_API NodeTable : public Table {
 public:
     NodeTable(const StorageManager* storageManager,
         const catalog::NodeTableCatalogEntry* nodeTableEntry, MemoryManager* mm);
@@ -119,6 +119,11 @@ public:
     void initScanState(transaction::Transaction* transaction, TableScanState& scanState,
         common::table_id_t tableID, common::offset_t startOffset) const;
 
+    // Virtual method for operator-level scan coordination initialization
+    // Called once per scan operation (not per scan state)
+    virtual void initializeScanCoordination(
+        [[maybe_unused]] const transaction::Transaction* transaction) {}
+
     bool scanInternal(transaction::Transaction* transaction, TableScanState& scanState) override;
     template<bool lock = true>
     bool lookup(const transaction::Transaction* transaction, const TableScanState& scanState) const;

package/lbug-source/src/include/storage/table/parquet_node_table.h

@@ -0,0 +1,103 @@
+#pragma once
+
+#include <mutex>
+#include <vector>
+
+#include "catalog/catalog_entry/node_table_catalog_entry.h"
+#include "common/exception/runtime.h"
+#include "common/types/internal_id_util.h"
+#include "common/types/value/value.h"
+#include "processor/operator/persistent/reader/parquet/parquet_reader.h"
+#include "storage/table/node_table.h"
+
+namespace lbug {
+namespace storage {
+
+struct ParquetNodeTableScanState final : NodeTableScanState {
+    std::unique_ptr<processor::ParquetReader> parquetReader;
+    std::unique_ptr<processor::ParquetReaderScanState> parquetScanState;
+    bool initialized = false;
+    bool scanCompleted = false; // Track if this scan state has finished reading
+    bool dataRead = false;
+    std::vector<std::vector<std::unique_ptr<common::Value>>> allData;
+    size_t totalRows = 0;
+    size_t nextRowToDistribute = 0;
+    uint64_t lastQueryId = 0; // Track the last query ID to detect new queries
+
+    ParquetNodeTableScanState([[maybe_unused]] MemoryManager& mm, common::ValueVector* nodeIDVector,
+        std::vector<common::ValueVector*> outputVectors,
+        std::shared_ptr<common::DataChunkState> outChunkState)
+        : NodeTableScanState{nodeIDVector, std::move(outputVectors), std::move(outChunkState)} {
+        parquetScanState = std::make_unique<processor::ParquetReaderScanState>();
+    }
+};
+
+// Shared state to coordinate row group assignment across parallel scan states
+struct ParquetNodeTableSharedState {
+    std::mutex mtx;
+    common::node_group_idx_t currentRowGroupIdx = 0;
+    common::node_group_idx_t numRowGroups = 0;
+
+    void reset(common::node_group_idx_t totalRowGroups) {
+        std::lock_guard<std::mutex> lock(mtx);
+        currentRowGroupIdx = 0;
+        numRowGroups = totalRowGroups;
+    }
+
+    bool getNextRowGroup(common::node_group_idx_t& assignedRowGroupIdx) {
+        std::lock_guard<std::mutex> lock(mtx);
+        if (currentRowGroupIdx < numRowGroups) {
+            assignedRowGroupIdx = currentRowGroupIdx++;
+            return true;
+        }
+        return false;
+    }
+};
+
+class ParquetNodeTable final : public NodeTable {
+public:
+    ParquetNodeTable(const StorageManager* storageManager,
+        const catalog::NodeTableCatalogEntry* nodeTableEntry, MemoryManager* memoryManager);
+
+    void initScanState(transaction::Transaction* transaction, TableScanState& scanState,
+        bool resetCachedBoundNodeSelVec = true) const override;
+
+    // Override to reset shared state for row group coordination at the start of each scan operation
+    void initializeScanCoordination(const transaction::Transaction* transaction) override;
+
+    bool scanInternal(transaction::Transaction* transaction, TableScanState& scanState) override;
+
+    // For parquet-backed tables, we don't support modifications
+    void insert([[maybe_unused]] transaction::Transaction* transaction,
+        [[maybe_unused]] TableInsertState& insertState) override {
+        throw common::RuntimeException("Cannot insert into parquet-backed node table");
+    }
+    void update([[maybe_unused]] transaction::Transaction* transaction,
+        [[maybe_unused]] TableUpdateState& updateState) override {
+        throw common::RuntimeException("Cannot update parquet-backed node table");
+    }
+    bool delete_([[maybe_unused]] transaction::Transaction* transaction,
+        [[maybe_unused]] TableDeleteState& deleteState) override {
+        throw common::RuntimeException("Cannot delete from parquet-backed node table");
+        return false;
+    }
+
+    common::row_idx_t getNumTotalRows(const transaction::Transaction* transaction) override;
+
+    const std::string& getParquetFilePath() const { return parquetFilePath; }
+
+    // Note: Cannot override getNumCommittedNodeGroups since it's not virtual in base class
+    // Will need a different approach
+
+private:
+    std::string parquetFilePath;
+    const catalog::NodeTableCatalogEntry* nodeTableCatalogEntry;
+    mutable std::unique_ptr<ParquetNodeTableSharedState> sharedState;
+
+    void initializeParquetReader(transaction::Transaction* transaction) const;
+    void initParquetScanForRowGroup(transaction::Transaction* transaction,
+        ParquetNodeTableScanState& scanState) const;
+};
+
+} // namespace storage
+} // namespace lbug

package/lbug-source/src/include/storage/table/parquet_rel_table.h

@@ -0,0 +1,99 @@
+#pragma once
+
+#include "catalog/catalog_entry/rel_group_catalog_entry.h"
+#include "common/exception/runtime.h"
+#include "common/types/internal_id_util.h"
+#include "processor/operator/persistent/reader/parquet/parquet_reader.h"
+#include "storage/table/rel_table.h"
+#include "transaction/transaction.h"
+
+namespace lbug {
+namespace storage {
+
+struct ParquetRelTableScanState final : RelTableScanState {
+    std::unique_ptr<processor::ParquetReaderScanState> parquetScanState;
+    // For CSR format: store matching rows for current bound node
+    size_t nextRowToProcess = 0;
+
+    // Row group range for morsel-driven parallelism
+    uint64_t startRowGroup = 0;
+    uint64_t endRowGroup = 0;
+    uint64_t currentRowGroup = 0;
+
+    // Per-scan-state readers for thread safety
+    std::unique_ptr<processor::ParquetReader> nodeMappingReader;
+    std::unique_ptr<processor::ParquetReader> indicesReader;
+    std::unique_ptr<processor::ParquetReader> indptrReader;
+
+    ParquetRelTableScanState(MemoryManager& mm, common::ValueVector* nodeIDVector,
+        std::vector<common::ValueVector*> outputVectors,
+        std::shared_ptr<common::DataChunkState> outChunkState)
+        : RelTableScanState{mm, nodeIDVector, std::move(outputVectors), std::move(outChunkState)} {
+        parquetScanState = std::make_unique<processor::ParquetReaderScanState>();
+    }
+
+    void setToTable(const transaction::Transaction* transaction, Table* table_,
+        std::vector<common::column_id_t> columnIDs_,
+        std::vector<ColumnPredicateSet> columnPredicateSets_,
+        common::RelDataDirection direction_) override;
+};
+
+class ParquetRelTable final : public RelTable {
+public:
+    ParquetRelTable(catalog::RelGroupCatalogEntry* relGroupEntry, common::table_id_t fromTableID,
+        common::table_id_t toTableID, const StorageManager* storageManager,
+        MemoryManager* memoryManager, std::string fromNodeTableName);
+
+    void initScanState(transaction::Transaction* transaction, TableScanState& scanState,
+        bool resetCachedBoundNodeSelVec = true) const override;
+
+    bool scanInternal(transaction::Transaction* transaction, TableScanState& scanState) override;
+
+    // For parquet-backed tables, we don't support modifications
+    void insert([[maybe_unused]] transaction::Transaction* transaction,
+        [[maybe_unused]] TableInsertState& insertState) override {
+        throw common::RuntimeException("Cannot insert into parquet-backed rel table");
+    }
+    void update([[maybe_unused]] transaction::Transaction* transaction,
+        [[maybe_unused]] TableUpdateState& updateState) override {
+        throw common::RuntimeException("Cannot update parquet-backed rel table");
+    }
+    bool delete_([[maybe_unused]] transaction::Transaction* transaction,
+        [[maybe_unused]] TableDeleteState& deleteState) override {
+        throw common::RuntimeException("Cannot delete from parquet-backed rel table");
+        return false;
+    }
+
+    common::row_idx_t getNumTotalRows(const transaction::Transaction* transaction) override;
+
+private:
+    catalog::RelGroupCatalogEntry* relGroupEntry; // Store reference to table schema
+    std::string nodeMappingFilePath;
+    std::string indicesFilePath;
+    std::string indptrFilePath;
+    mutable std::unique_ptr<processor::ParquetReader> nodeMappingReader;
+    mutable std::unique_ptr<processor::ParquetReader> indicesReader;
+    mutable std::unique_ptr<processor::ParquetReader> indptrReader;
+    mutable std::mutex parquetReaderMutex;
+    mutable std::mutex indptrDataMutex;
+    mutable std::vector<common::offset_t> indptrData; // Cached indptr data for CSR format
+    mutable common::internal_id_map_t<common::offset_t>
+        nodeMapping; // Maps node IDs to CSR node IDs
+    mutable std::unordered_map<common::offset_t, common::offset_t>
+        csrToNodeTableIdMap; // Reverse mapping: CSR node ID to node table ID
+
+    void initializeParquetReaders(transaction::Transaction* transaction) const;
+    void initializeIndptrReader(transaction::Transaction* transaction) const;
+    void loadIndptrData(transaction::Transaction* transaction) const;
+    void loadNodeMappingData(transaction::Transaction* transaction) const;
+    bool scanInternalByRowGroups(transaction::Transaction* transaction,
+        ParquetRelTableScanState& parquetRelScanState);
+    bool scanRowGroupForBoundNodes(transaction::Transaction* transaction,
+        ParquetRelTableScanState& parquetRelScanState,
+        const std::vector<uint64_t>& rowGroupsToProcess,
+        const std::unordered_set<common::offset_t>& boundNodeOffsets);
+    common::offset_t findSourceNodeForRow(common::offset_t globalRowIdx) const;
+};
+
+} // namespace storage
+} // namespace lbug

package/lbug-source/src/include/storage/table/rel_table.h

@@ -48,7 +48,7 @@ struct RelTableScanState : TableScanState {
         nodeGroupScanState = std::make_unique<CSRNodeGroupScanState>();
     }
 
-    void setToTable(const transaction::Transaction* transaction, Table* table_,
+    virtual void setToTable(const transaction::Transaction* transaction, Table* table_,
         std::vector<common::column_id_t> columnIDs_,
         std::vector<ColumnPredicateSet> columnPredicateSets_,
         common::RelDataDirection direction_) override;
@@ -138,7 +138,7 @@ struct LBUG_API RelTableDeleteState final : TableDeleteState {
           relIDVector{relIDVector}, detachDeleteDirection{detachDeleteDirection} {}
 };
 
-class LBUG_API RelTable final : public Table {
+class LBUG_API RelTable : public Table {
 public:
     using rel_multiplicity_constraint_throw_func_t =
         std::function<void(const std::string&, common::offset_t, common::RelDataDirection)>;

package/lbug-source/src/include/transaction/transaction.h

@@ -130,6 +130,8 @@ public:
         return getMinUncommittedNodeOffset(tableID) + localRowIdx;
     }
 
+    main::ClientContext* getClientContext() const { return clientContext; }
+
     void pushCreateDropCatalogEntry(catalog::CatalogSet& catalogSet,
         catalog::CatalogEntry& catalogEntry, bool isInternal, bool skipLoggingToWAL = false);
     void pushAlterCatalogEntry(catalog::CatalogSet& catalogSet, catalog::CatalogEntry& catalogEntry,

package/lbug-source/src/parser/transform/transform_ddl.cpp

@@ -81,7 +81,12 @@ std::unique_ptr<Statement> Transformer::transformCreateNodeTable(
     } else {
         createTableInfo.propertyDefinitions =
             transformPropertyDefinitions(*ctx.kU_PropertyDefinitions());
-        createTableInfo.extraInfo = std::make_unique<ExtraCreateNodeTableInfo>(getPKName(ctx));
+        options_t options;
+        if (ctx.kU_Options()) {
+            options = transformOptions(*ctx.kU_Options());
+        }
+        createTableInfo.extraInfo =
+            std::make_unique<ExtraCreateNodeTableInfo>(getPKName(ctx), std::move(options));
         return std::make_unique<CreateTable>(std::move(createTableInfo));
     }
 }

package/lbug-source/src/processor/operator/persistent/reader/parquet/parquet_reader.cpp

@@ -340,6 +340,10 @@ std::unique_ptr<ColumnReader> ParquetReader::createReader() {
         throw CopyException{"Root element of Parquet file must be a struct"};
     }
     // LCOV_EXCL_STOP
+    // Clear existing column metadata before populating (in case createReader is called multiple
+    // times)
+    columnNames.clear();
+    columnTypes.clear();
     for (auto& field : StructType::getFields(rootReader->getDataType())) {
         columnNames.push_back(field.getName());
         columnTypes.push_back(field.getType().copy());

package/lbug-source/src/processor/operator/scan/scan_multi_rel_tables.cpp

@@ -2,6 +2,7 @@
 
 #include "processor/execution_context.h"
 #include "storage/local_storage/local_storage.h"
+#include "storage/table/parquet_rel_table.h"
 
 using namespace lbug::common;
 using namespace lbug::storage;
@@ -54,8 +55,29 @@ void ScanMultiRelTable::initLocalStateInternal(ResultSet* resultSet, ExecutionCo
     auto clientContext = context->clientContext;
     boundNodeIDVector = resultSet->getValueVector(opInfo.nodeIDPos).get();
     auto nbrNodeIDVector = outVectors[0];
-    scanState = std::make_unique<RelTableScanState>(*MemoryManager::Get(*clientContext),
-        boundNodeIDVector, outVectors, nbrNodeIDVector->state);
+
+    // Check if any table in any scanner is a ParquetRelTable
+    bool hasParquetTable = false;
+    for (auto& [_, scanner] : scanners) {
+        for (auto& relInfo : scanner.relInfos) {
+            if (dynamic_cast<storage::ParquetRelTable*>(relInfo.table) != nullptr) {
+                hasParquetTable = true;
+                break;
+            }
+        }
+        if (hasParquetTable)
+            break;
+    }
+
+    // Create appropriate scan state type
+    if (hasParquetTable) {
+        scanState =
+            std::make_unique<storage::ParquetRelTableScanState>(*MemoryManager::Get(*clientContext),
+                boundNodeIDVector, outVectors, nbrNodeIDVector->state);
+    } else {
+        scanState = std::make_unique<RelTableScanState>(*MemoryManager::Get(*clientContext),
+            boundNodeIDVector, outVectors, nbrNodeIDVector->state);
+    }
     for (auto& [_, scanner] : scanners) {
         for (auto& relInfo : scanner.relInfos) {
             if (directionInfo.directionPos.isValid()) {

package/lbug-source/src/processor/operator/scan/scan_node_table.cpp

@@ -2,8 +2,10 @@
 
 #include "binder/expression/expression_util.h"
 #include "processor/execution_context.h"
+#include "storage/buffer_manager/memory_manager.h"
 #include "storage/local_storage/local_node_table.h"
 #include "storage/local_storage/local_storage.h"
+#include "storage/table/parquet_node_table.h"
 
 using namespace lbug::common;
 using namespace lbug::storage;
@@ -35,7 +37,23 @@ void ScanNodeTableSharedState::initialize(const transaction::Transaction* transa
     this->table = table;
     this->currentCommittedGroupIdx = 0;
     this->currentUnCommittedGroupIdx = 0;
-    this->numCommittedNodeGroups = table->getNumCommittedNodeGroups();
+
+    // Initialize table-specific scan coordination (e.g., for ParquetNodeTable)
+    table->initializeScanCoordination(transaction);
+
+    if (const auto parquetTable = dynamic_cast<ParquetNodeTable*>(table)) {
+        // For parquet tables, set numCommittedNodeGroups to number of row groups
+        std::vector<bool> columnSkips;
+        try {
+            auto tempReader = std::make_unique<processor::ParquetReader>(
+                parquetTable->getParquetFilePath(), columnSkips, transaction->getClientContext());
+            this->numCommittedNodeGroups = tempReader->getNumRowsGroups();
+        } catch (const std::exception& e) {
+            this->numCommittedNodeGroups = 1;
+        }
+    } else {
+        this->numCommittedNodeGroups = table->getNumCommittedNodeGroups();
+    }
     if (transaction->isWriteTransaction()) {
         if (const auto localTable =
                 transaction->getLocalStorage()->getLocalTable(this->table->getTableID())) {
@@ -46,21 +64,23 @@ void ScanNodeTableSharedState::initialize(const transaction::Transaction* transa
     progressSharedState.numGroups += numCommittedNodeGroups;
 }
 
-void ScanNodeTableSharedState::nextMorsel(NodeTableScanState& scanState,
+void ScanNodeTableSharedState::nextMorsel(TableScanState& scanState,
     ScanNodeTableProgressSharedState& progressSharedState) {
     std::unique_lock lck{mtx};
+    // Cast to NodeTableScanState since we know this is for node tables
+    auto& nodeScanState = scanState.cast<NodeTableScanState>();
     if (currentCommittedGroupIdx < numCommittedNodeGroups) {
-        scanState.nodeGroupIdx = currentCommittedGroupIdx++;
+        nodeScanState.nodeGroupIdx = currentCommittedGroupIdx++;
         progressSharedState.numGroupsScanned++;
-        scanState.source = TableScanSource::COMMITTED;
+        nodeScanState.source = TableScanSource::COMMITTED;
         return;
     }
     if (currentUnCommittedGroupIdx < numUnCommittedNodeGroups) {
-        scanState.nodeGroupIdx = currentUnCommittedGroupIdx++;
-        scanState.source = TableScanSource::UNCOMMITTED;
+        nodeScanState.nodeGroupIdx = currentUnCommittedGroupIdx++;
+        nodeScanState.source = TableScanSource::UNCOMMITTED;
         return;
     }
-    scanState.source = TableScanSource::NONE;
+    nodeScanState.source = TableScanSource::NONE;
 }
 
 table_id_map_t<SemiMask*> ScanNodeTable::getSemiMasks() const {
@@ -82,7 +102,18 @@ void ScanNodeTableInfo::initScanState(TableScanState& scanState,
 void ScanNodeTable::initLocalStateInternal(ResultSet* resultSet, ExecutionContext* context) {
     ScanTable::initLocalStateInternal(resultSet, context);
     auto nodeIDVector = resultSet->getValueVector(opInfo.nodeIDPos).get();
-    scanState = std::make_unique<NodeTableScanState>(nodeIDVector, outVectors, nodeIDVector->state);
+
+    // Check if the first table is a ParquetNodeTable and create appropriate scan state
+    auto* parquetTable = dynamic_cast<ParquetNodeTable*>(tableInfos[0].table);
+    if (parquetTable) {
+        scanState = std::make_unique<ParquetNodeTableScanState>(
+            *MemoryManager::Get(*context->clientContext), nodeIDVector, outVectors,
+            nodeIDVector->state);
+    } else {
+        scanState =
+            std::make_unique<NodeTableScanState>(nodeIDVector, outVectors, nodeIDVector->state);
+    }
+
     currentTableIdx = 0;
     initCurrentTable(context);
 }
@@ -91,6 +122,11 @@ void ScanNodeTable::initCurrentTable(ExecutionContext* context) {
     auto& currentInfo = tableInfos[currentTableIdx];
     currentInfo.initScanState(*scanState, outVectors, context->clientContext);
     scanState->semiMask = sharedStates[currentTableIdx]->getSemiMask();
+    // Call table->initScanState for ParquetNodeTable
+    if (dynamic_cast<ParquetNodeTable*>(tableInfos[currentTableIdx].table)) {
+        auto transaction = transaction::Transaction::Get(*context->clientContext);
+        tableInfos[currentTableIdx].table->initScanState(transaction, *scanState);
+    }
 }
 
 void ScanNodeTable::initGlobalStateInternal(ExecutionContext* context) {

package/lbug-source/src/processor/operator/scan/scan_rel_table.cpp

@@ -2,7 +2,9 @@
 
 #include "binder/expression/expression_util.h"
 #include "processor/execution_context.h"
+#include "storage/buffer_manager/memory_manager.h"
 #include "storage/local_storage/local_rel_table.h"
+#include "storage/table/parquet_rel_table.h"
 
 using namespace lbug::common;
 using namespace lbug::storage;
@@ -66,8 +68,16 @@ void ScanRelTable::initLocalStateInternal(ResultSet* resultSet, ExecutionContext
     auto clientContext = context->clientContext;
     auto boundNodeIDVector = resultSet->getValueVector(opInfo.nodeIDPos).get();
     auto nbrNodeIDVector = outVectors[0];
-    scanState = std::make_unique<RelTableScanState>(*MemoryManager::Get(*clientContext),
-        boundNodeIDVector, outVectors, nbrNodeIDVector->state);
+    // Check if this is a ParquetRelTable and create appropriate scan state
+    auto* parquetTable = dynamic_cast<storage::ParquetRelTable*>(tableInfo.table);
+    if (parquetTable) {
+        scanState =
+            std::make_unique<storage::ParquetRelTableScanState>(*MemoryManager::Get(*clientContext),
+                boundNodeIDVector, outVectors, nbrNodeIDVector->state);
+    } else {
+        scanState = std::make_unique<RelTableScanState>(*MemoryManager::Get(*clientContext),
+            boundNodeIDVector, outVectors, nbrNodeIDVector->state);
+    }
     tableInfo.initScanState(*scanState, outVectors, clientContext);
 }
 

package/lbug-source/src/storage/storage_manager.cpp

@@ -13,6 +13,8 @@
 #include "storage/buffer_manager/memory_manager.h"
 #include "storage/checkpointer.h"
 #include "storage/table/node_table.h"
+#include "storage/table/parquet_node_table.h"
+#include "storage/table/parquet_rel_table.h"
 #include "storage/table/rel_table.h"
 #include "storage/wal/wal_replayer.h"
 #include "transaction/transaction.h"
@@ -77,15 +79,31 @@ void StorageManager::recover(main::ClientContext& clientContext, bool throwOnWal
 }
 
 void StorageManager::createNodeTable(NodeTableCatalogEntry* entry) {
-    tables[entry->getTableID()] = std::make_unique<NodeTable>(this, entry, &memoryManager);
+    tableNameCache[entry->getTableID()] = entry->getName();
+    if (!entry->getStorage().empty()) {
+        // Create parquet-backed node table
+        tables[entry->getTableID()] =
+            std::make_unique<ParquetNodeTable>(this, entry, &memoryManager);
+    } else {
+        // Create regular node table
+        tables[entry->getTableID()] = std::make_unique<NodeTable>(this, entry, &memoryManager);
+    }
 }
 
 // TODO(Guodong): This API is added since storageManager doesn't provide an API to add a single
 // rel table. We may have to refactor the existing StorageManager::createTable(TableCatalogEntry*
 // entry).
 void StorageManager::addRelTable(RelGroupCatalogEntry* entry, const RelTableCatalogInfo& info) {
-    tables[info.oid] = std::make_unique<RelTable>(entry, info.nodePair.srcTableID,
-        info.nodePair.dstTableID, this, &memoryManager);
+    if (!entry->getStorage().empty()) {
+        // Create parquet-backed rel table
+        std::string fromNodeTableName = tableNameCache.at(info.nodePair.srcTableID);
+        tables[info.oid] = std::make_unique<ParquetRelTable>(entry, info.nodePair.srcTableID,
+            info.nodePair.dstTableID, this, &memoryManager, fromNodeTableName);
+    } else {
+        // Create regular rel table
+        tables[info.oid] = std::make_unique<RelTable>(entry, info.nodePair.srcTableID,
+            info.nodePair.dstTableID, this, &memoryManager);
+    }
 }
 
 void StorageManager::createRelTableGroup(RelGroupCatalogEntry* entry) {
@@ -257,7 +275,14 @@ void StorageManager::deserialize(main::ClientContext* context, const Catalog* ca
         KU_ASSERT(!tables.contains(tableID));
         auto tableEntry = catalog->getTableCatalogEntry(&DUMMY_TRANSACTION, tableID)
                               ->ptrCast<NodeTableCatalogEntry>();
-        tables[tableID] = std::make_unique<NodeTable>(this, tableEntry, &memoryManager);
+        tableNameCache[tableID] = tableEntry->getName();
+        if (!tableEntry->getStorage().empty()) {
+            // Create parquet-backed node table
+            tables[tableID] = std::make_unique<ParquetNodeTable>(this, tableEntry, &memoryManager);
+        } else {
+            // Create regular node table
+            tables[tableID] = std::make_unique<NodeTable>(this, tableEntry, &memoryManager);
+        }
         tables[tableID]->deserialize(context, this, deSer);
     }
     deSer.validateDebuggingInfo(key, "num_rel_groups");
@@ -279,8 +304,17 @@ void StorageManager::deserialize(main::ClientContext* context, const Catalog* ca
         for (auto k = 0u; k < numInnerRelTables; k++) {
             RelTableCatalogInfo info = RelTableCatalogInfo::deserialize(deSer);
             KU_ASSERT(!tables.contains(info.oid));
-            tables[info.oid] = std::make_unique<RelTable>(relGroupEntry, info.nodePair.srcTableID,
-                info.nodePair.dstTableID, this, &memoryManager);
+            if (!relGroupEntry->getStorage().empty()) {
+                // Create parquet-backed rel table
+                std::string fromNodeTableName = tableNameCache.at(info.nodePair.srcTableID);
+                tables[info.oid] =
+                    std::make_unique<ParquetRelTable>(relGroupEntry, info.nodePair.srcTableID,
+                        info.nodePair.dstTableID, this, &memoryManager, fromNodeTableName);
+            } else {
+                // Create regular rel table
+                tables[info.oid] = std::make_unique<RelTable>(relGroupEntry,
+                    info.nodePair.srcTableID, info.nodePair.dstTableID, this, &memoryManager);
+            }
             tables.at(info.oid)->deserialize(context, this, deSer);
         }
     }

package/lbug-source/src/storage/table/CMakeLists.txt

@@ -22,6 +22,8 @@ add_library(lbug_storage_store
         node_group_collection.cpp
         node_table.cpp
         null_column.cpp
+        parquet_node_table.cpp
+        parquet_rel_table.cpp
         rel_table.cpp
         rel_table_data.cpp
         string_chunk_data.cpp