lbug 0.12.3-dev.8 → 0.13.0

package/lbug-source/src/processor/map/map_count_rel_table.cpp

@@ -0,0 +1,55 @@
+#include "planner/operator/scan/logical_count_rel_table.h"
+#include "processor/operator/scan/count_rel_table.h"
+#include "processor/plan_mapper.h"
+#include "storage/storage_manager.h"
+
+using namespace lbug::common;
+using namespace lbug::planner;
+using namespace lbug::storage;
+
+namespace lbug {
+namespace processor {
+
+std::unique_ptr<PhysicalOperator> PlanMapper::mapCountRelTable(
+    const LogicalOperator* logicalOperator) {
+    auto& logicalCountRelTable = logicalOperator->constCast<LogicalCountRelTable>();
+    auto outSchema = logicalCountRelTable.getSchema();
+
+    auto storageManager = StorageManager::Get(*clientContext);
+
+    // Get the node tables for scanning bound nodes
+    std::vector<NodeTable*> nodeTables;
+    for (auto tableID : logicalCountRelTable.getBoundNodeTableIDs()) {
+        nodeTables.push_back(storageManager->getTable(tableID)->ptrCast<NodeTable>());
+    }
+
+    // Get the rel tables
+    std::vector<RelTable*> relTables;
+    for (auto tableID : logicalCountRelTable.getRelTableIDs()) {
+        relTables.push_back(storageManager->getTable(tableID)->ptrCast<RelTable>());
+    }
+
+    // Determine rel data direction from extend direction
+    auto extendDirection = logicalCountRelTable.getDirection();
+    RelDataDirection relDirection;
+    if (extendDirection == ExtendDirection::FWD) {
+        relDirection = RelDataDirection::FWD;
+    } else if (extendDirection == ExtendDirection::BWD) {
+        relDirection = RelDataDirection::BWD;
+    } else {
+        // For BOTH, we'll scan FWD (shouldn't reach here as optimizer filters BOTH)
+        relDirection = RelDataDirection::FWD;
+    }
+
+    // Get the output position for the count expression
+    auto countOutputPos = getDataPos(*logicalCountRelTable.getCountExpr(), *outSchema);
+
+    auto printInfo = std::make_unique<CountRelTablePrintInfo>(
+        logicalCountRelTable.getRelGroupEntry()->getName());
+
+    return std::make_unique<CountRelTable>(std::move(nodeTables), std::move(relTables),
+        relDirection, countOutputPos, getOperatorID(), std::move(printInfo));
+}
+
+} // namespace processor
+} // namespace lbug

package/lbug-source/src/processor/map/plan_mapper.cpp

@@ -62,6 +62,9 @@ std::unique_ptr<PhysicalOperator> PlanMapper::mapOperator(const LogicalOperator*
     case LogicalOperatorType::COPY_TO: {
         physicalOperator = mapCopyTo(logicalOperator);
     } break;
+    case LogicalOperatorType::COUNT_REL_TABLE: {
+        physicalOperator = mapCountRelTable(logicalOperator);
+    } break;
     case LogicalOperatorType::CREATE_MACRO: {
         physicalOperator = mapCreateMacro(logicalOperator);
     } break;

package/lbug-source/src/processor/operator/index_lookup.cpp

@@ -28,7 +28,6 @@ std::optional<WarningSourceData> getWarningSourceData(
     return ret;
 }
 
-// TODO(Guodong): Add short path for unfiltered case.
 bool checkNullKey(ValueVector* keyVector, offset_t vectorOffset,
     BatchInsertErrorHandler* errorHandler, const std::vector<ValueVector*>& warningDataVectors) {
     bool isNull = keyVector->isNull(vectorOffset);
@@ -71,27 +70,17 @@ struct OffsetVectorManager {
     offset_t insertOffset;
 };
 
-// TODO(Guodong): Add short path for unfiltered case.
 template<bool hasNoNullsGuarantee>
-void fillOffsetArraysFromVector(transaction::Transaction* transaction, const IndexLookupInfo& info,
-    ValueVector* keyVector, ValueVector* resultVector,
-    const std::vector<ValueVector*>& warningDataVectors, BatchInsertErrorHandler* errorHandler) {
-    KU_ASSERT(resultVector->dataType.getPhysicalType() == PhysicalTypeID::INT64);
+void fillOffsetArraysFromVectorInternal(transaction::Transaction* transaction,
+    const IndexLookupInfo& info, ValueVector* keyVector, ValueVector* resultVector,
+    const std::vector<ValueVector*>& warningDataVectors, BatchInsertErrorHandler* errorHandler,
+    const sel_t* selVector, sel_t numKeys) {
     TypeUtils::visit(
         keyVector->dataType.getPhysicalType(),
         [&]<IndexHashable T>(T) {
-            auto numKeys = keyVector->state->getSelVector().getSelSize();
-
-            // fetch all the selection pos at the start
-            // since we may modify the selection vector in the middle of the lookup
-            std::vector<sel_t> lookupPos(numKeys);
-            for (idx_t i = 0; i < numKeys; ++i) {
-                lookupPos[i] = (keyVector->state->getSelVector()[i]);
-            }
-
             OffsetVectorManager resultManager{resultVector, errorHandler};
-            for (auto i = 0u; i < numKeys; i++) {
-                auto pos = lookupPos[i];
+            for (sel_t i = 0u; i < numKeys; i++) {
+                auto pos = selVector ? selVector[i] : i;
                 if constexpr (!hasNoNullsGuarantee) {
                     if (!checkNullKey(keyVector, pos, errorHandler, warningDataVectors)) {
                         continue;
@@ -99,12 +88,9 @@ void fillOffsetArraysFromVector(transaction::Transaction* transaction, const Ind
                 }
                 offset_t lookupOffset = 0;
                 if (!info.nodeTable->lookupPK(transaction, keyVector, pos, lookupOffset)) {
-                    TypeUtils::visit(keyVector->dataType, [&]<typename type>(type) {
-                        errorHandler->handleError(
-                            ExceptionMessage::nonExistentPKException(
-                                TypeUtils::toString(keyVector->getValue<type>(pos), keyVector)),
-                            getWarningSourceData(warningDataVectors, pos));
-                    });
+                    errorHandler->handleError(ExceptionMessage::nonExistentPKException(
+                                                  keyVector->getAsValue(pos)->toString()),
+                        getWarningSourceData(warningDataVectors, pos));
                 } else {
                     resultManager.insertEntry(lookupOffset, pos);
                 }
@@ -112,6 +98,28 @@ void fillOffsetArraysFromVector(transaction::Transaction* transaction, const Ind
         },
         [&](auto) { KU_UNREACHABLE; });
 }
+
+template<bool hasNoNullsGuarantee>
+void fillOffsetArraysFromVector(transaction::Transaction* transaction, const IndexLookupInfo& info,
+    ValueVector* keyVector, ValueVector* resultVector,
+    const std::vector<ValueVector*>& warningDataVectors, BatchInsertErrorHandler* errorHandler) {
+    KU_ASSERT(resultVector->dataType.getPhysicalType() == PhysicalTypeID::INT64);
+    auto& selVector = keyVector->state->getSelVector();
+    auto numKeys = selVector.getSelSize();
+    if (selVector.isUnfiltered()) {
+        // Fast path: selection vector is unfiltered - pass a null selection vector
+        fillOffsetArraysFromVectorInternal<hasNoNullsGuarantee>(transaction, info, keyVector,
+            resultVector, warningDataVectors, errorHandler, nullptr /* selVector */, numKeys);
+    } else {
+        // Filtered case: copy selection positions since we may modify the selection vector
+        std::vector<sel_t> lookupPos(numKeys);
+        for (idx_t i = 0; i < numKeys; ++i) {
+            lookupPos[i] = selVector[i];
+        }
+        fillOffsetArraysFromVectorInternal<hasNoNullsGuarantee>(transaction, info, keyVector,
+            resultVector, warningDataVectors, errorHandler, lookupPos.data(), numKeys);
+    }
+}
 } // namespace
 
 std::string IndexLookupPrintInfo::toString() const {

package/lbug-source/src/processor/operator/persistent/reader/parquet/parquet_reader.cpp

@@ -340,6 +340,10 @@ std::unique_ptr<ColumnReader> ParquetReader::createReader() {
         throw CopyException{"Root element of Parquet file must be a struct"};
     }
     // LCOV_EXCL_STOP
+    // Clear existing column metadata before populating (in case createReader is called multiple
+    // times)
+    columnNames.clear();
+    columnTypes.clear();
     for (auto& field : StructType::getFields(rootReader->getDataType())) {
         columnNames.push_back(field.getName());
         columnTypes.push_back(field.getType().copy());

package/lbug-source/src/processor/operator/physical_operator.cpp

@@ -27,6 +27,8 @@ std::string PhysicalOperatorUtils::operatorTypeToString(PhysicalOperatorType ope
         return "BATCH_INSERT";
     case PhysicalOperatorType::COPY_TO:
         return "COPY_TO";
+    case PhysicalOperatorType::COUNT_REL_TABLE:
+        return "COUNT_REL_TABLE";
     case PhysicalOperatorType::CREATE_MACRO:
         return "CREATE_MACRO";
     case PhysicalOperatorType::CREATE_SEQUENCE:

package/lbug-source/src/processor/operator/scan/CMakeLists.txt

@@ -1,5 +1,6 @@
 add_library(lbug_processor_operator_scan
         OBJECT
+        count_rel_table.cpp
         primary_key_scan_node_table.cpp
         scan_multi_rel_tables.cpp
         scan_node_table.cpp

package/lbug-source/src/processor/operator/scan/count_rel_table.cpp

@@ -0,0 +1,137 @@
+#include "processor/operator/scan/count_rel_table.h"
+
+#include "common/system_config.h"
+#include "main/client_context.h"
+#include "main/database.h"
+#include "processor/execution_context.h"
+#include "storage/buffer_manager/memory_manager.h"
+#include "storage/local_storage/local_rel_table.h"
+#include "storage/local_storage/local_storage.h"
+#include "storage/table/column.h"
+#include "storage/table/column_chunk_data.h"
+#include "storage/table/csr_chunked_node_group.h"
+#include "storage/table/csr_node_group.h"
+#include "storage/table/rel_table_data.h"
+#include "transaction/transaction.h"
+
+using namespace lbug::common;
+using namespace lbug::storage;
+using namespace lbug::transaction;
+
+namespace lbug {
+namespace processor {
+
+void CountRelTable::initLocalStateInternal(ResultSet* resultSet, ExecutionContext* /*context*/) {
+    countVector = resultSet->getValueVector(countOutputPos).get();
+    hasExecuted = false;
+    totalCount = 0;
+}
+
+// Count rels by using CSR metadata, accounting for deletions and uncommitted data.
+// This is more efficient than scanning through all edges.
+bool CountRelTable::getNextTuplesInternal(ExecutionContext* context) {
+    if (hasExecuted) {
+        return false;
+    }
+
+    auto transaction = Transaction::Get(*context->clientContext);
+    auto* memoryManager = context->clientContext->getDatabase()->getMemoryManager();
+
+    for (auto* relTable : relTables) {
+        // Get the RelTableData for the specified direction
+        auto* relTableData = relTable->getDirectedTableData(direction);
+        auto numNodeGroups = relTableData->getNumNodeGroups();
+        auto* csrLengthColumn = relTableData->getCSRLengthColumn();
+
+        // For each node group in the rel table
+        for (node_group_idx_t nodeGroupIdx = 0; nodeGroupIdx < numNodeGroups; nodeGroupIdx++) {
+            auto* nodeGroup = relTableData->getNodeGroup(nodeGroupIdx);
+            if (!nodeGroup) {
+                continue;
+            }
+
+            auto& csrNodeGroup = nodeGroup->cast<CSRNodeGroup>();
+
+            // Count from persistent (checkpointed) data
+            if (auto* persistentGroup = csrNodeGroup.getPersistentChunkedGroup()) {
+                // Sum the actual relationship lengths from the CSR header instead of using
+                // getNumRows() which includes dummy rows added for CSR offset array gaps
+                auto& csrPersistentGroup = persistentGroup->cast<ChunkedCSRNodeGroup>();
+                auto& csrHeader = csrPersistentGroup.getCSRHeader();
+
+                // Get the number of nodes in this CSR header
+                auto numNodes = csrHeader.length->getNumValues();
+                if (numNodes == 0) {
+                    continue;
+                }
+
+                // Create an in-memory chunk to scan the CSR length column into
+                auto lengthChunk =
+                    ColumnChunkFactory::createColumnChunkData(*memoryManager, LogicalType::UINT64(),
+                        false /*enableCompression*/, StorageConfig::NODE_GROUP_SIZE,
+                        ResidencyState::IN_MEMORY, false /*initializeToZero*/);
+
+                // Initialize scan state and scan the length column from disk
+                ChunkState chunkState;
+                csrHeader.length->initializeScanState(chunkState, csrLengthColumn);
+                csrLengthColumn->scan(chunkState, lengthChunk.get(), 0 /*offsetInChunk*/, numNodes);
+
+                // Sum all the lengths
+                auto* lengthData = reinterpret_cast<const uint64_t*>(lengthChunk->getData());
+                row_idx_t groupRelCount = 0;
+                for (offset_t i = 0; i < numNodes; ++i) {
+                    groupRelCount += lengthData[i];
+                }
+                totalCount += groupRelCount;
+
+                // Subtract deletions from persistent data
+                if (persistentGroup->hasVersionInfo()) {
+                    auto numDeletions =
+                        persistentGroup->getNumDeletions(transaction, 0, groupRelCount);
+                    totalCount -= numDeletions;
+                }
+            }
+
+            // Count in-memory committed data (not yet checkpointed)
+            // This data is stored in chunkedGroups within the NodeGroup
+            auto numChunkedGroups = csrNodeGroup.getNumChunkedGroups();
+            for (node_group_idx_t i = 0; i < numChunkedGroups; i++) {
+                auto* chunkedGroup = csrNodeGroup.getChunkedNodeGroup(i);
+                if (chunkedGroup) {
+                    auto numRows = chunkedGroup->getNumRows();
+                    totalCount += numRows;
+                    // Subtract deletions from in-memory committed data
+                    if (chunkedGroup->hasVersionInfo()) {
+                        auto numDeletions = chunkedGroup->getNumDeletions(transaction, 0, numRows);
+                        totalCount -= numDeletions;
+                    }
+                }
+            }
+        }
+
+        // Add uncommitted insertions from local storage
+        if (transaction->isWriteTransaction()) {
+            if (auto* localTable =
+                    transaction->getLocalStorage()->getLocalTable(relTable->getTableID())) {
+                auto& localRelTable = localTable->cast<LocalRelTable>();
+                // Count entries in the CSR index for this direction.
+                // We can't use getNumTotalRows() because it includes deleted rows.
+                auto& csrIndex = localRelTable.getCSRIndex(direction);
+                for (const auto& [nodeOffset, rowIndices] : csrIndex) {
+                    totalCount += rowIndices.size();
+                }
+            }
+        }
+    }
+
+    hasExecuted = true;
+
+    // Write the count to the output vector (single value)
+    countVector->state->getSelVectorUnsafe().setToUnfiltered(1);
+    countVector->setValue<int64_t>(0, static_cast<int64_t>(totalCount));
+
+    return true;
+}
+
+} // namespace processor
+} // namespace lbug

package/lbug-source/src/processor/operator/scan/scan_multi_rel_tables.cpp

@@ -2,6 +2,7 @@
 
 #include "processor/execution_context.h"
 #include "storage/local_storage/local_storage.h"
+#include "storage/table/parquet_rel_table.h"
 
 using namespace lbug::common;
 using namespace lbug::storage;
@@ -54,8 +55,29 @@ void ScanMultiRelTable::initLocalStateInternal(ResultSet* resultSet, ExecutionCo
     auto clientContext = context->clientContext;
     boundNodeIDVector = resultSet->getValueVector(opInfo.nodeIDPos).get();
     auto nbrNodeIDVector = outVectors[0];
-    scanState = std::make_unique<RelTableScanState>(*MemoryManager::Get(*clientContext),
-        boundNodeIDVector, outVectors, nbrNodeIDVector->state);
+
+    // Check if any table in any scanner is a ParquetRelTable
+    bool hasParquetTable = false;
+    for (auto& [_, scanner] : scanners) {
+        for (auto& relInfo : scanner.relInfos) {
+            if (dynamic_cast<storage::ParquetRelTable*>(relInfo.table) != nullptr) {
+                hasParquetTable = true;
+                break;
+            }
+        }
+        if (hasParquetTable)
+            break;
+    }
+
+    // Create appropriate scan state type
+    if (hasParquetTable) {
+        scanState =
+            std::make_unique<storage::ParquetRelTableScanState>(*MemoryManager::Get(*clientContext),
+                boundNodeIDVector, outVectors, nbrNodeIDVector->state);
+    } else {
+        scanState = std::make_unique<RelTableScanState>(*MemoryManager::Get(*clientContext),
+            boundNodeIDVector, outVectors, nbrNodeIDVector->state);
+    }
     for (auto& [_, scanner] : scanners) {
         for (auto& relInfo : scanner.relInfos) {
             if (directionInfo.directionPos.isValid()) {

package/lbug-source/src/processor/operator/scan/scan_node_table.cpp

@@ -2,8 +2,10 @@
 
 #include "binder/expression/expression_util.h"
 #include "processor/execution_context.h"
+#include "storage/buffer_manager/memory_manager.h"
 #include "storage/local_storage/local_node_table.h"
 #include "storage/local_storage/local_storage.h"
+#include "storage/table/parquet_node_table.h"
 
 using namespace lbug::common;
 using namespace lbug::storage;
@@ -35,7 +37,23 @@ void ScanNodeTableSharedState::initialize(const transaction::Transaction* transa
     this->table = table;
     this->currentCommittedGroupIdx = 0;
     this->currentUnCommittedGroupIdx = 0;
-    this->numCommittedNodeGroups = table->getNumCommittedNodeGroups();
+
+    // Initialize table-specific scan coordination (e.g., for ParquetNodeTable)
+    table->initializeScanCoordination(transaction);
+
+    if (const auto parquetTable = dynamic_cast<ParquetNodeTable*>(table)) {
+        // For parquet tables, set numCommittedNodeGroups to number of row groups
+        std::vector<bool> columnSkips;
+        try {
+            auto tempReader = std::make_unique<processor::ParquetReader>(
+                parquetTable->getParquetFilePath(), columnSkips, transaction->getClientContext());
+            this->numCommittedNodeGroups = tempReader->getNumRowsGroups();
+        } catch (const std::exception& e) {
+            this->numCommittedNodeGroups = 1;
+        }
+    } else {
+        this->numCommittedNodeGroups = table->getNumCommittedNodeGroups();
+    }
     if (transaction->isWriteTransaction()) {
         if (const auto localTable =
                 transaction->getLocalStorage()->getLocalTable(this->table->getTableID())) {
@@ -46,21 +64,23 @@ void ScanNodeTableSharedState::initialize(const transaction::Transaction* transa
     progressSharedState.numGroups += numCommittedNodeGroups;
 }
 
-void ScanNodeTableSharedState::nextMorsel(NodeTableScanState& scanState,
+void ScanNodeTableSharedState::nextMorsel(TableScanState& scanState,
     ScanNodeTableProgressSharedState& progressSharedState) {
     std::unique_lock lck{mtx};
+    // Cast to NodeTableScanState since we know this is for node tables
+    auto& nodeScanState = scanState.cast<NodeTableScanState>();
     if (currentCommittedGroupIdx < numCommittedNodeGroups) {
-        scanState.nodeGroupIdx = currentCommittedGroupIdx++;
+        nodeScanState.nodeGroupIdx = currentCommittedGroupIdx++;
         progressSharedState.numGroupsScanned++;
-        scanState.source = TableScanSource::COMMITTED;
+        nodeScanState.source = TableScanSource::COMMITTED;
         return;
     }
     if (currentUnCommittedGroupIdx < numUnCommittedNodeGroups) {
-        scanState.nodeGroupIdx = currentUnCommittedGroupIdx++;
-        scanState.source = TableScanSource::UNCOMMITTED;
+        nodeScanState.nodeGroupIdx = currentUnCommittedGroupIdx++;
+        nodeScanState.source = TableScanSource::UNCOMMITTED;
         return;
     }
-    scanState.source = TableScanSource::NONE;
+    nodeScanState.source = TableScanSource::NONE;
 }
 
 table_id_map_t<SemiMask*> ScanNodeTable::getSemiMasks() const {
@@ -82,7 +102,18 @@ void ScanNodeTableInfo::initScanState(TableScanState& scanState,
 void ScanNodeTable::initLocalStateInternal(ResultSet* resultSet, ExecutionContext* context) {
     ScanTable::initLocalStateInternal(resultSet, context);
     auto nodeIDVector = resultSet->getValueVector(opInfo.nodeIDPos).get();
-    scanState = std::make_unique<NodeTableScanState>(nodeIDVector, outVectors, nodeIDVector->state);
+
+    // Check if the first table is a ParquetNodeTable and create appropriate scan state
+    auto* parquetTable = dynamic_cast<ParquetNodeTable*>(tableInfos[0].table);
+    if (parquetTable) {
+        scanState = std::make_unique<ParquetNodeTableScanState>(
+            *MemoryManager::Get(*context->clientContext), nodeIDVector, outVectors,
+            nodeIDVector->state);
+    } else {
+        scanState =
+            std::make_unique<NodeTableScanState>(nodeIDVector, outVectors, nodeIDVector->state);
+    }
+
     currentTableIdx = 0;
     initCurrentTable(context);
 }
@@ -91,6 +122,11 @@ void ScanNodeTable::initCurrentTable(ExecutionContext* context) {
     auto& currentInfo = tableInfos[currentTableIdx];
     currentInfo.initScanState(*scanState, outVectors, context->clientContext);
     scanState->semiMask = sharedStates[currentTableIdx]->getSemiMask();
+    // Call table->initScanState for ParquetNodeTable
+    if (dynamic_cast<ParquetNodeTable*>(tableInfos[currentTableIdx].table)) {
+        auto transaction = transaction::Transaction::Get(*context->clientContext);
+        tableInfos[currentTableIdx].table->initScanState(transaction, *scanState);
+    }
 }
 
 void ScanNodeTable::initGlobalStateInternal(ExecutionContext* context) {

package/lbug-source/src/processor/operator/scan/scan_rel_table.cpp

@@ -2,7 +2,10 @@
 
 #include "binder/expression/expression_util.h"
 #include "processor/execution_context.h"
+#include "storage/buffer_manager/memory_manager.h"
 #include "storage/local_storage/local_rel_table.h"
+#include "storage/table/foreign_rel_table.h"
+#include "storage/table/parquet_rel_table.h"
 
 using namespace lbug::common;
 using namespace lbug::storage;
@@ -66,8 +69,21 @@ void ScanRelTable::initLocalStateInternal(ResultSet* resultSet, ExecutionContext
     auto clientContext = context->clientContext;
     auto boundNodeIDVector = resultSet->getValueVector(opInfo.nodeIDPos).get();
     auto nbrNodeIDVector = outVectors[0];
-    scanState = std::make_unique<RelTableScanState>(*MemoryManager::Get(*clientContext),
-        boundNodeIDVector, outVectors, nbrNodeIDVector->state);
+    // Check if this is a ParquetRelTable or ForeignRelTable and create appropriate scan state
+    auto* parquetTable = dynamic_cast<storage::ParquetRelTable*>(tableInfo.table);
+    auto* foreignTable = dynamic_cast<storage::ForeignRelTable*>(tableInfo.table);
+    if (parquetTable) {
+        scanState =
+            std::make_unique<storage::ParquetRelTableScanState>(*MemoryManager::Get(*clientContext),
+                boundNodeIDVector, outVectors, nbrNodeIDVector->state);
+    } else if (foreignTable) {
+        scanState =
+            std::make_unique<storage::ForeignRelTableScanState>(*MemoryManager::Get(*clientContext),
+                boundNodeIDVector, outVectors, nbrNodeIDVector->state);
+    } else {
+        scanState = std::make_unique<RelTableScanState>(*MemoryManager::Get(*clientContext),
+            boundNodeIDVector, outVectors, nbrNodeIDVector->state);
+    }
     tableInfo.initScanState(*scanState, outVectors, clientContext);
 }
 

package/lbug-source/src/storage/storage_manager.cpp

@@ -12,7 +12,10 @@
 #include "storage/buffer_manager/buffer_manager.h"
 #include "storage/buffer_manager/memory_manager.h"
 #include "storage/checkpointer.h"
+#include "storage/table/foreign_rel_table.h"
 #include "storage/table/node_table.h"
+#include "storage/table/parquet_node_table.h"
+#include "storage/table/parquet_rel_table.h"
 #include "storage/table/rel_table.h"
 #include "storage/wal/wal_replayer.h"
 #include "transaction/transaction.h"
@@ -77,15 +80,35 @@ void StorageManager::recover(main::ClientContext& clientContext, bool throwOnWal
 }
 
 void StorageManager::createNodeTable(NodeTableCatalogEntry* entry) {
-    tables[entry->getTableID()] = std::make_unique<NodeTable>(this, entry, &memoryManager);
+    tableNameCache[entry->getTableID()] = entry->getName();
+    if (!entry->getStorage().empty()) {
+        // Create parquet-backed node table
+        tables[entry->getTableID()] =
+            std::make_unique<ParquetNodeTable>(this, entry, &memoryManager);
+    } else {
+        // Create regular node table
+        tables[entry->getTableID()] = std::make_unique<NodeTable>(this, entry, &memoryManager);
+    }
 }
 
 // TODO(Guodong): This API is added since storageManager doesn't provide an API to add a single
 // rel table. We may have to refactor the existing StorageManager::createTable(TableCatalogEntry*
 // entry).
 void StorageManager::addRelTable(RelGroupCatalogEntry* entry, const RelTableCatalogInfo& info) {
-    tables[info.oid] = std::make_unique<RelTable>(entry, info.nodePair.srcTableID,
-        info.nodePair.dstTableID, this, &memoryManager);
+    if (entry->getScanFunction().has_value()) {
+        // Create foreign-backed rel table
+        tables[info.oid] = std::make_unique<ForeignRelTable>(entry, info.nodePair.srcTableID,
+            info.nodePair.dstTableID, this, &memoryManager, *entry->getScanFunction(),
+            std::move(entry->getScanBindData().value()));
+    } else if (!entry->getStorage().empty()) {
+        // Create parquet-backed rel table
+        tables[info.oid] = std::make_unique<ParquetRelTable>(entry, info.nodePair.srcTableID,
+            info.nodePair.dstTableID, this, &memoryManager);
+    } else {
+        // Create regular rel table
+        tables[info.oid] = std::make_unique<RelTable>(entry, info.nodePair.srcTableID,
+            info.nodePair.dstTableID, this, &memoryManager);
+    }
 }
 
 void StorageManager::createRelTableGroup(RelGroupCatalogEntry* entry) {
@@ -257,7 +280,14 @@ void StorageManager::deserialize(main::ClientContext* context, const Catalog* ca
         KU_ASSERT(!tables.contains(tableID));
         auto tableEntry = catalog->getTableCatalogEntry(&DUMMY_TRANSACTION, tableID)
                               ->ptrCast<NodeTableCatalogEntry>();
-        tables[tableID] = std::make_unique<NodeTable>(this, tableEntry, &memoryManager);
+        tableNameCache[tableID] = tableEntry->getName();
+        if (!tableEntry->getStorage().empty()) {
+            // Create parquet-backed node table
+            tables[tableID] = std::make_unique<ParquetNodeTable>(this, tableEntry, &memoryManager);
+        } else {
+            // Create regular node table
+            tables[tableID] = std::make_unique<NodeTable>(this, tableEntry, &memoryManager);
+        }
         tables[tableID]->deserialize(context, this, deSer);
     }
     deSer.validateDebuggingInfo(key, "num_rel_groups");
@@ -279,8 +309,15 @@ void StorageManager::deserialize(main::ClientContext* context, const Catalog* ca
         for (auto k = 0u; k < numInnerRelTables; k++) {
             RelTableCatalogInfo info = RelTableCatalogInfo::deserialize(deSer);
             KU_ASSERT(!tables.contains(info.oid));
-            tables[info.oid] = std::make_unique<RelTable>(relGroupEntry, info.nodePair.srcTableID,
-                info.nodePair.dstTableID, this, &memoryManager);
+            if (!relGroupEntry->getStorage().empty()) {
+                // Create parquet-backed rel table
+                tables[info.oid] = std::make_unique<ParquetRelTable>(relGroupEntry,
+                    info.nodePair.srcTableID, info.nodePair.dstTableID, this, &memoryManager);
+            } else {
+                // Create regular rel table
+                tables[info.oid] = std::make_unique<RelTable>(relGroupEntry,
+                    info.nodePair.srcTableID, info.nodePair.dstTableID, this, &memoryManager);
+            }
             tables.at(info.oid)->deserialize(context, this, deSer);
         }
     }

package/lbug-source/src/storage/table/CMakeLists.txt

@@ -13,6 +13,7 @@ add_library(lbug_storage_store
         compression_flush_buffer.cpp
         dictionary_chunk.cpp
         dictionary_column.cpp
+        foreign_rel_table.cpp
         in_mem_chunked_node_group_collection.cpp
         in_memory_exception_chunk.cpp
         lazy_segment_scanner.cpp
@@ -22,6 +23,8 @@ add_library(lbug_storage_store
         node_group_collection.cpp
         node_table.cpp
         null_column.cpp
+        parquet_node_table.cpp
+        parquet_rel_table.cpp
         rel_table.cpp
         rel_table_data.cpp
         string_chunk_data.cpp

package/lbug-source/src/storage/table/foreign_rel_table.cpp

@@ -0,0 +1,63 @@
+#include "storage/table/foreign_rel_table.h"
+
+#include "function/table/table_function.h"
+#include "processor/operator/scan/scan_rel_table.h"
+#include "storage/storage_manager.h"
+#include "transaction/transaction.h"
+
+namespace lbug {
+namespace storage {
+
+ForeignRelTableScanState::ForeignRelTableScanState(MemoryManager& mm,
+    common::ValueVector* nodeIDVector, std::vector<common::ValueVector*> outputVectors,
+    std::shared_ptr<common::DataChunkState> outChunkState)
+    : RelTableScanState{mm, nodeIDVector, std::move(outputVectors), std::move(outChunkState)} {
+    dataChunk.valueVectors.resize(this->outputVectors.size());
+    for (size_t i = 0; i < this->outputVectors.size(); ++i) {
+        dataChunk.valueVectors[i] = std::shared_ptr<common::ValueVector>(this->outputVectors[i],
+            [](common::ValueVector*) {});
+    }
+    dataChunk.state = this->outState;
+}
+
+ForeignRelTable::ForeignRelTable(catalog::RelGroupCatalogEntry* relGroupEntry,
+    common::table_id_t fromTableID, common::table_id_t toTableID,
+    const StorageManager* storageManager, MemoryManager* memoryManager,
+    function::TableFunction scanFunction, std::shared_ptr<function::TableFuncBindData> scanBindData)
+    : RelTable{relGroupEntry, fromTableID, toTableID, storageManager, memoryManager},
+      scanFunction{std::move(scanFunction)}, scanBindData{std::move(scanBindData)} {}
+
+void ForeignRelTable::initScanState([[maybe_unused]] transaction::Transaction* transaction,
+    TableScanState& scanState, [[maybe_unused]] bool resetCachedBoundNodeSelVec) const {
+    // For foreign tables, we don't need node group initialization
+    // RelTable::initScanState(transaction, scanState, resetCachedBoundNodeSelVec);
+    auto& foreignRelScanState = static_cast<ForeignRelTableScanState&>(scanState);
+    function::TableFuncInitSharedStateInput sharedInput{scanBindData.get(), nullptr /* context */};
+    foreignRelScanState.sharedState = scanFunction.initSharedStateFunc(sharedInput);
+    function::TableFuncInitLocalStateInput localInput{*foreignRelScanState.sharedState,
+        *scanBindData, nullptr /* clientContext */};
+    foreignRelScanState.localState = scanFunction.initLocalStateFunc(localInput);
+}
+
+bool ForeignRelTable::scanInternal([[maybe_unused]] transaction::Transaction* transaction,
+    TableScanState& scanState) {
+    auto& foreignRelScanState = static_cast<ForeignRelTableScanState&>(scanState);
+    function::TableFuncInput input{scanBindData.get(), foreignRelScanState.localState.get(),
+        foreignRelScanState.sharedState.get(), nullptr /* clientContext */};
+    common::DataChunk dc;
+    dc.valueVectors = foreignRelScanState.dataChunk.valueVectors;
+    dc.state = foreignRelScanState.dataChunk.state;
+    function::TableFuncOutput output{std::move(dc)};
+    auto numTuples = scanFunction.tableFunc(input, output);
+    return numTuples > 0;
+}
+
+common::row_idx_t ForeignRelTable::getNumTotalRows(
+    [[maybe_unused]] const transaction::Transaction* transaction) {
+    // For foreign tables, we might need to query the foreign table for row count
+    // For now, return 0 or implement proper counting
+    return 0;
+}
+
+} // namespace storage
+} // namespace lbug