lbug 0.12.3-dev.16 → 0.12.3-dev.17

package/lbug-source/src/parser/transform/transform_ddl.cpp

@@ -81,7 +81,12 @@ std::unique_ptr<Statement> Transformer::transformCreateNodeTable(
     } else {
         createTableInfo.propertyDefinitions =
             transformPropertyDefinitions(*ctx.kU_PropertyDefinitions());
-        createTableInfo.extraInfo = std::make_unique<ExtraCreateNodeTableInfo>(getPKName(ctx));
+        options_t options;
+        if (ctx.kU_Options()) {
+            options = transformOptions(*ctx.kU_Options());
+        }
+        createTableInfo.extraInfo =
+            std::make_unique<ExtraCreateNodeTableInfo>(getPKName(ctx), std::move(options));
         return std::make_unique<CreateTable>(std::move(createTableInfo));
     }
 }

package/lbug-source/src/processor/operator/persistent/reader/parquet/parquet_reader.cpp

@@ -340,6 +340,10 @@ std::unique_ptr<ColumnReader> ParquetReader::createReader() {
         throw CopyException{"Root element of Parquet file must be a struct"};
     }
     // LCOV_EXCL_STOP
+    // Clear existing column metadata before populating (in case createReader is called multiple
+    // times)
+    columnNames.clear();
+    columnTypes.clear();
     for (auto& field : StructType::getFields(rootReader->getDataType())) {
         columnNames.push_back(field.getName());
         columnTypes.push_back(field.getType().copy());

package/lbug-source/src/processor/operator/scan/scan_multi_rel_tables.cpp

@@ -2,6 +2,7 @@
 
 #include "processor/execution_context.h"
 #include "storage/local_storage/local_storage.h"
+#include "storage/table/parquet_rel_table.h"
 
 using namespace lbug::common;
 using namespace lbug::storage;
@@ -54,8 +55,29 @@ void ScanMultiRelTable::initLocalStateInternal(ResultSet* resultSet, ExecutionCo
     auto clientContext = context->clientContext;
     boundNodeIDVector = resultSet->getValueVector(opInfo.nodeIDPos).get();
     auto nbrNodeIDVector = outVectors[0];
-    scanState = std::make_unique<RelTableScanState>(*MemoryManager::Get(*clientContext),
-        boundNodeIDVector, outVectors, nbrNodeIDVector->state);
+
+    // Check if any table in any scanner is a ParquetRelTable
+    bool hasParquetTable = false;
+    for (auto& [_, scanner] : scanners) {
+        for (auto& relInfo : scanner.relInfos) {
+            if (dynamic_cast<storage::ParquetRelTable*>(relInfo.table) != nullptr) {
+                hasParquetTable = true;
+                break;
+            }
+        }
+        if (hasParquetTable)
+            break;
+    }
+
+    // Create appropriate scan state type
+    if (hasParquetTable) {
+        scanState =
+            std::make_unique<storage::ParquetRelTableScanState>(*MemoryManager::Get(*clientContext),
+                boundNodeIDVector, outVectors, nbrNodeIDVector->state);
+    } else {
+        scanState = std::make_unique<RelTableScanState>(*MemoryManager::Get(*clientContext),
+            boundNodeIDVector, outVectors, nbrNodeIDVector->state);
+    }
     for (auto& [_, scanner] : scanners) {
         for (auto& relInfo : scanner.relInfos) {
             if (directionInfo.directionPos.isValid()) {

package/lbug-source/src/processor/operator/scan/scan_node_table.cpp

@@ -2,8 +2,10 @@
 
 #include "binder/expression/expression_util.h"
 #include "processor/execution_context.h"
+#include "storage/buffer_manager/memory_manager.h"
 #include "storage/local_storage/local_node_table.h"
 #include "storage/local_storage/local_storage.h"
+#include "storage/table/parquet_node_table.h"
 
 using namespace lbug::common;
 using namespace lbug::storage;
@@ -35,7 +37,23 @@ void ScanNodeTableSharedState::initialize(const transaction::Transaction* transa
     this->table = table;
     this->currentCommittedGroupIdx = 0;
     this->currentUnCommittedGroupIdx = 0;
-    this->numCommittedNodeGroups = table->getNumCommittedNodeGroups();
+
+    // Initialize table-specific scan coordination (e.g., for ParquetNodeTable)
+    table->initializeScanCoordination(transaction);
+
+    if (const auto parquetTable = dynamic_cast<ParquetNodeTable*>(table)) {
+        // For parquet tables, set numCommittedNodeGroups to number of row groups
+        std::vector<bool> columnSkips;
+        try {
+            auto tempReader = std::make_unique<processor::ParquetReader>(
+                parquetTable->getParquetFilePath(), columnSkips, transaction->getClientContext());
+            this->numCommittedNodeGroups = tempReader->getNumRowsGroups();
+        } catch (const std::exception& e) {
+            this->numCommittedNodeGroups = 1;
+        }
+    } else {
+        this->numCommittedNodeGroups = table->getNumCommittedNodeGroups();
+    }
     if (transaction->isWriteTransaction()) {
         if (const auto localTable =
                 transaction->getLocalStorage()->getLocalTable(this->table->getTableID())) {
@@ -46,21 +64,23 @@ void ScanNodeTableSharedState::initialize(const transaction::Transaction* transa
     progressSharedState.numGroups += numCommittedNodeGroups;
 }
 
-void ScanNodeTableSharedState::nextMorsel(NodeTableScanState& scanState,
+void ScanNodeTableSharedState::nextMorsel(TableScanState& scanState,
     ScanNodeTableProgressSharedState& progressSharedState) {
     std::unique_lock lck{mtx};
+    // Cast to NodeTableScanState since we know this is for node tables
+    auto& nodeScanState = scanState.cast<NodeTableScanState>();
     if (currentCommittedGroupIdx < numCommittedNodeGroups) {
-        scanState.nodeGroupIdx = currentCommittedGroupIdx++;
+        nodeScanState.nodeGroupIdx = currentCommittedGroupIdx++;
         progressSharedState.numGroupsScanned++;
-        scanState.source = TableScanSource::COMMITTED;
+        nodeScanState.source = TableScanSource::COMMITTED;
         return;
     }
     if (currentUnCommittedGroupIdx < numUnCommittedNodeGroups) {
-        scanState.nodeGroupIdx = currentUnCommittedGroupIdx++;
-        scanState.source = TableScanSource::UNCOMMITTED;
+        nodeScanState.nodeGroupIdx = currentUnCommittedGroupIdx++;
+        nodeScanState.source = TableScanSource::UNCOMMITTED;
         return;
     }
-    scanState.source = TableScanSource::NONE;
+    nodeScanState.source = TableScanSource::NONE;
 }
 
 table_id_map_t<SemiMask*> ScanNodeTable::getSemiMasks() const {
@@ -82,7 +102,18 @@ void ScanNodeTableInfo::initScanState(TableScanState& scanState,
 void ScanNodeTable::initLocalStateInternal(ResultSet* resultSet, ExecutionContext* context) {
     ScanTable::initLocalStateInternal(resultSet, context);
     auto nodeIDVector = resultSet->getValueVector(opInfo.nodeIDPos).get();
-    scanState = std::make_unique<NodeTableScanState>(nodeIDVector, outVectors, nodeIDVector->state);
+
+    // Check if the first table is a ParquetNodeTable and create appropriate scan state
+    auto* parquetTable = dynamic_cast<ParquetNodeTable*>(tableInfos[0].table);
+    if (parquetTable) {
+        scanState = std::make_unique<ParquetNodeTableScanState>(
+            *MemoryManager::Get(*context->clientContext), nodeIDVector, outVectors,
+            nodeIDVector->state);
+    } else {
+        scanState =
+            std::make_unique<NodeTableScanState>(nodeIDVector, outVectors, nodeIDVector->state);
+    }
+
     currentTableIdx = 0;
     initCurrentTable(context);
 }
@@ -91,6 +122,11 @@ void ScanNodeTable::initCurrentTable(ExecutionContext* context) {
     auto& currentInfo = tableInfos[currentTableIdx];
     currentInfo.initScanState(*scanState, outVectors, context->clientContext);
     scanState->semiMask = sharedStates[currentTableIdx]->getSemiMask();
+    // Call table->initScanState for ParquetNodeTable
+    if (dynamic_cast<ParquetNodeTable*>(tableInfos[currentTableIdx].table)) {
+        auto transaction = transaction::Transaction::Get(*context->clientContext);
+        tableInfos[currentTableIdx].table->initScanState(transaction, *scanState);
+    }
 }
 
 void ScanNodeTable::initGlobalStateInternal(ExecutionContext* context) {

package/lbug-source/src/processor/operator/scan/scan_rel_table.cpp

@@ -2,7 +2,9 @@
 
 #include "binder/expression/expression_util.h"
 #include "processor/execution_context.h"
+#include "storage/buffer_manager/memory_manager.h"
 #include "storage/local_storage/local_rel_table.h"
+#include "storage/table/parquet_rel_table.h"
 
 using namespace lbug::common;
 using namespace lbug::storage;
@@ -66,8 +68,16 @@ void ScanRelTable::initLocalStateInternal(ResultSet* resultSet, ExecutionContext
     auto clientContext = context->clientContext;
     auto boundNodeIDVector = resultSet->getValueVector(opInfo.nodeIDPos).get();
     auto nbrNodeIDVector = outVectors[0];
-    scanState = std::make_unique<RelTableScanState>(*MemoryManager::Get(*clientContext),
-        boundNodeIDVector, outVectors, nbrNodeIDVector->state);
+    // Check if this is a ParquetRelTable and create appropriate scan state
+    auto* parquetTable = dynamic_cast<storage::ParquetRelTable*>(tableInfo.table);
+    if (parquetTable) {
+        scanState =
+            std::make_unique<storage::ParquetRelTableScanState>(*MemoryManager::Get(*clientContext),
+                boundNodeIDVector, outVectors, nbrNodeIDVector->state);
+    } else {
+        scanState = std::make_unique<RelTableScanState>(*MemoryManager::Get(*clientContext),
+            boundNodeIDVector, outVectors, nbrNodeIDVector->state);
+    }
     tableInfo.initScanState(*scanState, outVectors, clientContext);
 }
 

package/lbug-source/src/storage/storage_manager.cpp

@@ -13,6 +13,8 @@
 #include "storage/buffer_manager/memory_manager.h"
 #include "storage/checkpointer.h"
 #include "storage/table/node_table.h"
+#include "storage/table/parquet_node_table.h"
+#include "storage/table/parquet_rel_table.h"
 #include "storage/table/rel_table.h"
 #include "storage/wal/wal_replayer.h"
 #include "transaction/transaction.h"
@@ -77,15 +79,31 @@ void StorageManager::recover(main::ClientContext& clientContext, bool throwOnWal
 }
 
 void StorageManager::createNodeTable(NodeTableCatalogEntry* entry) {
-    tables[entry->getTableID()] = std::make_unique<NodeTable>(this, entry, &memoryManager);
+    tableNameCache[entry->getTableID()] = entry->getName();
+    if (!entry->getStorage().empty()) {
+        // Create parquet-backed node table
+        tables[entry->getTableID()] =
+            std::make_unique<ParquetNodeTable>(this, entry, &memoryManager);
+    } else {
+        // Create regular node table
+        tables[entry->getTableID()] = std::make_unique<NodeTable>(this, entry, &memoryManager);
+    }
 }
 
 // TODO(Guodong): This API is added since storageManager doesn't provide an API to add a single
 // rel table. We may have to refactor the existing StorageManager::createTable(TableCatalogEntry*
 // entry).
 void StorageManager::addRelTable(RelGroupCatalogEntry* entry, const RelTableCatalogInfo& info) {
-    tables[info.oid] = std::make_unique<RelTable>(entry, info.nodePair.srcTableID,
-        info.nodePair.dstTableID, this, &memoryManager);
+    if (!entry->getStorage().empty()) {
+        // Create parquet-backed rel table
+        std::string fromNodeTableName = tableNameCache.at(info.nodePair.srcTableID);
+        tables[info.oid] = std::make_unique<ParquetRelTable>(entry, info.nodePair.srcTableID,
+            info.nodePair.dstTableID, this, &memoryManager, fromNodeTableName);
+    } else {
+        // Create regular rel table
+        tables[info.oid] = std::make_unique<RelTable>(entry, info.nodePair.srcTableID,
+            info.nodePair.dstTableID, this, &memoryManager);
+    }
 }
 
 void StorageManager::createRelTableGroup(RelGroupCatalogEntry* entry) {
@@ -257,7 +275,14 @@ void StorageManager::deserialize(main::ClientContext* context, const Catalog* ca
         KU_ASSERT(!tables.contains(tableID));
         auto tableEntry = catalog->getTableCatalogEntry(&DUMMY_TRANSACTION, tableID)
                               ->ptrCast<NodeTableCatalogEntry>();
-        tables[tableID] = std::make_unique<NodeTable>(this, tableEntry, &memoryManager);
+        tableNameCache[tableID] = tableEntry->getName();
+        if (!tableEntry->getStorage().empty()) {
+            // Create parquet-backed node table
+            tables[tableID] = std::make_unique<ParquetNodeTable>(this, tableEntry, &memoryManager);
+        } else {
+            // Create regular node table
+            tables[tableID] = std::make_unique<NodeTable>(this, tableEntry, &memoryManager);
+        }
         tables[tableID]->deserialize(context, this, deSer);
     }
     deSer.validateDebuggingInfo(key, "num_rel_groups");
@@ -279,8 +304,17 @@ void StorageManager::deserialize(main::ClientContext* context, const Catalog* ca
         for (auto k = 0u; k < numInnerRelTables; k++) {
             RelTableCatalogInfo info = RelTableCatalogInfo::deserialize(deSer);
             KU_ASSERT(!tables.contains(info.oid));
-            tables[info.oid] = std::make_unique<RelTable>(relGroupEntry, info.nodePair.srcTableID,
-                info.nodePair.dstTableID, this, &memoryManager);
+            if (!relGroupEntry->getStorage().empty()) {
+                // Create parquet-backed rel table
+                std::string fromNodeTableName = tableNameCache.at(info.nodePair.srcTableID);
+                tables[info.oid] =
+                    std::make_unique<ParquetRelTable>(relGroupEntry, info.nodePair.srcTableID,
+                        info.nodePair.dstTableID, this, &memoryManager, fromNodeTableName);
+            } else {
+                // Create regular rel table
+                tables[info.oid] = std::make_unique<RelTable>(relGroupEntry,
+                    info.nodePair.srcTableID, info.nodePair.dstTableID, this, &memoryManager);
+            }
             tables.at(info.oid)->deserialize(context, this, deSer);
         }
     }

package/lbug-source/src/storage/table/CMakeLists.txt

@@ -22,6 +22,8 @@ add_library(lbug_storage_store
         node_group_collection.cpp
         node_table.cpp
         null_column.cpp
+        parquet_node_table.cpp
+        parquet_rel_table.cpp
         rel_table.cpp
         rel_table_data.cpp
         string_chunk_data.cpp

package/lbug-source/src/storage/table/parquet_node_table.cpp

@@ -0,0 +1,338 @@
+#include "storage/table/parquet_node_table.h"
+
+#include <mutex>
+
+#include "catalog/catalog_entry/node_table_catalog_entry.h"
+#include "common/data_chunk/sel_vector.h"
+#include "common/exception/runtime.h"
+#include "common/file_system/virtual_file_system.h"
+#include "common/types/value/value.h"
+#include "main/client_context.h"
+#include "processor/operator/persistent/reader/parquet/parquet_reader.h"
+#include "storage/buffer_manager/memory_manager.h"
+#include "storage/storage_manager.h"
+#include "storage/storage_utils.h"
+#include "storage/table/column.h"
+#include "transaction/transaction.h"
+
+using namespace lbug::catalog;
+using namespace lbug::common;
+using namespace lbug::processor;
+using namespace lbug::transaction;
+
+namespace lbug {
+namespace storage {
+
+ParquetNodeTable::ParquetNodeTable(const StorageManager* storageManager,
+    const NodeTableCatalogEntry* nodeTableEntry, MemoryManager* memoryManager)
+    : NodeTable{storageManager, nodeTableEntry, memoryManager},
+      nodeTableCatalogEntry{nodeTableEntry} {
+    std::string prefix = nodeTableEntry->getStorage();
+    if (prefix.empty()) {
+        throw RuntimeException("Parquet file prefix is empty for parquet-backed node table");
+    }
+
+    // Get the table name for multi-table directory support
+    std::string tableName = nodeTableEntry->getName();
+
+    // For node tables with multi-table support:
+    // prefix_nodes_{tableName}.parquet (e.g., demo_nodes_city.parquet)
+    parquetFilePath = prefix + "_nodes_" + tableName + ".parquet";
+    sharedState = std::make_unique<ParquetNodeTableSharedState>();
+}
+
+void ParquetNodeTable::initScanState(Transaction* transaction, TableScanState& scanState,
+    [[maybe_unused]] bool resetCachedBoundNodeSelVec) const {
+    // Set up the scan state similar to how NodeTable does it
+    auto& nodeScanState = scanState.cast<NodeTableScanState>();
+    nodeScanState.source = TableScanSource::COMMITTED;
+
+    // Note: Don't set nodeGroupIdx here - it's set by the morsel-driven parallelism system
+
+    auto& parquetNodeScanState = static_cast<ParquetNodeTableScanState&>(nodeScanState);
+
+    // Reset scan state for each scan to allow multiple scans of the same table in one query
+    parquetNodeScanState.dataRead = false;
+    parquetNodeScanState.allData.clear();
+    parquetNodeScanState.totalRows = 0;
+    parquetNodeScanState.nextRowToDistribute = 0;
+
+    // Reset scan completion flag for this scan state
+    parquetNodeScanState.scanCompleted = false;
+
+    // Each scan state gets its own parquet reader for thread safety
+    if (!parquetNodeScanState.initialized) {
+        auto context = transaction->getClientContext();
+        if (!context) {
+            throw RuntimeException("Invalid client context for parquet scan state initialization");
+        }
+
+        std::vector<bool> columnSkips;
+        try {
+            parquetNodeScanState.parquetReader =
+                std::make_unique<ParquetReader>(parquetFilePath, columnSkips, context);
+            parquetNodeScanState.initialized = true;
+        } catch (const std::exception& e) {
+            throw RuntimeException("Failed to initialize parquet reader for file '" +
+                                   parquetFilePath + "': " + e.what());
+        }
+    }
+
+    // Set nodeGroupIdx to invalid initially - will be assigned by getNextRowGroup
+    parquetNodeScanState.nodeGroupIdx = INVALID_NODE_GROUP_IDX;
+
+    // Initialize scan state for the current row group (assigned via shared state)
+    initParquetScanForRowGroup(transaction, parquetNodeScanState);
+}
+
+void ParquetNodeTable::initializeScanCoordination(const Transaction* transaction) {
+    // Reset shared state at the start of each scan operation
+    // This is called once per scan operation by the ScanNodeTable operator
+    // Create a temporary reader to get the number of row groups
+    auto context = transaction->getClientContext();
+    if (!context) {
+        return;
+    }
+
+    std::vector<bool> columnSkips;
+    try {
+        auto tempReader = std::make_unique<ParquetReader>(parquetFilePath, columnSkips, context);
+        auto numRowGroups = tempReader->getNumRowsGroups();
+        sharedState->reset(numRowGroups);
+    } catch (const std::exception& e) {
+        // If we can't read the file, set to 1 row group as fallback
+        sharedState->reset(1);
+    }
+}
+
+void ParquetNodeTable::initParquetScanForRowGroup(Transaction* transaction,
+    ParquetNodeTableScanState& scanState) const {
+    auto context = transaction->getClientContext();
+    if (!context) {
+        return;
+    }
+
+    auto vfs = VirtualFileSystem::GetUnsafe(*context);
+    if (!vfs) {
+        return;
+    }
+
+    // Defensive check: ensure parquet reader exists
+    if (!scanState.parquetReader) {
+        return;
+    }
+
+    // Defensive check: ensure parquet scan state exists
+    if (!scanState.parquetScanState) {
+        return;
+    }
+
+    std::vector<uint64_t> groupsToRead;
+
+    // Use shared state to get the next available row group for this scan state
+    if (scanState.nodeGroupIdx == INVALID_NODE_GROUP_IDX) {
+        common::node_group_idx_t assignedRowGroup;
+        if (sharedState->getNextRowGroup(assignedRowGroup)) {
+            scanState.nodeGroupIdx = assignedRowGroup;
+            groupsToRead.push_back(assignedRowGroup);
+        } else {
+            // No more row groups available - mark scan as completed
+            scanState.scanCompleted = true;
+            // Still need to initialize the scan state with empty groups so reader is in valid state
+            scanState.parquetReader->initializeScan(*scanState.parquetScanState, groupsToRead, vfs);
+            return;
+        }
+    } else {
+        // Row group already assigned (e.g., by external morsel system or re-initialization)
+        groupsToRead.push_back(scanState.nodeGroupIdx);
+    }
+
+    // Re-initialize scan for the specific row groups
+    // Note: initializeScan can be called multiple times; the first call populates column metadata
+    scanState.parquetReader->initializeScan(*scanState.parquetScanState, groupsToRead, vfs);
+}
+
+bool ParquetNodeTable::scanInternal(Transaction* transaction, TableScanState& scanState) {
+    auto& parquetScanState = static_cast<ParquetNodeTableScanState&>(scanState);
+
+    // Check if this particular scan state has already completed
+    if (parquetScanState.scanCompleted) {
+        return false;
+    }
+
+    scanState.resetOutVectors();
+
+    // Read all data once into scan state
+    if (!parquetScanState.dataRead) {
+        // Only the first thread reads the parquet data
+        if (!parquetScanState.initialized) {
+            return false;
+        }
+
+        // Create a data chunk for reading parquet data
+        auto numColumns = parquetScanState.parquetReader->getNumColumns();
+
+        // Defensive check: ensure parquet file has at least one column
+        if (numColumns == 0) {
+            throw RuntimeException("Parquet file '" + parquetFilePath + "' has no columns");
+        }
+
+        DataChunk parquetDataChunk(numColumns, scanState.outState);
+
+        // Create vectors with parquet types
+        // Defensive check: ensure parquet file has enough columns for what we expect
+        // Always create the data chunk to match the exact number of parquet columns
+        // to prevent crashes in the parquet reader when accessing result vectors
+        for (uint32_t i = 0; i < numColumns; ++i) {
+            const auto& parquetColumnType = parquetScanState.parquetReader->getColumnType(i);
+            auto columnType = parquetColumnType.copy();
+            auto vector = std::make_shared<ValueVector>(std::move(columnType),
+                MemoryManager::Get(*transaction->getClientContext()), scanState.outState);
+            parquetDataChunk.insert(i, vector);
+        }
+
+        // Read from parquet
+        parquetScanState.parquetReader->scan(*parquetScanState.parquetScanState, parquetDataChunk);
+
+        auto selSize = parquetDataChunk.state->getSelVector().getSelSize();
+        if (selSize > 0) {
+            parquetScanState.allData.resize(selSize);
+            for (size_t row = 0; row < selSize; ++row) {
+                parquetScanState.allData[row].resize(
+                    scanState.outputVectors
+                        .size()); // Use output vector count, not parquet column count
+
+                // Map parquet columns to correct output vector positions by name
+                // Defensive check: ensure we don't access more columns than available in the chunk
+                auto maxParquetCol = std::min(static_cast<size_t>(numColumns),
+                    static_cast<size_t>(parquetDataChunk.getNumValueVectors()));
+
+                for (size_t parquetCol = 0; parquetCol < maxParquetCol; ++parquetCol) {
+                    // Defensive check: ensure the column index is valid for the data chunk
+                    if (parquetCol >= parquetDataChunk.getNumValueVectors()) {
+                        continue;
+                    }
+
+                    auto& srcVector = parquetDataChunk.getValueVectorMutable(parquetCol);
+
+                    // Get parquet column name and find its corresponding column ID
+                    std::string parquetColumnName =
+                        parquetScanState.parquetReader->getColumnName(parquetCol);
+                    auto nodeTableEntry = this->nodeTableCatalogEntry;
+
+                    // Check if the column exists first before calling getColumnID
+                    if (!nodeTableEntry->containsProperty(parquetColumnName)) {
+                        // Column doesn't exist in table schema, skip it
+                        continue;
+                    }
+
+                    // Find the column ID for this property name
+                    column_id_t parquetColumnID = nodeTableEntry->getColumnID(parquetColumnName);
+
+                    // Find which output vector position corresponds to this column ID
+                    size_t outputCol = INVALID_COLUMN_ID;
+                    for (size_t outCol = 0; outCol < scanState.columnIDs.size(); ++outCol) {
+                        if (scanState.columnIDs[outCol] == parquetColumnID) {
+                            outputCol = outCol;
+                            break;
+                        }
+                    }
+
+                    // Only copy data if we found a matching output position
+                    if (outputCol != INVALID_COLUMN_ID &&
+                        outputCol < parquetScanState.allData[row].size()) {
+                        // Defensive check: ensure the row index is valid for the source vector
+                        if (row >= srcVector.state->getSelVector().getSelSize()) {
+                            continue;
+                        }
+
+                        if (srcVector.isNull(row)) {
+                            parquetScanState.allData[row][outputCol] =
+                                std::make_unique<Value>(Value::createNullValue());
+                        } else {
+                            parquetScanState.allData[row][outputCol] =
+                                std::make_unique<Value>(*srcVector.getAsValue(row));
+                        }
+                    }
+                }
+            }
+            parquetScanState.totalRows = selSize;
+        }
+        parquetScanState.dataRead = true;
+    }
+
+    // Now distribute one row to this scan state
+    if (parquetScanState.nextRowToDistribute >= parquetScanState.totalRows) {
+        parquetScanState.scanCompleted = true;
+        return false; // No more rows to distribute
+    }
+
+    size_t rowIndex = parquetScanState.nextRowToDistribute++;
+
+    // Copy one row to output vectors
+    // Defensive checks: ensure valid row index and handle empty data gracefully
+    if (rowIndex >= parquetScanState.allData.size()) {
+        parquetScanState.scanCompleted = true;
+        return false;
+    }
+
+    auto numColumns =
+        std::min(scanState.outputVectors.size(), parquetScanState.allData[rowIndex].size());
+    for (size_t col = 0; col < numColumns; ++col) {
+        // Defensive check: ensure output vector exists
+        if (col >= scanState.outputVectors.size() || !scanState.outputVectors[col]) {
+            continue;
+        }
+
+        auto& dstVector = *scanState.outputVectors[col];
+
+        // Defensive check: ensure value exists for this column
+        if (col >= parquetScanState.allData[rowIndex].size() ||
+            !parquetScanState.allData[rowIndex][col]) {
+            dstVector.setNull(0, true);
+            continue;
+        }
+
+        auto& value = *parquetScanState.allData[rowIndex][col];
+
+        if (value.isNull()) {
+            dstVector.setNull(0, true);
+        } else {
+            dstVector.copyFromValue(0, value);
+        }
+    }
+
+    // Set node ID for this row
+    auto tableID = this->getTableID();
+    auto& nodeID = scanState.nodeIDVector->getValue<nodeID_t>(0);
+    nodeID.tableID = tableID;
+    nodeID.offset = rowIndex; // Use the actual row index from parquet
+
+    scanState.outState->getSelVectorUnsafe().setSelSize(1); // Return exactly one row
+    return true;
+}
+
+row_idx_t ParquetNodeTable::getNumTotalRows(const transaction::Transaction* transaction) {
+    // Create a temporary reader to get metadata
+    auto context = transaction->getClientContext();
+    if (!context) {
+        return 0;
+    }
+
+    std::vector<bool> columnSkips;
+
+    try {
+        auto tempReader = std::make_unique<ParquetReader>(parquetFilePath, columnSkips, context);
+        if (!tempReader) {
+            return 0;
+        }
+        auto metadata = tempReader->getMetadata();
+        return metadata ? metadata->num_rows : 0;
+    } catch (const std::exception& e) {
+        // If parquet file is corrupted or invalid, return 0 instead of crashing
+        return 0;
+    }
+}
+
+} // namespace storage
+} // namespace lbug