lbug 0.12.3-dev.26 → 0.12.3-dev.28

package/lbug-source/src/optimizer/order_by_push_down_optimizer.cpp

@@ -0,0 +1,123 @@
+#include "optimizer/order_by_push_down_optimizer.h"
+
+#include "binder/expression/expression.h"
+#include "binder/expression/expression_util.h"
+#include "binder/expression/property_expression.h"
+#include "binder/expression/variable_expression.h"
+#include "common/exception/runtime.h"
+#include "planner/operator/logical_order_by.h"
+#include "planner/operator/logical_table_function_call.h"
+
+using namespace lbug::binder;
+using namespace lbug::common;
+using namespace lbug::planner;
+
+namespace lbug {
+namespace optimizer {
+
+// This ensures that ORDER BY can be pushed down only through operators that support it.
+// It should not be pushed down for things like RECURSIVE_EXTEND etc.
+bool isPushDownSupported(LogicalOperator* op) {
+    switch (op->getOperatorType()) {
+    case LogicalOperatorType::TABLE_FUNCTION_CALL: {
+        return op->cast<LogicalTableFunctionCall>().getTableFunc().supportsPushDownFunc();
+    }
+    case LogicalOperatorType::MULTIPLICITY_REDUCER:
+    case LogicalOperatorType::EXPLAIN:
+    case LogicalOperatorType::ACCUMULATE:
+    case LogicalOperatorType::FILTER:
+    case LogicalOperatorType::PROJECTION:
+    case LogicalOperatorType::LIMIT: {
+        if (op->getNumChildren() == 0) {
+            return false;
+        }
+        return isPushDownSupported(op->getChild(0).get());
+    }
+    default:
+        return false;
+    }
+}
+
+void OrderByPushDownOptimizer::rewrite(LogicalPlan* plan) {
+    plan->setLastOperator(visitOperator(plan->getLastOperator()));
+}
+
+std::shared_ptr<LogicalOperator> OrderByPushDownOptimizer::visitOperator(
+    std::shared_ptr<LogicalOperator> op, std::string currentOrderBy) {
+    switch (op->getOperatorType()) {
+    case LogicalOperatorType::ORDER_BY: {
+        auto& orderBy = op->constCast<LogicalOrderBy>();
+        std::string newOrderBy = currentOrderBy;
+        if (!currentOrderBy.empty()) {
+            newOrderBy += ", ";
+        }
+        newOrderBy +=
+            buildOrderByString(orderBy.getExpressionsToOrderBy(), orderBy.getIsAscOrders());
+        auto newChild = visitOperator(orderBy.getChild(0), newOrderBy);
+        if (isPushDownSupported(newChild.get())) {
+            return newChild;
+        }
+        return std::make_shared<LogicalOrderBy>(orderBy.getExpressionsToOrderBy(),
+            orderBy.getIsAscOrders(), newChild);
+    }
+    case LogicalOperatorType::MULTIPLICITY_REDUCER:
+    case LogicalOperatorType::EXPLAIN:
+    case LogicalOperatorType::ACCUMULATE:
+    case LogicalOperatorType::FILTER:
+    case LogicalOperatorType::PROJECTION:
+    case LogicalOperatorType::LIMIT: {
+        for (auto i = 0u; i < op->getNumChildren(); ++i) {
+            op->setChild(i, visitOperator(op->getChild(i), currentOrderBy));
+        }
+        return op;
+    }
+    case LogicalOperatorType::TABLE_FUNCTION_CALL: {
+        if (!currentOrderBy.empty()) {
+            auto& tableFunc = op->cast<LogicalTableFunctionCall>();
+            if (tableFunc.getTableFunc().supportsPushDownFunc()) {
+                tableFunc.setOrderBy(currentOrderBy);
+            }
+        }
+        return op;
+    }
+    default:
+        return op;
+    }
+}
+
+std::string OrderByPushDownOptimizer::buildOrderByString(
+    const binder::expression_vector& expressions, const std::vector<bool>& isAscOrders) {
+    if (expressions.empty()) {
+        return "";
+    }
+    std::string result = " ORDER BY ";
+    bool first = true;
+    for (size_t i = 0; i < expressions.size(); ++i) {
+        auto& expr = expressions[i];
+        std::string colName;
+        if (expr->expressionType == common::ExpressionType::VARIABLE) {
+            auto& var = expr->constCast<binder::VariableExpression>();
+            colName = var.getVariableName();
+        } else if (expr->expressionType == common::ExpressionType::PROPERTY) {
+            auto& prop = expr->constCast<binder::PropertyExpression>();
+            colName = prop.getPropertyName();
+        } else {
+            // Skip expressions that cannot be pushed down
+            continue;
+        }
+        if (!first) {
+            result += ", ";
+        }
+        result += colName;
+        result += isAscOrders[i] ? " ASC" : " DESC";
+        first = false;
+    }
+    if (first) {
+        // No expressions could be pushed down
+        return "";
+    }
+    return result;
+}
+
+} // namespace optimizer
+} // namespace lbug

package/lbug-source/src/optimizer/projection_push_down_optimizer.cpp

@@ -261,7 +261,11 @@ void ProjectionPushDownOptimizer::visitTableFunctionCall(LogicalOperator* op) {
     auto& tableFunctionCall = op->cast<LogicalTableFunctionCall>();
     std::vector<bool> columnSkips;
     for (auto& column : tableFunctionCall.getBindData()->columns) {
-        columnSkips.push_back(!variablesInUse.contains(column));
+        // Check both variablesInUse and propertiesInUse since foreign table columns
+        // may be referenced as properties in the query (e.g., a.id) but represented
+        // as variables in the table function bind data
+        columnSkips.push_back(
+            !variablesInUse.contains(column) && !propertiesInUse.contains(column));
     }
     tableFunctionCall.setColumnSkips(std::move(columnSkips));
 }

package/lbug-source/src/parser/transform/transform_graph_pattern.cpp

@@ -202,7 +202,12 @@ std::vector<std::string> Transformer::transformNodeLabels(CypherParser::OC_NodeL
 }
 
 std::string Transformer::transformLabelName(CypherParser::OC_LabelNameContext& ctx) {
-    return transformSchemaName(*ctx.oC_SchemaName());
+    auto schemaNames = ctx.oC_SchemaName();
+    if (schemaNames.size() == 1) {
+        return transformSchemaName(*schemaNames[0]);
+    }
+    // Qualified name: db.table
+    return transformSchemaName(*schemaNames[0]) + "." + transformSchemaName(*schemaNames[1]);
 }
 
 std::string Transformer::transformRelTypeName(CypherParser::OC_RelTypeNameContext& ctx) {

package/lbug-source/src/planner/join_order/cardinality_estimator.cpp

@@ -1,6 +1,7 @@
 #include "planner/join_order/cardinality_estimator.h"
 
 #include "binder/expression/property_expression.h"
+#include "catalog/catalog_entry/table_catalog_entry.h"
 #include "main/client_context.h"
 #include "planner/join_order/join_order_util.h"
 #include "planner/operator/logical_aggregate.h"
@@ -39,7 +40,12 @@ void CardinalityEstimator::init(const NodeExpression& node) {
     cardinality_t numNodes = 0u;
     auto storageManager = storage::StorageManager::Get(*context);
     auto transaction = transaction::Transaction::Get(*context);
-    for (auto tableID : node.getTableIDs()) {
+    for (auto entry : node.getEntries()) {
+        // Skip foreign tables - they don't have storage in the local database
+        if (entry->getType() == catalog::CatalogEntryType::FOREIGN_TABLE_ENTRY) {
+            continue;
+        }
+        auto tableID = entry->getTableID();
         auto stats =
             storageManager->getTable(tableID)->cast<storage::NodeTable>().getStats(transaction);
         numNodes += stats.getTableCard();
@@ -195,7 +201,10 @@ uint64_t CardinalityEstimator::getNumNodes(const Transaction*,
     const std::vector<table_id_t>& tableIDs) const {
     cardinality_t numNodes = 0u;
     for (auto& tableID : tableIDs) {
-        KU_ASSERT(nodeTableStats.contains(tableID));
+        // Skip foreign tables - they won't be in nodeTableStats
+        if (!nodeTableStats.contains(tableID)) {
+            continue;
+        }
         numNodes += nodeTableStats.at(tableID).getTableCard();
     }
     return atLeastOne(numNodes);

package/lbug-source/src/planner/operator/logical_table_function_call.cpp

@@ -19,5 +19,9 @@ void LogicalTableFunctionCall::computeFactorizedSchema() {
     }
 }
 
+std::unique_ptr<OPPrintInfo> LogicalTableFunctionCall::getPrintInfo() const {
+    return std::make_unique<LogicalTableFunctionCallPrintInfo>(getExpressionsForPrinting());
+}
+
 } // namespace planner
 } // namespace lbug

package/lbug-source/src/planner/plan/plan_join_order.cpp

@@ -1,8 +1,11 @@
 #include <cmath>
 
+#include "binder/bound_scan_source.h"
 #include "binder/expression_visitor.h"
+#include "catalog/catalog_entry/catalog_entry_type.h"
 #include "common/enums/join_type.h"
 #include "common/enums/rel_direction.h"
+#include "common/enums/table_type.h"
 #include "common/utils.h"
 #include "planner/join_order/cost_model.h"
 #include "planner/join_order/join_plan_solver.h"
@@ -246,7 +249,19 @@ void Planner::planNodeScan(uint32_t nodePos) {
     newSubgraph.addQueryNode(nodePos);
     auto plan = LogicalPlan();
     auto properties = getProperties(*node);
-    appendScanNodeTable(node->getInternalID(), node->getTableIDs(), properties, plan);
+    if (node->getEntries().size() == 1 &&
+        node->getEntries()[0]->getType() == catalog::CatalogEntryType::FOREIGN_TABLE_ENTRY) {
+        auto boundScanInfo =
+            node->getEntries()[0]->getBoundScanInfo(clientContext, node->getUniqueName());
+        if (boundScanInfo != nullptr) {
+            // Use table function call for foreign tables
+            appendTableFunctionCall(*boundScanInfo, plan);
+        } else {
+            appendScanNodeTable(node->getInternalID(), node->getTableIDs(), properties, plan);
+        }
+    } else {
+        appendScanNodeTable(node->getInternalID(), node->getTableIDs(), properties, plan);
+    }
     auto predicates = getNewlyMatchedExprs(context.getEmptySubqueryGraph(), newSubgraph,
         context.getWhereExpressions());
     appendFilters(predicates, plan);

package/lbug-source/src/processor/operator/index_lookup.cpp

@@ -28,7 +28,6 @@ std::optional<WarningSourceData> getWarningSourceData(
     return ret;
 }
 
-// TODO(Guodong): Add short path for unfiltered case.
 bool checkNullKey(ValueVector* keyVector, offset_t vectorOffset,
     BatchInsertErrorHandler* errorHandler, const std::vector<ValueVector*>& warningDataVectors) {
     bool isNull = keyVector->isNull(vectorOffset);
@@ -71,27 +70,17 @@ struct OffsetVectorManager {
     offset_t insertOffset;
 };
 
-// TODO(Guodong): Add short path for unfiltered case.
 template<bool hasNoNullsGuarantee>
-void fillOffsetArraysFromVector(transaction::Transaction* transaction, const IndexLookupInfo& info,
-    ValueVector* keyVector, ValueVector* resultVector,
-    const std::vector<ValueVector*>& warningDataVectors, BatchInsertErrorHandler* errorHandler) {
-    KU_ASSERT(resultVector->dataType.getPhysicalType() == PhysicalTypeID::INT64);
+void fillOffsetArraysFromVectorInternal(transaction::Transaction* transaction,
+    const IndexLookupInfo& info, ValueVector* keyVector, ValueVector* resultVector,
+    const std::vector<ValueVector*>& warningDataVectors, BatchInsertErrorHandler* errorHandler,
+    const sel_t* selVector, sel_t numKeys) {
     TypeUtils::visit(
         keyVector->dataType.getPhysicalType(),
         [&]<IndexHashable T>(T) {
-            auto numKeys = keyVector->state->getSelVector().getSelSize();
-
-            // fetch all the selection pos at the start
-            // since we may modify the selection vector in the middle of the lookup
-            std::vector<sel_t> lookupPos(numKeys);
-            for (idx_t i = 0; i < numKeys; ++i) {
-                lookupPos[i] = (keyVector->state->getSelVector()[i]);
-            }
-
             OffsetVectorManager resultManager{resultVector, errorHandler};
-            for (auto i = 0u; i < numKeys; i++) {
-                auto pos = lookupPos[i];
+            for (sel_t i = 0u; i < numKeys; i++) {
+                auto pos = selVector ? selVector[i] : i;
                 if constexpr (!hasNoNullsGuarantee) {
                     if (!checkNullKey(keyVector, pos, errorHandler, warningDataVectors)) {
                         continue;
@@ -99,12 +88,9 @@ void fillOffsetArraysFromVector(transaction::Transaction* transaction, const Ind
                 }
                 offset_t lookupOffset = 0;
                 if (!info.nodeTable->lookupPK(transaction, keyVector, pos, lookupOffset)) {
-                    TypeUtils::visit(keyVector->dataType, [&]<typename type>(type) {
-                        errorHandler->handleError(
-                            ExceptionMessage::nonExistentPKException(
-                                TypeUtils::toString(keyVector->getValue<type>(pos), keyVector)),
-                            getWarningSourceData(warningDataVectors, pos));
-                    });
+                    errorHandler->handleError(ExceptionMessage::nonExistentPKException(
+                                                  keyVector->getAsValue(pos)->toString()),
+                        getWarningSourceData(warningDataVectors, pos));
                 } else {
                     resultManager.insertEntry(lookupOffset, pos);
                 }
@@ -112,6 +98,28 @@ void fillOffsetArraysFromVector(transaction::Transaction* transaction, const Ind
         },
         [&](auto) { KU_UNREACHABLE; });
 }
+
+template<bool hasNoNullsGuarantee>
+void fillOffsetArraysFromVector(transaction::Transaction* transaction, const IndexLookupInfo& info,
+    ValueVector* keyVector, ValueVector* resultVector,
+    const std::vector<ValueVector*>& warningDataVectors, BatchInsertErrorHandler* errorHandler) {
+    KU_ASSERT(resultVector->dataType.getPhysicalType() == PhysicalTypeID::INT64);
+    auto& selVector = keyVector->state->getSelVector();
+    auto numKeys = selVector.getSelSize();
+    if (selVector.isUnfiltered()) {
+        // Fast path: selection vector is unfiltered - pass a null selection vector
+        fillOffsetArraysFromVectorInternal<hasNoNullsGuarantee>(transaction, info, keyVector,
+            resultVector, warningDataVectors, errorHandler, nullptr /* selVector */, numKeys);
+    } else {
+        // Filtered case: copy selection positions since we may modify the selection vector
+        std::vector<sel_t> lookupPos(numKeys);
+        for (idx_t i = 0; i < numKeys; ++i) {
+            lookupPos[i] = selVector[i];
+        }
+        fillOffsetArraysFromVectorInternal<hasNoNullsGuarantee>(transaction, info, keyVector,
+            resultVector, warningDataVectors, errorHandler, lookupPos.data(), numKeys);
+    }
+}
 } // namespace
 
 std::string IndexLookupPrintInfo::toString() const {