lbug 0.12.3-dev.9 → 0.13.0

package/lbug-source/src/binder/bind/bind_ddl.cpp

@@ -1,3 +1,5 @@
+#include <optional>
+
 #include "binder/binder.h"
 #include "binder/ddl/bound_alter.h"
 #include "binder/ddl/bound_create_sequence.h"
@@ -16,7 +18,9 @@
 #include "common/types/types.h"
 #include "function/cast/functions/cast_from_string_functions.h"
 #include "function/sequence/sequence_functions.h"
+#include "function/table/table_function.h"
 #include "main/client_context.h"
+#include "main/database_manager.h"
 #include "parser/ddl/alter.h"
 #include "parser/ddl/create_sequence.h"
 #include "parser/ddl/create_table.h"
@@ -136,18 +140,9 @@ BoundCreateTableInfo Binder::bindCreateTableInfo(const CreateTableInfo* info) {
     }
 }
 
-BoundCreateTableInfo Binder::bindCreateNodeTableInfo(const CreateTableInfo* info) {
-    auto propertyDefinitions = bindPropertyDefinitions(info->propertyDefinitions, info->tableName);
-    auto& extraInfo = info->extraInfo->constCast<ExtraCreateNodeTableInfo>();
-    validatePrimaryKey(extraInfo.pKName, propertyDefinitions);
-    auto boundExtraInfo = std::make_unique<BoundExtraCreateNodeTableInfo>(extraInfo.pKName,
-        std::move(propertyDefinitions));
-    return BoundCreateTableInfo(CatalogEntryType::NODE_TABLE_ENTRY, info->tableName,
-        info->onConflict, std::move(boundExtraInfo), clientContext->useInternalCatalogEntry());
-}
-
 void Binder::validateNodeTableType(const TableCatalogEntry* entry) {
-    if (entry->getType() != CatalogEntryType::NODE_TABLE_ENTRY) {
+    if (entry->getType() != CatalogEntryType::NODE_TABLE_ENTRY &&
+        entry->getType() != CatalogEntryType::FOREIGN_TABLE_ENTRY) {
         throw BinderException(stringFormat("{} is not of type NODE.", entry->getName()));
     }
 }
@@ -168,6 +163,13 @@ void Binder::validateColumnExistence(const TableCatalogEntry* entry,
     }
 }
 
+static std::string getStorage(const case_insensitive_map_t<Value>& options) {
+    if (options.contains(TableOptionConstants::REL_STORAGE_OPTION)) {
+        return options.at(TableOptionConstants::REL_STORAGE_OPTION).toString();
+    }
+    return "";
+}
+
 static ExtendDirection getStorageDirection(const case_insensitive_map_t<Value>& options) {
     if (options.contains(TableOptionConstants::REL_STORAGE_DIRECTION_OPTION)) {
         return ExtendDirectionUtil::fromString(
@@ -176,6 +178,18 @@ static ExtendDirection getStorageDirection(const case_insensitive_map_t<Value>&
     return DEFAULT_EXTEND_DIRECTION;
 }
 
+BoundCreateTableInfo Binder::bindCreateNodeTableInfo(const CreateTableInfo* info) {
+    auto propertyDefinitions = bindPropertyDefinitions(info->propertyDefinitions, info->tableName);
+    auto& extraInfo = info->extraInfo->constCast<ExtraCreateNodeTableInfo>();
+    validatePrimaryKey(extraInfo.pKName, propertyDefinitions);
+    auto boundOptions = bindParsingOptions(extraInfo.options);
+    auto storage = getStorage(boundOptions);
+    auto boundExtraInfo = std::make_unique<BoundExtraCreateNodeTableInfo>(extraInfo.pKName,
+        std::move(propertyDefinitions), std::move(storage));
+    return BoundCreateTableInfo(CatalogEntryType::NODE_TABLE_ENTRY, info->tableName,
+        info->onConflict, std::move(boundExtraInfo), clientContext->useInternalCatalogEntry());
+}
+
 std::vector<PropertyDefinition> Binder::bindRelPropertyDefinitions(const CreateTableInfo& info) {
     std::vector<PropertyDefinition> propertyDefinitions;
     propertyDefinitions.emplace_back(
@@ -193,15 +207,82 @@ BoundCreateTableInfo Binder::bindCreateRelTableGroupInfo(const CreateTableInfo*
     auto dstMultiplicity = RelMultiplicityUtils::getBwd(extraInfo.relMultiplicity);
     auto boundOptions = bindParsingOptions(extraInfo.options);
     auto storageDirection = getStorageDirection(boundOptions);
+    auto storage = getStorage(boundOptions);
+    std::optional<function::TableFunction> scanFunction = std::nullopt;
+    std::optional<std::unique_ptr<function::TableFuncBindData>> scanBindData = std::nullopt;
+    if (!storage.empty()) {
+        auto dotPos = storage.find('.');
+        if (dotPos != std::string::npos) {
+            // schema.table format, check if it's a foreign table
+            std::string dbName = storage.substr(0, dotPos);
+            std::string tableName = storage.substr(dotPos + 1);
+            auto transaction = transaction::Transaction::Get(*clientContext);
+            if (!dbName.empty()) {
+                auto attachedDB =
+                    main::DatabaseManager::Get(*clientContext)->getAttachedDatabase(dbName);
+                if (attachedDB && attachedDB->getCatalog()->containsTable(transaction, tableName,
+                                      clientContext->useInternalCatalogEntry())) {
+                    auto tableEntry = attachedDB->getCatalog()->getTableCatalogEntry(transaction,
+                        tableName, clientContext->useInternalCatalogEntry());
+                    if (tableEntry->getType() == CatalogEntryType::FOREIGN_TABLE_ENTRY) {
+                        scanFunction = tableEntry->getScanFunction();
+                        auto boundScanInfo =
+                            tableEntry->getBoundScanInfo(clientContext, "" /* nodeUniqueName */);
+                        scanBindData = std::move(boundScanInfo->bindData);
+                    }
+                }
+            }
+        }
+    }
     // Bind from to pairs
     node_table_id_pair_set_t nodePairsSet;
     std::vector<NodeTableIDPair> nodePairs;
+    std::string foreignDatabaseName;
+    std::string foreignDatabaseType;
     for (auto& [srcTableName, dstTableName] : extraInfo.srcDstTablePairs) {
         auto srcEntry = bindNodeTableEntry(srcTableName);
         validateNodeTableType(srcEntry);
         auto dstEntry = bindNodeTableEntry(dstTableName);
         validateNodeTableType(dstEntry);
-        NodeTableIDPair pair{srcEntry->getTableID(), dstEntry->getTableID()};
+
+        // For foreign-backed rel tables, validate that FROM and TO are from same database
+        if (scanFunction.has_value()) {
+            // Both must be foreign tables
+            if (srcEntry->getType() != CatalogEntryType::FOREIGN_TABLE_ENTRY ||
+                dstEntry->getType() != CatalogEntryType::FOREIGN_TABLE_ENTRY) {
+                throw BinderException("Foreign-backed rel tables require both FROM and TO tables "
+                                      "to be foreign tables.");
+            }
+            // Extract database names from qualified table names
+            auto srcDotPos = srcTableName.find('.');
+            auto dstDotPos = dstTableName.find('.');
+            if (srcDotPos == std::string::npos || dstDotPos == std::string::npos) {
+                throw BinderException(
+                    "Foreign-backed rel tables require qualified table names (database.table).");
+            }
+            auto srcDbName = srcTableName.substr(0, srcDotPos);
+            auto dstDbName = dstTableName.substr(0, dstDotPos);
+            if (srcDbName != dstDbName) {
+                throw BinderException(
+                    stringFormat("Cannot create rel table with FROM and TO tables from different "
+                                 "databases. FROM is from '{}', TO is from '{}'.",
+                        srcDbName, dstDbName));
+            }
+            // Get database type for display
+            auto attachedDB =
+                main::DatabaseManager::Get(*clientContext)->getAttachedDatabase(srcDbName);
+            if (attachedDB) {
+                foreignDatabaseName = stringFormat("{}({})", srcDbName, attachedDB->getDBType());
+            }
+        }
+
+        // For foreign-backed rel tables, use FOREIGN_TABLE_ID since we don't reference local node
+        // tables
+        auto srcTableID =
+            scanFunction.has_value() ? common::FOREIGN_TABLE_ID : srcEntry->getTableID();
+        auto dstTableID =
+            scanFunction.has_value() ? common::FOREIGN_TABLE_ID : dstEntry->getTableID();
+        NodeTableIDPair pair{srcTableID, dstTableID};
         if (nodePairsSet.contains(pair)) {
             throw BinderException(
                 stringFormat("Found duplicate FROM-TO {}-{} pairs.", srcTableName, dstTableName));
@@ -209,9 +290,10 @@ BoundCreateTableInfo Binder::bindCreateRelTableGroupInfo(const CreateTableInfo*
         nodePairsSet.insert(pair);
         nodePairs.emplace_back(pair);
     }
-    auto boundExtraInfo =
-        std::make_unique<BoundExtraCreateRelTableGroupInfo>(std::move(propertyDefinitions),
-            srcMultiplicity, dstMultiplicity, storageDirection, std::move(nodePairs));
+    auto boundExtraInfo = std::make_unique<BoundExtraCreateRelTableGroupInfo>(
+        std::move(propertyDefinitions), srcMultiplicity, dstMultiplicity, storageDirection,
+        std::move(nodePairs), std::move(storage), std::move(scanFunction), std::move(scanBindData),
+        std::move(foreignDatabaseName));
     return BoundCreateTableInfo(CatalogEntryType::REL_GROUP_ENTRY, info->tableName,
         info->onConflict, std::move(boundExtraInfo), clientContext->useInternalCatalogEntry());
 }

package/lbug-source/src/binder/bind/bind_graph_pattern.cpp

@@ -15,6 +15,7 @@
 #include "function/rewrite_function.h"
 #include "function/schema/vector_node_rel_functions.h"
 #include "main/client_context.h"
+#include "main/database_manager.h"
 #include "transaction/transaction.h"
 
 using namespace lbug::common;
@@ -644,7 +645,8 @@ std::vector<TableCatalogEntry*> Binder::bindNodeTableEntries(
     } else {
         for (auto& name : tableNames) {
             auto entry = bindNodeTableEntry(name);
-            if (entry->getType() != CatalogEntryType::NODE_TABLE_ENTRY) {
+            if (entry->getType() != CatalogEntryType::NODE_TABLE_ENTRY &&
+                entry->getType() != CatalogEntryType::FOREIGN_TABLE_ENTRY) {
                 throw BinderException(
                     stringFormat("Cannot bind {} as a node pattern label.", entry->getName()));
             }
@@ -658,10 +660,35 @@ TableCatalogEntry* Binder::bindNodeTableEntry(const std::string& name) const {
     auto transaction = transaction::Transaction::Get(*clientContext);
     auto catalog = Catalog::Get(*clientContext);
     auto useInternal = clientContext->useInternalCatalogEntry();
-    if (!catalog->containsTable(transaction, name, useInternal)) {
+
+    std::string dbName;
+    std::string tableName = name;
+    auto dotPos = name.find('.');
+    if (dotPos != std::string::npos) {
+        dbName = name.substr(0, dotPos);
+        tableName = name.substr(dotPos + 1);
+    }
+
+    if (!dbName.empty()) {
+        // Qualified name: db.table
+        auto attachedDB = main::DatabaseManager::Get(*clientContext)->getAttachedDatabase(dbName);
+        if (!attachedDB) {
+            throw BinderException(stringFormat("Attached database {} does not exist.", dbName));
+        }
+        auto attachedCatalog = attachedDB->getCatalog();
+        if (!attachedCatalog->containsTable(transaction, tableName, useInternal)) {
+            throw BinderException(stringFormat("Table {} does not exist in attached database {}.",
+                tableName, dbName));
+        }
+        return attachedCatalog->getTableCatalogEntry(transaction, tableName, useInternal);
+    } else {
+        // Unqualified name: only search main catalog
+        // Foreign tables require qualified names (db.table) to avoid ambiguity
+        if (catalog->containsTable(transaction, name, useInternal)) {
+            return catalog->getTableCatalogEntry(transaction, name, useInternal);
+        }
         throw BinderException(stringFormat("Table {} does not exist.", name));
     }
-    return catalog->getTableCatalogEntry(transaction, name, useInternal);
 }
 
 std::vector<TableCatalogEntry*> Binder::bindRelGroupEntries(

package/lbug-source/src/catalog/catalog.cpp

@@ -190,9 +190,10 @@ CatalogEntry* Catalog::createRelGroupEntry(Transaction* transaction,
     for (auto& nodePair : extraInfo->nodePairs) {
         relTableInfos.emplace_back(nodePair, tables->getNextOID());
     }
-    auto relGroupEntry =
-        std::make_unique<RelGroupCatalogEntry>(info.tableName, extraInfo->srcMultiplicity,
-            extraInfo->dstMultiplicity, extraInfo->storageDirection, std::move(relTableInfos));
+    auto relGroupEntry = std::make_unique<RelGroupCatalogEntry>(info.tableName,
+        extraInfo->srcMultiplicity, extraInfo->dstMultiplicity, extraInfo->storageDirection,
+        std::move(relTableInfos), extraInfo->storage, extraInfo->scanFunction,
+        std::move(extraInfo->scanBindData), extraInfo->foreignDatabaseName);
     for (auto& definition : extraInfo->propertyDefinitions) {
         relGroupEntry->addProperty(definition);
     }
@@ -541,7 +542,8 @@ CatalogEntry* Catalog::createTableEntry(Transaction* transaction,
 CatalogEntry* Catalog::createNodeTableEntry(Transaction* transaction,
     const BoundCreateTableInfo& info) {
     const auto extraInfo = info.extraInfo->constPtrCast<BoundExtraCreateNodeTableInfo>();
-    auto entry = std::make_unique<NodeTableCatalogEntry>(info.tableName, extraInfo->primaryKeyName);
+    auto entry = std::make_unique<NodeTableCatalogEntry>(info.tableName, extraInfo->primaryKeyName,
+        extraInfo->storage);
     for (auto& definition : extraInfo->propertyDefinitions) {
         entry->addProperty(definition);
     }

package/lbug-source/src/catalog/catalog_entry/node_table_catalog_entry.cpp

@@ -21,16 +21,22 @@ void NodeTableCatalogEntry::serialize(common::Serializer& serializer) const {
     TableCatalogEntry::serialize(serializer);
     serializer.writeDebuggingInfo("primaryKeyName");
     serializer.write(primaryKeyName);
+    serializer.writeDebuggingInfo("storage");
+    serializer.write(storage);
 }
 
 std::unique_ptr<NodeTableCatalogEntry> NodeTableCatalogEntry::deserialize(
     common::Deserializer& deserializer) {
     std::string debuggingInfo;
     std::string primaryKeyName;
+    std::string storage;
     deserializer.validateDebuggingInfo(debuggingInfo, "primaryKeyName");
     deserializer.deserializeValue(primaryKeyName);
+    deserializer.validateDebuggingInfo(debuggingInfo, "storage");
+    deserializer.deserializeValue(storage);
     auto nodeTableEntry = std::make_unique<NodeTableCatalogEntry>();
     nodeTableEntry->primaryKeyName = primaryKeyName;
+    nodeTableEntry->storage = storage;
     return nodeTableEntry;
 }
 
@@ -42,6 +48,7 @@ std::string NodeTableCatalogEntry::toCypher(const ToCypherInfo& /*info*/) const
 std::unique_ptr<TableCatalogEntry> NodeTableCatalogEntry::copy() const {
     auto other = std::make_unique<NodeTableCatalogEntry>();
     other->primaryKeyName = primaryKeyName;
+    other->storage = storage;
     other->copyFrom(*this);
     return other;
 }
@@ -49,7 +56,7 @@ std::unique_ptr<TableCatalogEntry> NodeTableCatalogEntry::copy() const {
 std::unique_ptr<BoundExtraCreateCatalogEntryInfo> NodeTableCatalogEntry::getBoundExtraCreateInfo(
     transaction::Transaction*) const {
     return std::make_unique<BoundExtraCreateNodeTableInfo>(primaryKeyName,
-        copyVector(getProperties()));
+        copyVector(getProperties()), storage);
 }
 
 } // namespace catalog

package/lbug-source/src/catalog/catalog_entry/rel_group_catalog_entry.cpp

@@ -74,7 +74,10 @@ const RelTableCatalogInfo* RelGroupCatalogEntry::getRelEntryInfo(table_id_t srcT
 std::unordered_set<table_id_t> RelGroupCatalogEntry::getSrcNodeTableIDSet() const {
     std::unordered_set<table_id_t> result;
     for (auto& info : relTableInfos) {
-        result.insert(info.nodePair.srcTableID);
+        // Skip FOREIGN_TABLE_ID for foreign-backed rel tables
+        if (info.nodePair.srcTableID != FOREIGN_TABLE_ID) {
+            result.insert(info.nodePair.srcTableID);
+        }
     }
     return result;
 }
@@ -82,7 +85,10 @@ std::unordered_set<table_id_t> RelGroupCatalogEntry::getSrcNodeTableIDSet() cons
 std::unordered_set<table_id_t> RelGroupCatalogEntry::getDstNodeTableIDSet() const {
     std::unordered_set<table_id_t> result;
     for (auto& info : relTableInfos) {
-        result.insert(info.nodePair.dstTableID);
+        // Skip FOREIGN_TABLE_ID for foreign-backed rel tables
+        if (info.nodePair.dstTableID != FOREIGN_TABLE_ID) {
+            result.insert(info.nodePair.dstTableID);
+        }
     }
     return result;
 }
@@ -95,6 +101,13 @@ void RelGroupCatalogEntry::serialize(Serializer& serializer) const {
     serializer.serializeValue(dstMultiplicity);
     serializer.writeDebuggingInfo("storageDirection");
     serializer.serializeValue(storageDirection);
+    serializer.writeDebuggingInfo("storage");
+    serializer.serializeValue(storage);
+    serializer.writeDebuggingInfo("scanFunction");
+    serializer.serializeValue(scanFunction.has_value());
+    if (scanFunction.has_value()) {
+        // TODO: serialize TableFunction
+    }
     serializer.writeDebuggingInfo("relTableInfos");
     serializer.serializeVector(relTableInfos);
 }
@@ -105,6 +118,7 @@ std::unique_ptr<RelGroupCatalogEntry> RelGroupCatalogEntry::deserialize(
     auto srcMultiplicity = RelMultiplicity::MANY;
     auto dstMultiplicity = RelMultiplicity::MANY;
     auto storageDirection = ExtendDirection::BOTH;
+    std::string storage;
     std::vector<RelTableCatalogInfo> relTableInfos;
     deserializer.validateDebuggingInfo(debuggingInfo, "srcMultiplicity");
     deserializer.deserializeValue(srcMultiplicity);
@@ -112,20 +126,40 @@ std::unique_ptr<RelGroupCatalogEntry> RelGroupCatalogEntry::deserialize(
     deserializer.deserializeValue(dstMultiplicity);
     deserializer.validateDebuggingInfo(debuggingInfo, "storageDirection");
     deserializer.deserializeValue(storageDirection);
+    deserializer.validateDebuggingInfo(debuggingInfo, "storage");
+    deserializer.deserializeValue(storage);
+    deserializer.validateDebuggingInfo(debuggingInfo, "scanFunction");
+    bool hasScanFunction;
+    deserializer.deserializeValue(hasScanFunction);
+    std::optional<function::TableFunction> scanFunction = std::nullopt;
+    if (hasScanFunction) {
+        // TODO: deserialize TableFunction
+    }
     deserializer.validateDebuggingInfo(debuggingInfo, "relTableInfos");
     deserializer.deserializeVector(relTableInfos);
     auto relGroupEntry = std::make_unique<RelGroupCatalogEntry>();
     relGroupEntry->srcMultiplicity = srcMultiplicity;
     relGroupEntry->dstMultiplicity = dstMultiplicity;
     relGroupEntry->storageDirection = storageDirection;
+    relGroupEntry->storage = storage;
+    relGroupEntry->scanFunction = scanFunction;
     relGroupEntry->relTableInfos = relTableInfos;
     return relGroupEntry;
 }
 
 static std::string getFromToStr(const NodeTableIDPair& pair, const Catalog* catalog,
-    const transaction::Transaction* transaction) {
-    auto srcTableName = catalog->getTableCatalogEntry(transaction, pair.srcTableID)->getName();
-    auto dstTableName = catalog->getTableCatalogEntry(transaction, pair.dstTableID)->getName();
+    const transaction::Transaction* transaction, const std::string& storage) {
+    std::string srcTableName, dstTableName;
+    // For foreign-backed rel tables, node table IDs are FOREIGN_TABLE_ID
+    if (pair.srcTableID == common::FOREIGN_TABLE_ID && !storage.empty()) {
+        auto dotPos = storage.find('.');
+        srcTableName =
+            dotPos != std::string::npos ? storage.substr(0, dotPos) + ".nodes" : "foreign_table";
+        dstTableName = srcTableName;
+    } else {
+        srcTableName = catalog->getTableCatalogEntry(transaction, pair.srcTableID)->getName();
+        dstTableName = catalog->getTableCatalogEntry(transaction, pair.dstTableID)->getName();
+    }
     return stringFormat("FROM `{}` TO `{}`", srcTableName, dstTableName);
 }
 
@@ -136,9 +170,10 @@ std::string RelGroupCatalogEntry::toCypher(const ToCypherInfo& info) const {
     std::stringstream ss;
     ss << stringFormat("CREATE REL TABLE `{}` (", getName());
     KU_ASSERT(!relTableInfos.empty());
-    ss << getFromToStr(relTableInfos[0].nodePair, catalog, transaction);
+    ss << getFromToStr(relTableInfos[0].nodePair, catalog, transaction, storage);
     for (auto i = 1u; i < relTableInfos.size(); ++i) {
-        ss << stringFormat(", {}", getFromToStr(relTableInfos[i].nodePair, catalog, transaction));
+        ss << stringFormat(", {}",
+            getFromToStr(relTableInfos[i].nodePair, catalog, transaction, storage));
     }
     ss << ", " << propertyCollection.toCypher() << RelMultiplicityUtils::toString(srcMultiplicity)
        << "_" << RelMultiplicityUtils::toString(dstMultiplicity) << ");";
@@ -167,6 +202,10 @@ std::unique_ptr<TableCatalogEntry> RelGroupCatalogEntry::copy() const {
     other->srcMultiplicity = srcMultiplicity;
     other->dstMultiplicity = dstMultiplicity;
     other->storageDirection = storageDirection;
+    other->storage = storage;
+    other->scanFunction = scanFunction;
+    other->scanBindData = std::nullopt; // TODO: implement copy for bindData if needed
+    other->foreignDatabaseName = foreignDatabaseName;
     other->relTableInfos = relTableInfos;
     other->copyFrom(*this);
     return other;

package/lbug-source/src/catalog/catalog_set.cpp

@@ -246,6 +246,7 @@ void CatalogSet::serialize(Serializer serializer) const {
         case CatalogEntryType::COPY_FUNCTION_ENTRY:
         case CatalogEntryType::TABLE_FUNCTION_ENTRY:
         case CatalogEntryType::STANDALONE_TABLE_FUNCTION_ENTRY:
+        case CatalogEntryType::FOREIGN_TABLE_ENTRY:
             continue;
         default: {
             auto committedEntry = getCommittedEntryNoLock(entry.get());

package/lbug-source/src/function/function_collection.cpp

@@ -228,7 +228,8 @@ FunctionCollection* FunctionCollection::getFunctions() {
         TABLE_FUNCTION(StatsInfoFunction), TABLE_FUNCTION(StorageInfoFunction),
         TABLE_FUNCTION(ShowAttachedDatabasesFunction), TABLE_FUNCTION(ShowSequencesFunction),
         TABLE_FUNCTION(ShowFunctionsFunction), TABLE_FUNCTION(BMInfoFunction),
-        TABLE_FUNCTION(FileInfoFunction), TABLE_FUNCTION(ShowLoadedExtensionsFunction),
+        TABLE_FUNCTION(FileInfoFunction), TABLE_FUNCTION(DiskSizeInfoFunction),
+        TABLE_FUNCTION(ShowLoadedExtensionsFunction),
         TABLE_FUNCTION(ShowOfficialExtensionsFunction), TABLE_FUNCTION(ShowIndexesFunction),
         TABLE_FUNCTION(ShowProjectedGraphsFunction), TABLE_FUNCTION(ProjectedGraphInfoFunction),
         TABLE_FUNCTION(ShowMacrosFunction),

package/lbug-source/src/function/table/CMakeLists.txt

@@ -8,6 +8,7 @@ add_library(lbug_table_function
         clear_warnings.cpp
         current_setting.cpp
         db_version.cpp
+        disk_size_info.cpp
         drop_project_graph.cpp
         file_info.cpp
         free_space_info.cpp