duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/src/function/table/system/duckdb_functions.cpp

@@ -222,7 +222,7 @@ struct AggregateFunctionExtractor {
 
 struct MacroExtractor {
 	static idx_t FunctionCount(ScalarMacroCatalogEntry &entry) {
-		return 1;
+		return entry.macros.size();
 	}
 
 	static Value GetFunctionType() {
@@ -235,12 +235,13 @@ struct MacroExtractor {
 
 	static vector<Value> GetParameters(ScalarMacroCatalogEntry &entry, idx_t offset) {
 		vector<Value> results;
-		for (auto &param : entry.function->parameters) {
+		auto &macro_entry = *entry.macros[offset];
+		for (auto &param : macro_entry.parameters) {
 			D_ASSERT(param->type == ExpressionType::COLUMN_REF);
 			auto &colref = param->Cast<ColumnRefExpression>();
 			results.emplace_back(colref.GetColumnName());
 		}
-		for (auto &param_entry : entry.function->default_parameters) {
+		for (auto &param_entry : macro_entry.default_parameters) {
 			results.emplace_back(param_entry.first);
 		}
 		return results;
@@ -248,10 +249,11 @@ struct MacroExtractor {
 
 	static Value GetParameterTypes(ScalarMacroCatalogEntry &entry, idx_t offset) {
 		vector<Value> results;
-		for (idx_t i = 0; i < entry.function->parameters.size(); i++) {
+		auto &macro_entry = *entry.macros[offset];
+		for (idx_t i = 0; i < macro_entry.parameters.size(); i++) {
 			results.emplace_back(LogicalType::VARCHAR);
 		}
-		for (idx_t i = 0; i < entry.function->default_parameters.size(); i++) {
+		for (idx_t i = 0; i < macro_entry.default_parameters.size(); i++) {
 			results.emplace_back(LogicalType::VARCHAR);
 		}
 		return Value::LIST(LogicalType::VARCHAR, std::move(results));
@@ -262,8 +264,9 @@ struct MacroExtractor {
 	}
 
 	static Value GetMacroDefinition(ScalarMacroCatalogEntry &entry, idx_t offset) {
-		D_ASSERT(entry.function->type == MacroType::SCALAR_MACRO);
-		auto &func = entry.function->Cast<ScalarMacroFunction>();
+		auto &macro_entry = *entry.macros[offset];
+		D_ASSERT(macro_entry.type == MacroType::SCALAR_MACRO);
+		auto &func = macro_entry.Cast<ScalarMacroFunction>();
 		return func.expression->ToString();
 	}
 
@@ -278,7 +281,7 @@ struct MacroExtractor {
 
 struct TableMacroExtractor {
 	static idx_t FunctionCount(TableMacroCatalogEntry &entry) {
-		return 1;
+		return entry.macros.size();
 	}
 
 	static Value GetFunctionType() {
@@ -291,12 +294,13 @@ struct TableMacroExtractor {
 
 	static vector<Value> GetParameters(TableMacroCatalogEntry &entry, idx_t offset) {
 		vector<Value> results;
-		for (auto &param : entry.function->parameters) {
+		auto &macro_entry = *entry.macros[offset];
+		for (auto &param : macro_entry.parameters) {
 			D_ASSERT(param->type == ExpressionType::COLUMN_REF);
 			auto &colref = param->Cast<ColumnRefExpression>();
 			results.emplace_back(colref.GetColumnName());
 		}
-		for (auto &param_entry : entry.function->default_parameters) {
+		for (auto &param_entry : macro_entry.default_parameters) {
 			results.emplace_back(param_entry.first);
 		}
 		return results;
@@ -304,10 +308,11 @@ struct TableMacroExtractor {
 
 	static Value GetParameterTypes(TableMacroCatalogEntry &entry, idx_t offset) {
 		vector<Value> results;
-		for (idx_t i = 0; i < entry.function->parameters.size(); i++) {
+		auto &macro_entry = *entry.macros[offset];
+		for (idx_t i = 0; i < macro_entry.parameters.size(); i++) {
 			results.emplace_back(LogicalType::VARCHAR);
 		}
-		for (idx_t i = 0; i < entry.function->default_parameters.size(); i++) {
+		for (idx_t i = 0; i < macro_entry.default_parameters.size(); i++) {
 			results.emplace_back(LogicalType::VARCHAR);
 		}
 		return Value::LIST(LogicalType::VARCHAR, std::move(results));
@@ -318,8 +323,9 @@ struct TableMacroExtractor {
 	}
 
 	static Value GetMacroDefinition(TableMacroCatalogEntry &entry, idx_t offset) {
-		if (entry.function->type == MacroType::TABLE_MACRO) {
-			auto &func = entry.function->Cast<TableMacroFunction>();
+		auto &macro_entry = *entry.macros[offset];
+		if (macro_entry.type == MacroType::TABLE_MACRO) {
+			auto &func = macro_entry.Cast<TableMacroFunction>();
 			return func.query_node->ToString();
 		}
 		return Value();

package/src/duckdb/src/function/table/system/duckdb_indexes.cpp

@@ -75,6 +75,20 @@ unique_ptr<GlobalTableFunctionState> DuckDBIndexesInit(ClientContext &context, T
 	return std::move(result);
 }
 
+Value GetIndexExpressions(IndexCatalogEntry &index) {
+	auto create_info = index.GetInfo();
+	auto &create_index_info = create_info->Cast<CreateIndexInfo>();
+
+	auto vec = create_index_info.ExpressionsToList();
+
+	vector<Value> content;
+	content.reserve(vec.size());
+	for (auto &item : vec) {
+		content.push_back(Value(item));
+	}
+	return Value::LIST(LogicalType::VARCHAR, std::move(content));
+}
+
 void DuckDBIndexesFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) {
 	auto &data = data_p.global_state->Cast<DuckDBIndexesData>();
 	if (data.offset >= data.entries.size()) {
@@ -119,7 +133,7 @@ void DuckDBIndexesFunction(ClientContext &context, TableFunctionInput &data_p, D
 		// is_primary, BOOLEAN
 		output.SetValue(col++, count, Value::BOOLEAN(index.IsPrimary()));
 		// expressions, VARCHAR
-		output.SetValue(col++, count, Value());
+		output.SetValue(col++, count, GetIndexExpressions(index).ToString());
 		// sql, VARCHAR
 		auto sql = index.ToSQL();
 		output.SetValue(col++, count, sql.empty() ? Value() : Value(std::move(sql)));

package/src/duckdb/src/function/table/system/duckdb_variables.cpp

@@ -0,0 +1,84 @@
+#include "duckdb/function/table/system_functions.hpp"
+
+#include "duckdb/catalog/catalog.hpp"
+#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp"
+#include "duckdb/catalog/catalog_entry/type_catalog_entry.hpp"
+#include "duckdb/common/enum_util.hpp"
+#include "duckdb/common/exception.hpp"
+#include "duckdb/main/client_context.hpp"
+#include "duckdb/main/client_config.hpp"
+#include "duckdb/main/client_data.hpp"
+
+namespace duckdb {
+
+struct VariableData {
+	string name;
+	Value value;
+};
+
+struct DuckDBVariablesData : public GlobalTableFunctionState {
+	DuckDBVariablesData() : offset(0) {
+	}
+
+	vector<VariableData> variables;
+	idx_t offset;
+};
+
+static unique_ptr<FunctionData> DuckDBVariablesBind(ClientContext &context, TableFunctionBindInput &input,
+                                                    vector<LogicalType> &return_types, vector<string> &names) {
+	names.emplace_back("name");
+	return_types.emplace_back(LogicalType::VARCHAR);
+
+	names.emplace_back("value");
+	return_types.emplace_back(LogicalType::VARCHAR);
+
+	names.emplace_back("type");
+	return_types.emplace_back(LogicalType::VARCHAR);
+
+	return nullptr;
+}
+
+unique_ptr<GlobalTableFunctionState> DuckDBVariablesInit(ClientContext &context, TableFunctionInitInput &input) {
+	auto result = make_uniq<DuckDBVariablesData>();
+	auto &config = ClientConfig::GetConfig(context);
+
+	for (auto &entry : config.user_variables) {
+		VariableData data;
+		data.name = entry.first;
+		data.value = entry.second;
+		result->variables.push_back(std::move(data));
+	}
+	return std::move(result);
+}
+
+void DuckDBVariablesFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) {
+	auto &data = data_p.global_state->Cast<DuckDBVariablesData>();
+	if (data.offset >= data.variables.size()) {
+		// finished returning values
+		return;
+	}
+	// start returning values
+	// either fill up the chunk or return all the remaining columns
+	idx_t count = 0;
+	while (data.offset < data.variables.size() && count < STANDARD_VECTOR_SIZE) {
+		auto &variable_entry = data.variables[data.offset++];
+
+		// return values:
+		idx_t col = 0;
+		// name, VARCHAR
+		output.SetValue(col++, count, Value(variable_entry.name));
+		// value, BIGINT
+		output.SetValue(col++, count, Value(variable_entry.value.ToString()));
+		// type, VARCHAR
+		output.SetValue(col, count, Value(variable_entry.value.type().ToString()));
+		count++;
+	}
+	output.SetCardinality(count);
+}
+
+void DuckDBVariablesFun::RegisterFunction(BuiltinFunctions &set) {
+	set.AddFunction(
+	    TableFunction("duckdb_variables", {}, DuckDBVariablesFunction, DuckDBVariablesBind, DuckDBVariablesInit));
+}
+
+} // namespace duckdb

package/src/duckdb/src/function/table/system/test_all_types.cpp

@@ -31,6 +31,7 @@ vector<TestType> TestAllTypesFun::GetTestTypes(bool use_large_enum) {
 	result.emplace_back(LogicalType::USMALLINT, "usmallint");
 	result.emplace_back(LogicalType::UINTEGER, "uint");
 	result.emplace_back(LogicalType::UBIGINT, "ubigint");
+	result.emplace_back(LogicalType::VARINT, "varint");
 	result.emplace_back(LogicalType::DATE, "date");
 	result.emplace_back(LogicalType::TIME, "time");
 	result.emplace_back(LogicalType::TIMESTAMP, "timestamp");

package/src/duckdb/src/function/table/system/test_vector_types.cpp

@@ -78,6 +78,14 @@ struct TestVectorFlat {
 			break;
 		}
 		case PhysicalType::LIST: {
+			if (type.id() == LogicalTypeId::MAP) {
+				auto &child_type = ListType::GetChildType(type);
+				auto child_values = GenerateValues(info, child_type);
+				result.push_back(Value::MAP(child_type, {child_values[0]}));
+				result.push_back(Value(type));
+				result.push_back(Value::MAP(child_type, {child_values[1]}));
+				break;
+			}
 			auto &child_type = ListType::GetChildType(type);
 			auto child_values = GenerateValues(info, child_type);
 
@@ -157,6 +165,9 @@ struct TestVectorSequence {
 		case LogicalTypeId::UINTEGER:
 		case LogicalTypeId::UBIGINT:
 			result.Sequence(3, 2, 3);
+#if STANDARD_VECTOR_SIZE <= 2
+			result.Flatten(3);
+#endif
 			return;
 		default:
 			break;
@@ -170,6 +181,7 @@ struct TestVectorSequence {
 			break;
 		}
 		case PhysicalType::LIST: {
+			D_ASSERT(type.id() != LogicalTypeId::MAP);
 			auto data = FlatVector::GetData<list_entry_t>(result);
 			data[0].offset = 0;
 			data[0].length = 2;
@@ -196,15 +208,33 @@ struct TestVectorSequence {
 	}
 
 	static void Generate(TestVectorInfo &info) {
-#if STANDARD_VECTOR_SIZE > 2
+		static constexpr const idx_t SEQ_CARDINALITY = 3;
+
 		auto result = make_uniq<DataChunk>();
-		result->Initialize(Allocator::DefaultAllocator(), info.types);
+		result->Initialize(Allocator::DefaultAllocator(), info.types,
+		                   MaxValue<idx_t>(SEQ_CARDINALITY, STANDARD_VECTOR_SIZE));
 
 		for (idx_t c = 0; c < info.types.size(); c++) {
+			if (info.types[c].id() == LogicalTypeId::MAP) {
+				// FIXME: we don't support MAP in the TestVectorSequence
+				return;
+			}
 			GenerateVector(info, info.types[c], result->data[c]);
 		}
-		result->SetCardinality(3);
+		result->SetCardinality(SEQ_CARDINALITY);
+#if STANDARD_VECTOR_SIZE > 2
 		info.entries.push_back(std::move(result));
+#else
+		// vsize = 2, split into two smaller data chunks
+		for (idx_t offset = 0; offset < SEQ_CARDINALITY; offset += STANDARD_VECTOR_SIZE) {
+			auto new_result = make_uniq<DataChunk>();
+			new_result->Initialize(Allocator::DefaultAllocator(), info.types);
+
+			idx_t copy_count = MinValue<idx_t>(STANDARD_VECTOR_SIZE, SEQ_CARDINALITY - offset);
+			result->Copy(*new_result, *FlatVector::IncrementalSelectionVector(), offset + copy_count, offset);
+
+			info.entries.push_back(std::move(new_result));
+		}
 #endif
 	}
 };

package/src/duckdb/src/function/table/system_functions.cpp

@@ -36,6 +36,7 @@ void BuiltinFunctions::RegisterSQLiteFunctions() {
 	DuckDBTablesFun::RegisterFunction(*this);
 	DuckDBTemporaryFilesFun::RegisterFunction(*this);
 	DuckDBTypesFun::RegisterFunction(*this);
+	DuckDBVariablesFun::RegisterFunction(*this);
 	DuckDBViewsFun::RegisterFunction(*this);
 	TestAllTypesFun::RegisterFunction(*this);
 	TestVectorTypesFun::RegisterFunction(*this);

package/src/duckdb/src/function/table/table_scan.cpp

@@ -86,7 +86,6 @@ static unique_ptr<LocalTableFunctionState> TableScanInitLocal(ExecutionContext &
 }
 
 unique_ptr<GlobalTableFunctionState> TableScanInitGlobal(ClientContext &context, TableFunctionInitInput &input) {
-
 	D_ASSERT(input.bind_data);
 	auto &bind_data = input.bind_data->Cast<TableScanBindData>();
 	auto result = make_uniq<TableScanGlobalState>(context, input.bind_data.get());
@@ -167,7 +166,7 @@ double TableScanProgress(ClientContext &context, const FunctionData *bind_data_p
 	}
 	idx_t scanned_rows = gstate.state.scan_state.processed_rows;
 	scanned_rows += gstate.state.local_state.processed_rows;
-	auto percentage = 100 * (double(scanned_rows) / total_rows);
+	auto percentage = 100 * (static_cast<double>(scanned_rows) / static_cast<double>(total_rows));
 	if (percentage > 100) {
 		//! In case the last chunk has less elements than STANDARD_VECTOR_SIZE, if our percentage is over 100
 		//! It means we finished this table.
@@ -211,10 +210,13 @@ unique_ptr<NodeStatistics> TableScanCardinality(ClientContext &context, const Fu
 // Index Scan
 //===--------------------------------------------------------------------===//
 struct IndexScanGlobalState : public GlobalTableFunctionState {
-	explicit IndexScanGlobalState(data_ptr_t row_id_data) : row_ids(LogicalType::ROW_TYPE, row_id_data) {
+	IndexScanGlobalState(const data_ptr_t row_id_data, const idx_t count)
+	    : row_ids(LogicalType::ROW_TYPE, row_id_data), row_ids_count(count), row_ids_offset(0) {
 	}
 
-	Vector row_ids;
+	const Vector row_ids;
+	const idx_t row_ids_count;
+	idx_t row_ids_offset;
 	ColumnFetchState fetch_state;
 	TableScanState local_storage_state;
 	vector<storage_t> column_ids;
@@ -223,19 +225,21 @@ struct IndexScanGlobalState : public GlobalTableFunctionState {
 
 static unique_ptr<GlobalTableFunctionState> IndexScanInitGlobal(ClientContext &context, TableFunctionInitInput &input) {
 	auto &bind_data = input.bind_data->Cast<TableScanBindData>();
+
 	data_ptr_t row_id_data = nullptr;
-	if (!bind_data.result_ids.empty()) {
-		row_id_data = (data_ptr_t)&bind_data.result_ids[0]; // NOLINT - this is not pretty
+	if (!bind_data.row_ids.empty()) {
+		row_id_data = (data_ptr_t)&bind_data.row_ids[0]; // NOLINT - this is not pretty
 	}
-	auto result = make_uniq<IndexScanGlobalState>(row_id_data);
+
+	auto result = make_uniq<IndexScanGlobalState>(row_id_data, bind_data.row_ids.size());
 	auto &local_storage = LocalStorage::Get(context, bind_data.table.catalog);
 
 	result->local_storage_state.options.force_fetch_row = ClientConfig::GetConfig(context).force_fetch_row;
-
 	result->column_ids.reserve(input.column_ids.size());
 	for (auto &id : input.column_ids) {
 		result->column_ids.push_back(GetStorageIndex(bind_data.table, id));
 	}
+
 	result->local_storage_state.Initialize(result->column_ids, input.filters.get());
 	local_storage.InitializeScan(bind_data.table.GetStorage(), result->local_storage_state.local_state, input.filters);
 
@@ -250,9 +254,17 @@ static void IndexScanFunction(ClientContext &context, TableFunctionInput &data_p
 	auto &local_storage = LocalStorage::Get(transaction);
 
 	if (!state.finished) {
-		bind_data.table.GetStorage().Fetch(transaction, output, state.column_ids, state.row_ids,
-		                                   bind_data.result_ids.size(), state.fetch_state);
-		state.finished = true;
+		auto remaining = state.row_ids_count - state.row_ids_offset;
+		auto scan_count = remaining < STANDARD_VECTOR_SIZE ? remaining : STANDARD_VECTOR_SIZE;
+
+		Vector row_ids(state.row_ids, state.row_ids_offset, state.row_ids_offset + scan_count);
+		bind_data.table.GetStorage().Fetch(transaction, output, state.column_ids, row_ids, scan_count,
+		                                   state.fetch_state);
+
+		state.row_ids_offset += scan_count;
+		if (state.row_ids_offset == state.row_ids_count) {
+			state.finished = true;
+		}
 	}
 	if (output.size() == 0) {
 		local_storage.Scan(state.local_storage_state.local_state, state.column_ids, output);
@@ -265,10 +277,11 @@ static void RewriteIndexExpression(Index &index, LogicalGet &get, Expression &ex
 		// bound column ref: rewrite to fit in the current set of bound column ids
 		bound_colref.binding.table_index = get.table_index;
 		auto &column_ids = index.GetColumnIds();
+		auto &get_column_ids = get.GetColumnIds();
 		column_t referenced_column = column_ids[bound_colref.binding.column_index];
 		// search for the referenced column in the set of column_ids
-		for (idx_t i = 0; i < get.column_ids.size(); i++) {
-			if (get.column_ids[i] == referenced_column) {
+		for (idx_t i = 0; i < get_column_ids.size(); i++) {
+			if (get_column_ids[i] == referenced_column) {
 				bound_colref.binding.column_index = i;
 				return;
 			}
@@ -310,7 +323,6 @@ void TableScanPushdownComplexFilter(ClientContext &context, LogicalGet &get, Fun
 
 	auto checkpoint_lock = storage.GetSharedCheckpointLock();
 	auto &info = storage.GetDataTableInfo();
-	auto &transaction = Transaction::Get(context, bind_data.table.catalog);
 
 	// bind and scan any ART indexes
 	info->GetIndexes().BindAndScan<ART>(context, *info, [&](ART &art_index) {
@@ -328,17 +340,28 @@ void TableScanPushdownComplexFilter(ClientContext &context, LogicalGet &get, Fun
 			return false;
 		}
 
-		// try to find a matching index for any of the filter expressions
+		// Try to find a matching index for any of the filter expressions.
 		for (auto &filter : filters) {
-			auto index_state = art_index.TryInitializeScan(transaction, *index_expression, *filter);
+			auto index_state = art_index.TryInitializeScan(*index_expression, *filter);
 			if (index_state != nullptr) {
-				if (art_index.Scan(transaction, storage, *index_state, STANDARD_VECTOR_SIZE, bind_data.result_ids)) {
-					// use an index scan!
+
+				auto &db_config = DBConfig::GetConfig(context);
+				auto index_scan_percentage = db_config.options.index_scan_percentage;
+				auto index_scan_max_count = db_config.options.index_scan_max_count;
+
+				auto total_rows = storage.GetTotalRows();
+				auto total_rows_from_percentage = LossyNumericCast<idx_t>(double(total_rows) * index_scan_percentage);
+				auto max_count = MaxValue(index_scan_max_count, total_rows_from_percentage);
+
+				// Check if we can use an index scan, and already retrieve the matching row ids.
+				if (art_index.Scan(*index_state, max_count, bind_data.row_ids)) {
 					bind_data.is_index_scan = true;
 					get.function = TableScanFunction::GetIndexScanFunction();
-				} else {
-					bind_data.result_ids.clear();
+					return true;
 				}
+
+				// Clear the row ids in case we exceeded the maximum count and stopped scanning.
+				bind_data.row_ids.clear();
 				return true;
 			}
 		}
@@ -360,7 +383,7 @@ static void TableScanSerialize(Serializer &serializer, const optional_ptr<Functi
 	serializer.WriteProperty(102, "table", bind_data.table.name);
 	serializer.WriteProperty(103, "is_index_scan", bind_data.is_index_scan);
 	serializer.WriteProperty(104, "is_create_index", bind_data.is_create_index);
-	serializer.WriteProperty(105, "result_ids", bind_data.result_ids);
+	serializer.WriteProperty(105, "result_ids", bind_data.row_ids);
 }
 
 static unique_ptr<FunctionData> TableScanDeserialize(Deserializer &deserializer, TableFunction &function) {
@@ -375,7 +398,7 @@ static unique_ptr<FunctionData> TableScanDeserialize(Deserializer &deserializer,
 	auto result = make_uniq<TableScanBindData>(catalog_entry.Cast<DuckTableEntry>());
 	deserializer.ReadProperty(103, "is_index_scan", result->is_index_scan);
 	deserializer.ReadProperty(104, "is_create_index", result->is_create_index);
-	deserializer.ReadProperty(105, "result_ids", result->result_ids);
+	deserializer.ReadProperty(105, "result_ids", result->row_ids);
 	return std::move(result);
 }
 

package/src/duckdb/src/function/table/unnest.cpp

@@ -47,7 +47,7 @@ static unique_ptr<FunctionData> UnnestBind(ClientContext &context, TableFunction
 		throw BinderException("UNNEST requires a single list as input");
 	}
 	return_types.push_back(ListType::GetChildType(input.input_table_types[0]));
-	names.push_back(input.input_table_names[0]);
+	names.push_back("unnest");
 	return make_uniq<UnnestBindData>(input.input_table_types[0]);
 }
 
@@ -78,7 +78,7 @@ static OperatorResultType UnnestFunction(ExecutionContext &context, TableFunctio
 }
 
 void UnnestTableFunction::RegisterFunction(BuiltinFunctions &set) {
-	TableFunction unnest_function("unnest", {LogicalTypeId::TABLE}, nullptr, UnnestBind, UnnestInit, UnnestLocalInit);
+	TableFunction unnest_function("unnest", {LogicalType::ANY}, nullptr, UnnestBind, UnnestInit, UnnestLocalInit);
 	unnest_function.in_out_function = UnnestFunction;
 	set.AddFunction(unnest_function);
 }

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -2,21 +2,21 @@
 #define DUCKDB_PATCH_VERSION "0"
 #endif
 #ifndef DUCKDB_MINOR_VERSION
-#define DUCKDB_MINOR_VERSION 0
+#define DUCKDB_MINOR_VERSION 1
 #endif
 #ifndef DUCKDB_MAJOR_VERSION
 #define DUCKDB_MAJOR_VERSION 1
 #endif
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "v1.0.0"
+#define DUCKDB_VERSION "v1.1.0"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "1f98600c2c"
+#define DUCKDB_SOURCE_ID "fa5c2fe15f"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"
 #include "duckdb/common/string_util.hpp"
-#include "duckdb/common/platform.h"
+#include "duckdb/common/platform.hpp"
 
 #include <cstdint>
 

package/src/duckdb/src/function/table_function.cpp

@@ -18,8 +18,9 @@ TableFunction::TableFunction(string name, vector<LogicalType> arguments, table_f
       init_global(init_global), init_local(init_local), function(function), in_out_function(nullptr),
       in_out_function_final(nullptr), statistics(nullptr), dependency(nullptr), cardinality(nullptr),
       pushdown_complex_filter(nullptr), to_string(nullptr), table_scan_progress(nullptr), get_batch_index(nullptr),
-      get_bind_info(nullptr), type_pushdown(nullptr), get_multi_file_reader(nullptr), serialize(nullptr),
-      deserialize(nullptr), projection_pushdown(false), filter_pushdown(false), filter_prune(false) {
+      get_bind_info(nullptr), type_pushdown(nullptr), get_multi_file_reader(nullptr), supports_pushdown_type(nullptr),
+      serialize(nullptr), deserialize(nullptr), projection_pushdown(false), filter_pushdown(false),
+      filter_prune(false) {
 }
 
 TableFunction::TableFunction(const vector<LogicalType> &arguments, table_function_t function,
@@ -32,8 +33,8 @@ TableFunction::TableFunction()
       init_local(nullptr), function(nullptr), in_out_function(nullptr), statistics(nullptr), dependency(nullptr),
       cardinality(nullptr), pushdown_complex_filter(nullptr), to_string(nullptr), table_scan_progress(nullptr),
       get_batch_index(nullptr), get_bind_info(nullptr), type_pushdown(nullptr), get_multi_file_reader(nullptr),
-      serialize(nullptr), deserialize(nullptr), projection_pushdown(false), filter_pushdown(false),
-      filter_prune(false) {
+      supports_pushdown_type(nullptr), serialize(nullptr), deserialize(nullptr), projection_pushdown(false),
+      filter_pushdown(false), filter_prune(false) {
 }
 
 bool TableFunction::Equal(const TableFunction &rhs) const {

package/src/duckdb/src/function/table_macro_function.cpp

@@ -27,8 +27,8 @@ unique_ptr<MacroFunction> TableMacroFunction::Copy() const {
 	return std::move(result);
 }
 
-string TableMacroFunction::ToSQL(const string &schema, const string &name) const {
-	return MacroFunction::ToSQL(schema, name) + StringUtil::Format("TABLE (%s);", query_node->ToString());
+string TableMacroFunction::ToSQL() const {
+	return MacroFunction::ToSQL() + StringUtil::Format("TABLE (%s)", query_node->ToString());
 }
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/catalog/catalog.hpp

@@ -14,6 +14,7 @@
 #include "duckdb/catalog/catalog_transaction.hpp"
 #include "duckdb/common/reference_map.hpp"
 #include "duckdb/common/atomic.hpp"
+#include "duckdb/common/map.hpp"
 #include "duckdb/common/optional_ptr.hpp"
 #include "duckdb/common/enums/on_entry_not_found.hpp"
 #include "duckdb/common/error_data.hpp"
@@ -102,10 +103,13 @@ public:
 	bool IsSystemCatalog() const;
 	bool IsTemporaryCatalog() const;
 
-	//! Returns the current version of the catalog (incremented whenever anything changes, not stored between restarts)
-	DUCKDB_API idx_t GetCatalogVersion();
-	//! Trigger a modification in the catalog, increasing the catalog version and returning the previous version
-	DUCKDB_API idx_t ModifyCatalog();
+	//! Returns a version number that uniquely characterizes the current catalog snapshot.
+	//! If there are transaction-local changes, the version returned is >= TRANSACTION_START, o.w. it is a simple number
+	//! starting at 0 that is incremented at each commit that has had catalog changes.
+	//! If the catalog does not support versioning, no index is returned.
+	DUCKDB_API virtual optional_idx GetCatalogVersion(ClientContext &context) {
+		return {}; // don't return anything by default
+	}
 
 	//! Returns the catalog name - based on how the catalog was attached
 	DUCKDB_API const string &GetName() const;

package/src/duckdb/src/include/duckdb/catalog/catalog_entry/duck_index_entry.hpp

@@ -11,6 +11,7 @@
 #include "duckdb/catalog/catalog_entry/index_catalog_entry.hpp"
 
 namespace duckdb {
+class TableCatalogEntry;
 
 //! Wrapper class to allow copying a DuckIndexEntry (for altering the DuckIndexEntry metadata such as comments)
 struct IndexDataTableInfo {
@@ -27,13 +28,15 @@ struct IndexDataTableInfo {
 class DuckIndexEntry : public IndexCatalogEntry {
 public:
 	//! Create a DuckIndexEntry
-	DuckIndexEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateIndexInfo &info);
+	DuckIndexEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateIndexInfo &create_info,
+	               TableCatalogEntry &table);
+	DuckIndexEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateIndexInfo &create_info,
+	               shared_ptr<IndexDataTableInfo> storage_info);
 
 	unique_ptr<CatalogEntry> Copy(ClientContext &context) const override;
 
 	//! The indexed table information
 	shared_ptr<IndexDataTableInfo> info;
-
 	//! We need the initial size of the index after the CREATE INDEX statement,
 	//! as it is necessary to determine the auto checkpoint threshold
 	idx_t initial_index_size;

package/src/duckdb/src/include/duckdb/catalog/catalog_entry/duck_schema_entry.hpp

@@ -68,6 +68,9 @@ public:
 
 	void Verify(Catalog &catalog) override;
 
+private:
+	void OnDropEntry(CatalogTransaction transaction, CatalogEntry &entry);
+
 private:
 	//! Get the catalog set for the specified type
 	CatalogSet &GetCatalogSet(CatalogType type);

package/src/duckdb/src/include/duckdb/catalog/catalog_entry/index_catalog_entry.hpp

@@ -50,9 +50,9 @@ public:
 	virtual string GetTableName() const = 0;
 
 	//! Returns true, if this index is UNIQUE
-	bool IsUnique();
+	bool IsUnique() const;
 	//! Returns true, if this index is a PRIMARY KEY
-	bool IsPrimary();
+	bool IsPrimary() const;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/catalog/catalog_entry/macro_catalog_entry.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/catalog/catalog_entry/function_entry.hpp"
 #include "duckdb/function/macro_function.hpp"
 #include "duckdb/parser/parsed_data/create_macro_info.hpp"
+#include "duckdb/function/function_set.hpp"
 
 namespace duckdb {
 
@@ -21,14 +22,12 @@ public:
 	MacroCatalogEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateMacroInfo &info);
 
 	//! The macro function
-	unique_ptr<MacroFunction> function;
+	vector<unique_ptr<MacroFunction>> macros;
 
 public:
 	unique_ptr<CreateInfo> GetInfo() const override;
 
-	string ToSQL() const override {
-		return function->ToSQL(schema.name, name);
-	}
+	string ToSQL() const override;
 };
 
 } // namespace duckdb