duckdb 1.1.4-dev11.0 → 1.1.4-dev14.0

package/src/duckdb/src/planner/binder/tableref/bind_showref.cpp

@@ -99,26 +99,31 @@ unique_ptr<BoundTableRef> Binder::BindShowQuery(ShowRef &ref) {
 	for (idx_t column_idx = 0; column_idx < plan.types.size(); column_idx++) {
 		// check if we can trace the column to a base table so that we can figure out constraint information
 		auto result = FindBaseTableColumn(*plan.plan, column_idx);
+		idx_t row_index = output.size();
+		auto &alias = plan.names[column_idx];
 		if (result.table) {
 			// we can! emit the information from the base table directly
-			PragmaTableInfo::GetColumnInfo(*result.table, *result.column, output, output.size());
+			PragmaTableInfo::GetColumnInfo(*result.table, *result.column, output, row_index);
+			// Override the base column name with the alias if one is specified.
+			if (alias != result.column->Name()) {
+				output.SetValue(0, row_index, Value(alias));
+			}
 		} else {
 			// we cannot - read the type/name from the plan instead
 			auto type = plan.types[column_idx];
-			auto &name = plan.names[column_idx];
 
 			// "name", TypeId::VARCHAR
-			output.SetValue(0, output.size(), Value(name));
+			output.SetValue(0, row_index, Value(alias));
 			// "type", TypeId::VARCHAR
-			output.SetValue(1, output.size(), Value(type.ToString()));
+			output.SetValue(1, row_index, Value(type.ToString()));
 			// "null", TypeId::VARCHAR
-			output.SetValue(2, output.size(), Value("YES"));
+			output.SetValue(2, row_index, Value("YES"));
 			// "pk", TypeId::BOOL
-			output.SetValue(3, output.size(), Value());
+			output.SetValue(3, row_index, Value());
 			// "dflt_value", TypeId::VARCHAR
-			output.SetValue(4, output.size(), Value());
+			output.SetValue(4, row_index, Value());
 			// "extra", TypeId::VARCHAR
-			output.SetValue(5, output.size(), Value());
+			output.SetValue(5, row_index, Value());
 		}
 
 		output.SetCardinality(output.size() + 1);

package/src/duckdb/src/planner/binder/tableref/bind_table_function.cpp

@@ -126,8 +126,7 @@ bool Binder::BindTableFunctionParameters(TableFunctionCatalogEntry &table_functi
 		    child->GetExpressionType() == ExpressionType::SUBQUERY) {
 			D_ASSERT(table_function.functions.Size() == 1);
 			auto fun = table_function.functions.GetFunctionByOffset(0);
-			if (table_function.functions.Size() != 1 || fun.arguments.empty() ||
-			    fun.arguments[0].id() != LogicalTypeId::TABLE) {
+			if (table_function.functions.Size() != 1 || fun.arguments.empty()) {
 				throw BinderException(
 				    "Only table-in-out functions can have subquery parameters - %s only accepts constant parameters",
 				    fun.name);
@@ -203,7 +202,13 @@ unique_ptr<LogicalOperator> Binder::BindTableFunctionInternal(TableFunction &tab
 		                                  table_function.function_info.get(), this, table_function, ref);
 		if (table_function.bind_replace) {
 			auto new_plan = table_function.bind_replace(context, bind_input);
-			if (new_plan != nullptr) {
+			if (new_plan) {
+				if (!ref.alias.empty()) {
+					new_plan->alias = ref.alias;
+				}
+				if (!ref.column_name_alias.empty()) {
+					new_plan->column_name_alias = ref.column_name_alias;
+				}
 				return CreatePlan(*Bind(*new_plan));
 			} else if (!table_function.bind) {
 				throw BinderException("Failed to bind \"%s\": nullptr returned from bind_replace without bind function",

package/src/duckdb/src/planner/expression/bound_function_expression.cpp

@@ -109,12 +109,28 @@ void BoundFunctionExpression::Serialize(Serializer &serializer) const {
 unique_ptr<Expression> BoundFunctionExpression::Deserialize(Deserializer &deserializer) {
 	auto return_type = deserializer.ReadProperty<LogicalType>(200, "return_type");
 	auto children = deserializer.ReadProperty<vector<unique_ptr<Expression>>>(201, "children");
+
 	auto entry = FunctionSerializer::Deserialize<ScalarFunction, ScalarFunctionCatalogEntry>(
 	    deserializer, CatalogType::SCALAR_FUNCTION_ENTRY, children, return_type);
 	auto function_return_type = entry.first.return_type;
+
+	auto is_operator = deserializer.ReadProperty<bool>(202, "is_operator");
+
+	if (entry.first.bind_expression) {
+		// bind the function expression
+		auto &context = deserializer.Get<ClientContext &>();
+		auto bind_input = FunctionBindExpressionInput(context, entry.second, children);
+		// replace the function expression with the bound expression
+		auto bound_expression = entry.first.bind_expression(bind_input);
+		if (bound_expression) {
+			return bound_expression;
+		}
+		// Otherwise, fall thorugh and continue on normally
+	}
+
 	auto result = make_uniq<BoundFunctionExpression>(std::move(function_return_type), std::move(entry.first),
 	                                                 std::move(children), std::move(entry.second));
-	deserializer.ReadProperty(202, "is_operator", result->is_operator);
+	result->is_operator = is_operator;
 	if (result->return_type != return_type) {
 		// return type mismatch - push a cast
 		auto &context = deserializer.Get<ClientContext &>();

package/src/duckdb/src/planner/expression_binder/index_binder.cpp

@@ -62,6 +62,7 @@ void IndexBinder::InitCreateIndexInfo(LogicalGet &get, CreateIndexInfo &info, co
 	info.scan_types.emplace_back(LogicalType::ROW_TYPE);
 	info.names = get.names;
 	info.schema = schema;
+	info.catalog = get.GetTable()->catalog.GetName();
 	get.AddColumnId(COLUMN_IDENTIFIER_ROW_ID);
 }
 

package/src/duckdb/src/planner/filter/conjunction_filter.cpp

@@ -1,4 +1,5 @@
 #include "duckdb/planner/filter/conjunction_filter.hpp"
+
 #include "duckdb/planner/expression/bound_conjunction_expression.hpp"
 
 namespace duckdb {

package/src/duckdb/src/planner/filter/constant_filter.cpp

@@ -2,6 +2,8 @@
 #include "duckdb/storage/statistics/base_statistics.hpp"
 #include "duckdb/planner/expression/bound_comparison_expression.hpp"
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
+#include "duckdb/common/value_operations/value_operations.hpp"
+#include "duckdb/common/enum_util.hpp"
 
 namespace duckdb {
 
@@ -13,6 +15,25 @@ ConstantFilter::ConstantFilter(ExpressionType comparison_type_p, Value constant_
 	}
 }
 
+bool ConstantFilter::Compare(const Value &value) const {
+	switch (comparison_type) {
+	case ExpressionType::COMPARE_EQUAL:
+		return ValueOperations::Equals(value, constant);
+	case ExpressionType::COMPARE_NOTEQUAL:
+		return ValueOperations::NotEquals(value, constant);
+	case ExpressionType::COMPARE_GREATERTHAN:
+		return ValueOperations::GreaterThan(value, constant);
+	case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
+		return ValueOperations::GreaterThanEquals(value, constant);
+	case ExpressionType::COMPARE_LESSTHAN:
+		return ValueOperations::LessThan(value, constant);
+	case ExpressionType::COMPARE_LESSTHANOREQUALTO:
+		return ValueOperations::LessThanEquals(value, constant);
+	default:
+		throw InternalException("unknown comparison type for ConstantFilter: " + EnumUtil::ToString(comparison_type));
+	}
+}
+
 FilterPropagateResult ConstantFilter::CheckStatistics(BaseStatistics &stats) {
 	if (!stats.CanHaveNoNull()) {
 		// no non-null values are possible: always false

package/src/duckdb/src/planner/filter/in_filter.cpp

@@ -1,15 +1,12 @@
 #include "duckdb/planner/filter/in_filter.hpp"
+
 #include "duckdb/storage/statistics/base_statistics.hpp"
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
 #include "duckdb/planner/expression/bound_operator_expression.hpp"
 
 namespace duckdb {
 
-InFilter::InFilter(vector<Value> values_p) : InFilter(std::move(values_p), false) {
-}
-
-InFilter::InFilter(vector<Value> values_p, bool origin_is_hash_join)
-    : TableFilter(TableFilterType::IN_FILTER), values(std::move(values_p)), origin_is_hash_join(origin_is_hash_join) {
+InFilter::InFilter(vector<Value> values_p) : TableFilter(TableFilterType::IN_FILTER), values(std::move(values_p)) {
 	for (auto &val : values) {
 		if (val.IsNull()) {
 			throw InternalException("InFilter constant cannot be NULL - use IsNullFilter instead");
@@ -74,11 +71,11 @@ bool InFilter::Equals(const TableFilter &other_p) const {
 		return false;
 	}
 	auto &other = other_p.Cast<InFilter>();
-	return other.values == values && other.origin_is_hash_join == origin_is_hash_join;
+	return other.values == values;
 }
 
 unique_ptr<TableFilter> InFilter::Copy() const {
-	return make_uniq<InFilter>(values, origin_is_hash_join);
+	return make_uniq<InFilter>(values);
 }
 
 } // namespace duckdb

package/src/duckdb/src/planner/logical_operator.cpp

@@ -161,7 +161,7 @@ void LogicalOperator::Verify(ClientContext &context) {
 		if (expressions[expr_idx]->HasParameter()) {
 			continue;
 		}
-		MemoryStream stream;
+		MemoryStream stream(Allocator::Get(context));
 		// We are serializing a query plan
 		try {
 			BinarySerializer::Serialize(*expressions[expr_idx], stream);
@@ -215,8 +215,10 @@ vector<idx_t> LogicalOperator::GetTableIndex() const {
 }
 
 unique_ptr<LogicalOperator> LogicalOperator::Copy(ClientContext &context) const {
-	MemoryStream stream;
-	BinarySerializer serializer(stream);
+	MemoryStream stream(Allocator::Get(context));
+	SerializationOptions options;
+	options.serialization_compatibility = SerializationCompatibility::Latest();
+	BinarySerializer serializer(stream, options);
 	try {
 		serializer.Begin();
 		this->Serialize(serializer);

package/src/duckdb/src/planner/planner.cpp

@@ -184,7 +184,7 @@ void Planner::VerifyPlan(ClientContext &context, unique_ptr<LogicalOperator> &op
 
 	// format (de)serialization of this operator
 	try {
-		MemoryStream stream;
+		MemoryStream stream(Allocator::Get(context));
 
 		SerializationOptions options;
 		if (config.options.serialization_compatibility.manually_set) {

package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp

@@ -401,6 +401,7 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 				// only right has correlation: push into right
 				plan->children[1] = PushDownDependentJoinInternal(std::move(plan->children[1]),
 				                                                  parent_propagate_null_values, lateral_depth);
+				delim_offset += plan->children[0]->GetColumnBindings().size();
 				// Remove the correlated columns coming from outside for current join node
 				return plan;
 			}
@@ -419,6 +420,7 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 				// only right has correlation: push into right
 				plan->children[1] = PushDownDependentJoinInternal(std::move(plan->children[1]),
 				                                                  parent_propagate_null_values, lateral_depth);
+				delim_offset += plan->children[0]->GetColumnBindings().size();
 				return plan;
 			}
 		} else if (join.join_type == JoinType::MARK) {

package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp

@@ -60,7 +60,7 @@ void SingleFileTableDataWriter::FinalizeTable(const TableStatistics &global_stat
 	auto pointer = table_data_writer.GetMetaBlockPointer();
 
 	// Serialize statistics as a single unit
-	BinarySerializer stats_serializer(table_data_writer);
+	BinarySerializer stats_serializer(table_data_writer, serializer.GetOptions());
 	stats_serializer.Begin();
 	global_stats.Serialize(stats_serializer);
 	stats_serializer.End();
@@ -75,7 +75,7 @@ void SingleFileTableDataWriter::FinalizeTable(const TableStatistics &global_stat
 		}
 
 		// Each RowGroup is its own unit
-		BinarySerializer row_group_serializer(table_data_writer);
+		BinarySerializer row_group_serializer(table_data_writer, serializer.GetOptions());
 		row_group_serializer.Begin();
 		RowGroup::Serialize(row_group_pointer, row_group_serializer);
 		row_group_serializer.End();
@@ -86,8 +86,7 @@ void SingleFileTableDataWriter::FinalizeTable(const TableStatistics &global_stat
 	serializer.WriteProperty(101, "table_pointer", pointer);
 	serializer.WriteProperty(102, "total_rows", total_rows);
 
-	auto db_options = checkpoint_manager.db.GetDatabase().config.options;
-	auto v1_0_0_storage = db_options.serialization_compatibility.serialization_version < 3;
+	auto v1_0_0_storage = serializer.GetOptions().serialization_compatibility.serialization_version < 3;
 	case_insensitive_map_t<Value> options;
 	if (!v1_0_0_storage) {
 		options.emplace("v1_0_0_storage", v1_0_0_storage);

package/src/duckdb/src/storage/checkpoint_manager.cpp

@@ -178,11 +178,7 @@ void SingleFileCheckpointWriter::CreateCheckpoint() {
 	        ]
 	    }
 	 */
-	SerializationOptions serialization_options;
-
-	serialization_options.serialization_compatibility = config.options.serialization_compatibility;
-
-	BinarySerializer serializer(*metadata_writer, serialization_options);
+	BinarySerializer serializer(*metadata_writer, SerializationOptions(db));
 	serializer.Begin();
 	serializer.WriteList(100, "catalog_entries", catalog_entries.size(), [&](Serializer::List &list, idx_t i) {
 		auto &entry = catalog_entries[i];
@@ -259,11 +255,13 @@ void SingleFileCheckpointWriter::CreateCheckpoint() {
 
 void CheckpointReader::LoadCheckpoint(CatalogTransaction transaction, MetadataReader &reader) {
 	BinaryDeserializer deserializer(reader);
+	deserializer.Set<Catalog &>(catalog);
 	deserializer.Begin();
 	deserializer.ReadList(100, "catalog_entries", [&](Deserializer::List &list, idx_t i) {
 		return list.ReadObject([&](Deserializer &obj) { ReadEntry(transaction, obj); });
 	});
 	deserializer.End();
+	deserializer.Unset<Catalog>();
 }
 
 MetadataManager &SingleFileCheckpointReader::GetMetadataManager() {

package/src/duckdb/src/storage/compression/dictionary/decompression.cpp

@@ -43,16 +43,16 @@ void CompressedStringScanState::Initialize(ColumnSegment &segment, bool initiali
 	block_size = segment.GetBlockManager().GetBlockSize();
 
 	dict = DictionaryCompression::GetDictionary(segment, *handle);
-	dictionary = make_buffer<Vector>(segment.type, index_buffer_count);
-	dictionary_size = index_buffer_count;
-
 	if (!initialize_dictionary) {
 		// Used by fetch, as fetch will never produce a DictionaryVector
 		return;
 	}
 
+	dictionary = make_buffer<Vector>(segment.type, index_buffer_count);
+	dictionary_size = index_buffer_count;
 	auto dict_child_data = FlatVector::GetData<string_t>(*(dictionary));
-	for (uint32_t i = 0; i < index_buffer_count; i++) {
+	FlatVector::SetNull(*dictionary, 0, true);
+	for (uint32_t i = 1; i < index_buffer_count; i++) {
 		// NOTE: the passing of dict_child_vector, will not be used, its for big strings
 		uint16_t str_len = GetStringLength(i);
 		dict_child_data[i] = FetchStringFromDict(UnsafeNumericCast<int32_t>(index_buffer_ptr[i]), str_len);

package/src/duckdb/src/storage/compression/fsst.cpp

@@ -658,7 +658,7 @@ void FSSTStorage::StringScanPartial(ColumnSegment &segment, ColumnScanState &sta
 		for (idx_t i = 0; i < scan_count; i++) {
 			uint32_t string_length = bitunpack_buffer[i + offsets.scan_offset];
 			result_data[i] = UncompressedStringStorage::FetchStringFromDict(
-			    segment, dict, result, baseptr,
+			    segment, dict.end, result, baseptr,
 			    UnsafeNumericCast<int32_t>(delta_decode_buffer[i + offsets.unused_delta_decoded_values]),
 			    string_length);
 			FSSTVector::SetCount(result, scan_count);
@@ -736,7 +736,7 @@ void FSSTStorage::StringFetchRow(ColumnSegment &segment, ColumnFetchState &state
 	uint32_t string_length = bitunpack_buffer[offsets.scan_offset];
 
 	string_t compressed_string = UncompressedStringStorage::FetchStringFromDict(
-	    segment, dict, result, base_ptr,
+	    segment, dict.end, result, base_ptr,
 	    UnsafeNumericCast<int32_t>(delta_decode_buffer[offsets.unused_delta_decoded_values]), string_length);
 
 	vector<unsigned char> uncompress_buffer;

package/src/duckdb/src/storage/compression/roaring/common.cpp

@@ -167,9 +167,14 @@ void SetInvalidRange(ValidityMask &result, idx_t start, idx_t end) {
 }
 
 unique_ptr<AnalyzeState> RoaringInitAnalyze(ColumnData &col_data, PhysicalType type) {
+	// check if the storage version we are writing to supports roaring
+	auto &storage = col_data.GetStorageManager();
+	if (storage.GetStorageVersion() < 4) {
+		// compatibility mode with old versions - disable roaring
+		return nullptr;
+	}
 	CompressionInfo info(col_data.GetBlockManager().GetBlockSize());
 	auto state = make_uniq<RoaringAnalyzeState>(info);
-
 	return std::move(state);
 }
 
@@ -220,6 +225,10 @@ void RoaringScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t sc
 }
 
 void RoaringScan(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result) {
+	if (result.GetVectorType() == VectorType::DICTIONARY_VECTOR) {
+		// dictionary encoding handles the validity itself
+		return;
+	}
 	RoaringScanPartial(segment, state, scan_count, result, 0);
 }
 

package/src/duckdb/src/storage/compression/string_uncompressed.cpp

@@ -94,7 +94,7 @@ void UncompressedStringStorage::StringScanPartial(ColumnSegment &segment, Column
 	auto start = segment.GetRelativeIndex(state.row_index);
 
 	auto baseptr = scan_state.handle.Ptr() + segment.GetBlockOffset();
-	auto dict = GetDictionary(segment, scan_state.handle);
+	auto dict_end = GetDictionaryEnd(segment, scan_state.handle);
 	auto base_data = reinterpret_cast<int32_t *>(baseptr + DICTIONARY_HEADER_SIZE);
 	auto result_data = FlatVector::GetData<string_t>(result);
 
@@ -105,7 +105,7 @@ void UncompressedStringStorage::StringScanPartial(ColumnSegment &segment, Column
 		auto current_offset = base_data[start + i];
 		auto string_length = UnsafeNumericCast<uint32_t>(std::abs(current_offset) - std::abs(previous_offset));
 		result_data[result_offset + i] =
-		    FetchStringFromDict(segment, dict, result, baseptr, current_offset, string_length);
+		    FetchStringFromDict(segment, dict_end, result, baseptr, current_offset, string_length);
 		previous_offset = base_data[start + i];
 	}
 }
@@ -125,7 +125,7 @@ void UncompressedStringStorage::Select(ColumnSegment &segment, ColumnScanState &
 	auto start = segment.GetRelativeIndex(state.row_index);
 
 	auto baseptr = scan_state.handle.Ptr() + segment.GetBlockOffset();
-	auto dict = GetDictionary(segment, scan_state.handle);
+	auto dict_end = GetDictionaryEnd(segment, scan_state.handle);
 	auto base_data = reinterpret_cast<int32_t *>(baseptr + DICTIONARY_HEADER_SIZE);
 	auto result_data = FlatVector::GetData<string_t>(result);
 
@@ -134,7 +134,7 @@ void UncompressedStringStorage::Select(ColumnSegment &segment, ColumnScanState &
 		auto current_offset = base_data[index];
 		auto prev_offset = index > 0 ? base_data[index - 1] : 0;
 		auto string_length = UnsafeNumericCast<uint32_t>(std::abs(current_offset) - std::abs(prev_offset));
-		result_data[i] = FetchStringFromDict(segment, dict, result, baseptr, current_offset, string_length);
+		result_data[i] = FetchStringFromDict(segment, dict_end, result, baseptr, current_offset, string_length);
 	}
 }
 
@@ -164,7 +164,7 @@ void UncompressedStringStorage::StringFetchRow(ColumnSegment &segment, ColumnFet
 	auto &handle = state.GetOrInsertHandle(segment);
 
 	auto baseptr = handle.Ptr() + segment.GetBlockOffset();
-	auto dict = GetDictionary(segment, handle);
+	auto dict_end = GetDictionaryEnd(segment, handle);
 	auto base_data = reinterpret_cast<int32_t *>(baseptr + DICTIONARY_HEADER_SIZE);
 	auto result_data = FlatVector::GetData<string_t>(result);
 
@@ -176,7 +176,7 @@ void UncompressedStringStorage::StringFetchRow(ColumnSegment &segment, ColumnFet
 	} else {
 		string_length = NumericCast<uint32_t>(std::abs(dict_offset) - std::abs(base_data[row_id - 1]));
 	}
-	result_data[result_idx] = FetchStringFromDict(segment, dict, result, baseptr, dict_offset, string_length);
+	result_data[result_idx] = FetchStringFromDict(segment, dict_end, result, baseptr, dict_offset, string_length);
 }
 
 //===--------------------------------------------------------------------===//
@@ -301,6 +301,11 @@ StringDictionaryContainer UncompressedStringStorage::GetDictionary(ColumnSegment
 	return container;
 }
 
+uint32_t UncompressedStringStorage::GetDictionaryEnd(ColumnSegment &segment, BufferHandle &handle) {
+	auto startptr = handle.Ptr() + segment.GetBlockOffset();
+	return Load<uint32_t>(startptr + sizeof(uint32_t));
+}
+
 idx_t UncompressedStringStorage::RemainingSpace(ColumnSegment &segment, BufferHandle &handle) {
 	auto dictionary = GetDictionary(segment, handle);
 	D_ASSERT(dictionary.end == segment.SegmentSize());

package/src/duckdb/src/storage/compression/validity_uncompressed.cpp

@@ -401,6 +401,10 @@ void ValidityScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t s
 }
 
 void ValidityScan(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result) {
+	if (result.GetVectorType() == VectorType::DICTIONARY_VECTOR) {
+		// dictionary encoding handles the validity itself
+		return;
+	}
 	result.Flatten(scan_count);
 
 	auto start = segment.GetRelativeIndex(state.row_index);

package/src/duckdb/src/storage/compression/zstd.cpp

@@ -140,6 +140,12 @@ public:
 };
 
 unique_ptr<AnalyzeState> ZSTDStorage::StringInitAnalyze(ColumnData &col_data, PhysicalType type) {
+	// check if the storage version we are writing to supports sztd
+	auto &storage = col_data.GetStorageManager();
+	if (storage.GetStorageVersion() < 4) {
+		// compatibility mode with old versions - disable zstd
+		return nullptr;
+	}
 	CompressionInfo info(col_data.GetBlockManager().GetBlockSize());
 	auto &data_table_info = col_data.info;
 	auto &attached_db = data_table_info.GetDB();