duckdb 0.8.2-dev37.0 → 0.8.2-dev3949.0

package/src/duckdb/src/common/row_operations/row_aggregate.cpp

@@ -76,6 +76,10 @@ void RowOperations::CombineStates(RowOperationsState &state, TupleDataLayout &la
 	//	Move to the first aggregate states
 	VectorOperations::AddInPlace(sources, layout.GetAggrOffset(), count);
 	VectorOperations::AddInPlace(targets, layout.GetAggrOffset(), count);
+
+	// Keep track of the offset
+	idx_t offset = layout.GetAggrOffset();
+
 	for (auto &aggr : layout.GetAggregates()) {
 		D_ASSERT(aggr.function.combine);
 		AggregateInputData aggr_input_data(aggr.GetFunctionData(), state.allocator);
@@ -84,23 +88,34 @@ void RowOperations::CombineStates(RowOperationsState &state, TupleDataLayout &la
 		// Move to the next aggregate states
 		VectorOperations::AddInPlace(sources, aggr.payload_size, count);
 		VectorOperations::AddInPlace(targets, aggr.payload_size, count);
+
+		// Increment the offset
+		offset += aggr.payload_size;
 	}
+
+	// Now subtract the offset to get back to the original position
+	VectorOperations::AddInPlace(sources, -offset, count);
+	VectorOperations::AddInPlace(targets, -offset, count);
 }
 
 void RowOperations::FinalizeStates(RowOperationsState &state, TupleDataLayout &layout, Vector &addresses,
                                    DataChunk &result, idx_t aggr_idx) {
+	// Copy the addresses
+	Vector addresses_copy(LogicalType::POINTER);
+	VectorOperations::Copy(addresses, addresses_copy, result.size(), 0, 0);
+
 	//	Move to the first aggregate state
-	VectorOperations::AddInPlace(addresses, layout.GetAggrOffset(), result.size());
+	VectorOperations::AddInPlace(addresses_copy, layout.GetAggrOffset(), result.size());
 
 	auto &aggregates = layout.GetAggregates();
 	for (idx_t i = 0; i < aggregates.size(); i++) {
 		auto &target = result.data[aggr_idx + i];
 		auto &aggr = aggregates[i];
 		AggregateInputData aggr_input_data(aggr.GetFunctionData(), state.allocator);
-		aggr.function.finalize(addresses, aggr_input_data, target, result.size(), 0);
+		aggr.function.finalize(addresses_copy, aggr_input_data, target, result.size(), 0);
 
 		// Move to the next aggregate state
-		VectorOperations::AddInPlace(addresses, aggr.payload_size, result.size());
+		VectorOperations::AddInPlace(addresses_copy, aggr.payload_size, result.size());
 	}
 }
 

package/src/duckdb/src/common/row_operations/row_external.cpp

@@ -99,7 +99,7 @@ void RowOperations::UnswizzleHeapPointer(const RowLayout &layout, const data_ptr
 
 static inline void VerifyUnswizzledString(const RowLayout &layout, const idx_t &col_idx, const data_ptr_t &row_ptr) {
 #ifdef DEBUG
-	if (layout.GetTypes()[col_idx] == LogicalTypeId::BLOB) {
+	if (layout.GetTypes()[col_idx].id() != LogicalTypeId::VARCHAR) {
 		return;
 	}
 	idx_t entry_idx;

package/src/duckdb/src/common/serializer/binary_deserializer.cpp

@@ -2,142 +2,132 @@
 
 namespace duckdb {
 
-void BinaryDeserializer::SetTag(const char *tag) {
-	current_tag = tag;
-	stack.back().read_field_count++;
-}
-
-//===--------------------------------------------------------------------===//
-// Nested Types Hooks
-//===--------------------------------------------------------------------===//
-void BinaryDeserializer::OnObjectBegin() {
-	auto expected_field_count = ReadPrimitive<uint32_t>();
-	auto expected_size = ReadPrimitive<uint64_t>();
-	D_ASSERT(expected_field_count > 0);
-	D_ASSERT(expected_size > 0);
-
-	stack.emplace_back(expected_field_count, expected_size);
-}
-
-void BinaryDeserializer::OnObjectEnd() {
-	auto &frame = stack.back();
-	if (frame.read_field_count < frame.expected_field_count) {
-		throw SerializationException("Not all fields were read. This file might have been written with a newer version "
-		                             "of DuckDB and is incompatible with this version of DuckDB.");
+//-------------------------------------------------------------------------
+// Nested Type Hooks
+//-------------------------------------------------------------------------
+void BinaryDeserializer::OnPropertyBegin(const field_id_t field_id, const char *) {
+	auto field = NextField();
+	if (field != field_id) {
+		throw InternalException("Failed to deserialize: field id mismatch, expected: %d, got: %d", field_id, field);
 	}
-	stack.pop_back();
-}
-
-idx_t BinaryDeserializer::OnListBegin() {
-	return ReadPrimitive<idx_t>();
-}
-
-void BinaryDeserializer::OnListEnd() {
-}
-
-// Deserialize maps as [ { key: ..., value: ... } ]
-idx_t BinaryDeserializer::OnMapBegin() {
-	return ReadPrimitive<idx_t>();
-}
-
-void BinaryDeserializer::OnMapEntryBegin() {
 }
 
-void BinaryDeserializer::OnMapKeyBegin() {
+void BinaryDeserializer::OnPropertyEnd() {
 }
 
-void BinaryDeserializer::OnMapValueBegin() {
+bool BinaryDeserializer::OnOptionalPropertyBegin(const field_id_t field_id, const char *s) {
+	auto next_field = PeekField();
+	auto present = next_field == field_id;
+	if (present) {
+		ConsumeField();
+	}
+	return present;
 }
 
-void BinaryDeserializer::OnMapEntryEnd() {
+void BinaryDeserializer::OnOptionalPropertyEnd(bool present) {
 }
 
-void BinaryDeserializer::OnMapEnd() {
+void BinaryDeserializer::OnObjectBegin() {
+	nesting_level++;
 }
 
-void BinaryDeserializer::OnPairBegin() {
+void BinaryDeserializer::OnObjectEnd() {
+	auto next_field = NextField();
+	if (next_field != MESSAGE_TERMINATOR_FIELD_ID) {
+		throw InternalException("Failed to deserialize: expected end of object, but found field id: %d", next_field);
+	}
+	nesting_level--;
 }
 
-void BinaryDeserializer::OnPairKeyBegin() {
+idx_t BinaryDeserializer::OnListBegin() {
+	return VarIntDecode<idx_t>();
 }
 
-void BinaryDeserializer::OnPairValueBegin() {
+void BinaryDeserializer::OnListEnd() {
 }
 
-void BinaryDeserializer::OnPairEnd() {
+bool BinaryDeserializer::OnNullableBegin() {
+	return ReadBool();
 }
 
-bool BinaryDeserializer::OnOptionalBegin() {
-	return ReadPrimitive<bool>();
+void BinaryDeserializer::OnNullableEnd() {
 }
 
-//===--------------------------------------------------------------------===//
+//-------------------------------------------------------------------------
 // Primitive Types
-//===--------------------------------------------------------------------===//
+//-------------------------------------------------------------------------
 bool BinaryDeserializer::ReadBool() {
-	return ReadPrimitive<bool>();
+	return static_cast<bool>(ReadPrimitive<uint8_t>());
+}
+
+char BinaryDeserializer::ReadChar() {
+	return ReadPrimitive<char>();
 }
 
 int8_t BinaryDeserializer::ReadSignedInt8() {
-	return ReadPrimitive<int8_t>();
+	return VarIntDecode<int8_t>();
 }
 
 uint8_t BinaryDeserializer::ReadUnsignedInt8() {
-	return ReadPrimitive<uint8_t>();
+	return VarIntDecode<uint8_t>();
 }
 
 int16_t BinaryDeserializer::ReadSignedInt16() {
-	return ReadPrimitive<int16_t>();
+	return VarIntDecode<int16_t>();
 }
 
 uint16_t BinaryDeserializer::ReadUnsignedInt16() {
-	return ReadPrimitive<uint16_t>();
+	return VarIntDecode<uint16_t>();
 }
 
 int32_t BinaryDeserializer::ReadSignedInt32() {
-	return ReadPrimitive<int32_t>();
+	return VarIntDecode<int32_t>();
 }
 
 uint32_t BinaryDeserializer::ReadUnsignedInt32() {
-	return ReadPrimitive<uint32_t>();
+	return VarIntDecode<uint32_t>();
 }
 
 int64_t BinaryDeserializer::ReadSignedInt64() {
-	return ReadPrimitive<int64_t>();
+	return VarIntDecode<int64_t>();
 }
 
 uint64_t BinaryDeserializer::ReadUnsignedInt64() {
-	return ReadPrimitive<uint64_t>();
+	return VarIntDecode<uint64_t>();
 }
 
 float BinaryDeserializer::ReadFloat() {
-	return ReadPrimitive<float>();
+	auto value = ReadPrimitive<float>();
+	return value;
 }
 
 double BinaryDeserializer::ReadDouble() {
-	return ReadPrimitive<double>();
+	auto value = ReadPrimitive<double>();
+	return value;
 }
 
 string BinaryDeserializer::ReadString() {
-	uint32_t size = ReadPrimitive<uint32_t>();
-	if (size == 0) {
+	auto len = VarIntDecode<uint32_t>();
+	if (len == 0) {
 		return string();
 	}
-	auto buffer = make_unsafe_uniq_array<data_t>(size);
-	ReadData(buffer.get(), size);
-	return string(const_char_ptr_cast(buffer.get()), size);
-}
-
-interval_t BinaryDeserializer::ReadInterval() {
-	return ReadPrimitive<interval_t>();
+	auto buffer = make_unsafe_uniq_array<data_t>(len);
+	ReadData(buffer.get(), len);
+	return string(const_char_ptr_cast(buffer.get()), len);
 }
 
 hugeint_t BinaryDeserializer::ReadHugeInt() {
-	return ReadPrimitive<hugeint_t>();
+	auto upper = VarIntDecode<int64_t>();
+	auto lower = VarIntDecode<uint64_t>();
+	return hugeint_t(upper, lower);
 }
 
-void BinaryDeserializer::ReadDataPtr(data_ptr_t &ptr, idx_t count) {
-	ReadData(ptr, count);
+void BinaryDeserializer::ReadDataPtr(data_ptr_t &ptr_p, idx_t count) {
+	auto len = VarIntDecode<uint64_t>();
+	if (len != count) {
+		throw SerializationException("Tried to read blob of %d size, but only %d elements are available", count, len);
+	}
+	ReadData(ptr_p, count);
 }
 
 } // namespace duckdb

package/src/duckdb/src/common/serializer/binary_serializer.cpp

@@ -1,116 +1,134 @@
 #include "duckdb/common/serializer/binary_serializer.hpp"
 
-namespace duckdb {
-
-void BinarySerializer::SetTag(const char *tag) {
-	current_tag = tag;
-
-	// Increment the number of fields
-	stack.back().field_count++;
-}
-
-//===--------------------------------------------------------------------===//
-// Nested types
-//===--------------------------------------------------------------------===//
-void BinarySerializer::OnOptionalBegin(bool present) {
-	Write(present);
-}
-
-void BinarySerializer::OnListBegin(idx_t count) {
-	Write(count);
-}
-
-void BinarySerializer::OnListEnd(idx_t count) {
-}
+#ifdef DEBUG
+#include "duckdb/common/string_util.hpp"
+#endif
 
-// Serialize maps as arrays of objects with "key" and "value" properties.
-void BinarySerializer::OnMapBegin(idx_t count) {
-	Write(count);
-}
+namespace duckdb {
 
-void BinarySerializer::OnMapEntryBegin() {
-}
+void BinarySerializer::OnPropertyBegin(const field_id_t field_id, const char *tag) {
+	// Just write the field id straight up
+	Write<field_id_t>(field_id);
+#ifdef DEBUG
+	// Check that the tag is unique
+	auto &state = debug_stack.back();
+	auto &seen_field_ids = state.seen_field_ids;
+	auto &seen_field_tags = state.seen_field_tags;
+	auto &seen_fields = state.seen_fields;
+
+	if (seen_field_ids.find(field_id) != seen_field_ids.end() || seen_field_tags.find(tag) != seen_field_tags.end()) {
+		string all_fields;
+		for (auto &field : seen_fields) {
+			all_fields += StringUtil::Format("\"%s\":%d ", field.first, field.second);
+		}
+		throw InternalException("Duplicate field id/tag in field: \"%s\":%d, other fields: %s", tag, field_id,
+		                        all_fields);
+	}
 
-void BinarySerializer::OnMapKeyBegin() {
+	seen_field_ids.insert(field_id);
+	seen_field_tags.insert(tag);
+	seen_fields.emplace_back(tag, field_id);
+#else
+	(void)tag;
+#endif
 }
 
-void BinarySerializer::OnMapValueBegin() {
+void BinarySerializer::OnPropertyEnd() {
+	// Nothing to do here
 }
 
-void BinarySerializer::OnMapEntryEnd() {
+void BinarySerializer::OnOptionalPropertyBegin(const field_id_t field_id, const char *tag, bool present) {
+	// Dont write anything at all if the property is not present
+	if (present) {
+		OnPropertyBegin(field_id, tag);
+	}
 }
 
-void BinarySerializer::OnMapEnd(idx_t count) {
+void BinarySerializer::OnOptionalPropertyEnd(bool present) {
+	// Nothing to do here
 }
 
+//-------------------------------------------------------------------------
+// Nested Type Hooks
+//-------------------------------------------------------------------------
 void BinarySerializer::OnObjectBegin() {
-	stack.push_back(State({0, 0, data.size()}));
-	Write<uint32_t>(0); // Placeholder for the field count
-	Write<uint64_t>(0); // Placeholder for the size
+#ifdef DEBUG
+	debug_stack.emplace_back();
+#endif
 }
 
 void BinarySerializer::OnObjectEnd() {
-	auto &frame = stack.back();
-	// Patch the field count and size
-	auto message_start = &data[frame.offset];
-	Store<uint32_t>(frame.field_count, message_start);
-	Store<uint64_t>(frame.size, message_start + sizeof(uint32_t));
-	stack.pop_back();
+#ifdef DEBUG
+	debug_stack.pop_back();
+#endif
+	// Write object terminator
+	Write<field_id_t>(MESSAGE_TERMINATOR_FIELD_ID);
 }
 
-void BinarySerializer::OnPairBegin() {
+void BinarySerializer::OnListBegin(idx_t count) {
+	VarIntEncode(count);
 }
 
-void BinarySerializer::OnPairKeyBegin() {
+void BinarySerializer::OnListEnd() {
 }
 
-void BinarySerializer::OnPairValueBegin() {
+void BinarySerializer::OnNullableBegin(bool present) {
+	WriteValue(present);
 }
 
-void BinarySerializer::OnPairEnd() {
+void BinarySerializer::OnNullableEnd() {
 }
 
-//===--------------------------------------------------------------------===//
-// Primitive types
-//===--------------------------------------------------------------------===//
+//-------------------------------------------------------------------------
+// Primitive Types
+//-------------------------------------------------------------------------
 void BinarySerializer::WriteNull() {
 	// This should never be called, optional writes should be handled by OnOptionalBegin
 }
 
+void BinarySerializer::WriteValue(bool value) {
+	Write<uint8_t>(value);
+}
+
 void BinarySerializer::WriteValue(uint8_t value) {
+	VarIntEncode(value);
+}
+
+void BinarySerializer::WriteValue(char value) {
 	Write(value);
 }
 
 void BinarySerializer::WriteValue(int8_t value) {
-	Write(value);
+	VarIntEncode(value);
 }
 
 void BinarySerializer::WriteValue(uint16_t value) {
-	Write(value);
+	VarIntEncode(value);
 }
 
 void BinarySerializer::WriteValue(int16_t value) {
-	Write(value);
+	VarIntEncode(value);
 }
 
 void BinarySerializer::WriteValue(uint32_t value) {
-	Write(value);
+	VarIntEncode(value);
 }
 
 void BinarySerializer::WriteValue(int32_t value) {
-	Write(value);
+	VarIntEncode(value);
 }
 
 void BinarySerializer::WriteValue(uint64_t value) {
-	Write(value);
+	VarIntEncode(value);
 }
 
 void BinarySerializer::WriteValue(int64_t value) {
-	Write(value);
+	VarIntEncode(value);
 }
 
 void BinarySerializer::WriteValue(hugeint_t value) {
-	Write(value);
+	VarIntEncode(value.upper);
+	VarIntEncode(value.lower);
 }
 
 void BinarySerializer::WriteValue(float value) {
@@ -121,40 +139,27 @@ void BinarySerializer::WriteValue(double value) {
 	Write(value);
 }
 
-void BinarySerializer::WriteValue(interval_t value) {
-	Write(value);
-}
-
 void BinarySerializer::WriteValue(const string &value) {
-	auto len = value.length();
-	Write<uint32_t>((uint32_t)len);
-	if (len > 0) {
-		WriteData(value.c_str(), len);
-	}
+	uint32_t len = value.length();
+	VarIntEncode(len);
+	WriteDataInternal(value.c_str(), len);
 }
 
 void BinarySerializer::WriteValue(const string_t value) {
-	auto len = value.GetSize();
-	Write<uint32_t>((uint32_t)len);
-	if (len > 0) {
-		WriteData(value.GetDataUnsafe(), len);
-	}
+	uint32_t len = value.GetSize();
+	VarIntEncode(len);
+	WriteDataInternal(value.GetDataUnsafe(), len);
 }
 
 void BinarySerializer::WriteValue(const char *value) {
-	auto len = strlen(value);
-	Write<uint32_t>((uint32_t)len);
-	if (len > 0) {
-		WriteData(value, len);
-	}
-}
-
-void BinarySerializer::WriteValue(bool value) {
-	Write(value);
+	uint32_t len = strlen(value);
+	VarIntEncode(len);
+	WriteDataInternal(value, len);
 }
 
 void BinarySerializer::WriteDataPtr(const_data_ptr_t ptr, idx_t count) {
-	WriteData(ptr, count);
+	VarIntEncode(static_cast<uint64_t>(count));
+	WriteDataInternal(ptr, count);
 }
 
 } // namespace duckdb

package/src/duckdb/src/common/serializer/buffered_file_reader.cpp

@@ -63,13 +63,4 @@ ClientContext &BufferedFileReader::GetContext() {
 	return *context;
 }
 
-optional_ptr<Catalog> BufferedFileReader::GetCatalog() {
-	return catalog;
-}
-
-void BufferedFileReader::SetCatalog(Catalog &catalog_p) {
-	D_ASSERT(!catalog);
-	this->catalog = &catalog_p;
-}
-
 } // namespace duckdb

package/src/duckdb/src/common/serializer/format_serializer.cpp

@@ -0,0 +1,15 @@
+#include "duckdb/common/serializer/format_serializer.hpp"
+
+namespace duckdb {
+
+template <>
+void FormatSerializer::WriteValue(const vector<bool> &vec) {
+	auto count = vec.size();
+	OnListBegin(count);
+	for (auto item : vec) {
+		WriteValue(item);
+	}
+	OnListEnd();
+}
+
+} // namespace duckdb

package/src/duckdb/src/common/sort/merge_sorter.cpp

@@ -273,16 +273,13 @@ void MergeSorter::ComputeMerge(const idx_t &count, bool left_smaller[]) {
 			break;
 		}
 		// Pin the radix sorting data
-		if (!l_done) {
-			left->PinRadix(l.block_idx);
-			l_radix_ptr = left->RadixPtr();
-		}
-		if (!r_done) {
-			right->PinRadix(r.block_idx);
-			r_radix_ptr = right->RadixPtr();
-		}
-		const idx_t &l_count = !l_done ? l_sorted_block.radix_sorting_data[l.block_idx]->count : 0;
-		const idx_t &r_count = !r_done ? r_sorted_block.radix_sorting_data[r.block_idx]->count : 0;
+		left->PinRadix(l.block_idx);
+		l_radix_ptr = left->RadixPtr();
+		right->PinRadix(r.block_idx);
+		r_radix_ptr = right->RadixPtr();
+
+		const idx_t l_count = l_sorted_block.radix_sorting_data[l.block_idx]->count;
+		const idx_t r_count = r_sorted_block.radix_sorting_data[r.block_idx]->count;
 		// Compute the merge
 		if (sort_layout.all_constant) {
 			// All sorting columns are constant size
@@ -298,12 +295,8 @@ void MergeSorter::ComputeMerge(const idx_t &count, bool left_smaller[]) {
 			}
 		} else {
 			// Pin the blob data
-			if (!l_done) {
-				left->PinData(*l_sorted_block.blob_sorting_data);
-			}
-			if (!r_done) {
-				right->PinData(*r_sorted_block.blob_sorting_data);
-			}
+			left->PinData(*l_sorted_block.blob_sorting_data);
+			right->PinData(*r_sorted_block.blob_sorting_data);
 			// Merge with variable size sorting columns
 			for (; compared < count && l.entry_idx < l_count && r.entry_idx < r_count; compared++) {
 				left_smaller[compared] =