duckdb 0.8.2-dev2700.0 → 0.8.2-dev2842.0

package/src/duckdb/src/common/types/row/row_data_collection_scanner.cpp

@@ -4,6 +4,8 @@
 #include "duckdb/common/types/row/row_data_collection.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
 
+#include <numeric>
+
 namespace duckdb {
 
 void RowDataCollectionScanner::AlignHeapBlocks(RowDataCollection &swizzled_block_collection,
@@ -155,6 +157,31 @@ RowDataCollectionScanner::RowDataCollectionScanner(RowDataCollection &rows_p, Ro
 	ValidateUnscannedBlock();
 }
 
+RowDataCollectionScanner::RowDataCollectionScanner(RowDataCollection &rows_p, RowDataCollection &heap_p,
+                                                   const RowLayout &layout_p, bool external_p, idx_t block_idx,
+                                                   bool flush_p)
+    : rows(rows_p), heap(heap_p), layout(layout_p), read_state(*this), total_count(rows.count), total_scanned(0),
+      external(external_p), flush(flush_p), unswizzling(!layout.AllConstant() && external && !heap.keep_pinned) {
+
+	if (unswizzling) {
+		D_ASSERT(rows.blocks.size() == heap.blocks.size());
+	}
+
+	D_ASSERT(block_idx < rows.blocks.size());
+	read_state.block_idx = block_idx;
+	read_state.entry_idx = 0;
+
+	//	Pretend that we have scanned up to the start block
+	//	and will stop at the end
+	auto begin = rows.blocks.begin();
+	auto end = begin + block_idx;
+	total_scanned =
+	    std::accumulate(begin, end, idx_t(0), [&](idx_t c, const unique_ptr<RowDataBlock> &b) { return c + b->count; });
+	total_count = total_scanned + (*end)->count;
+
+	ValidateUnscannedBlock();
+}
+
 void RowDataCollectionScanner::SwizzleBlock(RowDataBlock &data_block, RowDataBlock &heap_block) {
 	// Pin the data block and swizzle the pointers within the rows
 	D_ASSERT(!data_block.block->IsSwizzled());
@@ -190,7 +217,7 @@ void RowDataCollectionScanner::ReSwizzle() {
 }
 
 void RowDataCollectionScanner::ValidateUnscannedBlock() const {
-	if (unswizzling && read_state.block_idx < rows.blocks.size()) {
+	if (unswizzling && read_state.block_idx < rows.blocks.size() && Remaining()) {
 		D_ASSERT(rows.blocks[read_state.block_idx]->block->IsSwizzled());
 	}
 }
@@ -202,6 +229,9 @@ void RowDataCollectionScanner::Scan(DataChunk &chunk) {
 		return;
 	}
 
+	//	Only flush blocks we processed.
+	const auto flush_block_idx = read_state.block_idx;
+
 	const idx_t &row_width = layout.GetRowWidth();
 	// Set up a batch of pointers to scan data from
 	idx_t scanned = 0;
@@ -227,6 +257,8 @@ void RowDataCollectionScanner::Scan(DataChunk &chunk) {
 		}
 		// Update state indices
 		read_state.entry_idx += next;
+		scanned += next;
+		total_scanned += next;
 		if (read_state.entry_idx == data_block->count) {
 			// Pin completed blocks so we don't lose them
 			pinned_blocks.emplace_back(rows.buffer_manager.Pin(data_block->block));
@@ -238,7 +270,6 @@ void RowDataCollectionScanner::Scan(DataChunk &chunk) {
 			read_state.entry_idx = 0;
 			ValidateUnscannedBlock();
 		}
-		scanned += next;
 	}
 	D_ASSERT(scanned == count);
 	// Deserialize the payload data
@@ -248,14 +279,13 @@ void RowDataCollectionScanner::Scan(DataChunk &chunk) {
 	}
 	chunk.SetCardinality(count);
 	chunk.Verify();
-	total_scanned += scanned;
 
 	//	Switch to a new set of pinned blocks
 	read_state.pinned_blocks.swap(pinned_blocks);
 
 	if (flush) {
 		// Release blocks we have passed.
-		for (idx_t i = 0; i < read_state.block_idx; ++i) {
+		for (idx_t i = flush_block_idx; i < read_state.block_idx; ++i) {
 			rows.blocks[i]->block = nullptr;
 			if (unswizzling) {
 				heap.blocks[i]->block = nullptr;
@@ -263,7 +293,7 @@ void RowDataCollectionScanner::Scan(DataChunk &chunk) {
 		}
 	} else if (unswizzling) {
 		// Reswizzle blocks we have passed so they can be flushed safely.
-		for (idx_t i = 0; i < read_state.block_idx; ++i) {
+		for (idx_t i = flush_block_idx; i < read_state.block_idx; ++i) {
 			auto &data_block = rows.blocks[i];
 			if (data_block->block && !data_block->block->IsSwizzled()) {
 				SwizzleBlock(*data_block, *heap.blocks[i]);

package/src/duckdb/src/common/types/value.cpp

@@ -1763,55 +1763,55 @@ Value Value::Deserialize(Deserializer &main_source) {
 }
 
 void Value::FormatSerialize(FormatSerializer &serializer) const {
-	serializer.WriteProperty("type", type_);
-	serializer.WriteProperty("is_null", is_null);
+	serializer.WriteProperty(100, "type", type_);
+	serializer.WriteProperty(101, "is_null", is_null);
 	if (!IsNull()) {
 		switch (type_.InternalType()) {
 		case PhysicalType::BOOL:
-			serializer.WriteProperty("value", value_.boolean);
+			serializer.WriteProperty(100, "value", value_.boolean);
 			break;
 		case PhysicalType::INT8:
-			serializer.WriteProperty("value", value_.tinyint);
+			serializer.WriteProperty(100, "value", value_.tinyint);
 			break;
 		case PhysicalType::INT16:
-			serializer.WriteProperty("value", value_.smallint);
+			serializer.WriteProperty(100, "value", value_.smallint);
 			break;
 		case PhysicalType::INT32:
-			serializer.WriteProperty("value", value_.integer);
+			serializer.WriteProperty(100, "value", value_.integer);
 			break;
 		case PhysicalType::INT64:
-			serializer.WriteProperty("value", value_.bigint);
+			serializer.WriteProperty(100, "value", value_.bigint);
 			break;
 		case PhysicalType::UINT8:
-			serializer.WriteProperty("value", value_.utinyint);
+			serializer.WriteProperty(100, "value", value_.utinyint);
 			break;
 		case PhysicalType::UINT16:
-			serializer.WriteProperty("value", value_.usmallint);
+			serializer.WriteProperty(100, "value", value_.usmallint);
 			break;
 		case PhysicalType::UINT32:
-			serializer.WriteProperty("value", value_.uinteger);
+			serializer.WriteProperty(100, "value", value_.uinteger);
 			break;
 		case PhysicalType::UINT64:
-			serializer.WriteProperty("value", value_.ubigint);
+			serializer.WriteProperty(100, "value", value_.ubigint);
 			break;
 		case PhysicalType::INT128:
-			serializer.WriteProperty("value", value_.hugeint);
+			serializer.WriteProperty(100, "value", value_.hugeint);
 			break;
 		case PhysicalType::FLOAT:
-			serializer.WriteProperty("value", value_.float_);
+			serializer.WriteProperty(100, "value", value_.float_);
 			break;
 		case PhysicalType::DOUBLE:
-			serializer.WriteProperty("value", value_.double_);
+			serializer.WriteProperty(100, "value", value_.double_);
 			break;
 		case PhysicalType::INTERVAL:
-			serializer.WriteProperty("value", value_.interval);
+			serializer.WriteProperty(100, "value", value_.interval);
 			break;
 		case PhysicalType::VARCHAR:
 			if (type_.id() == LogicalTypeId::BLOB) {
 				auto blob_str = Blob::ToString(StringValue::Get(*this));
-				serializer.WriteProperty("value", blob_str);
+				serializer.WriteProperty(100, "value", blob_str);
 			} else {
-				serializer.WriteProperty("value", StringValue::Get(*this));
+				serializer.WriteProperty(100, "value", StringValue::Get(*this));
 			}
 			break;
 		default: {
@@ -1824,8 +1824,8 @@ void Value::FormatSerialize(FormatSerializer &serializer) const {
 }
 
 Value Value::FormatDeserialize(FormatDeserializer &deserializer) {
-	auto type = deserializer.ReadProperty<LogicalType>("type");
-	auto is_null = deserializer.ReadProperty<bool>("is_null");
+	auto type = deserializer.ReadProperty<LogicalType>(100, "type");
+	auto is_null = deserializer.ReadProperty<bool>(101, "is_null");
 	Value new_value = Value(type);
 	if (is_null) {
 		return new_value;
@@ -1833,46 +1833,46 @@ Value Value::FormatDeserialize(FormatDeserializer &deserializer) {
 	new_value.is_null = false;
 	switch (type.InternalType()) {
 	case PhysicalType::BOOL:
-		new_value.value_.boolean = deserializer.ReadProperty<bool>("value");
+		new_value.value_.boolean = deserializer.ReadProperty<bool>(100, "value");
 		break;
 	case PhysicalType::UINT8:
-		new_value.value_.utinyint = deserializer.ReadProperty<uint8_t>("value");
+		new_value.value_.utinyint = deserializer.ReadProperty<uint8_t>(100, "value");
 		break;
 	case PhysicalType::INT8:
-		new_value.value_.tinyint = deserializer.ReadProperty<int8_t>("value");
+		new_value.value_.tinyint = deserializer.ReadProperty<int8_t>(100, "value");
 		break;
 	case PhysicalType::UINT16:
-		new_value.value_.usmallint = deserializer.ReadProperty<uint16_t>("value");
+		new_value.value_.usmallint = deserializer.ReadProperty<uint16_t>(100, "value");
 		break;
 	case PhysicalType::INT16:
-		new_value.value_.smallint = deserializer.ReadProperty<int16_t>("value");
+		new_value.value_.smallint = deserializer.ReadProperty<int16_t>(100, "value");
 		break;
 	case PhysicalType::UINT32:
-		new_value.value_.uinteger = deserializer.ReadProperty<uint32_t>("value");
+		new_value.value_.uinteger = deserializer.ReadProperty<uint32_t>(100, "value");
 		break;
 	case PhysicalType::INT32:
-		new_value.value_.integer = deserializer.ReadProperty<int32_t>("value");
+		new_value.value_.integer = deserializer.ReadProperty<int32_t>(100, "value");
 		break;
 	case PhysicalType::UINT64:
-		new_value.value_.ubigint = deserializer.ReadProperty<uint64_t>("value");
+		new_value.value_.ubigint = deserializer.ReadProperty<uint64_t>(100, "value");
 		break;
 	case PhysicalType::INT64:
-		new_value.value_.bigint = deserializer.ReadProperty<int64_t>("value");
+		new_value.value_.bigint = deserializer.ReadProperty<int64_t>(100, "value");
 		break;
 	case PhysicalType::INT128:
-		new_value.value_.hugeint = deserializer.ReadProperty<hugeint_t>("value");
+		new_value.value_.hugeint = deserializer.ReadProperty<hugeint_t>(100, "value");
 		break;
 	case PhysicalType::FLOAT:
-		new_value.value_.float_ = deserializer.ReadProperty<float>("value");
+		new_value.value_.float_ = deserializer.ReadProperty<float>(100, "value");
 		break;
 	case PhysicalType::DOUBLE:
-		new_value.value_.double_ = deserializer.ReadProperty<double>("value");
+		new_value.value_.double_ = deserializer.ReadProperty<double>(100, "value");
 		break;
 	case PhysicalType::INTERVAL:
-		new_value.value_.interval = deserializer.ReadProperty<interval_t>("value");
+		new_value.value_.interval = deserializer.ReadProperty<interval_t>(100, "value");
 		break;
 	case PhysicalType::VARCHAR: {
-		auto str = deserializer.ReadProperty<string>("value");
+		auto str = deserializer.ReadProperty<string>(100, "value");
 		if (type.id() == LogicalTypeId::BLOB) {
 			new_value.value_info_ = make_shared<StringValueInfo>(Blob::ToBlob(str));
 		} else {

package/src/duckdb/src/common/types/vector.cpp

@@ -999,14 +999,14 @@ void Vector::FormatSerialize(FormatSerializer &serializer, idx_t count) {
 	ToUnifiedFormat(count, vdata);
 
 	const bool all_valid = (count > 0) && !vdata.validity.AllValid();
-	serializer.WriteProperty("all_valid", all_valid);
+	serializer.WriteProperty(100, "all_valid", all_valid);
 	if (all_valid) {
 		ValidityMask flat_mask(count);
 		for (idx_t i = 0; i < count; ++i) {
 			auto row_idx = vdata.sel->get_index(i);
 			flat_mask.Set(i, vdata.validity.RowIsValid(row_idx));
 		}
-		serializer.WriteProperty("validity", const_data_ptr_cast(flat_mask.GetData()),
+		serializer.WriteProperty(101, "validity", const_data_ptr_cast(flat_mask.GetData()),
 		                         flat_mask.ValidityMaskSize(count));
 	}
 	if (TypeIsConstantSize(logical_type.InternalType())) {
@@ -1014,14 +1014,14 @@ void Vector::FormatSerialize(FormatSerializer &serializer, idx_t count) {
 		idx_t write_size = GetTypeIdSize(logical_type.InternalType()) * count;
 		auto ptr = make_unsafe_uniq_array<data_t>(write_size);
 		VectorOperations::WriteToStorage(*this, count, ptr.get());
-		serializer.WriteProperty("data", ptr.get(), write_size);
+		serializer.WriteProperty(102, "data", ptr.get(), write_size);
 	} else {
 		switch (logical_type.InternalType()) {
 		case PhysicalType::VARCHAR: {
 			auto strings = UnifiedVectorFormat::GetData<string_t>(vdata);
 
 			// Serialize data as a list
-			serializer.SetTag("data");
+			serializer.SetTag(102, "data");
 			serializer.OnListBegin(count);
 			for (idx_t i = 0; i < count; i++) {
 				auto idx = vdata.sel->get_index(i);
@@ -1036,7 +1036,7 @@ void Vector::FormatSerialize(FormatSerializer &serializer, idx_t count) {
 			auto &entries = StructVector::GetEntries(*this);
 
 			// Serialize entries as a list
-			serializer.SetTag("children");
+			serializer.SetTag(103, "children");
 			serializer.OnListBegin(entries.size());
 			for (auto &entry : entries) {
 				serializer.OnObjectBegin();
@@ -1059,17 +1059,17 @@ void Vector::FormatSerialize(FormatSerializer &serializer, idx_t count) {
 				entries[i].offset = source.offset;
 				entries[i].length = source.length;
 			}
-			serializer.WriteProperty("list_size", list_size);
-			serializer.SetTag("entries");
+			serializer.WriteProperty(104, "list_size", list_size);
+			serializer.SetTag(105, "entries");
 			serializer.OnListBegin(count);
 			for (idx_t i = 0; i < count; i++) {
 				serializer.OnObjectBegin();
-				serializer.WriteProperty("offset", entries[i].offset);
-				serializer.WriteProperty("length", entries[i].length);
+				serializer.WriteProperty(100, "offset", entries[i].offset);
+				serializer.WriteProperty(101, "length", entries[i].length);
 				serializer.OnObjectEnd();
 			}
 			serializer.OnListEnd(count);
-			serializer.SetTag("child");
+			serializer.SetTag(106, "child");
 			serializer.OnObjectBegin();
 			child.FormatSerialize(serializer, list_size);
 			serializer.OnObjectEnd();
@@ -1086,24 +1086,24 @@ void Vector::FormatDeserialize(FormatDeserializer &deserializer, idx_t count) {
 
 	auto &validity = FlatVector::Validity(*this);
 	validity.Reset();
-	const auto has_validity = deserializer.ReadProperty<bool>("all_valid");
+	const auto has_validity = deserializer.ReadProperty<bool>(100, "all_valid");
 	if (has_validity) {
 		validity.Initialize(count);
-		deserializer.ReadProperty("validity", data_ptr_cast(validity.GetData()), validity.ValidityMaskSize(count));
+		deserializer.ReadProperty(101, "validity", data_ptr_cast(validity.GetData()), validity.ValidityMaskSize(count));
 	}
 
 	if (TypeIsConstantSize(logical_type.InternalType())) {
 		// constant size type: read fixed amount of data
 		auto column_size = GetTypeIdSize(logical_type.InternalType()) * count;
 		auto ptr = make_unsafe_uniq_array<data_t>(column_size);
-		deserializer.ReadProperty("data", ptr.get(), column_size);
+		deserializer.ReadProperty(102, "data", ptr.get(), column_size);
 
 		VectorOperations::ReadFromStorage(ptr.get(), count, *this);
 	} else {
 		switch (logical_type.InternalType()) {
 		case PhysicalType::VARCHAR: {
 			auto strings = FlatVector::GetData<string_t>(*this);
-			deserializer.SetTag("data");
+			deserializer.SetTag(102, "data");
 			auto read_count = deserializer.OnListBegin();
 			D_ASSERT(read_count == count);
 			(void)read_count; // otherwise unused variable error in release mode
@@ -1122,7 +1122,7 @@ void Vector::FormatDeserialize(FormatDeserializer &deserializer, idx_t count) {
 		case PhysicalType::STRUCT: {
 			auto &entries = StructVector::GetEntries(*this);
 			// Deserialize entries as a list
-			deserializer.SetTag("children");
+			deserializer.SetTag(103, "children");
 			auto read_size = deserializer.OnListBegin();
 			D_ASSERT(read_size == entries.size());
 			(void)read_size;
@@ -1136,26 +1136,26 @@ void Vector::FormatDeserialize(FormatDeserializer &deserializer, idx_t count) {
 		}
 		case PhysicalType::LIST: {
 			// Read the list size
-			auto list_size = deserializer.ReadProperty<uint64_t>("list_size");
+			auto list_size = deserializer.ReadProperty<uint64_t>(104, "list_size");
 			ListVector::Reserve(*this, list_size);
 			ListVector::SetListSize(*this, list_size);
 
 			// Read the entries
 			auto list_entries = FlatVector::GetData<list_entry_t>(*this);
-			deserializer.SetTag("entries");
+			deserializer.SetTag(105, "entries");
 			auto entries_count = deserializer.OnListBegin();
 			D_ASSERT(entries_count == count);
 			(void)entries_count;
 			for (idx_t i = 0; i < count; i++) {
 				deserializer.OnObjectBegin();
-				deserializer.ReadProperty("offset", list_entries[i].offset);
-				deserializer.ReadProperty("length", list_entries[i].length);
+				deserializer.ReadProperty(100, "offset", list_entries[i].offset);
+				deserializer.ReadProperty(101, "length", list_entries[i].length);
 				deserializer.OnObjectEnd();
 			}
 			deserializer.OnListEnd();
 
 			// Read the child vector
-			deserializer.SetTag("child");
+			deserializer.SetTag(106, "child");
 			auto &child = ListVector::GetEntry(*this);
 			deserializer.OnObjectBegin();
 			child.FormatDeserialize(deserializer, list_size);

package/src/duckdb/src/core_functions/aggregate/holistic/approximate_quantile.cpp

@@ -55,12 +55,12 @@ struct ApproximateQuantileBindData : public FunctionData {
 	static void FormatSerialize(FormatSerializer &serializer, const optional_ptr<FunctionData> bind_data_p,
 	                            const AggregateFunction &function) {
 		auto &bind_data = bind_data_p->Cast<ApproximateQuantileBindData>();
-		serializer.WriteProperty("quantiles", bind_data.quantiles);
+		serializer.WriteProperty(100, "quantiles", bind_data.quantiles);
 	}
 
 	static unique_ptr<FunctionData> FormatDeserialize(FormatDeserializer &deserializer, AggregateFunction &function) {
 		auto result = make_uniq<ApproximateQuantileBindData>();
-		deserializer.ReadProperty("quantiles", result->quantiles);
+		deserializer.ReadProperty(100, "quantiles", result->quantiles);
 		return std::move(result);
 	}
 

package/src/duckdb/src/core_functions/aggregate/holistic/quantile.cpp

@@ -461,16 +461,16 @@ struct QuantileBindData : public FunctionData {
 	static void FormatSerialize(FormatSerializer &serializer, const optional_ptr<FunctionData> bind_data_p,
 	                            const AggregateFunction &function) {
 		auto &bind_data = bind_data_p->Cast<QuantileBindData>();
-		serializer.WriteProperty("quantiles", bind_data.quantiles);
-		serializer.WriteProperty("order", bind_data.order);
-		serializer.WriteProperty("desc", bind_data.desc);
+		serializer.WriteProperty(100, "quantiles", bind_data.quantiles);
+		serializer.WriteProperty(101, "order", bind_data.order);
+		serializer.WriteProperty(102, "desc", bind_data.desc);
 	}
 
 	static unique_ptr<FunctionData> FormatDeserialize(FormatDeserializer &deserializer, AggregateFunction &function) {
 		auto result = make_uniq<QuantileBindData>();
-		deserializer.ReadProperty("quantiles", result->quantiles);
-		deserializer.ReadProperty("order", result->order);
-		deserializer.ReadProperty("desc", result->desc);
+		deserializer.ReadProperty(100, "quantiles", result->quantiles);
+		deserializer.ReadProperty(101, "order", result->order);
+		deserializer.ReadProperty(102, "desc", result->desc);
 		return std::move(result);
 	}
 

package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp

@@ -87,14 +87,14 @@ struct ReservoirQuantileBindData : public FunctionData {
 	static void FormatSerialize(FormatSerializer &serializer, const optional_ptr<FunctionData> bind_data_p,
 	                            const AggregateFunction &function) {
 		auto &bind_data = bind_data_p->Cast<ReservoirQuantileBindData>();
-		serializer.WriteProperty("quantiles", bind_data.quantiles);
-		serializer.WriteProperty("sample_size", bind_data.sample_size);
+		serializer.WriteProperty(100, "quantiles", bind_data.quantiles);
+		serializer.WriteProperty(101, "sample_size", bind_data.sample_size);
 	}
 
 	static unique_ptr<FunctionData> FormatDeserialize(FormatDeserializer &deserializer, AggregateFunction &function) {
 		auto result = make_uniq<ReservoirQuantileBindData>();
-		deserializer.ReadProperty("quantiles", result->quantiles);
-		deserializer.ReadProperty("sample_size", result->sample_size);
+		deserializer.ReadProperty(100, "quantiles", result->quantiles);
+		deserializer.ReadProperty(101, "sample_size", result->sample_size);
 		return std::move(result);
 	}
 

package/src/duckdb/src/core_functions/scalar/list/list_lambdas.cpp

@@ -34,13 +34,13 @@ public:
 	static void FormatSerialize(FormatSerializer &serializer, const optional_ptr<FunctionData> bind_data_p,
 	                            const ScalarFunction &function) {
 		auto &bind_data = bind_data_p->Cast<ListLambdaBindData>();
-		serializer.WriteProperty("stype", bind_data.stype);
-		serializer.WriteOptionalProperty("lambda_expr", bind_data.lambda_expr);
+		serializer.WriteProperty(100, "stype", bind_data.stype);
+		serializer.WriteOptionalProperty(101, "lambda_expr", bind_data.lambda_expr);
 	}
 
 	static unique_ptr<FunctionData> FormatDeserialize(FormatDeserializer &deserializer, ScalarFunction &function) {
-		auto stype = deserializer.ReadProperty<LogicalType>("stype");
-		auto lambda_expr = deserializer.ReadOptionalProperty<unique_ptr<Expression>>("lambda_expr");
+		auto stype = deserializer.ReadProperty<LogicalType>(100, "stype");
+		auto lambda_expr = deserializer.ReadOptionalProperty<unique_ptr<Expression>>(101, "lambda_expr");
 		return make_uniq<ListLambdaBindData>(stype, std::move(lambda_expr));
 	}
 };