duckdb 0.8.2-dev4126.0 → 0.8.2-dev4203.0

package/src/duckdb/src/planner/planner.cpp

@@ -1,7 +1,8 @@
 #include "duckdb/planner/planner.hpp"
 
-#include "duckdb/common/serializer.hpp"
-#include "duckdb/common/serializer/buffered_deserializer.hpp"
+#include "duckdb/common/serializer/binary_deserializer.hpp"
+#include "duckdb/common/serializer/binary_serializer.hpp"
+#include "duckdb/common/serializer/memory_stream.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/client_data.hpp"
@@ -11,8 +12,6 @@
 #include "duckdb/planner/binder.hpp"
 #include "duckdb/planner/expression/bound_parameter_expression.hpp"
 #include "duckdb/transaction/meta_transaction.hpp"
-#include "duckdb/common/serializer/binary_serializer.hpp"
-#include "duckdb/common/serializer/binary_deserializer.hpp"
 
 namespace duckdb {
 
@@ -169,29 +168,21 @@ void Planner::VerifyPlan(ClientContext &context, unique_ptr<LogicalOperator> &op
 
 	// format (de)serialization of this operator
 	try {
-		auto blob = BinarySerializer::Serialize(*op, true);
+		MemoryStream stream;
+		BinarySerializer::Serialize(*op, stream, true);
+		stream.Rewind();
 		bound_parameter_map_t parameters;
-		auto result = BinaryDeserializer::Deserialize<LogicalOperator>(context, parameters, blob.data(), blob.size());
+		auto new_plan = BinaryDeserializer::Deserialize<LogicalOperator>(stream, context, parameters);
+
+		if (map) {
+			*map = std::move(parameters);
+		}
+		op = std::move(new_plan);
 	} catch (SerializationException &ex) {
 		// pass
-	}
-
-	BufferedSerializer serializer;
-	try {
-		op->Serialize(serializer);
 	} catch (NotImplementedException &ex) {
-		// ignore for now (FIXME)
-		return;
-	}
-	auto data = serializer.GetData();
-	auto deserializer = BufferedContextDeserializer(context, data.data.get(), data.size);
-
-	PlanDeserializationState state(context);
-	auto new_plan = LogicalOperator::Deserialize(deserializer, state);
-	if (map) {
-		*map = std::move(state.parameter_data);
+		// pass
 	}
-	op = std::move(new_plan);
 }
 
 } // namespace duckdb

package/src/duckdb/src/planner/table_filter.cpp

@@ -1,4 +1,3 @@
-#include "duckdb/common/field_writer.hpp"
 #include "duckdb/planner/table_filter.hpp"
 #include "duckdb/planner/filter/conjunction_filter.hpp"
 #include "duckdb/planner/filter/constant_filter.hpp"
@@ -25,62 +24,4 @@ void TableFilterSet::PushFilter(idx_t column_index, unique_ptr<TableFilter> filt
 	}
 }
 
-//! Serializes a LogicalType to a stand-alone binary blob
-void TableFilterSet::Serialize(Serializer &serializer) const {
-	serializer.Write<idx_t>(filters.size());
-	for (auto &entry : filters) {
-		serializer.Write<idx_t>(entry.first);
-		entry.second->Serialize(serializer);
-	}
-}
-
-//! Deserializes a blob back into an LogicalType
-unique_ptr<TableFilterSet> TableFilterSet::Deserialize(Deserializer &source) {
-	auto len = source.Read<idx_t>();
-	auto res = make_uniq<TableFilterSet>();
-	for (idx_t i = 0; i < len; i++) {
-		auto key = source.Read<idx_t>();
-		auto value = TableFilter::Deserialize(source);
-		res->filters[key] = std::move(value);
-	}
-	return res;
-}
-
-//! Serializes a LogicalType to a stand-alone binary blob
-void TableFilter::Serialize(Serializer &serializer) const {
-	FieldWriter writer(serializer);
-	writer.WriteField<TableFilterType>(filter_type);
-	Serialize(writer);
-	writer.Finalize();
-}
-
-//! Deserializes a blob back into an LogicalType
-unique_ptr<TableFilter> TableFilter::Deserialize(Deserializer &source) {
-	unique_ptr<TableFilter> result;
-
-	FieldReader reader(source);
-	auto filter_type = reader.ReadRequired<TableFilterType>();
-	switch (filter_type) {
-	case TableFilterType::CONSTANT_COMPARISON:
-		result = ConstantFilter::Deserialize(reader);
-		break;
-	case TableFilterType::CONJUNCTION_AND:
-		result = ConjunctionAndFilter::Deserialize(reader);
-		break;
-	case TableFilterType::CONJUNCTION_OR:
-		result = ConjunctionOrFilter::Deserialize(reader);
-		break;
-	case TableFilterType::IS_NOT_NULL:
-		result = IsNotNullFilter::Deserialize(reader);
-		break;
-	case TableFilterType::IS_NULL:
-		result = IsNullFilter::Deserialize(reader);
-		break;
-	default:
-		throw NotImplementedException("Unsupported table filter type for deserialization");
-	}
-	reader.Finalize();
-	return result;
-}
-
 } // namespace duckdb

package/src/duckdb/src/storage/checkpoint/row_group_writer.cpp

@@ -1,9 +1,7 @@
 #include "duckdb/storage/checkpoint/table_data_writer.hpp"
-
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
-#include "duckdb/common/vector_operations/vector_operations.hpp"
-#include "duckdb/common/serializer/buffered_serializer.hpp"
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
+#include "duckdb/common/serializer/binary_serializer.hpp"
 
 namespace duckdb {
 
@@ -19,21 +17,13 @@ PartialBlockAllocation RowGroupWriter::GetBlockAllocation(uint32_t segment_size)
 	return partial_block_manager.GetBlockAllocation(segment_size);
 }
 
-void SingleFileRowGroupWriter::WriteColumnDataPointers(ColumnCheckpointState &column_checkpoint_state) {
-	auto &meta_writer = table_data_writer;
+void SingleFileRowGroupWriter::WriteColumnDataPointers(ColumnCheckpointState &column_checkpoint_state,
+                                                       Serializer &serializer) {
 	const auto &data_pointers = column_checkpoint_state.data_pointers;
-
-	meta_writer.Write<idx_t>(data_pointers.size());
-	// then write the data pointers themselves
-	for (idx_t k = 0; k < data_pointers.size(); k++) {
-		auto &data_pointer = data_pointers[k];
-		meta_writer.Write<idx_t>(data_pointer.row_start);
-		meta_writer.Write<idx_t>(data_pointer.tuple_count);
-		meta_writer.Write<block_id_t>(data_pointer.block_pointer.block_id);
-		meta_writer.Write<uint32_t>(data_pointer.block_pointer.offset);
-		meta_writer.Write<CompressionType>(data_pointer.compression_type);
-		data_pointer.statistics.Serialize(meta_writer);
-	}
+	serializer.WriteList(100, "data_pointers", data_pointers.size(), [&](Serializer::List &list, idx_t i) {
+		auto &data_pointer = data_pointers[i];
+		list.WriteElement(data_pointer);
+	});
 }
 
 MetadataWriter &SingleFileRowGroupWriter::GetPayloadWriter() {

package/src/duckdb/src/storage/checkpoint/table_data_reader.cpp

@@ -1,7 +1,7 @@
 #include "duckdb/storage/checkpoint/table_data_reader.hpp"
 #include "duckdb/storage/metadata/metadata_reader.hpp"
 #include "duckdb/common/types/null_value.hpp"
-
+#include "duckdb/common/serializer/binary_deserializer.hpp"
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 
 #include "duckdb/planner/parsed_data/bound_create_table_info.hpp"
@@ -18,10 +18,13 @@ void TableDataReader::ReadTableData() {
 	auto &columns = info.Base().columns;
 	D_ASSERT(!columns.empty());
 
-	// deserialize the total table statistics
-	info.data->table_stats.Deserialize(reader, columns);
+	// We stored the table statistics as a unit in FinalizeTable.
+	BinaryDeserializer stats_deserializer(reader);
+	stats_deserializer.Begin();
+	info.data->table_stats.Deserialize(stats_deserializer, columns);
+	stats_deserializer.End();
 
-	// deserialize each of the individual row groups
+	// Deserialize the row group pointers (lazily, just set the count and the pointer to them for now)
 	info.data->row_group_count = reader.Read<uint64_t>();
 	info.data->block_pointer = reader.GetMetaBlockPointer();
 }

package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp

@@ -2,10 +2,9 @@
 
 #include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
-#include "duckdb/common/vector_operations/vector_operations.hpp"
-#include "duckdb/common/serializer/buffered_serializer.hpp"
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
 #include "duckdb/storage/table/table_statistics.hpp"
+#include "duckdb/common/serializer/binary_serializer.hpp"
 
 namespace duckdb {
 
@@ -16,9 +15,9 @@ TableDataWriter::TableDataWriter(TableCatalogEntry &table_p) : table(table_p.Cas
 TableDataWriter::~TableDataWriter() {
 }
 
-void TableDataWriter::WriteTableData() {
+void TableDataWriter::WriteTableData(Serializer &metadata_serializer) {
 	// start scanning the table and append the data to the uncompressed segments
-	table.GetStorage().Checkpoint(*this);
+	table.GetStorage().Checkpoint(*this, metadata_serializer);
 }
 
 CompressionType TableDataWriter::GetColumnCompressionType(idx_t i) {
@@ -31,22 +30,25 @@ void TableDataWriter::AddRowGroup(RowGroupPointer &&row_group_pointer, unique_pt
 }
 
 SingleFileTableDataWriter::SingleFileTableDataWriter(SingleFileCheckpointWriter &checkpoint_manager,
-                                                     TableCatalogEntry &table, MetadataWriter &table_data_writer,
-                                                     MetadataWriter &meta_data_writer)
-    : TableDataWriter(table), checkpoint_manager(checkpoint_manager), table_data_writer(table_data_writer),
-      meta_data_writer(meta_data_writer) {
+                                                     TableCatalogEntry &table, MetadataWriter &table_data_writer)
+    : TableDataWriter(table), checkpoint_manager(checkpoint_manager), table_data_writer(table_data_writer) {
 }
 
 unique_ptr<RowGroupWriter> SingleFileTableDataWriter::GetRowGroupWriter(RowGroup &row_group) {
 	return make_uniq<SingleFileRowGroupWriter>(table, checkpoint_manager.partial_block_manager, table_data_writer);
 }
 
-void SingleFileTableDataWriter::FinalizeTable(TableStatistics &&global_stats, DataTableInfo *info) {
+void SingleFileTableDataWriter::FinalizeTable(TableStatistics &&global_stats, DataTableInfo *info,
+                                              Serializer &metadata_serializer) {
 	// store the current position in the metadata writer
 	// this is where the row groups for this table start
 	auto pointer = table_data_writer.GetMetaBlockPointer();
 
-	global_stats.Serialize(table_data_writer);
+	// Serialize statistics as a single unit
+	BinarySerializer stats_serializer(table_data_writer);
+	stats_serializer.Begin();
+	global_stats.Serialize(stats_serializer);
+	stats_serializer.End();
 
 	// now start writing the row group pointers to disk
 	table_data_writer.Write<uint64_t>(row_group_pointers.size());
@@ -56,22 +58,22 @@ void SingleFileTableDataWriter::FinalizeTable(TableStatistics &&global_stats, Da
 		if (row_group_count > total_rows) {
 			total_rows = row_group_count;
 		}
-		RowGroup::Serialize(row_group_pointer, table_data_writer);
-	}
 
-	// Pointer to the table itself goes to the metadata stream.
-	meta_data_writer.Write<idx_t>(pointer.block_pointer);
-	meta_data_writer.Write<uint64_t>(pointer.offset);
-	meta_data_writer.Write<idx_t>(total_rows);
+		// Each RowGroup is its own unit
+		BinarySerializer row_group_serializer(table_data_writer);
+		row_group_serializer.Begin();
+		RowGroup::Serialize(row_group_pointer, row_group_serializer);
+		row_group_serializer.End();
+	}
 
-	// Now we serialize indexes in the table_metadata_writer
+	// TODO: Serialize this:
 	auto index_pointers = info->indexes.SerializeIndexes(table_data_writer);
 
-	// Write-off to metadata block ids and offsets of indexes
-	meta_data_writer.Write<idx_t>(index_pointers.size());
-	for (const auto &index_pointer : index_pointers) {
-		meta_data_writer.Write<BlockPointer>(index_pointer);
-	}
+	// Now begin the metadata as a unit
+	// Pointer to the table itself goes to the metadata stream.
+	metadata_serializer.WriteProperty(101, "table_pointer", pointer);
+	metadata_serializer.WriteProperty(102, "total_rows", total_rows);
+	metadata_serializer.WriteProperty(103, "index_pointers", index_pointers);
 }
 
 } // namespace duckdb