duckdb 0.8.2-dev3458.0 → 0.8.2-dev3949.0

package/src/duckdb/src/storage/table/row_group.cpp

@@ -1,6 +1,5 @@
 #include "duckdb/storage/table/row_group.hpp"
 #include "duckdb/common/types/vector.hpp"
-#include "duckdb/transaction/transaction.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/field_writer.hpp"
 #include "duckdb/storage/table/column_data.hpp"
@@ -18,6 +17,8 @@
 #include "duckdb/transaction/duck_transaction.hpp"
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
 
 namespace duckdb {
 
@@ -927,6 +928,72 @@ RowGroupPointer RowGroup::Deserialize(Deserializer &main_source, const vector<Lo
 	return result;
 }
 
+void RowGroup::FormatSerialize(RowGroupPointer &pointer, FormatSerializer &serializer) {
+	serializer.WriteProperty(100, "row_start", pointer.row_start);
+	serializer.WriteProperty(101, "tuple_count", pointer.tuple_count);
+	serializer.WriteProperty(102, "data_pointers", pointer.data_pointers);
+
+	// Checkpoint deletes
+	auto versions = pointer.versions.get();
+
+	if (!versions) {
+		// no version information: write nothing
+		serializer.WriteProperty(103, "versions_count", 0);
+		return;
+	}
+	// first count how many ChunkInfo's we need to deserialize
+	idx_t chunk_info_count = 0;
+	idx_t idx_map[ROW_GROUP_VECTOR_COUNT];
+	for (idx_t vector_idx = 0; vector_idx < RowGroup::ROW_GROUP_VECTOR_COUNT; vector_idx++) {
+		auto chunk_info = versions->info[vector_idx].get();
+		if (!chunk_info) {
+			continue;
+		}
+		idx_map[chunk_info_count++] = vector_idx;
+	}
+
+	// now serialize the actual version information
+	serializer.WriteProperty(103, "versions_count", chunk_info_count);
+	serializer.WriteList(104, "versions", chunk_info_count, [&](FormatSerializer::List &list, idx_t i) {
+		auto vector_idx = idx_map[i];
+		auto chunk_info = versions->info[vector_idx].get();
+		list.WriteObject([&](FormatSerializer &obj) {
+			obj.WriteProperty(100, "vector_index", vector_idx);
+			obj.WriteProperty(101, "chunk_info", const_cast<const ChunkInfo *>(chunk_info));
+		});
+	});
+}
+
+RowGroupPointer RowGroup::FormatDeserialize(FormatDeserializer &deserializer) {
+	RowGroupPointer result;
+	result.row_start = deserializer.ReadProperty<uint64_t>(100, "row_start");
+	result.tuple_count = deserializer.ReadProperty<uint64_t>(101, "tuple_count");
+	result.data_pointers = deserializer.ReadProperty<vector<MetaBlockPointer>>(102, "data_pointers");
+	result.versions = nullptr;
+	// Deserialize Deletes
+	auto chunk_count = deserializer.ReadProperty<idx_t>(103, "versions_count");
+	if (chunk_count == 0) {
+		// no deletes
+		return result;
+	}
+
+	auto version_info = make_shared<VersionNode>();
+	deserializer.ReadList(104, "versions", [&](FormatDeserializer::List &list, idx_t i) {
+		list.ReadObject([&](FormatDeserializer &obj) {
+			auto vector_index = obj.ReadProperty<idx_t>(100, "vector_index");
+			if (vector_index >= RowGroup::ROW_GROUP_VECTOR_COUNT) {
+				throw Exception("In DeserializeDeletes, vector_index is out of range for the row group. Corrupted "
+				                "file?");
+			}
+			version_info->info[vector_index] = obj.ReadProperty<unique_ptr<ChunkInfo>>(101, "chunk_info");
+		});
+	});
+
+	result.versions = version_info;
+
+	return result;
+}
+
 //===--------------------------------------------------------------------===//
 // GetColumnSegmentInfo
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/storage/table/table_statistics.cpp

@@ -1,5 +1,7 @@
 #include "duckdb/storage/table/table_statistics.hpp"
 #include "duckdb/storage/table/persistent_table_data.hpp"
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
 
 namespace duckdb {
 
@@ -112,6 +114,25 @@ void TableStatistics::Deserialize(Deserializer &source, ColumnList &columns) {
 	}
 }
 
+void TableStatistics::FormatSerialize(FormatSerializer &serializer) {
+	auto column_count = column_stats.size();
+	serializer.WriteList(100, "column_stats", column_count,
+	                     [&](FormatSerializer::List &list, idx_t i) { list.WriteElement(column_stats[i]); });
+}
+
+void TableStatistics::FormatDeserialize(FormatDeserializer &deserializer, ColumnList &columns) {
+	auto physical_columns = columns.Physical();
+	auto iter = physical_columns.begin();
+	deserializer.ReadList(100, "column_stats", [&](FormatDeserializer::List &list, idx_t i) {
+		auto &col = *iter.operator++();
+		auto type = col.GetType();
+		deserializer.Set<LogicalType &>(type);
+		auto stats = ColumnStatistics::FormatDeserialize(deserializer);
+		deserializer.Unset<LogicalType>();
+		column_stats.push_back(std::move(stats));
+	});
+}
+
 unique_ptr<TableStatisticsLock> TableStatistics::GetLock() {
 	return make_uniq<TableStatisticsLock>(stats_lock);
 }

package/src/duckdb/src/storage/wal_replay.cpp

@@ -53,7 +53,7 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
 			}
 		}
 	} catch (std::exception &ex) { // LCOV_EXCL_START
-		Printer::Print(StringUtil::Format("Exception in WAL playback during initial read: %s\n", ex.what()));
+		Printer::PrintF("Exception in WAL playback during initial read: %s\n", ex.what());
 		return false;
 	} catch (...) {
 		Printer::Print("Unknown Exception in WAL playback during initial read");
@@ -99,7 +99,7 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
 		}
 	} catch (std::exception &ex) { // LCOV_EXCL_START
 		// FIXME: this should report a proper warning in the connection
-		Printer::Print(StringUtil::Format("Exception in WAL playback: %s\n", ex.what()));
+		Printer::PrintF("Exception in WAL playback: %s\n", ex.what());
 		// exception thrown in WAL replay: rollback
 		con.Rollback();
 	} catch (...) {

package/src/duckdb/src/verification/deserialized_statement_verifier_v2.cpp

@@ -11,10 +11,24 @@ DeserializedStatementVerifierV2::DeserializedStatementVerifierV2(unique_ptr<SQLS
 unique_ptr<StatementVerifier> DeserializedStatementVerifierV2::Create(const SQLStatement &statement) {
 	auto &select_stmt = statement.Cast<SelectStatement>();
 
-	auto blob = BinarySerializer::Serialize(select_stmt);
+	auto blob = BinarySerializer::Serialize(select_stmt, true);
 	auto result = BinaryDeserializer::Deserialize<SelectStatement>(blob.data(), blob.size());
 
 	return make_uniq<DeserializedStatementVerifierV2>(std::move(result));
 }
 
+DeserializedStatementVerifierNoDefaultV2::DeserializedStatementVerifierNoDefaultV2(unique_ptr<SQLStatement> statement_p)
+    : StatementVerifier(VerificationType::DESERIALIZED_V2_NO_DEFAULT, "Deserialized V2 without default values",
+                        std::move(statement_p)) {
+}
+
+unique_ptr<StatementVerifier> DeserializedStatementVerifierNoDefaultV2::Create(const SQLStatement &statement) {
+	auto &select_stmt = statement.Cast<SelectStatement>();
+
+	auto blob = BinarySerializer::Serialize(select_stmt, false);
+	auto result = BinaryDeserializer::Deserialize<SelectStatement>(blob.data(), blob.size());
+
+	return make_uniq<DeserializedStatementVerifierNoDefaultV2>(std::move(result));
+}
+
 } // namespace duckdb

package/src/duckdb/src/verification/statement_verifier.cpp

@@ -35,6 +35,8 @@ unique_ptr<StatementVerifier> StatementVerifier::Create(VerificationType type, c
 		return DeserializedStatementVerifier::Create(statement_p);
 	case VerificationType::DESERIALIZED_V2:
 		return DeserializedStatementVerifierV2::Create(statement_p);
+	case VerificationType::DESERIALIZED_V2_NO_DEFAULT:
+		return DeserializedStatementVerifierNoDefaultV2::Create(statement_p);
 	case VerificationType::PARSED:
 		return ParsedStatementVerifier::Create(statement_p);
 	case VerificationType::UNOPTIMIZED:

package/src/duckdb/third_party/utf8proc/include/utf8proc_wrapper.hpp

@@ -1,3 +1,11 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// utf8proc_wrapper.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
 #pragma once
 
 #include <string>

package/src/duckdb/ub_src_execution.cpp

@@ -12,8 +12,6 @@
 
 #include "src/execution/join_hashtable.cpp"
 
-#include "src/execution/partitionable_hashtable.cpp"
-
 #include "src/execution/perfect_aggregate_hashtable.cpp"
 
 #include "src/execution/physical_operator.cpp"

package/src/duckdb/ub_src_execution_operator_csv_scanner.cpp

@@ -0,0 +1,18 @@
+#include "src/execution/operator/csv_scanner/base_csv_reader.cpp"
+
+#include "src/execution/operator/csv_scanner/buffered_csv_reader.cpp"
+
+#include "src/execution/operator/csv_scanner/csv_buffer.cpp"
+
+#include "src/execution/operator/csv_scanner/csv_buffer_manager.cpp"
+
+#include "src/execution/operator/csv_scanner/csv_file_handle.cpp"
+
+#include "src/execution/operator/csv_scanner/csv_reader_options.cpp"
+
+#include "src/execution/operator/csv_scanner/csv_state_machine.cpp"
+
+#include "src/execution/operator/csv_scanner/csv_state_machine_cache.cpp"
+
+#include "src/execution/operator/csv_scanner/parallel_csv_reader.cpp"
+

package/src/duckdb/ub_src_execution_operator_csv_scanner_sniffer.cpp

@@ -0,0 +1,12 @@
+#include "src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp"
+
+#include "src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp"
+
+#include "src/execution/operator/csv_scanner/sniffer/header_detection.cpp"
+
+#include "src/execution/operator/csv_scanner/sniffer/type_detection.cpp"
+
+#include "src/execution/operator/csv_scanner/sniffer/type_refinement.cpp"
+
+#include "src/execution/operator/csv_scanner/sniffer/type_replacement.cpp"
+

package/src/duckdb/ub_src_execution_operator_persistent.cpp

@@ -1,15 +1,3 @@
-#include "src/execution/operator/persistent/base_csv_reader.cpp"
-
-#include "src/execution/operator/persistent/buffered_csv_reader.cpp"
-
-#include "src/execution/operator/persistent/parallel_csv_reader.cpp"
-
-#include "src/execution/operator/persistent/csv_buffer.cpp"
-
-#include "src/execution/operator/persistent/csv_file_handle.cpp"
-
-#include "src/execution/operator/persistent/csv_reader_options.cpp"
-
 #include "src/execution/operator/persistent/csv_rejects_table.cpp"
 
 #include "src/execution/operator/persistent/physical_fixed_batch_copy.cpp"

package/src/duckdb/ub_src_storage_serialization.cpp

@@ -26,3 +26,5 @@
 
 #include "src/storage/serialization/serialize_types.cpp"
 
+#include "src/storage/serialization/serialize_storage.cpp"
+