duckdb 0.7.2-dev717.0 → 0.7.2-dev865.0

package/src/duckdb/src/common/types/value.cpp

@@ -30,6 +30,9 @@
 #include "duckdb/function/cast/cast_function_set.hpp"
 #include "duckdb/main/error_manager.hpp"
 
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
+
 #include <utility>
 #include <cmath>
 
@@ -1230,99 +1233,6 @@ interval_t Value::GetValueUnsafe() const {
 	return value_.interval;
 }
 
-//===--------------------------------------------------------------------===//
-// GetReferenceUnsafe
-//===--------------------------------------------------------------------===//
-template <>
-int8_t &Value::GetReferenceUnsafe() {
-	D_ASSERT(type_.InternalType() == PhysicalType::INT8 || type_.InternalType() == PhysicalType::BOOL);
-	return value_.tinyint;
-}
-
-template <>
-int16_t &Value::GetReferenceUnsafe() {
-	D_ASSERT(type_.InternalType() == PhysicalType::INT16);
-	return value_.smallint;
-}
-
-template <>
-int32_t &Value::GetReferenceUnsafe() {
-	D_ASSERT(type_.InternalType() == PhysicalType::INT32);
-	return value_.integer;
-}
-
-template <>
-int64_t &Value::GetReferenceUnsafe() {
-	D_ASSERT(type_.InternalType() == PhysicalType::INT64);
-	return value_.bigint;
-}
-
-template <>
-hugeint_t &Value::GetReferenceUnsafe() {
-	D_ASSERT(type_.InternalType() == PhysicalType::INT128);
-	return value_.hugeint;
-}
-
-template <>
-uint8_t &Value::GetReferenceUnsafe() {
-	D_ASSERT(type_.InternalType() == PhysicalType::UINT8);
-	return value_.utinyint;
-}
-
-template <>
-uint16_t &Value::GetReferenceUnsafe() {
-	D_ASSERT(type_.InternalType() == PhysicalType::UINT16);
-	return value_.usmallint;
-}
-
-template <>
-uint32_t &Value::GetReferenceUnsafe() {
-	D_ASSERT(type_.InternalType() == PhysicalType::UINT32);
-	return value_.uinteger;
-}
-
-template <>
-uint64_t &Value::GetReferenceUnsafe() {
-	D_ASSERT(type_.InternalType() == PhysicalType::UINT64);
-	return value_.ubigint;
-}
-
-template <>
-float &Value::GetReferenceUnsafe() {
-	D_ASSERT(type_.InternalType() == PhysicalType::FLOAT);
-	return value_.float_;
-}
-
-template <>
-double &Value::GetReferenceUnsafe() {
-	D_ASSERT(type_.InternalType() == PhysicalType::DOUBLE);
-	return value_.double_;
-}
-
-template <>
-date_t &Value::GetReferenceUnsafe() {
-	D_ASSERT(type_.InternalType() == PhysicalType::INT32);
-	return value_.date;
-}
-
-template <>
-dtime_t &Value::GetReferenceUnsafe() {
-	D_ASSERT(type_.InternalType() == PhysicalType::INT64);
-	return value_.time;
-}
-
-template <>
-timestamp_t &Value::GetReferenceUnsafe() {
-	D_ASSERT(type_.InternalType() == PhysicalType::INT64);
-	return value_.timestamp;
-}
-
-template <>
-interval_t &Value::GetReferenceUnsafe() {
-	D_ASSERT(type_.InternalType() == PhysicalType::INTERVAL);
-	return value_.interval;
-}
-
 //===--------------------------------------------------------------------===//
 // Hash
 //===--------------------------------------------------------------------===//
@@ -1800,6 +1710,120 @@ Value Value::Deserialize(Deserializer &main_source) {
 	return new_value;
 }
 
+void Value::FormatSerialize(FormatSerializer &serializer) const {
+	serializer.WriteProperty("type", type_);
+	serializer.WriteProperty("is_null", is_null);
+	if (!IsNull()) {
+		switch (type_.InternalType()) {
+		case PhysicalType::BOOL:
+			serializer.WriteProperty("value", value_.boolean);
+			break;
+		case PhysicalType::INT8:
+			serializer.WriteProperty("value", value_.tinyint);
+			break;
+		case PhysicalType::INT16:
+			serializer.WriteProperty("value", value_.smallint);
+			break;
+		case PhysicalType::INT32:
+			serializer.WriteProperty("value", value_.integer);
+			break;
+		case PhysicalType::INT64:
+			serializer.WriteProperty("value", value_.bigint);
+			break;
+		case PhysicalType::UINT8:
+			serializer.WriteProperty("value", value_.utinyint);
+			break;
+		case PhysicalType::UINT16:
+			serializer.WriteProperty("value", value_.usmallint);
+			break;
+		case PhysicalType::UINT32:
+			serializer.WriteProperty("value", value_.uinteger);
+			break;
+		case PhysicalType::UINT64:
+			serializer.WriteProperty("value", value_.ubigint);
+			break;
+		case PhysicalType::INT128:
+			serializer.WriteProperty("value", value_.hugeint);
+			break;
+		case PhysicalType::FLOAT:
+			serializer.WriteProperty("value", value_.float_);
+			break;
+		case PhysicalType::DOUBLE:
+			serializer.WriteProperty("value", value_.double_);
+			break;
+		case PhysicalType::INTERVAL:
+			serializer.WriteProperty("value", value_.interval);
+			break;
+		case PhysicalType::VARCHAR:
+			serializer.WriteProperty("value", StringValue::Get(*this));
+			break;
+		default: {
+			Vector v(*this);
+			v.FormatSerialize(serializer, 1);
+			break;
+		}
+		}
+	}
+}
+
+Value Value::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto type = deserializer.ReadProperty<LogicalType>("type");
+	auto is_null = deserializer.ReadProperty<bool>("is_null");
+	Value new_value = Value(type);
+	if (is_null) {
+		return new_value;
+	}
+	new_value.is_null = false;
+	switch (type.InternalType()) {
+	case PhysicalType::BOOL:
+		new_value.value_.boolean = deserializer.ReadProperty<bool>("value");
+		break;
+	case PhysicalType::UINT8:
+		new_value.value_.utinyint = deserializer.ReadProperty<uint8_t>("value");
+		break;
+	case PhysicalType::INT8:
+		new_value.value_.tinyint = deserializer.ReadProperty<int8_t>("value");
+		break;
+	case PhysicalType::UINT16:
+		new_value.value_.usmallint = deserializer.ReadProperty<uint16_t>("value");
+		break;
+	case PhysicalType::INT16:
+		new_value.value_.smallint = deserializer.ReadProperty<int16_t>("value");
+		break;
+	case PhysicalType::UINT32:
+		new_value.value_.uinteger = deserializer.ReadProperty<uint32_t>("value");
+		break;
+	case PhysicalType::INT32:
+		new_value.value_.integer = deserializer.ReadProperty<int32_t>("value");
+		break;
+	case PhysicalType::UINT64:
+		new_value.value_.ubigint = deserializer.ReadProperty<uint64_t>("value");
+		break;
+	case PhysicalType::INT64:
+		new_value.value_.bigint = deserializer.ReadProperty<int64_t>("value");
+		break;
+	case PhysicalType::FLOAT:
+		new_value.value_.float_ = deserializer.ReadProperty<float>("value");
+		break;
+	case PhysicalType::DOUBLE:
+		new_value.value_.double_ = deserializer.ReadProperty<double>("value");
+		break;
+	case PhysicalType::INTERVAL:
+		new_value.value_.interval = deserializer.ReadProperty<interval_t>("value");
+		break;
+	case PhysicalType::VARCHAR:
+		new_value.value_info_ = make_shared<StringValueInfo>(deserializer.ReadProperty<string>("value"));
+		break;
+	default: {
+		Vector v(type);
+		v.FormatDeserialize(deserializer, 1);
+		new_value = v.GetValue(0);
+		break;
+	}
+	}
+	return new_value;
+}
+
 void Value::Print() const {
 	Printer::Print(ToString());
 }

package/src/duckdb/src/common/types/vector.cpp

@@ -18,6 +18,9 @@
 #include "duckdb/common/fsst.hpp"
 #include "fsst.h"
 
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
+
 #include <cstring> // strlen() on Solaris
 
 namespace duckdb {
@@ -241,7 +244,8 @@ struct DataArrays {
 	idx_t type_size;
 	bool is_nested;
 	DataArrays(Vector &vec, data_ptr_t data, VectorBuffer *buffer, idx_t type_size, bool is_nested)
-	    : vec(vec), data(data), buffer(buffer), type_size(type_size), is_nested(is_nested) {};
+	    : vec(vec), data(data), buffer(buffer), type_size(type_size), is_nested(is_nested) {
+	}
 };
 
 void FindChildren(std::vector<DataArrays> &to_resize, VectorBuffer &auxiliary) {
@@ -962,6 +966,141 @@ void Vector::Serialize(idx_t count, Serializer &serializer) {
 	}
 }
 
+void Vector::FormatSerialize(FormatSerializer &serializer, idx_t count) {
+	auto &type = GetType();
+
+	UnifiedVectorFormat vdata;
+	ToUnifiedFormat(count, vdata);
+
+	const auto write_validity = (count > 0) && !vdata.validity.AllValid();
+	serializer.WriteProperty("has_validity", write_validity);
+	if (write_validity) {
+		ValidityMask flat_mask(count);
+		for (idx_t i = 0; i < count; ++i) {
+			auto row_idx = vdata.sel->get_index(i);
+			flat_mask.Set(i, vdata.validity.RowIsValid(row_idx));
+		}
+		serializer.WriteProperty("validity_mask", (const_data_ptr_t)flat_mask.GetData(),
+		                         flat_mask.ValidityMaskSize(count));
+	}
+	if (TypeIsConstantSize(type.InternalType())) {
+		// constant size type: simple copy
+		idx_t write_size = GetTypeIdSize(type.InternalType()) * count;
+		auto ptr = unique_ptr<data_t[]>(new data_t[write_size]);
+		VectorOperations::WriteToStorage(*this, count, ptr.get());
+		serializer.WriteProperty("data", write_size);
+	} else {
+		switch (type.InternalType()) {
+		case PhysicalType::VARCHAR: {
+			auto strings = (string_t *)vdata.data;
+			for (idx_t i = 0; i < count; i++) {
+				auto idx = vdata.sel->get_index(i);
+				auto source = !vdata.validity.RowIsValid(idx) ? NullValue<string_t>() : strings[idx];
+				string_t str = string_t(source.GetDataUnsafe(), source.GetSize());
+				serializer.WriteProperty("data", str);
+			}
+			break;
+		}
+		case PhysicalType::STRUCT: {
+			Flatten(count);
+			auto &entries = StructVector::GetEntries(*this);
+			for (auto &entry : entries) {
+				entry->FormatSerialize(serializer, count);
+			}
+			break;
+		}
+		case PhysicalType::LIST: {
+			auto &child = ListVector::GetEntry(*this);
+			auto list_size = ListVector::GetListSize(*this);
+
+			// serialize the list entries in a flat array
+			auto data = unique_ptr<list_entry_t[]>(new list_entry_t[count]);
+			auto source_array = (list_entry_t *)vdata.data;
+			for (idx_t i = 0; i < count; i++) {
+				auto idx = vdata.sel->get_index(i);
+				auto source = source_array[idx];
+				data[i].offset = source.offset;
+				data[i].length = source.length;
+			}
+
+			// write the list size
+			serializer.WriteProperty("list_size", list_size);
+			serializer.WriteProperty("data", (data_ptr_t)data.get(), count * sizeof(list_entry_t));
+			child.FormatSerialize(serializer, list_size);
+			break;
+		}
+		default:
+			throw InternalException("Unimplemented variable width type for Vector::Serialize!");
+		}
+	}
+}
+
+void Vector::FormatDeserialize(FormatDeserializer &deserializer, idx_t count) {
+	/*
+	auto &type = GetType();
+
+	auto &validity = FlatVector::Validity(*this);
+	validity.Reset();
+	const auto has_validity = deserializer.ReadProperty<bool>("has_validity");
+	if (has_validity) {
+	    validity.Initialize(count);
+	    source.ReadData((data_ptr_t)validity.GetData(), validity.ValidityMaskSize(count));
+	}
+
+	if (TypeIsConstantSize(type.InternalType())) {
+	    // constant size type: read fixed amount of data from
+	    auto column_size = GetTypeIdSize(type.InternalType()) * count;
+	    auto ptr = unique_ptr<data_t[]>(new data_t[column_size]);
+	    source.ReadData(ptr.get(), column_size);
+
+	    VectorOperations::ReadFromStorage(ptr.get(), count, *this);
+	} else {
+	    switch (type.InternalType()) {
+	    case PhysicalType::VARCHAR: {
+	        auto strings = FlatVector::GetData<string_t>(*this);
+	        for (idx_t i = 0; i < count; i++) {
+	            // read the strings
+	            auto str = source.Read<string>();
+	            // now add the string to the StringHeap of the vector
+	            // and write the pointer into the vector
+	            if (validity.RowIsValid(i)) {
+	                strings[i] = StringVector::AddStringOrBlob(*this, str);
+	            }
+	        }
+	        break;
+	    }
+	    case PhysicalType::STRUCT: {
+	        auto &entries = StructVector::GetEntries(*this);
+	        for (auto &entry : entries) {
+	            entry->FormatDeserialize(deserializer, count);
+	        }
+	        break;
+	    }
+	    case PhysicalType::LIST: {
+	        // read the list size
+	        auto list_size = deserializer.ReadProperty<idx_t>("list_size");
+	        ListVector::Reserve(*this, list_size);
+	        ListVector::SetListSize(*this, list_size);
+
+	        // read the list entry
+	        auto list_entries = FlatVector::GetData(*this);
+	        source.ReadData(list_entries, count * sizeof(list_entry_t));
+
+	        // deserialize the child vector
+	        auto &child = ListVector::GetEntry(*this);
+	        child.Deserialize(list_size, source);
+
+	        break;
+	    }
+	    default:
+	        throw InternalException("Unimplemented variable width type for Vector::Deserialize!");
+	    }
+	}
+	 */
+
+	throw NotImplementedException("TODO: Implement deserialization for DuckDB Vectors");
+}
+
 void Vector::Deserialize(idx_t count, Deserializer &source) {
 	auto &type = GetType();