duckdb 0.8.2-dev87.0 → 0.9.0

package/src/duckdb/extension/json/json_serializer.cpp

@@ -19,19 +19,32 @@ void JsonSerializer::PushValue(yyjson_mut_val *val) {
 	}
 }
 
-void JsonSerializer::SetTag(const char *tag) {
+void JsonSerializer::OnPropertyBegin(const field_id_t, const char *tag) {
 	current_tag = yyjson_mut_strcpy(doc, tag);
 }
 
-//===--------------------------------------------------------------------===//
-// Nested types
-//===--------------------------------------------------------------------===//
-void JsonSerializer::OnOptionalBegin(bool present) {
+void JsonSerializer::OnPropertyEnd() {
+}
+
+void JsonSerializer::OnOptionalPropertyBegin(const field_id_t, const char *tag, bool) {
+	current_tag = yyjson_mut_strcpy(doc, tag);
+}
+
+void JsonSerializer::OnOptionalPropertyEnd(bool) {
+}
+
+//-------------------------------------------------------------------------
+// Nested Types
+//-------------------------------------------------------------------------
+void JsonSerializer::OnNullableBegin(bool present) {
 	if (!present && !skip_if_null) {
 		WriteNull();
 	}
 }
 
+void JsonSerializer::OnNullableEnd() {
+}
+
 void JsonSerializer::OnListBegin(idx_t count) {
 	auto new_value = yyjson_mut_arr(doc);
 	// We always push a value to the stack, we just don't add it as a child to the current value
@@ -43,38 +56,7 @@ void JsonSerializer::OnListBegin(idx_t count) {
 	stack.push_back(new_value);
 }
 
-void JsonSerializer::OnListEnd(idx_t count) {
-	stack.pop_back();
-}
-
-// Serialize maps as arrays of objects with "key" and "value" properties.
-void JsonSerializer::OnMapBegin(idx_t count) {
-	auto new_value = yyjson_mut_arr(doc);
-	if (!(count == 0 && skip_if_empty)) {
-		PushValue(new_value);
-	}
-	stack.push_back(new_value);
-}
-
-void JsonSerializer::OnMapEntryBegin() {
-	auto new_value = yyjson_mut_obj(doc);
-	PushValue(new_value);
-	stack.push_back(new_value);
-}
-
-void JsonSerializer::OnMapKeyBegin() {
-	SetTag("key");
-}
-
-void JsonSerializer::OnMapValueBegin() {
-	SetTag("value");
-}
-
-void JsonSerializer::OnMapEntryEnd() {
-	stack.pop_back();
-}
-
-void JsonSerializer::OnMapEnd(idx_t count) {
+void JsonSerializer::OnListEnd() {
 	stack.pop_back();
 }
 
@@ -120,28 +102,13 @@ void JsonSerializer::OnObjectEnd() {
 	}
 }
 
-void JsonSerializer::OnPairBegin() {
-	auto new_value = yyjson_mut_obj(doc);
-	PushValue(new_value);
-	stack.push_back(new_value);
-}
-
-void JsonSerializer::OnPairKeyBegin() {
-	SetTag("key");
-}
-
-void JsonSerializer::OnPairValueBegin() {
-	SetTag("value");
-}
-
-void JsonSerializer::OnPairEnd() {
-	stack.pop_back();
-}
-
-//===--------------------------------------------------------------------===//
-// Primitive types
-//===--------------------------------------------------------------------===//
+//-------------------------------------------------------------------------
+// Primitive Types
+//-------------------------------------------------------------------------
 void JsonSerializer::WriteNull() {
+	if (skip_if_null) {
+		return;
+	}
 	auto val = yyjson_mut_null(doc);
 	PushValue(val);
 }
@@ -172,7 +139,7 @@ void JsonSerializer::WriteValue(uint32_t value) {
 }
 
 void JsonSerializer::WriteValue(int32_t value) {
-	auto val = yyjson_mut_int(doc, value);
+	auto val = yyjson_mut_sint(doc, value);
 	PushValue(val);
 }
 
@@ -190,8 +157,8 @@ void JsonSerializer::WriteValue(hugeint_t value) {
 	auto val = yyjson_mut_obj(doc);
 	PushValue(val);
 	stack.push_back(val);
-	WriteProperty("upper", value.upper);
-	WriteProperty("lower", value.lower);
+	WriteProperty(100, "upper", value.upper);
+	WriteProperty(101, "lower", value.lower);
 	stack.pop_back();
 }
 
@@ -205,16 +172,6 @@ void JsonSerializer::WriteValue(double value) {
 	PushValue(val);
 }
 
-void JsonSerializer::WriteValue(interval_t value) {
-	auto val = yyjson_mut_obj(doc);
-	PushValue(val);
-	stack.push_back(val);
-	WriteProperty("months", value.months);
-	WriteProperty("days", value.days);
-	WriteProperty("micros", value.micros);
-	stack.pop_back();
-}
-
 void JsonSerializer::WriteValue(const string &value) {
 	if (skip_if_empty && value.empty()) {
 		return;
@@ -245,7 +202,7 @@ void JsonSerializer::WriteValue(bool value) {
 }
 
 void JsonSerializer::WriteDataPtr(const_data_ptr_t ptr, idx_t count) {
-	auto blob = Blob::ToBlob(string_t((const char *)ptr, count));
+	auto blob = Blob::ToBlob(string_t(const_char_ptr_cast(ptr), count));
 	auto val = yyjson_mut_strcpy(doc, blob.c_str());
 	PushValue(val);
 }

package/src/duckdb/extension/json/serialize_json.cpp

@@ -0,0 +1,92 @@
+//===----------------------------------------------------------------------===//
+// This file is automatically generated by scripts/generate_serialization.py
+// Do not edit this file manually, your changes will be overwritten
+//===----------------------------------------------------------------------===//
+
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
+#include "buffered_json_reader.hpp"
+#include "json_transform.hpp"
+#include "json_scan.hpp"
+
+namespace duckdb {
+
+void BufferedJSONReaderOptions::Serialize(Serializer &serializer) const {
+	serializer.WriteProperty<JSONFormat>(100, "format", format);
+	serializer.WriteProperty<JSONRecordType>(101, "record_type", record_type);
+	serializer.WriteProperty<FileCompressionType>(102, "compression", compression);
+	serializer.WriteProperty<MultiFileReaderOptions>(103, "file_options", file_options);
+}
+
+BufferedJSONReaderOptions BufferedJSONReaderOptions::Deserialize(Deserializer &deserializer) {
+	BufferedJSONReaderOptions result;
+	deserializer.ReadProperty<JSONFormat>(100, "format", result.format);
+	deserializer.ReadProperty<JSONRecordType>(101, "record_type", result.record_type);
+	deserializer.ReadProperty<FileCompressionType>(102, "compression", result.compression);
+	deserializer.ReadProperty<MultiFileReaderOptions>(103, "file_options", result.file_options);
+	return result;
+}
+
+void JSONScanData::Serialize(Serializer &serializer) const {
+	serializer.WriteProperty<JSONScanType>(100, "json_type", type);
+	serializer.WriteProperty<BufferedJSONReaderOptions>(101, "options", options);
+	serializer.WriteProperty<MultiFileReaderBindData>(102, "reader_bind", reader_bind);
+	serializer.WritePropertyWithDefault<vector<string>>(103, "files", files);
+	serializer.WritePropertyWithDefault<bool>(104, "ignore_errors", ignore_errors);
+	serializer.WritePropertyWithDefault<idx_t>(105, "maximum_object_size", maximum_object_size);
+	serializer.WritePropertyWithDefault<bool>(106, "auto_detect", auto_detect);
+	serializer.WritePropertyWithDefault<idx_t>(107, "sample_size", sample_size);
+	serializer.WritePropertyWithDefault<idx_t>(108, "max_depth", max_depth);
+	serializer.WriteProperty<JSONTransformOptions>(109, "transform_options", transform_options);
+	serializer.WritePropertyWithDefault<vector<string>>(110, "names", names);
+	serializer.WritePropertyWithDefault<string>(111, "date_format", GetDateFormat());
+	serializer.WritePropertyWithDefault<string>(112, "timestamp_format", GetTimestampFormat());
+}
+
+unique_ptr<JSONScanData> JSONScanData::Deserialize(Deserializer &deserializer) {
+	auto type = deserializer.ReadProperty<JSONScanType>(100, "json_type");
+	auto options = deserializer.ReadProperty<BufferedJSONReaderOptions>(101, "options");
+	auto reader_bind = deserializer.ReadProperty<MultiFileReaderBindData>(102, "reader_bind");
+	auto files = deserializer.ReadPropertyWithDefault<vector<string>>(103, "files");
+	auto ignore_errors = deserializer.ReadPropertyWithDefault<bool>(104, "ignore_errors");
+	auto maximum_object_size = deserializer.ReadPropertyWithDefault<idx_t>(105, "maximum_object_size");
+	auto auto_detect = deserializer.ReadPropertyWithDefault<bool>(106, "auto_detect");
+	auto sample_size = deserializer.ReadPropertyWithDefault<idx_t>(107, "sample_size");
+	auto max_depth = deserializer.ReadPropertyWithDefault<idx_t>(108, "max_depth");
+	auto transform_options = deserializer.ReadProperty<JSONTransformOptions>(109, "transform_options");
+	auto names = deserializer.ReadPropertyWithDefault<vector<string>>(110, "names");
+	auto date_format = deserializer.ReadPropertyWithDefault<string>(111, "date_format");
+	auto timestamp_format = deserializer.ReadPropertyWithDefault<string>(112, "timestamp_format");
+	auto result = duckdb::unique_ptr<JSONScanData>(new JSONScanData(deserializer.Get<ClientContext &>(), std::move(files), std::move(date_format), std::move(timestamp_format)));
+	result->type = type;
+	result->options = options;
+	result->reader_bind = reader_bind;
+	result->ignore_errors = ignore_errors;
+	result->maximum_object_size = maximum_object_size;
+	result->auto_detect = auto_detect;
+	result->sample_size = sample_size;
+	result->max_depth = max_depth;
+	result->transform_options = transform_options;
+	result->names = std::move(names);
+	return result;
+}
+
+void JSONTransformOptions::Serialize(Serializer &serializer) const {
+	serializer.WritePropertyWithDefault<bool>(100, "strict_cast", strict_cast);
+	serializer.WritePropertyWithDefault<bool>(101, "error_duplicate_key", error_duplicate_key);
+	serializer.WritePropertyWithDefault<bool>(102, "error_missing_key", error_missing_key);
+	serializer.WritePropertyWithDefault<bool>(103, "error_unknown_key", error_unknown_key);
+	serializer.WritePropertyWithDefault<bool>(104, "delay_error", delay_error);
+}
+
+JSONTransformOptions JSONTransformOptions::Deserialize(Deserializer &deserializer) {
+	JSONTransformOptions result;
+	deserializer.ReadPropertyWithDefault<bool>(100, "strict_cast", result.strict_cast);
+	deserializer.ReadPropertyWithDefault<bool>(101, "error_duplicate_key", result.error_duplicate_key);
+	deserializer.ReadPropertyWithDefault<bool>(102, "error_missing_key", result.error_missing_key);
+	deserializer.ReadPropertyWithDefault<bool>(103, "error_unknown_key", result.error_unknown_key);
+	deserializer.ReadPropertyWithDefault<bool>(104, "delay_error", result.delay_error);
+	return result;
+}
+
+} // namespace duckdb

package/src/duckdb/extension/parquet/column_reader.cpp

@@ -1,27 +1,25 @@
 #include "column_reader.hpp"
-#include "parquet_timestamp.hpp"
-#include "utf8proc_wrapper.hpp"
-#include "parquet_reader.hpp"
 
 #include "boolean_column_reader.hpp"
-#include "cast_column_reader.hpp"
-#include "row_number_column_reader.hpp"
 #include "callback_column_reader.hpp"
-#include "parquet_decimal_utils.hpp"
+#include "cast_column_reader.hpp"
+#include "duckdb.hpp"
 #include "list_column_reader.hpp"
+#include "miniz_wrapper.hpp"
+#include "parquet_decimal_utils.hpp"
+#include "parquet_reader.hpp"
+#include "parquet_timestamp.hpp"
+#include "row_number_column_reader.hpp"
+#include "snappy.h"
 #include "string_column_reader.hpp"
 #include "struct_column_reader.hpp"
 #include "templated_column_reader.hpp"
-
-#include "snappy.h"
-#include "miniz_wrapper.hpp"
+#include "utf8proc_wrapper.hpp"
 #include "zstd.h"
-#include <iostream>
 
-#include "duckdb.hpp"
 #ifndef DUCKDB_AMALGAMATION
-#include "duckdb/common/types/blob.hpp"
 #include "duckdb/common/types/bit.hpp"
+#include "duckdb/common/types/blob.hpp"
 #include "duckdb/common/types/chunk_collection.hpp"
 #endif
 
@@ -509,13 +507,13 @@ idx_t ColumnReader::Read(uint64_t num_values, parquet_filter_t &filter, data_ptr
 			// TODO keep this in the state
 			auto read_buf = make_shared<ResizeableBuffer>();
 
-			switch (type.InternalType()) {
-			case PhysicalType::INT32:
+			switch (schema.type) {
+			case duckdb_parquet::format::Type::INT32:
 				read_buf->resize(reader.allocator, sizeof(int32_t) * (read_now - null_count));
 				dbp_decoder->GetBatch<int32_t>(read_buf->ptr, read_now - null_count);
 
 				break;
-			case PhysicalType::INT64:
+			case duckdb_parquet::format::Type::INT64:
 				read_buf->resize(reader.allocator, sizeof(int64_t) * (read_now - null_count));
 				dbp_decoder->GetBatch<int64_t>(read_buf->ptr, read_now - null_count);
 				break;
@@ -614,7 +612,7 @@ uint32_t StringColumnReader::VerifyString(const char *str_data, uint32_t str_len
 
 void StringColumnReader::Dictionary(shared_ptr<ResizeableBuffer> data, idx_t num_entries) {
 	dict = std::move(data);
-	dict_strings = duckdb::unique_ptr<string_t[]>(new string_t[num_entries]);
+	dict_strings = unique_ptr<string_t[]>(new string_t[num_entries]);
 	for (idx_t dict_idx = 0; dict_idx < num_entries; dict_idx++) {
 		uint32_t str_len;
 		if (fixed_width_string_length == 0) {
@@ -873,7 +871,7 @@ idx_t ListColumnReader::Read(uint64_t num_values, parquet_filter_t &filter, data
 
 ListColumnReader::ListColumnReader(ParquetReader &reader, LogicalType type_p, const SchemaElement &schema_p,
                                    idx_t schema_idx_p, idx_t max_define_p, idx_t max_repeat_p,
-                                   duckdb::unique_ptr<ColumnReader> child_column_reader_p)
+                                   unique_ptr<ColumnReader> child_column_reader_p)
     : ColumnReader(reader, std::move(type_p), schema_p, schema_idx_p, max_define_p, max_repeat_p),
       child_column_reader(std::move(child_column_reader_p)),
       read_cache(reader.allocator, ListType::GetChildType(Type())), read_vector(read_cache), overflow_child_count(0) {
@@ -889,8 +887,8 @@ ListColumnReader::ListColumnReader(ParquetReader &reader, LogicalType type_p, co
 void ListColumnReader::ApplyPendingSkips(idx_t num_values) {
 	pending_skips -= num_values;
 
-	auto define_out = duckdb::unique_ptr<uint8_t[]>(new uint8_t[num_values]);
-	auto repeat_out = duckdb::unique_ptr<uint8_t[]>(new uint8_t[num_values]);
+	auto define_out = unique_ptr<uint8_t[]>(new uint8_t[num_values]);
+	auto repeat_out = unique_ptr<uint8_t[]>(new uint8_t[num_values]);
 
 	idx_t remaining = num_values;
 	idx_t read = 0;
@@ -953,7 +951,7 @@ idx_t RowNumberColumnReader::Read(uint64_t num_values, parquet_filter_t &filter,
 //===--------------------------------------------------------------------===//
 // Cast Column Reader
 //===--------------------------------------------------------------------===//
-CastColumnReader::CastColumnReader(duckdb::unique_ptr<ColumnReader> child_reader_p, LogicalType target_type_p)
+CastColumnReader::CastColumnReader(unique_ptr<ColumnReader> child_reader_p, LogicalType target_type_p)
     : ColumnReader(child_reader_p->Reader(), std::move(target_type_p), child_reader_p->Schema(),
                    child_reader_p->FileIdx(), child_reader_p->MaxDefine(), child_reader_p->MaxRepeat()),
       child_reader(std::move(child_reader_p)) {
@@ -1005,7 +1003,7 @@ idx_t CastColumnReader::GroupRowsAvailable() {
 //===--------------------------------------------------------------------===//
 StructColumnReader::StructColumnReader(ParquetReader &reader, LogicalType type_p, const SchemaElement &schema_p,
                                        idx_t schema_idx_p, idx_t max_define_p, idx_t max_repeat_p,
-                                       vector<duckdb::unique_ptr<ColumnReader>> child_readers_p)
+                                       vector<unique_ptr<ColumnReader>> child_readers_p)
     : ColumnReader(reader, std::move(type_p), schema_p, schema_idx_p, max_define_p, max_repeat_p),
       child_readers(std::move(child_readers_p)) {
 	D_ASSERT(type.InternalType() == PhysicalType::STRUCT);
@@ -1155,9 +1153,9 @@ protected:
 };
 
 template <bool FIXED_LENGTH>
-static duckdb::unique_ptr<ColumnReader> CreateDecimalReaderInternal(ParquetReader &reader, const LogicalType &type_p,
-                                                                    const SchemaElement &schema_p, idx_t file_idx_p,
-                                                                    idx_t max_define, idx_t max_repeat) {
+static unique_ptr<ColumnReader> CreateDecimalReaderInternal(ParquetReader &reader, const LogicalType &type_p,
+                                                            const SchemaElement &schema_p, idx_t file_idx_p,
+                                                            idx_t max_define, idx_t max_repeat) {
 	switch (type_p.InternalType()) {
 	case PhysicalType::INT16:
 		return make_uniq<DecimalColumnReader<int16_t, FIXED_LENGTH>>(reader, type_p, schema_p, file_idx_p, max_define,
@@ -1391,7 +1389,6 @@ unique_ptr<ColumnReader> ColumnReader::CreateReader(ParquetReader &reader, const
 		return make_uniq<CallbackColumnReader<int32_t, date_t, ParquetIntToDate>>(reader, type_p, schema_p, file_idx_p,
 		                                                                          max_define, max_repeat);
 	case LogicalTypeId::TIME:
-	case LogicalTypeId::TIME_TZ:
 		if (schema_p.__isset.logicalType && schema_p.logicalType.__isset.TIME) {
 			if (schema_p.logicalType.TIME.unit.__isset.MILLIS) {
 				return make_uniq<CallbackColumnReader<int32_t, dtime_t, ParquetIntToTimeMs>>(
@@ -1415,6 +1412,21 @@ unique_ptr<ColumnReader> ColumnReader::CreateReader(ParquetReader &reader, const
 				break;
 			}
 		}
+	case LogicalTypeId::TIME_TZ:
+		if (schema_p.__isset.logicalType && schema_p.logicalType.__isset.TIME) {
+			if (schema_p.logicalType.TIME.unit.__isset.MICROS) {
+				return make_uniq<CallbackColumnReader<int64_t, dtime_tz_t, ParquetIntToTimeTZ>>(
+				    reader, type_p, schema_p, file_idx_p, max_define, max_repeat);
+			}
+		} else if (schema_p.__isset.converted_type) {
+			switch (schema_p.converted_type) {
+			case ConvertedType::TIME_MICROS:
+				return make_uniq<CallbackColumnReader<int64_t, dtime_tz_t, ParquetIntToTimeTZ>>(
+				    reader, type_p, schema_p, file_idx_p, max_define, max_repeat);
+			default:
+				break;
+			}
+		}
 	case LogicalTypeId::BLOB:
 	case LogicalTypeId::VARCHAR:
 		return make_uniq<StringColumnReader>(reader, type_p, schema_p, file_idx_p, max_define, max_repeat);