duckdb 0.6.2-dev971.0 → 0.7.1-dev2.0

package/src/duckdb/extension/parquet/include/callback_column_reader.hpp

@@ -28,7 +28,7 @@ public:
 	                     idx_t max_define_p, idx_t max_repeat_p)
 	    : TemplatedColumnReader<DUCKDB_PHYSICAL_TYPE,
 	                            CallbackParquetValueConversion<PARQUET_PHYSICAL_TYPE, DUCKDB_PHYSICAL_TYPE, FUNC>>(
-	          reader, move(type_p), schema_p, file_idx_p, max_define_p, max_repeat_p) {
+	          reader, std::move(type_p), schema_p, file_idx_p, max_define_p, max_repeat_p) {
 	}
 
 protected:

package/src/duckdb/extension/parquet/include/column_reader.hpp

@@ -134,6 +134,8 @@ protected:
 
 	idx_t pending_skips = 0;
 
+	virtual void ResetPage();
+
 private:
 	void AllocateBlock(idx_t size);
 	void AllocateCompressed(idx_t size);

package/src/duckdb/extension/parquet/include/generated_column_reader.hpp

@@ -35,7 +35,7 @@ public:
 			string string = constant.ToString();
 			string_stats->Update(string);
 			string_stats->max_string_length = string.length();
-			return move(string_stats);
+			return std::move(string_stats);
 		}
 		default:
 			return nullptr;

package/src/duckdb/extension/parquet/include/parquet_reader.hpp

@@ -73,6 +73,7 @@ struct ParquetOptions {
 	bool filename = false;
 	bool file_row_number = false;
 	bool hive_partitioning = false;
+	bool union_by_name = false;
 
 public:
 	void Serialize(FieldWriter &writer) const;
@@ -84,19 +85,19 @@ public:
 	ParquetReader(Allocator &allocator, unique_ptr<FileHandle> file_handle_p,
 	              const vector<LogicalType> &expected_types_p, const string &initial_filename_p = string());
 	ParquetReader(Allocator &allocator, unique_ptr<FileHandle> file_handle_p)
-	    : ParquetReader(allocator, move(file_handle_p), vector<LogicalType>(), string()) {
+	    : ParquetReader(allocator, std::move(file_handle_p), vector<LogicalType>(), string()) {
 	}
 
 	ParquetReader(ClientContext &context, string file_name, const vector<string> &names,
 	              const vector<LogicalType> &expected_types_p, const vector<column_t> &column_ids,
 	              ParquetOptions parquet_options, const string &initial_filename = string());
 	ParquetReader(ClientContext &context, string file_name, ParquetOptions parquet_options)
-	    : ParquetReader(context, move(file_name), vector<string>(), vector<LogicalType>(), vector<column_t>(),
+	    : ParquetReader(context, std::move(file_name), vector<string>(), vector<LogicalType>(), vector<column_t>(),
 	                    parquet_options, string()) {
 	}
 	ParquetReader(ClientContext &context, string file_name, const vector<LogicalType> &expected_types_p,
 	              ParquetOptions parquet_options)
-	    : ParquetReader(context, move(file_name), vector<string>(), expected_types_p, vector<column_t>(),
+	    : ParquetReader(context, std::move(file_name), vector<string>(), expected_types_p, vector<column_t>(),
 	                    parquet_options, string()) {
 	}
 	~ParquetReader();
@@ -108,6 +109,19 @@ public:
 	vector<string> names;
 	shared_ptr<ParquetFileMetadataCache> metadata;
 	ParquetOptions parquet_options;
+	//! maps hive partition names to string columns
+	unique_ptr<std::map<string, string>> hive_map;
+
+	//! when reading multiple parquet files (with union by name option)
+	//! TableFunction might return more cols than any single parquet file. Even all parquet files have same
+	//! cols, those files might have cols at different positions and with different logical type.
+	//! e.g. p1.parquet (a INT , b VARCHAR) p2.parquet (c VARCHAR, a VARCHAR)
+	vector<idx_t> union_idx_map;
+	//! If the parquet file dont have union_cols5  union_null_cols[5] will be true.
+	//! some parquet files may not have all union cols.
+	vector<bool> union_null_cols;
+	//! All union cols will cast to same type.
+	vector<LogicalType> union_col_types;
 
 public:
 	void InitializeScan(ParquetReaderScanState &state, vector<column_t> column_ids, vector<idx_t> groups_to_read,
@@ -139,6 +153,7 @@ private:
 	uint64_t GetGroupSpan(ParquetReaderScanState &state);
 	void PrepareRowGroupBuffer(ParquetReaderScanState &state, idx_t out_col_idx);
 	LogicalType DeriveLogicalType(const SchemaElement &s_ele);
+	void RearrangeChildReaders(unique_ptr<duckdb::ColumnReader> &root_reader, vector<column_t> &column_ids);
 
 	template <typename... Args>
 	std::runtime_error FormatException(const string fmt_str, Args... params) {

package/src/duckdb/extension/parquet/include/parquet_rle_bp_decoder.hpp

@@ -67,7 +67,7 @@ public:
 			return 0;
 		}
 		uint8_t ret = 1;
-		while (((idx_t)(1 << ret) - 1) < val) {
+		while (((idx_t)(1u << ret) - 1) < val) {
 			ret++;
 		}
 		return ret;

package/src/duckdb/extension/parquet/include/templated_column_reader.hpp

@@ -33,7 +33,7 @@ class TemplatedColumnReader : public ColumnReader {
 public:
 	TemplatedColumnReader(ParquetReader &reader, LogicalType type_p, const SchemaElement &schema_p, idx_t schema_idx_p,
 	                      idx_t max_define_p, idx_t max_repeat_p)
-	    : ColumnReader(reader, move(type_p), schema_p, schema_idx_p, max_define_p, max_repeat_p) {};
+	    : ColumnReader(reader, std::move(type_p), schema_p, schema_idx_p, max_define_p, max_repeat_p) {};
 
 	shared_ptr<ResizeableBuffer> dict;
 
@@ -47,7 +47,7 @@ public:
 	}
 
 	void Dictionary(shared_ptr<ResizeableBuffer> data, idx_t num_entries) override {
-		dict = move(data);
+		dict = std::move(data);
 	}
 
 	void Offsets(uint32_t *offsets, uint8_t *defines, uint64_t num_values, parquet_filter_t &filter,
@@ -72,7 +72,7 @@ public:
 
 	void Plain(shared_ptr<ByteBuffer> plain_data, uint8_t *defines, uint64_t num_values, parquet_filter_t &filter,
 	           idx_t result_offset, Vector &result) override {
-		PlainTemplated<VALUE_TYPE, VALUE_CONVERSION>(move(plain_data), defines, num_values, filter, result_offset,
+		PlainTemplated<VALUE_TYPE, VALUE_CONVERSION>(std::move(plain_data), defines, num_values, filter, result_offset,
 		                                             result);
 	}
 };

package/src/duckdb/extension/parquet/parquet-extension.cpp

@@ -20,6 +20,7 @@
 #include "duckdb/common/field_writer.hpp"
 #include "duckdb/common/file_system.hpp"
 #include "duckdb/common/hive_partitioning.hpp"
+#include "duckdb/common/union_by_name.hpp"
 #include "duckdb/common/types/chunk_collection.hpp"
 #include "duckdb/function/copy_function.hpp"
 #include "duckdb/function/table_function.hpp"
@@ -32,6 +33,7 @@
 #include "duckdb/parser/tableref/table_function_ref.hpp"
 #include "duckdb/planner/operator/logical_get.hpp"
 #include "duckdb/storage/statistics/base_statistics.hpp"
+#include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp"
 #endif
 
 namespace duckdb {
@@ -45,6 +47,10 @@ struct ParquetReadBindData : public TableFunctionData {
 	vector<string> names;
 	vector<LogicalType> types;
 
+	// The union readers are created (when parquet union_by_name option is on) during binding
+	// Those readers can be re-used during ParquetParallelStateNext
+	vector<shared_ptr<ParquetReader>> union_readers;
+
 	// These come from the initial_reader, but need to be stored in case the initial_reader is removed by a filter
 	idx_t initial_file_cardinality;
 	idx_t initial_file_row_groups;
@@ -127,6 +133,7 @@ void ParquetOptions::Serialize(FieldWriter &writer) const {
 	writer.WriteField<bool>(filename);
 	writer.WriteField<bool>(file_row_number);
 	writer.WriteField<bool>(hive_partitioning);
+	writer.WriteField<bool>(union_by_name);
 }
 
 void ParquetOptions::Deserialize(FieldReader &reader) {
@@ -134,6 +141,7 @@ void ParquetOptions::Deserialize(FieldReader &reader) {
 	filename = reader.ReadRequired<bool>();
 	file_row_number = reader.ReadRequired<bool>();
 	hive_partitioning = reader.ReadRequired<bool>();
+	union_by_name = reader.ReadRequired<bool>();
 }
 
 BindInfo ParquetGetBatchInfo(const FunctionData *bind_data) {
@@ -148,6 +156,7 @@ BindInfo ParquetGetBatchInfo(const FunctionData *bind_data) {
 	bind_info.InsertOption("filename", Value::BOOLEAN(parquet_bind->parquet_options.filename));
 	bind_info.InsertOption("file_row_number", Value::BOOLEAN(parquet_bind->parquet_options.file_row_number));
 	bind_info.InsertOption("hive_partitioning", Value::BOOLEAN(parquet_bind->parquet_options.hive_partitioning));
+	bind_info.InsertOption("union_by_name", Value::BOOLEAN(parquet_bind->parquet_options.union_by_name));
 	return bind_info;
 }
 
@@ -164,6 +173,8 @@ public:
 		table_function.named_parameters["filename"] = LogicalType::BOOLEAN;
 		table_function.named_parameters["file_row_number"] = LogicalType::BOOLEAN;
 		table_function.named_parameters["hive_partitioning"] = LogicalType::BOOLEAN;
+		table_function.named_parameters["union_by_name"] = LogicalType::BOOLEAN;
+		table_function.named_parameters["compression"] = LogicalType::VARCHAR;
 		table_function.get_batch_index = ParquetScanGetBatchIndex;
 		table_function.serialize = ParquetScanSerialize;
 		table_function.deserialize = ParquetScanDeserialize;
@@ -180,6 +191,7 @@ public:
 		table_function.named_parameters["filename"] = LogicalType::BOOLEAN;
 		table_function.named_parameters["file_row_number"] = LogicalType::BOOLEAN;
 		table_function.named_parameters["hive_partitioning"] = LogicalType::BOOLEAN;
+		table_function.named_parameters["union_by_name"] = LogicalType::BOOLEAN;
 		set.AddFunction(table_function);
 		return set;
 	}
@@ -201,22 +213,31 @@ public:
 				parquet_options.file_row_number = true;
 			} else if (loption == "hive_partitioning") {
 				parquet_options.hive_partitioning = true;
+			} else if (loption == "union_by_name") {
+				parquet_options.union_by_name = true;
 			} else {
 				throw NotImplementedException("Unsupported option for COPY FROM parquet: %s", option.first);
 			}
 		}
-		auto result = make_unique<ParquetReadBindData>();
 
 		FileSystem &fs = FileSystem::GetFileSystem(context);
-		result->files = fs.Glob(info.file_path, context);
-		if (result->files.empty()) {
+		auto files = fs.Glob(info.file_path, context);
+		if (files.empty()) {
 			throw IOException("No files found that match the pattern \"%s\"", info.file_path);
 		}
-		result->SetInitialReader(
-		    make_shared<ParquetReader>(context, result->files[0], expected_types, parquet_options));
-		result->names = result->initial_reader->names;
-		result->types = result->initial_reader->return_types;
-		return move(result);
+
+		// The most likely path (Parquet read without union by name option)
+		if (!parquet_options.union_by_name) {
+			auto result = make_unique<ParquetReadBindData>();
+			result->files = std::move(files);
+			result->SetInitialReader(
+			    make_shared<ParquetReader>(context, result->files[0], expected_types, parquet_options));
+			result->names = result->initial_reader->names;
+			result->types = result->initial_reader->return_types;
+			return std::move(result);
+		} else {
+			return ParquetUnionNamesBind(context, files, expected_types, expected_names, parquet_options);
+		}
 	}
 
 	static unique_ptr<BaseStatistics> ParquetScanStats(ClientContext &context, const FunctionData *bind_data_p,
@@ -303,12 +324,41 @@ public:
 	                                                        vector<LogicalType> &return_types, vector<string> &names,
 	                                                        ParquetOptions parquet_options) {
 		auto result = make_unique<ParquetReadBindData>();
-		result->files = move(files);
 
-		result->SetInitialReader(make_shared<ParquetReader>(context, result->files[0], parquet_options));
-		return_types = result->types = result->initial_reader->return_types;
-		names = result->names = result->initial_reader->names;
-		return move(result);
+		// The most likely path (Parquet Scan without union by name option)
+		if (!parquet_options.union_by_name) {
+			result->files = std::move(files);
+			result->SetInitialReader(make_shared<ParquetReader>(context, result->files[0], parquet_options));
+			return_types = result->types = result->initial_reader->return_types;
+			names = result->names = result->initial_reader->names;
+			return std::move(result);
+		} else {
+			return ParquetUnionNamesBind(context, files, return_types, names, parquet_options);
+		}
+	}
+
+	static unique_ptr<FunctionData> ParquetUnionNamesBind(ClientContext &context, vector<string> files,
+	                                                      vector<LogicalType> &return_types, vector<string> &names,
+	                                                      ParquetOptions parquet_options) {
+		auto result = make_unique<ParquetReadBindData>();
+		result->files = std::move(files);
+
+		case_insensitive_map_t<idx_t> union_names_map;
+		vector<string> union_col_names;
+		vector<LogicalType> union_col_types;
+		auto dummy_readers = UnionByName<ParquetReader, ParquetOptions>::UnionCols(
+		    context, result->files, union_col_types, union_col_names, union_names_map, parquet_options);
+
+		dummy_readers = UnionByName<ParquetReader, ParquetOptions>::CreateUnionMap(
+		    std::move(dummy_readers), union_col_types, union_col_names, union_names_map);
+
+		std::move(dummy_readers.begin(), dummy_readers.end(), std::back_inserter(result->union_readers));
+		names.assign(union_col_names.begin(), union_col_names.end());
+		return_types.assign(union_col_types.begin(), union_col_types.end());
+		result->SetInitialReader(result->union_readers[0]);
+		D_ASSERT(names.size() == return_types.size());
+
+		return std::move(result);
 	}
 
 	static vector<string> ParquetGlob(FileSystem &fs, const string &glob, ClientContext &context) {
@@ -337,11 +387,13 @@ public:
 				parquet_options.file_row_number = BooleanValue::Get(kv.second);
 			} else if (loption == "hive_partitioning") {
 				parquet_options.hive_partitioning = BooleanValue::Get(kv.second);
+			} else if (loption == "union_by_name") {
+				parquet_options.union_by_name = BooleanValue::Get(kv.second);
 			}
 		}
 		FileSystem &fs = FileSystem::GetFileSystem(context);
 		auto files = ParquetGlob(fs, file_name, context);
-		return ParquetScanBindInternal(context, move(files), return_types, names, parquet_options);
+		return ParquetScanBindInternal(context, std::move(files), return_types, names, parquet_options);
 	}
 
 	static unique_ptr<FunctionData> ParquetScanBindList(ClientContext &context, TableFunctionBindInput &input,
@@ -370,9 +422,11 @@ public:
 				parquet_options.file_row_number = BooleanValue::Get(kv.second);
 			} else if (loption == "hive_partitioning") {
 				parquet_options.hive_partitioning = BooleanValue::Get(kv.second);
+			} else if (loption == "union_by_name") {
+				parquet_options.union_by_name = true;
 			}
 		}
-		return ParquetScanBindInternal(context, move(files), return_types, names, parquet_options);
+		return ParquetScanBindInternal(context, std::move(files), return_types, names, parquet_options);
 	}
 
 	static double ParquetProgress(ClientContext &context, const FunctionData *bind_data_p,
@@ -406,7 +460,7 @@ public:
 		if (!ParquetParallelStateNext(context.client, bind_data, *result, gstate)) {
 			return nullptr;
 		}
-		return move(result);
+		return std::move(result);
 	}
 
 	static unique_ptr<GlobalTableFunctionState> ParquetScanInitGlobal(ClientContext &context,
@@ -417,20 +471,24 @@ public:
 
 		result->file_opening = std::vector<bool>(bind_data.files.size(), false);
 		result->file_mutexes = std::unique_ptr<mutex[]>(new mutex[bind_data.files.size()]);
-		result->readers = std::vector<shared_ptr<ParquetReader>>(bind_data.files.size(), nullptr);
-
-		if (bind_data.initial_reader) {
-			result->initial_reader = bind_data.initial_reader;
-			result->readers[0] = bind_data.initial_reader;
-		} else {
-			if (bind_data.files.empty()) {
-				result->initial_reader = nullptr;
+		if (!bind_data.parquet_options.union_by_name) {
+			result->readers = std::vector<shared_ptr<ParquetReader>>(bind_data.files.size(), nullptr);
+			if (bind_data.initial_reader) {
+				result->initial_reader = bind_data.initial_reader;
+				result->readers[0] = bind_data.initial_reader;
 			} else {
-				result->initial_reader =
-				    make_shared<ParquetReader>(context, bind_data.files[0], bind_data.names, bind_data.types,
-				                               input.column_ids, bind_data.parquet_options, bind_data.files[0]);
-				result->readers[0] = result->initial_reader;
+				if (bind_data.files.empty()) {
+					result->initial_reader = nullptr;
+				} else {
+					result->initial_reader =
+					    make_shared<ParquetReader>(context, bind_data.files[0], bind_data.names, bind_data.types,
+					                               input.column_ids, bind_data.parquet_options, bind_data.files[0]);
+					result->readers[0] = result->initial_reader;
+				}
 			}
+		} else {
+			result->readers = std::move(bind_data.union_readers);
+			result->initial_reader = result->readers[0];
 		}
 
 		result->row_group_index = 0;
@@ -448,7 +506,7 @@ public:
 				}
 			}
 		}
-		return move(result);
+		return std::move(result);
 	}
 
 	static idx_t ParquetScanGetBatchIndex(ClientContext &context, const FunctionData *bind_data_p,
@@ -497,6 +555,9 @@ public:
 
 			bind_data.chunk_count++;
 			if (output.size() > 0) {
+				if (bind_data.parquet_options.union_by_name) {
+					UnionByName<ParquetReader, ParquetOptions>::SetNullUnionCols(output, data.reader->union_null_cols);
+				}
 				return;
 			}
 			if (!ParquetParallelStateNext(context, bind_data, data, gstate)) {
@@ -533,6 +594,12 @@ public:
 			D_ASSERT(parallel_state.initial_reader);
 
 			if (parallel_state.readers[parallel_state.file_index]) {
+				const auto &current_reader = parallel_state.readers[parallel_state.file_index];
+				if (current_reader->union_null_cols.empty()) {
+					current_reader->union_null_cols.resize(current_reader->return_types.size());
+					std::fill(current_reader->union_null_cols.begin(), current_reader->union_null_cols.end(), false);
+				}
+
 				if (parallel_state.row_group_index <
 				    parallel_state.readers[parallel_state.file_index]->NumRowGroups()) {
 					// The current reader has rowgroups left to be scanned
@@ -683,7 +750,7 @@ unique_ptr<FunctionData> ParquetWriteBind(ClientContext &context, CopyInfo &info
 	}
 	bind_data->sql_types = sql_types;
 	bind_data->column_names = names;
-	return move(bind_data);
+	return std::move(bind_data);
 }
 
 unique_ptr<GlobalFunctionData> ParquetWriteInitializeGlobal(ClientContext &context, FunctionData &bind_data,
@@ -695,7 +762,7 @@ unique_ptr<GlobalFunctionData> ParquetWriteInitializeGlobal(ClientContext &conte
 	global_state->writer =
 	    make_unique<ParquetWriter>(fs, file_path, FileSystem::GetFileOpener(context), parquet_bind.sql_types,
 	                               parquet_bind.column_names, parquet_bind.codec);
-	return move(global_state);
+	return std::move(global_state);
 }
 
 void ParquetWriteSink(ExecutionContext &context, FunctionData &bind_data_p, GlobalFunctionData &gstate,
@@ -744,8 +811,8 @@ bool ParquetWriteIsParallel(ClientContext &context, FunctionData &bind_data) {
 	return true;
 }
 
-unique_ptr<TableFunctionRef> ParquetScanReplacement(ClientContext &context, const string &table_name,
-                                                    ReplacementScanData *data) {
+unique_ptr<TableRef> ParquetScanReplacement(ClientContext &context, const string &table_name,
+                                            ReplacementScanData *data) {
 	auto lower_name = StringUtil::Lower(table_name);
 	if (!StringUtil::EndsWith(lower_name, ".parquet") && !StringUtil::Contains(lower_name, ".parquet?")) {
 		return nullptr;
@@ -753,8 +820,8 @@ unique_ptr<TableFunctionRef> ParquetScanReplacement(ClientContext &context, cons
 	auto table_function = make_unique<TableFunctionRef>();
 	vector<unique_ptr<ParsedExpression>> children;
 	children.push_back(make_unique<ConstantExpression>(Value(table_name)));
-	table_function->function = make_unique<FunctionExpression>("parquet_scan", move(children));
-	return table_function;
+	table_function->function = make_unique<FunctionExpression>("parquet_scan", std::move(children));
+	return std::move(table_function);
 }
 
 void ParquetExtension::Load(DuckDB &db) {

package/src/duckdb/extension/parquet/parquet_metadata.cpp

@@ -423,7 +423,7 @@ unique_ptr<FunctionData> ParquetMetaDataBind(ClientContext &context, TableFuncti
 	if (result->files.empty()) {
 		throw IOException("No files found that match the pattern \"%s\"", file_name);
 	}
-	return move(result);
+	return std::move(result);
 }
 
 template <bool SCHEMA>
@@ -438,7 +438,7 @@ unique_ptr<GlobalTableFunctionState> ParquetMetaDataInit(ClientContext &context,
 		result->LoadFileMetaData(context, bind_data.return_types, bind_data.files[0]);
 	}
 	result->file_index = 0;
-	return move(result);
+	return std::move(result);
 }
 
 template <bool SCHEMA>