duckdb 0.8.2-dev161.0 → 0.8.2-dev1764.0

package/src/duckdb/src/function/scalar/string/like.cpp

@@ -504,8 +504,7 @@ static void RegularLikeFunction(DataChunk &input, ExpressionState &state, Vector
 }
 void LikeFun::RegisterFunction(BuiltinFunctions &set) {
 	// like
-	set.AddFunction(ScalarFunction("~~", {LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::BOOLEAN,
-	                               RegularLikeFunction<LikeOperator, false>, LikeBindFunction));
+	set.AddFunction(GetLikeFunction());
 	// not like
 	set.AddFunction(ScalarFunction("!~~", {LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::BOOLEAN,
 	                               RegularLikeFunction<NotLikeOperator, true>, LikeBindFunction));
@@ -522,9 +521,13 @@ void LikeFun::RegisterFunction(BuiltinFunctions &set) {
 	                               nullptr, ILikePropagateStats<NotILikeOperatorASCII>));
 }
 
+ScalarFunction LikeFun::GetLikeFunction() {
+	return ScalarFunction("~~", {LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::BOOLEAN,
+	                      RegularLikeFunction<LikeOperator, false>, LikeBindFunction);
+}
+
 void LikeEscapeFun::RegisterFunction(BuiltinFunctions &set) {
-	set.AddFunction({"like_escape"}, ScalarFunction({LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR},
-	                                                LogicalType::BOOLEAN, LikeEscapeFunction<LikeEscapeOperator>));
+	set.AddFunction(GetLikeEscapeFun());
 	set.AddFunction({"not_like_escape"},
 	                ScalarFunction({LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR},
 	                               LogicalType::BOOLEAN, LikeEscapeFunction<NotLikeEscapeOperator>));
@@ -535,4 +538,9 @@ void LikeEscapeFun::RegisterFunction(BuiltinFunctions &set) {
 	                ScalarFunction({LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR},
 	                               LogicalType::BOOLEAN, LikeEscapeFunction<NotILikeEscapeOperator>));
 }
+
+ScalarFunction LikeEscapeFun::GetLikeEscapeFun() {
+	return ScalarFunction("like_escape", {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR},
+	                      LogicalType::BOOLEAN, LikeEscapeFunction<LikeEscapeOperator>);
+}
 } // namespace duckdb

package/src/duckdb/src/function/scalar/system/aggregate_export.cpp

@@ -1,11 +1,11 @@
 #include "duckdb/catalog/catalog_entry/aggregate_function_catalog_entry.hpp"
+#include "duckdb/function/function_binder.hpp"
 #include "duckdb/function/scalar/generic_functions.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/database.hpp"
 #include "duckdb/planner/expression/bound_aggregate_expression.hpp"
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
 #include "duckdb/planner/expression/bound_function_expression.hpp"
-#include "duckdb/function/function_binder.hpp"
 
 namespace duckdb {
 
@@ -39,11 +39,14 @@ struct CombineState : public FunctionLocalState {
 	unsafe_unique_array<data_t> state_buffer0, state_buffer1;
 	Vector state_vector0, state_vector1;
 
+	ArenaAllocator allocator;
+
 	explicit CombineState(idx_t state_size_p)
 	    : state_size(state_size_p), state_buffer0(make_unsafe_uniq_array<data_t>(state_size_p)),
 	      state_buffer1(make_unsafe_uniq_array<data_t>(state_size_p)),
 	      state_vector0(Value::POINTER(CastPointerToValue(state_buffer0.get()))),
-	      state_vector1(Value::POINTER(CastPointerToValue(state_buffer1.get()))) {
+	      state_vector1(Value::POINTER(CastPointerToValue(state_buffer1.get()))),
+	      allocator(Allocator::DefaultAllocator()) {
 	}
 };
 
@@ -58,10 +61,12 @@ struct FinalizeState : public FunctionLocalState {
 	unsafe_unique_array<data_t> state_buffer;
 	Vector addresses;
 
+	ArenaAllocator allocator;
+
 	explicit FinalizeState(idx_t state_size_p)
 	    : state_size(state_size_p),
 	      state_buffer(make_unsafe_uniq_array<data_t>(STANDARD_VECTOR_SIZE * AlignValue(state_size_p))),
-	      addresses(LogicalType::POINTER) {
+	      addresses(LogicalType::POINTER), allocator(Allocator::DefaultAllocator()) {
 	}
 };
 
@@ -74,6 +79,7 @@ static unique_ptr<FunctionLocalState> InitFinalizeState(ExpressionState &state,
 static void AggregateStateFinalize(DataChunk &input, ExpressionState &state_p, Vector &result) {
 	auto &bind_data = ExportAggregateBindData::GetFrom(state_p);
 	auto &local_state = ExecuteFunctionState::GetFunctionState(state_p)->Cast<FinalizeState>();
+	local_state.allocator.Reset();
 
 	D_ASSERT(bind_data.state_size == bind_data.aggr.state_size());
 	D_ASSERT(input.data.size() == 1);
@@ -100,7 +106,7 @@ static void AggregateStateFinalize(DataChunk &input, ExpressionState &state_p, V
 		state_vec_ptr[i] = data_ptr_cast(target_ptr);
 	}
 
-	AggregateInputData aggr_input_data(nullptr, Allocator::DefaultAllocator());
+	AggregateInputData aggr_input_data(nullptr, local_state.allocator);
 	bind_data.aggr.finalize(local_state.addresses, aggr_input_data, result, input.size(), 0);
 
 	for (idx_t i = 0; i < input.size(); i++) {
@@ -114,6 +120,7 @@ static void AggregateStateFinalize(DataChunk &input, ExpressionState &state_p, V
 static void AggregateStateCombine(DataChunk &input, ExpressionState &state_p, Vector &result) {
 	auto &bind_data = ExportAggregateBindData::GetFrom(state_p);
 	auto &local_state = ExecuteFunctionState::GetFunctionState(state_p)->Cast<CombineState>();
+	local_state.allocator.Reset();
 
 	D_ASSERT(bind_data.state_size == bind_data.aggr.state_size());
 
@@ -164,7 +171,7 @@ static void AggregateStateCombine(DataChunk &input, ExpressionState &state_p, Ve
 		memcpy(local_state.state_buffer0.get(), state0.GetData(), bind_data.state_size);
 		memcpy(local_state.state_buffer1.get(), state1.GetData(), bind_data.state_size);
 
-		AggregateInputData aggr_input_data(nullptr, Allocator::DefaultAllocator());
+		AggregateInputData aggr_input_data(nullptr, local_state.allocator);
 		bind_data.aggr.combine(local_state.state_vector0, local_state.state_vector1, aggr_input_data, 1);
 
 		result_ptr[i] = StringVector::AddStringOrBlob(result, const_char_ptr_cast(local_state.state_buffer1.get()),

package/src/duckdb/src/function/table/arrow.cpp

@@ -127,6 +127,25 @@ LogicalType ArrowTableFunction::GetArrowLogicalType(
 		}
 		return LogicalType::STRUCT(child_types);
 
+	} else if (format[0] == '+' && format[1] == 'u') {
+		if (format[2] != 's') {
+			throw NotImplementedException("Unsupported Internal Arrow Type: \"%c\" Union", format[2]);
+		}
+		D_ASSERT(format[3] == ':');
+
+		std::string prefix = "+us:";
+		// TODO: what are these type ids actually for?
+		auto type_ids = StringUtil::Split(format.substr(prefix.size()), ',');
+
+		child_list_t<LogicalType> members;
+		for (idx_t type_idx = 0; type_idx < (idx_t)schema.n_children; type_idx++) {
+			auto type = schema.children[type_idx];
+
+			members.emplace_back(type->name, GetArrowLogicalType(*type, arrow_convert_data, col_idx));
+		}
+
+		return LogicalType::UNION(members);
+
 	} else if (format == "+m") {
 		convert_data.variable_sz_type.emplace_back(ArrowVariableSizeType::NORMAL, 0);
 

package/src/duckdb/src/function/table/arrow_conversion.cpp

@@ -644,8 +644,42 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowScanLoca
 		}
 		break;
 	}
+	case LogicalTypeId::UNION: {
+		auto type_ids = ArrowBufferData<int8_t>(array, array.n_buffers == 1 ? 0 : 1);
+		D_ASSERT(type_ids);
+		auto members = UnionType::CopyMemberTypes(vector.GetType());
+
+		auto &validity_mask = FlatVector::Validity(vector);
+
+		duckdb::vector<Vector> children;
+		for (idx_t type_idx = 0; type_idx < (::idx_t)array.n_children; type_idx++) {
+			Vector child(members[type_idx].second);
+			auto arrow_array = array.children[type_idx];
+
+			SetValidityMask(child, *arrow_array, scan_state, size, nested_offset);
+
+			ColumnArrowToDuckDB(child, *arrow_array, scan_state, size, arrow_convert_data, col_idx, arrow_convert_idx,
+			                    nested_offset, &validity_mask);
+
+			children.push_back(std::move(child));
+		}
+
+		for (idx_t row_idx = 0; row_idx < size; row_idx++) {
+			auto tag = type_ids[row_idx];
+
+			auto out_of_range = tag < 0 || tag >= array.n_children;
+			if (out_of_range) {
+				throw InvalidInputException("Arrow union tag out of range: %d", tag);
+			}
+
+			const Value &value = children[tag].GetValue(row_idx);
+			vector.SetValue(row_idx, value.IsNull() ? Value() : Value::UNION(members, tag, value));
+		}
+
+		break;
+	}
 	default:
-		throw NotImplementedException("Unsupported type %s", vector.GetType().ToString());
+		throw NotImplementedException("Unsupported type for arrow conversion: %s", vector.GetType().ToString());
 	}
 }
 

package/src/duckdb/src/function/table/copy_csv.cpp

@@ -116,13 +116,20 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, CopyInfo &in
 	for (auto &option : info.options) {
 		auto loption = StringUtil::Lower(option.first);
 		auto &set = option.second;
-		options.SetReadOption(loption, ConvertVectorToValue(std::move(set)), expected_names);
+		options.SetReadOption(loption, ConvertVectorToValue(set), expected_names);
 	}
 	// verify the parsed options
 	if (options.force_not_null.empty()) {
 		// no FORCE_QUOTE specified: initialize to false
 		options.force_not_null.resize(expected_types.size(), false);
 	}
+
+	// Look for rejects table options last
+	named_parameter_map_t options_map;
+	for (auto &option : info.options) {
+		options_map[option.first] = ConvertVectorToValue(std::move(option.second));
+	}
+
 	bind_data->FinalizeRead(context);
 	if (!bind_data->single_threaded && options.auto_detect) {
 		options.file_path = bind_data->files[0];

package/src/duckdb/src/function/table/read_csv.cpp

@@ -14,6 +14,8 @@
 #include "duckdb/common/multi_file_reader.hpp"
 #include "duckdb/main/client_data.hpp"
 #include "duckdb/execution/operator/persistent/csv_line_info.hpp"
+#include "duckdb/execution/operator/persistent/csv_rejects_table.hpp"
+
 #include <limits>
 
 namespace duckdb {
@@ -42,6 +44,43 @@ void ReadCSVData::FinalizeRead(ClientContext &context) {
 		// not supported for parallel CSV reading
 		single_threaded = true;
 	}
+
+	// Validate rejects_table options
+	if (!options.rejects_table_name.empty()) {
+		if (!options.ignore_errors) {
+			throw BinderException("REJECTS_TABLE option is only supported when IGNORE_ERRORS is set to true");
+		}
+		if (options.file_options.union_by_name) {
+			throw BinderException("REJECTS_TABLE option is not supported when UNION_BY_NAME is set to true");
+		}
+	}
+
+	if (!options.rejects_recovery_columns.empty()) {
+		if (options.rejects_table_name.empty()) {
+			throw BinderException(
+			    "REJECTS_RECOVERY_COLUMNS option is only supported when REJECTS_TABLE is set to a table name");
+		}
+		for (auto &recovery_col : options.rejects_recovery_columns) {
+			bool found = false;
+			for (idx_t col_idx = 0; col_idx < return_names.size(); col_idx++) {
+				if (StringUtil::CIEquals(return_names[col_idx], recovery_col)) {
+					options.rejects_recovery_column_ids.push_back(col_idx);
+					found = true;
+					break;
+				}
+			}
+			if (!found) {
+				throw BinderException("Unsupported parameter for REJECTS_RECOVERY_COLUMNS: column \"%s\" not found",
+				                      recovery_col);
+			}
+		}
+	}
+
+	if (options.rejects_limit != 0) {
+		if (options.rejects_table_name.empty()) {
+			throw BinderException("REJECTS_LIMIT option is only supported when REJECTS_TABLE is set to a table name");
+		}
+	}
 }
 
 uint8_t GetCandidateSpecificity(const LogicalType &candidate_type) {
@@ -65,13 +104,14 @@ uint8_t GetCandidateSpecificity(const LogicalType &candidate_type) {
 
 static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, TableFunctionBindInput &input,
                                             vector<LogicalType> &return_types, vector<string> &names) {
+
 	auto result = make_uniq<ReadCSVData>();
 	auto &options = result->options;
 	result->files = MultiFileReader::GetFileList(context, input.inputs[0], "CSV");
 
 	bool explicitly_set_columns = false;
 	for (auto &kv : input.named_parameters) {
-		if (MultiFileReader::ParseOption(kv.first, kv.second, options.file_options)) {
+		if (MultiFileReader::ParseOption(kv.first, kv.second, options.file_options, context)) {
 			continue;
 		}
 		auto loption = StringUtil::Lower(kv.first);
@@ -180,9 +220,7 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, TableFunctio
 			options.SetReadOption(loption, kv.second, names);
 		}
 	}
-	if (options.file_options.auto_detect_hive_partitioning) {
-		options.file_options.hive_partitioning = MultiFileReaderOptions::AutoDetectHivePartitioning(result->files);
-	}
+	options.file_options.AutoDetectHivePartitioning(result->files, context);
 
 	if (!options.auto_detect && return_types.empty()) {
 		throw BinderException("read_csv requires columns to be specified through the 'columns' option. Use "
@@ -235,9 +273,15 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, TableFunctio
 	} else {
 		result->reader_bind = MultiFileReader::BindOptions(options.file_options, result->files, return_types, names);
 	}
+
 	result->return_types = return_types;
 	result->return_names = names;
 	result->FinalizeRead(context);
+
+	if (options.auto_detect) {
+		result->initial_reader->options = options;
+	}
+
 	return std::move(result);
 }
 
@@ -261,10 +305,6 @@ public:
 	      line_info(main_mutex, batch_to_tuple_end, tuple_start, tuple_end) {
 		file_handle->DisableReset();
 		current_file_path = files_path_p[0];
-		line_info.lines_read[0] = rows_to_skip;
-		if (has_header) {
-			line_info.lines_read[0]++;
-		}
 		file_size = file_handle->FileSize();
 		first_file_size = file_size;
 		on_disk_file = file_handle->OnDiskFile();
@@ -291,10 +331,17 @@ public:
 		// Initialize all the book-keeping variables
 		auto file_count = files_path_p.size();
 		line_info.current_batches.resize(file_count);
+		line_info.lines_read.resize(file_count);
 		tuple_start.resize(file_count);
 		tuple_end.resize(file_count);
 		tuple_end_to_batch.resize(file_count);
 		batch_to_tuple_end.resize(file_count);
+
+		// Initialize the lines read
+		line_info.lines_read[0][0] = rows_to_skip;
+		if (has_header) {
+			line_info.lines_read[0][0]++;
+		}
 	}
 	ParallelCSVGlobalState() : line_info(main_mutex, batch_to_tuple_end, tuple_start, tuple_end) {
 		running_threads = MaxThreads();
@@ -501,6 +548,9 @@ bool ParallelCSVGlobalState::Next(ClientContext &context, const ReadCSVData &bin
 			current_csv_position = 0;
 			file_number++;
 			local_batch_index = 0;
+
+			line_info.lines_read[file_number][local_batch_index] = (bind_data.options.has_header ? 1 : 0);
+
 			current_buffer =
 			    make_shared<CSVBuffer>(context, buffer_size, *file_handle, current_csv_position, file_number);
 			next_buffer = shared_ptr<CSVBuffer>(
@@ -512,7 +562,7 @@ bool ParallelCSVGlobalState::Next(ClientContext &context, const ReadCSVData &bin
 		}
 	}
 	// set up the current buffer
-	line_info.current_batches.back().insert(local_batch_index);
+	line_info.current_batches[file_number].insert(local_batch_index);
 	auto result = make_uniq<CSVBufferRead>(current_buffer, next_buffer, next_byte, next_byte + bytes_per_local_state,
 	                                       batch_index++, local_batch_index++, &line_info);
 	// move the byte index of the CSV reader to the next buffer
@@ -551,11 +601,12 @@ bool ParallelCSVGlobalState::Next(ClientContext &context, const ReadCSVData &bin
 		reader->options.file_path = current_file_path;
 		MultiFileReader::InitializeReader(*reader, bind_data.options.file_options, bind_data.reader_bind,
 		                                  bind_data.return_types, bind_data.return_names, column_ids, nullptr,
-		                                  bind_data.files.front());
+		                                  bind_data.files.front(), context);
 	} else {
 		// update the current reader
 		reader->SetBufferRead(std::move(result));
 	}
+
 	return true;
 }
 void ParallelCSVGlobalState::UpdateVerification(VerificationPositions positions, idx_t file_number_p, idx_t batch_idx) {
@@ -574,7 +625,7 @@ void ParallelCSVGlobalState::UpdateLinesRead(CSVBufferRead &buffer_read, idx_t f
 	auto lines_read = buffer_read.lines_read;
 	lock_guard<mutex> parallel_lock(main_mutex);
 	line_info.current_batches[file_idx].erase(batch_idx);
-	line_info.lines_read[batch_idx] += lines_read;
+	line_info.lines_read[file_idx][batch_idx] += lines_read;
 }
 
 bool LineInfo::CanItGetLine(idx_t file_idx, idx_t batch_idx) {
@@ -593,21 +644,36 @@ bool LineInfo::CanItGetLine(idx_t file_idx, idx_t batch_idx) {
 }
 
 // Returns the 1-indexed line number
-idx_t LineInfo::GetLine(idx_t batch_idx, idx_t line_error, idx_t file_idx, idx_t cur_start, bool verify) {
+idx_t LineInfo::GetLine(idx_t batch_idx, idx_t line_error, idx_t file_idx, idx_t cur_start, bool verify,
+                        bool stop_at_first) {
 	unique_ptr<lock_guard<mutex>> parallel_lock;
 	if (!verify) {
 		parallel_lock = duckdb::make_uniq<lock_guard<mutex>>(main_mutex);
 	}
 	idx_t line_count = 0;
+
+	if (!stop_at_first) {
+		// Figure out the amount of lines read in the current file
+		auto &file_batches = current_batches[file_idx];
+		for (auto &batch : file_batches) {
+			if (batch > batch_idx) {
+				break;
+			}
+			line_count += lines_read[file_idx][batch];
+		}
+		return line_count + line_error + 1;
+	}
+
+	// Otherwise, check if we already have an error on another thread
 	if (done) {
 		// line count is 0-indexed, but we want to return 1-indexed
 		return first_line + 1;
 	}
 	for (idx_t i = 0; i <= batch_idx; i++) {
-		if (lines_read.find(i) == lines_read.end() && i != batch_idx) {
+		if (lines_read[file_idx].find(i) == lines_read[file_idx].end() && i != batch_idx) {
 			throw InternalException("Missing batch index on Parallel CSV Reader GetLine");
 		}
-		line_count += lines_read[i];
+		line_count += lines_read[file_idx][i];
 	}
 
 	// before we are done, if this is not a call in Verify() we must check Verify up to this batch
@@ -782,7 +848,7 @@ private:
 			}
 			MultiFileReader::InitializeReader(*result, bind_data.options.file_options, bind_data.reader_bind,
 			                                  bind_data.return_types, bind_data.return_names, column_ids, nullptr,
-			                                  bind_data.files.front());
+			                                  bind_data.files.front(), context);
 		}
 		total_size = result->file_handle->FileSize();
 		return result;
@@ -831,14 +897,14 @@ static unique_ptr<GlobalTableFunctionState> SingleThreadedCSVInit(ClientContext
 	}
 	MultiFileReader::InitializeReader(*result->initial_reader, bind_data.options.file_options, bind_data.reader_bind,
 	                                  bind_data.return_types, bind_data.return_names, input.column_ids, input.filters,
-	                                  bind_data.files.front());
+	                                  bind_data.files.front(), context);
 	for (auto &reader : bind_data.union_readers) {
 		if (!reader) {
 			continue;
 		}
 		MultiFileReader::InitializeReader(*reader, bind_data.options.file_options, bind_data.reader_bind,
 		                                  bind_data.return_types, bind_data.return_names, input.column_ids,
-		                                  input.filters, bind_data.files.front());
+		                                  input.filters, bind_data.files.front(), context);
 	}
 	result->column_ids = input.column_ids;
 
@@ -914,6 +980,12 @@ static void SingleThreadedCSVFunction(ClientContext &context, TableFunctionInput
 //===--------------------------------------------------------------------===//
 static unique_ptr<GlobalTableFunctionState> ReadCSVInitGlobal(ClientContext &context, TableFunctionInitInput &input) {
 	auto &bind_data = input.bind_data->Cast<ReadCSVData>();
+
+	// Create the temporary rejects table
+	auto rejects_table = bind_data.options.rejects_table_name;
+	if (!rejects_table.empty()) {
+		CSVRejectsTable::GetOrCreate(context, rejects_table)->InitializeTable(context, bind_data);
+	}
 	if (bind_data.single_threaded) {
 		return SingleThreadedCSVInit(context, input);
 	} else {
@@ -974,6 +1046,9 @@ static void ReadCSVAddNamedParameters(TableFunction &table_function) {
 	table_function.named_parameters["max_line_size"] = LogicalType::VARCHAR;
 	table_function.named_parameters["maximum_line_size"] = LogicalType::VARCHAR;
 	table_function.named_parameters["ignore_errors"] = LogicalType::BOOLEAN;
+	table_function.named_parameters["rejects_table"] = LogicalType::VARCHAR;
+	table_function.named_parameters["rejects_limit"] = LogicalType::BIGINT;
+	table_function.named_parameters["rejects_recovery_columns"] = LogicalType::LIST(LogicalType::VARCHAR);
 	table_function.named_parameters["buffer_size"] = LogicalType::UBIGINT;
 	table_function.named_parameters["decimal_separator"] = LogicalType::VARCHAR;
 	table_function.named_parameters["parallel"] = LogicalType::BOOLEAN;
@@ -1062,6 +1137,10 @@ void BufferedCSVReaderOptions::Serialize(FieldWriter &writer) const {
 		csv_formats.push_back(format.second.format_specifier);
 	}
 	writer.WriteList<string>(csv_formats);
+	writer.WriteString(rejects_table_name);
+	writer.WriteField<idx_t>(rejects_limit);
+	writer.WriteList<string>(rejects_recovery_columns);
+	writer.WriteList<idx_t>(rejects_recovery_column_ids);
 }
 
 void BufferedCSVReaderOptions::Deserialize(FieldReader &reader) {
@@ -1109,6 +1188,10 @@ void BufferedCSVReaderOptions::Deserialize(FieldReader &reader) {
 		has_format[type] = true;
 		StrTimeFormat::ParseFormatSpecifier(format, date_format[type]);
 	}
+	rejects_table_name = reader.ReadRequired<string>();
+	rejects_limit = reader.ReadRequired<idx_t>();
+	rejects_recovery_columns = reader.ReadRequiredList<string>();
+	rejects_recovery_column_ids = reader.ReadRequiredList<idx_t>();
 }
 
 static void CSVReaderSerialize(FieldWriter &writer, const FunctionData *bind_data_p, const TableFunction &function) {

package/src/duckdb/src/function/table/system/test_all_types.cpp

@@ -204,6 +204,13 @@ vector<TestType> TestAllTypesFun::GetTestTypes(bool use_large_enum) {
 	auto max_map_value = Value::MAP(ListType::GetChildType(map_type), map_values);
 	result.emplace_back(map_type, "map", std::move(min_map_value), std::move(max_map_value));
 
+	// union
+	child_list_t<LogicalType> members = {{"name", LogicalType::VARCHAR}, {"age", LogicalType::SMALLINT}};
+	auto union_type = LogicalType::UNION(members);
+	const Value &min = Value::UNION(members, 0, Value("Frank"));
+	const Value &max = Value::UNION(members, 1, Value::SMALLINT(5));
+	result.emplace_back(union_type, "union", min, max);
+
 	return result;
 }
 

package/src/duckdb/src/function/table/system_functions.cpp

@@ -10,6 +10,7 @@ namespace duckdb {
 
 void BuiltinFunctions::RegisterSQLiteFunctions() {
 	PragmaVersion::RegisterFunction(*this);
+	PragmaPlatform::RegisterFunction(*this);
 	PragmaCollations::RegisterFunction(*this);
 	PragmaTableInfo::RegisterFunction(*this);
 	PragmaStorageInfo::RegisterFunction(*this);

package/src/duckdb/src/function/table/table_scan.cpp

@@ -281,6 +281,15 @@ void TableScanPushdownComplexFilter(ClientContext &context, LogicalGet &get, Fun
 	if (bind_data.is_index_scan) {
 		return;
 	}
+	if (!get.table_filters.filters.empty()) {
+		// if there were filters before we can't convert this to an index scan
+		return;
+	}
+	if (!get.projection_ids.empty()) {
+		// if columns were pruned by RemoveUnusedColumns we can't convert this to an index scan,
+		// because index scan does not support filter_prune (yet)
+		return;
+	}
 	if (filters.empty()) {
 		// no indexes or no filters: skip the pushdown
 		return;

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,11 +1,12 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.8.2-dev161"
+#define DUCKDB_VERSION "0.8.2-dev1764"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "563662b093"
+#define DUCKDB_SOURCE_ID "07b0b0a2a4"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"
+#include "duckdb/common/string_util.hpp"
 
 #include <cstdint>
 
@@ -63,6 +64,13 @@ const char *DuckDB::LibraryVersion() {
 }
 
 string DuckDB::Platform() {
+#if defined(DUCKDB_CUSTOM_PLATFORM)
+	return DUCKDB_QUOTE_DEFINE(DUCKDB_CUSTOM_PLATFORM);
+#endif
+#if defined(DUCKDB_WASM_VERSION)
+	// DuckDB-Wasm requires CUSTOM_PLATFORM to be defined
+	static_assert(0, "DUCKDB_WASM_VERSION should rely on CUSTOM_PLATFORM being provided");
+#endif
 	string os = "linux";
 #if INTPTR_MAX == INT64_MAX
 	string arch = "amd64";
@@ -97,4 +105,40 @@ string DuckDB::Platform() {
 	return os + "_" + arch + postfix;
 }
 
+struct PragmaPlatformData : public GlobalTableFunctionState {
+	PragmaPlatformData() : finished(false) {
+	}
+
+	bool finished;
+};
+
+static unique_ptr<FunctionData> PragmaPlatformBind(ClientContext &context, TableFunctionBindInput &input,
+                                                   vector<LogicalType> &return_types, vector<string> &names) {
+	names.emplace_back("platform");
+	return_types.emplace_back(LogicalType::VARCHAR);
+	return nullptr;
+}
+
+static unique_ptr<GlobalTableFunctionState> PragmaPlatformInit(ClientContext &context, TableFunctionInitInput &input) {
+	return make_uniq<PragmaPlatformData>();
+}
+
+static void PragmaPlatformFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) {
+	auto &data = data_p.global_state->Cast<PragmaPlatformData>();
+	if (data.finished) {
+		// finished returning values
+		return;
+	}
+	output.SetCardinality(1);
+	output.SetValue(0, 0, DuckDB::Platform());
+	data.finished = true;
+}
+
+void PragmaPlatform::RegisterFunction(BuiltinFunctions &set) {
+	TableFunction pragma_platform("pragma_platform", {}, PragmaPlatformFunction);
+	pragma_platform.bind = PragmaPlatformBind;
+	pragma_platform.init_global = PragmaPlatformInit;
+	set.AddFunction(pragma_platform);
+}
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp

@@ -87,4 +87,5 @@ AdbcStatusCode StatementExecutePartitions(struct AdbcStatement *statement, struc
 void SetError(struct AdbcError *error, const std::string &message);
 
 void InitiliazeADBCError(AdbcError *error);
+
 } // namespace duckdb_adbc

package/src/duckdb/src/include/duckdb/common/allocator.hpp

@@ -116,6 +116,8 @@ public:
 	DUCKDB_API static Allocator &DefaultAllocator();
 	DUCKDB_API static shared_ptr<Allocator> &DefaultAllocatorReference();
 
+	static void ThreadFlush(idx_t threshold);
+
 private:
 	allocate_function_ptr_t allocate_function;
 	free_function_ptr_t free_function;

package/src/duckdb/src/include/duckdb/common/bswap.hpp

@@ -0,0 +1,42 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/bswap.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+
+namespace duckdb {
+
+#define BSWAP16(x) ((uint16_t)((((uint16_t)(x)&0xff00) >> 8) | (((uint16_t)(x)&0x00ff) << 8)))
+
+#define BSWAP32(x)                                                                                                     \
+	((uint32_t)((((uint32_t)(x)&0xff000000) >> 24) | (((uint32_t)(x)&0x00ff0000) >> 8) |                               \
+	            (((uint32_t)(x)&0x0000ff00) << 8) | (((uint32_t)(x)&0x000000ff) << 24)))
+
+#define BSWAP64(x)                                                                                                     \
+	((uint64_t)((((uint64_t)(x)&0xff00000000000000ull) >> 56) | (((uint64_t)(x)&0x00ff000000000000ull) >> 40) |        \
+	            (((uint64_t)(x)&0x0000ff0000000000ull) >> 24) | (((uint64_t)(x)&0x000000ff00000000ull) >> 8) |         \
+	            (((uint64_t)(x)&0x00000000ff000000ull) << 8) | (((uint64_t)(x)&0x0000000000ff0000ull) << 24) |         \
+	            (((uint64_t)(x)&0x000000000000ff00ull) << 40) | (((uint64_t)(x)&0x00000000000000ffull) << 56)))
+
+template <class T>
+static inline T BSwap(const T &x) {
+	static_assert(sizeof(T) == 1 || sizeof(T) == 2 || sizeof(T) == 4 || sizeof(T) == 8,
+	              "Size of type must be 1, 2, 4, or 8 for BSwap");
+	if (sizeof(T) == 1) {
+		return x;
+	} else if (sizeof(T) == 2) {
+		return BSWAP16(x);
+	} else if (sizeof(T) == 4) {
+		return BSWAP32(x);
+	} else {
+		return BSWAP64(x);
+	}
+}
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/dl.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/common/windows.hpp"
 #include "duckdb/common/local_file_system.hpp"
+#include "duckdb/common/windows_util.hpp"
 
 #ifndef _WIN32
 #include <dlfcn.h>
@@ -24,7 +25,8 @@ namespace duckdb {
 
 inline void *dlopen(const char *file, int mode) {
 	D_ASSERT(file);
-	return (void *)LoadLibrary(file);
+	auto fpath = WindowsUtil::UTF8ToUnicode(file);
+	return (void *)LoadLibraryW(fpath.c_str());
 }
 
 inline void *dlsym(void *handle, const char *name) {