duckdb 0.8.2-dev157.0 → 0.8.2-dev1573.0

package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp

@@ -32,7 +32,7 @@ string PhysicalComparisonJoin::ParamsToString() const {
 	}
 	extra_info += "\n[INFOSEPARATOR]\n";
 	extra_info += StringUtil::Format("EC: %llu\n", estimated_props->GetCardinality<idx_t>());
-	extra_info += StringUtil::Format("Cost: %llu", (idx_t)estimated_props->GetCost());
+	extra_info += StringUtil::Format("Cost: %llu", estimated_props->GetCost<idx_t>());
 	return extra_info;
 }
 

package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp

@@ -96,7 +96,7 @@ public:
 class HashJoinLocalSinkState : public LocalSinkState {
 public:
 	HashJoinLocalSinkState(const PhysicalHashJoin &op, ClientContext &context) : build_executor(context) {
-		auto &allocator = Allocator::Get(context);
+		auto &allocator = BufferAllocator::Get(context);
 		if (!op.right_projection_map.empty()) {
 			build_chunk.Initialize(allocator, op.build_types);
 		}
@@ -124,7 +124,7 @@ public:
 unique_ptr<JoinHashTable> PhysicalHashJoin::InitializeHashTable(ClientContext &context) const {
 	auto result =
 	    make_uniq<JoinHashTable>(BufferManager::GetBufferManager(context), conditions, build_types, join_type);
-	result->max_ht_size = double(BufferManager::GetBufferManager(context).GetMaxMemory()) * 0.6;
+	result->max_ht_size = double(0.6) * BufferManager::GetBufferManager(context).GetMaxMemory();
 	if (!delim_types.empty() && join_type == JoinType::MARK) {
 		// correlated MARK join
 		if (delim_types.size() + 1 == conditions.size()) {
@@ -162,7 +162,7 @@ unique_ptr<JoinHashTable> PhysicalHashJoin::InitializeHashTable(ClientContext &c
 			payload_types.push_back(aggr->return_type);
 			info.correlated_aggregates.push_back(std::move(aggr));
 
-			auto &allocator = Allocator::Get(context);
+			auto &allocator = BufferAllocator::Get(context);
 			info.correlated_counts = make_uniq<GroupedAggregateHashTable>(context, allocator, delim_types,
 			                                                              payload_types, correlated_aggregates);
 			info.correlated_types = delim_types;
@@ -312,10 +312,10 @@ void HashJoinGlobalSinkState::InitializeProbeSpill() {
 	}
 }
 
-class HashJoinPartitionTask : public ExecutorTask {
+class HashJoinRepartitionTask : public ExecutorTask {
 public:
-	HashJoinPartitionTask(shared_ptr<Event> event_p, ClientContext &context, JoinHashTable &global_ht,
-	                      JoinHashTable &local_ht)
+	HashJoinRepartitionTask(shared_ptr<Event> event_p, ClientContext &context, JoinHashTable &global_ht,
+	                        JoinHashTable &local_ht)
 	    : ExecutorTask(context), event(std::move(event_p)), global_ht(global_ht), local_ht(local_ht) {
 	}
 
@@ -349,7 +349,7 @@ public:
 		partition_tasks.reserve(local_hts.size());
 		for (auto &local_ht : local_hts) {
 			partition_tasks.push_back(
-			    make_uniq<HashJoinPartitionTask>(shared_from_this(), context, *sink.hash_table, *local_ht));
+			    make_uniq<HashJoinRepartitionTask>(shared_from_this(), context, *sink.hash_table, *local_ht));
 		}
 		SetTasks(std::move(partition_tasks));
 	}
@@ -434,7 +434,7 @@ public:
 };
 
 unique_ptr<OperatorState> PhysicalHashJoin::GetOperatorState(ExecutionContext &context) const {
-	auto &allocator = Allocator::Get(context.client);
+	auto &allocator = BufferAllocator::Get(context.client);
 	auto &sink = sink_state->Cast<HashJoinGlobalSinkState>();
 	auto state = make_uniq<HashJoinOperatorState>(context.client);
 	if (sink.perfect_join_executor) {
@@ -532,7 +532,18 @@ public:
 	bool AssignTask(HashJoinGlobalSinkState &sink, HashJoinLocalSourceState &lstate);
 
 	idx_t MaxThreads() override {
-		return probe_count / ((idx_t)STANDARD_VECTOR_SIZE * parallel_scan_chunk_count);
+		D_ASSERT(op.sink_state);
+		auto &gstate = op.sink_state->Cast<HashJoinGlobalSinkState>();
+
+		idx_t count;
+		if (gstate.probe_spill) {
+			count = probe_count;
+		} else if (IsRightOuterJoin(op.join_type)) {
+			count = gstate.hash_table->Count();
+		} else {
+			return 0;
+		}
+		return count / ((idx_t)STANDARD_VECTOR_SIZE * parallel_scan_chunk_count);
 	}
 
 public:
@@ -611,7 +622,7 @@ unique_ptr<GlobalSourceState> PhysicalHashJoin::GetGlobalSourceState(ClientConte
 
 unique_ptr<LocalSourceState> PhysicalHashJoin::GetLocalSourceState(ExecutionContext &context,
                                                                    GlobalSourceState &gstate) const {
-	return make_uniq<HashJoinLocalSourceState>(*this, Allocator::Get(context.client));
+	return make_uniq<HashJoinLocalSourceState>(*this, BufferAllocator::Get(context.client));
 }
 
 HashJoinGlobalSourceState::HashJoinGlobalSourceState(const PhysicalHashJoin &op, ClientContext &context)

package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp

@@ -16,7 +16,7 @@
 
 namespace duckdb {
 
-PhysicalIEJoin::PhysicalIEJoin(LogicalOperator &op, unique_ptr<PhysicalOperator> left,
+PhysicalIEJoin::PhysicalIEJoin(LogicalComparisonJoin &op, unique_ptr<PhysicalOperator> left,
                                unique_ptr<PhysicalOperator> right, vector<JoinCondition> cond, JoinType join_type,
                                idx_t estimated_cardinality)
     : PhysicalRangeJoin(op, PhysicalOperatorType::IE_JOIN, std::move(left), std::move(right), std::move(cond),
@@ -641,6 +641,8 @@ public:
 	    : op(op), true_sel(STANDARD_VECTOR_SIZE), left_executor(context), right_executor(context),
 	      left_matches(nullptr), right_matches(nullptr) {
 		auto &allocator = Allocator::Get(context);
+		unprojected.Initialize(allocator, op.unprojected_types);
+
 		if (op.conditions.size() < 3) {
 			return;
 		}
@@ -696,6 +698,8 @@ public:
 	ExpressionExecutor right_executor;
 	DataChunk right_keys;
 
+	DataChunk unprojected;
+
 	// Outer joins
 	idx_t outer_idx;
 	idx_t outer_count;
@@ -703,13 +707,14 @@ public:
 	bool *right_matches;
 };
 
-void PhysicalIEJoin::ResolveComplexJoin(ExecutionContext &context, DataChunk &chunk, LocalSourceState &state_p) const {
+void PhysicalIEJoin::ResolveComplexJoin(ExecutionContext &context, DataChunk &result, LocalSourceState &state_p) const {
 	auto &state = state_p.Cast<IEJoinLocalSourceState>();
 	auto &ie_sink = sink_state->Cast<IEJoinGlobalState>();
 	auto &left_table = *ie_sink.tables[0];
 	auto &right_table = *ie_sink.tables[1];
 
 	const auto left_cols = children[0]->GetTypes().size();
+	auto &chunk = state.unprojected;
 	do {
 		SelectionVector lsel(STANDARD_VECTOR_SIZE);
 		SelectionVector rsel(STANDARD_VECTOR_SIZE);
@@ -720,6 +725,7 @@ void PhysicalIEJoin::ResolveComplexJoin(ExecutionContext &context, DataChunk &ch
 		}
 
 		// found matches: extract them
+
 		chunk.Reset();
 		SliceSortedPayload(chunk, left_table.global_sort_state, state.left_block_index, lsel, result_count, 0);
 		SliceSortedPayload(chunk, right_table.global_sort_state, state.right_block_index, rsel, result_count,
@@ -762,6 +768,10 @@ void PhysicalIEJoin::ResolveComplexJoin(ExecutionContext &context, DataChunk &ch
 			}
 		}
 
+		//	We need all of the data to compute other predicates,
+		//	but we only return what is in the projection map
+		ProjectResult(chunk, result);
+
 		// found matches: mark the found matches if required
 		if (left_table.found_match) {
 			for (idx_t i = 0; i < result_count; i++) {
@@ -773,8 +783,8 @@ void PhysicalIEJoin::ResolveComplexJoin(ExecutionContext &context, DataChunk &ch
 				right_table.found_match[state.right_base + rsel[sel->get_index(i)]] = true;
 			}
 		}
-		chunk.Verify();
-	} while (chunk.size() == 0);
+		result.Verify();
+	} while (result.size() == 0);
 }
 
 class IEJoinGlobalSourceState : public GlobalSourceState {
@@ -961,15 +971,18 @@ SourceResultType PhysicalIEJoin::GetData(ExecutionContext &context, DataChunk &r
 			ie_gstate.GetNextPair(context.client, ie_sink, ie_lstate);
 			continue;
 		}
-		SliceSortedPayload(result, ie_sink.tables[0]->global_sort_state, ie_lstate.left_block_index, ie_lstate.true_sel,
+		auto &chunk = ie_lstate.unprojected;
+		chunk.Reset();
+		SliceSortedPayload(chunk, ie_sink.tables[0]->global_sort_state, ie_lstate.left_block_index, ie_lstate.true_sel,
 		                   count);
 
 		// Fill in NULLs to the right
-		for (auto col_idx = left_cols; col_idx < result.ColumnCount(); ++col_idx) {
-			result.data[col_idx].SetVectorType(VectorType::CONSTANT_VECTOR);
-			ConstantVector::SetNull(result.data[col_idx], true);
+		for (auto col_idx = left_cols; col_idx < chunk.ColumnCount(); ++col_idx) {
+			chunk.data[col_idx].SetVectorType(VectorType::CONSTANT_VECTOR);
+			ConstantVector::SetNull(chunk.data[col_idx], true);
 		}
 
+		ProjectResult(chunk, result);
 		result.SetCardinality(count);
 		result.Verify();
 
@@ -984,15 +997,18 @@ SourceResultType PhysicalIEJoin::GetData(ExecutionContext &context, DataChunk &r
 			continue;
 		}
 
-		SliceSortedPayload(result, ie_sink.tables[1]->global_sort_state, ie_lstate.right_block_index,
-		                   ie_lstate.true_sel, count, left_cols);
+		auto &chunk = ie_lstate.unprojected;
+		chunk.Reset();
+		SliceSortedPayload(chunk, ie_sink.tables[1]->global_sort_state, ie_lstate.right_block_index, ie_lstate.true_sel,
+		                   count, left_cols);
 
 		// Fill in NULLs to the left
 		for (idx_t col_idx = 0; col_idx < left_cols; ++col_idx) {
-			result.data[col_idx].SetVectorType(VectorType::CONSTANT_VECTOR);
-			ConstantVector::SetNull(result.data[col_idx], true);
+			chunk.data[col_idx].SetVectorType(VectorType::CONSTANT_VECTOR);
+			ConstantVector::SetNull(chunk.data[col_idx], true);
 		}
 
+		ProjectResult(chunk, result);
 		result.SetCardinality(count);
 		result.Verify();
 

package/src/duckdb/src/execution/operator/join/physical_join.cpp

@@ -60,7 +60,7 @@ void PhysicalJoin::BuildJoinPipelines(Pipeline &current, MetaPipeline &meta_pipe
 	// Join can become a source operator if it's RIGHT/OUTER, or if the hash join goes out-of-core
 	bool add_child_pipeline = false;
 	auto &join_op = op.Cast<PhysicalJoin>();
-	if (IsRightOuterJoin(join_op.join_type) || join_op.type == PhysicalOperatorType::HASH_JOIN) {
+	if (join_op.IsSource()) {
 		add_child_pipeline = true;
 	}
 

package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp

@@ -14,7 +14,7 @@
 
 namespace duckdb {
 
-PhysicalPiecewiseMergeJoin::PhysicalPiecewiseMergeJoin(LogicalOperator &op, unique_ptr<PhysicalOperator> left,
+PhysicalPiecewiseMergeJoin::PhysicalPiecewiseMergeJoin(LogicalComparisonJoin &op, unique_ptr<PhysicalOperator> left,
                                                        unique_ptr<PhysicalOperator> right, vector<JoinCondition> cond,
                                                        JoinType join_type, idx_t estimated_cardinality)
     : PhysicalRangeJoin(op, PhysicalOperatorType::PIECEWISE_MERGE_JOIN, std::move(left), std::move(right),
@@ -208,6 +208,7 @@ public:
 	idx_t right_position;
 	idx_t right_chunk_index;
 	idx_t right_base;
+	idx_t prev_left_index;
 
 	// Secondary predicate shared data
 	SelectionVector sel;
@@ -431,7 +432,8 @@ void PhysicalPiecewiseMergeJoin::ResolveSimpleJoin(ExecutionContext &context, Da
 	}
 }
 
-static idx_t MergeJoinComplexBlocks(BlockMergeInfo &l, BlockMergeInfo &r, const ExpressionType comparison) {
+static idx_t MergeJoinComplexBlocks(BlockMergeInfo &l, BlockMergeInfo &r, const ExpressionType comparison,
+                                    idx_t &prev_left_index) {
 	const auto cmp = MergeJoinComparisonValue(comparison);
 
 	// The sort parameters should all be the same
@@ -465,6 +467,20 @@ static idx_t MergeJoinComplexBlocks(BlockMergeInfo &l, BlockMergeInfo &r, const
 
 	idx_t result_count = 0;
 	while (true) {
+		if (l.entry_idx < prev_left_index) {
+			// left side smaller: found match
+			l.result.set_index(result_count, sel_t(l.entry_idx));
+			r.result.set_index(result_count, sel_t(r.entry_idx));
+			result_count++;
+			// move left side forward
+			l.entry_idx++;
+			l_ptr += entry_size;
+			if (result_count == STANDARD_VECTOR_SIZE) {
+				// out of space!
+				break;
+			}
+			continue;
+		}
 		if (l.entry_idx < l.not_null) {
 			int comp_res;
 			if (all_constant) {
@@ -474,7 +490,6 @@ static idx_t MergeJoinComplexBlocks(BlockMergeInfo &l, BlockMergeInfo &r, const
 				rread.entry_idx = r.entry_idx;
 				comp_res = Comparators::CompareTuple(lread, rread, l_ptr, r_ptr, l.state.sort_layout, external);
 			}
-
 			if (comp_res <= cmp) {
 				// left side smaller: found match
 				l.result.set_index(result_count, sel_t(l.entry_idx));
@@ -490,6 +505,8 @@ static idx_t MergeJoinComplexBlocks(BlockMergeInfo &l, BlockMergeInfo &r, const
 				continue;
 			}
 		}
+
+		prev_left_index = l.entry_idx;
 		// right side smaller or equal, or left side exhausted: move
 		// right pointer forward reset left side to start
 		r.entry_idx++;
@@ -521,6 +538,7 @@ OperatorResultType PhysicalPiecewiseMergeJoin::ResolveComplexJoin(ExecutionConte
 			state.right_chunk_index = 0;
 			state.right_base = 0;
 			state.left_position = 0;
+			state.prev_left_index = 0;
 			state.right_position = 0;
 			state.first_fetch = false;
 			state.finished = false;
@@ -547,7 +565,8 @@ OperatorResultType PhysicalPiecewiseMergeJoin::ResolveComplexJoin(ExecutionConte
 		BlockMergeInfo right_info(gstate.table->global_sort_state, state.right_chunk_index, state.right_position,
 		                          rhs_not_null);
 
-		idx_t result_count = MergeJoinComplexBlocks(left_info, right_info, conditions[0].comparison);
+		idx_t result_count =
+		    MergeJoinComplexBlocks(left_info, right_info, conditions[0].comparison, state.prev_left_index);
 		if (result_count == 0) {
 			// exhausted this chunk on the right side
 			// move to the next right chunk

package/src/duckdb/src/execution/operator/join/physical_range_join.cpp

@@ -159,16 +159,16 @@ void PhysicalRangeJoin::GlobalSortedTable::Finalize(Pipeline &pipeline, Event &e
 	}
 }
 
-PhysicalRangeJoin::PhysicalRangeJoin(LogicalOperator &op, PhysicalOperatorType type, unique_ptr<PhysicalOperator> left,
-                                     unique_ptr<PhysicalOperator> right, vector<JoinCondition> cond, JoinType join_type,
-                                     idx_t estimated_cardinality)
+PhysicalRangeJoin::PhysicalRangeJoin(LogicalComparisonJoin &op, PhysicalOperatorType type,
+                                     unique_ptr<PhysicalOperator> left, unique_ptr<PhysicalOperator> right,
+                                     vector<JoinCondition> cond, JoinType join_type, idx_t estimated_cardinality)
     : PhysicalComparisonJoin(op, type, std::move(cond), join_type, estimated_cardinality) {
 	// Reorder the conditions so that ranges are at the front.
 	// TODO: use stats to improve the choice?
 	// TODO: Prefer fixed length types?
 	if (conditions.size() > 1) {
-		auto conditions_p = std::move(conditions);
-		conditions.resize(conditions_p.size());
+		vector<JoinCondition> conditions_p(conditions.size());
+		std::swap(conditions_p, conditions);
 		idx_t range_position = 0;
 		idx_t other_position = conditions_p.size();
 		for (idx_t i = 0; i < conditions_p.size(); ++i) {
@@ -188,6 +188,30 @@ PhysicalRangeJoin::PhysicalRangeJoin(LogicalOperator &op, PhysicalOperatorType t
 
 	children.push_back(std::move(left));
 	children.push_back(std::move(right));
+
+	//	Fill out the left projection map.
+	left_projection_map = op.left_projection_map;
+	if (left_projection_map.empty()) {
+		const auto left_count = children[0]->types.size();
+		left_projection_map.reserve(left_count);
+		for (column_t i = 0; i < left_count; ++i) {
+			left_projection_map.emplace_back(i);
+		}
+	}
+	//	Fill out the right projection map.
+	right_projection_map = op.right_projection_map;
+	if (right_projection_map.empty()) {
+		const auto right_count = children[1]->types.size();
+		right_projection_map.reserve(right_count);
+		for (column_t i = 0; i < right_count; ++i) {
+			right_projection_map.emplace_back(i);
+		}
+	}
+
+	//	Construct the unprojected type layout from the children's types
+	unprojected_types = children[0]->GetTypes();
+	auto &types = children[1]->GetTypes();
+	unprojected_types.insert(unprojected_types.end(), types.begin(), types.end());
 }
 
 idx_t PhysicalRangeJoin::LocalSortedTable::MergeNulls(const vector<JoinCondition> &conditions) {
@@ -266,6 +290,18 @@ idx_t PhysicalRangeJoin::LocalSortedTable::MergeNulls(const vector<JoinCondition
 	}
 }
 
+void PhysicalRangeJoin::ProjectResult(DataChunk &chunk, DataChunk &result) const {
+	const auto left_projected = left_projection_map.size();
+	for (idx_t i = 0; i < left_projected; ++i) {
+		result.data[i].Reference(chunk.data[left_projection_map[i]]);
+	}
+	const auto left_width = children[0]->types.size();
+	for (idx_t i = 0; i < right_projection_map.size(); ++i) {
+		result.data[left_projected + i].Reference(chunk.data[left_width + right_projection_map[i]]);
+	}
+	result.SetCardinality(chunk);
+}
+
 BufferHandle PhysicalRangeJoin::SliceSortedPayload(DataChunk &payload, GlobalSortState &state, const idx_t block_idx,
                                                    const SelectionVector &result, const idx_t result_count,
                                                    const idx_t left_cols) {

package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp

@@ -1,5 +1,4 @@
 #include "duckdb/execution/operator/persistent/base_csv_reader.hpp"
-
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 #include "duckdb/common/file_system.hpp"
 #include "duckdb/common/string_util.hpp"
@@ -10,6 +9,7 @@
 #include "duckdb/common/vector_operations/unary_executor.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/function/scalar/strftime_format.hpp"
+#include "duckdb/main/appender.hpp"
 #include "duckdb/main/database.hpp"
 #include "duckdb/parser/column_definition.hpp"
 #include "duckdb/storage/data_table.hpp"
@@ -18,7 +18,8 @@
 #include "duckdb/parser/keyword_helper.hpp"
 #include "duckdb/main/error_manager.hpp"
 #include "duckdb/execution/operator/persistent/parallel_csv_reader.hpp"
-
+#include "duckdb/execution/operator/persistent/csv_rejects_table.hpp"
+#include "duckdb/main/client_data.hpp"
 #include <algorithm>
 #include <cctype>
 #include <cstring>
@@ -448,6 +449,17 @@ bool TryCastFloatingVectorCommaSeparated(BufferedCSVReaderOptions &options, Vect
 	}
 }
 
+// Location of erroneous value in the current parse chunk
+struct ErrorLocation {
+	idx_t row_idx;
+	idx_t col_idx;
+	idx_t row_line;
+
+	ErrorLocation(idx_t row_idx, idx_t col_idx, idx_t row_line)
+	    : row_idx(row_idx), col_idx(col_idx), row_line(row_line) {
+	}
+};
+
 bool BaseCSVReader::Flush(DataChunk &insert_chunk, idx_t buffer_idx, bool try_add_line) {
 	if (parse_chunk.size() == 0) {
 		return true;
@@ -506,10 +518,7 @@ bool BaseCSVReader::Flush(DataChunk &insert_chunk, idx_t buffer_idx, bool try_ad
 			if (try_add_line) {
 				return false;
 			}
-			if (options.ignore_errors) {
-				conversion_error_ignored = true;
-				continue;
-			}
+
 			string col_name = to_string(col_idx);
 			if (col_idx < names.size()) {
 				col_name = "\"" + names[col_idx] + "\"";
@@ -527,16 +536,18 @@ bool BaseCSVReader::Flush(DataChunk &insert_chunk, idx_t buffer_idx, bool try_ad
 				}
 			}
 
-			idx_t error_line;
 			// The line_error must be summed with linenr (All lines emmited from this batch)
 			// But subtracted from the parse_chunk
 			D_ASSERT(line_error + linenr >= parse_chunk.size());
 			line_error += linenr;
 			line_error -= parse_chunk.size();
 
-			error_line = GetLineError(line_error, buffer_idx);
+			auto error_line = GetLineError(line_error, buffer_idx);
+
+			if (options.ignore_errors) {
+				conversion_error_ignored = true;
 
-			if (options.auto_detect) {
+			} else if (options.auto_detect) {
 				throw InvalidInputException("%s in column %s, at line %llu.\n\nParser "
 				                            "options:\n%s.\n\nConsider either increasing the sample size "
 				                            "(SAMPLE_SIZE=X [X rows] or SAMPLE_SIZE=-1 [all rows]), "
@@ -550,11 +561,19 @@ bool BaseCSVReader::Flush(DataChunk &insert_chunk, idx_t buffer_idx, bool try_ad
 	}
 	if (conversion_error_ignored) {
 		D_ASSERT(options.ignore_errors);
+
 		SelectionVector succesful_rows(parse_chunk.size());
 		idx_t sel_size = 0;
 
+		// Keep track of failed cells
+		vector<ErrorLocation> failed_cells;
+
 		for (idx_t row_idx = 0; row_idx < parse_chunk.size(); row_idx++) {
-			bool failed = false;
+
+			auto global_row_idx = row_idx + linenr - parse_chunk.size();
+			auto row_line = GetLineError(global_row_idx, buffer_idx, false);
+
+			bool row_failed = false;
 			for (idx_t c = 0; c < reader_data.column_ids.size(); c++) {
 				auto col_idx = reader_data.column_ids[c];
 				auto result_idx = reader_data.column_mapping[c];
@@ -564,14 +583,82 @@ bool BaseCSVReader::Flush(DataChunk &insert_chunk, idx_t buffer_idx, bool try_ad
 
 				bool was_already_null = FlatVector::IsNull(parse_vector, row_idx);
 				if (!was_already_null && FlatVector::IsNull(result_vector, row_idx)) {
-					failed = true;
-					break;
+					row_failed = true;
+					failed_cells.emplace_back(row_idx, col_idx, row_line);
 				}
 			}
-			if (!failed) {
+			if (!row_failed) {
 				succesful_rows.set_index(sel_size++, row_idx);
 			}
 		}
+
+		// Now do a second pass to produce the reject table entries
+		if (!failed_cells.empty() && !options.rejects_table_name.empty()) {
+			auto limit = options.rejects_limit;
+
+			auto rejects = CSVRejectsTable::GetOrCreate(context, options.rejects_table_name);
+			lock_guard<mutex> lock(rejects->write_lock);
+
+			// short circuit if we already have too many rejects
+			if (limit == 0 || rejects->count < limit) {
+				auto &table = rejects->GetTable(context);
+				InternalAppender appender(context, table);
+				auto file_name = GetFileName();
+
+				for (auto &cell : failed_cells) {
+					if (limit != 0 && rejects->count >= limit) {
+						break;
+					}
+					rejects->count++;
+
+					auto row_idx = cell.row_idx;
+					auto col_idx = cell.col_idx;
+					auto row_line = cell.row_line;
+
+					auto col_name = to_string(col_idx);
+					if (col_idx < names.size()) {
+						col_name = "\"" + names[col_idx] + "\"";
+					}
+
+					auto &parse_vector = parse_chunk.data[col_idx];
+					auto parsed_str = FlatVector::GetData<string_t>(parse_vector)[row_idx];
+					auto &type = insert_chunk.data[col_idx].GetType();
+					auto row_error_msg = StringUtil::Format("Could not convert string '%s' to '%s'",
+					                                        parsed_str.GetString(), type.ToString());
+
+					// Add the row to the rejects table
+					appender.BeginRow();
+					appender.Append(string_t(file_name));
+					appender.Append(row_line);
+					appender.Append(col_idx);
+					appender.Append(string_t(col_name));
+					appender.Append(parsed_str);
+
+					if (!options.rejects_recovery_columns.empty()) {
+						child_list_t<Value> recovery_key;
+						for (auto &key_idx : options.rejects_recovery_column_ids) {
+							// Figure out if the recovery key is valid.
+							// If not, error out for real.
+							auto &component_vector = parse_chunk.data[key_idx];
+							if (FlatVector::IsNull(component_vector, row_idx)) {
+								throw InvalidInputException("%s at line %llu in column %s. Parser options:\n%s ",
+								                            "Could not parse recovery column", row_line, col_name,
+								                            options.ToString());
+							}
+							auto component = Value(FlatVector::GetData<string_t>(component_vector)[row_idx]);
+							recovery_key.emplace_back(names[key_idx], component);
+						}
+						appender.Append(Value::STRUCT(recovery_key));
+					}
+
+					appender.Append(string_t(row_error_msg));
+					appender.EndRow();
+				}
+				appender.Close();
+			}
+		}
+
+		// Now slice the insert chunk to only include the succesful rows
 		insert_chunk.Slice(succesful_rows, sel_size);
 	}
 	parse_chunk.Reset();

package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp

@@ -13,7 +13,7 @@ CSVFileHandle::CSVFileHandle(FileSystem &fs, Allocator &allocator, unique_ptr<Fi
 
 unique_ptr<FileHandle> CSVFileHandle::OpenFileHandle(FileSystem &fs, Allocator &allocator, const string &path,
                                                      FileCompressionType compression) {
-	auto file_handle = fs.OpenFile(path.c_str(), FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK, compression);
+	auto file_handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK, compression);
 	if (file_handle->CanSeek()) {
 		file_handle->Reset();
 	}

package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp

@@ -179,6 +179,26 @@ void BufferedCSVReaderOptions::SetReadOption(const string &loption, const Value
 		allow_quoted_nulls = ParseBoolean(value, loption);
 	} else if (loption == "parallel") {
 		parallel_mode = ParseBoolean(value, loption) ? ParallelMode::PARALLEL : ParallelMode::SINGLE_THREADED;
+	} else if (loption == "rejects_table") {
+		// skip, handled in SetRejectsOptions
+		auto table_name = ParseString(value, loption);
+		if (table_name.empty()) {
+			throw BinderException("REJECTS_TABLE option cannot be empty");
+		}
+		rejects_table_name = table_name;
+	} else if (loption == "rejects_recovery_columns") {
+		// Get the list of columns to use as a recovery key
+		auto &children = ListValue::GetChildren(value);
+		for (auto &child : children) {
+			auto col_name = child.GetValue<string>();
+			rejects_recovery_columns.push_back(col_name);
+		}
+	} else if (loption == "rejects_limit") {
+		int64_t limit = ParseInteger(value, loption);
+		if (limit < 0) {
+			throw BinderException("Unsupported parameter for REJECTS_LIMIT: cannot be negative");
+		}
+		rejects_limit = limit;
 	} else {
 		throw BinderException("Unrecognized option for CSV reader \"%s\"", loption);
 	}

package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp

@@ -0,0 +1,48 @@
+#include "duckdb/main/appender.hpp"
+#include "duckdb/parser/parsed_data/create_table_info.hpp"
+#include "duckdb/function/table/read_csv.hpp"
+#include "duckdb/execution/operator/persistent/csv_rejects_table.hpp"
+#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
+
+namespace duckdb {
+
+TableCatalogEntry &CSVRejectsTable::GetTable(ClientContext &context) {
+	auto &temp_catalog = Catalog::GetCatalog(context, TEMP_CATALOG);
+	auto &table_entry = temp_catalog.GetEntry<TableCatalogEntry>(context, TEMP_CATALOG, DEFAULT_SCHEMA, name);
+	return table_entry;
+}
+
+shared_ptr<CSVRejectsTable> CSVRejectsTable::GetOrCreate(ClientContext &context, const string &name) {
+	auto key = "CSV_REJECTS_TABLE_CACHE_ENTRY_" + StringUtil::Upper(name);
+	auto &cache = ObjectCache::GetObjectCache(context);
+	return cache.GetOrCreate<CSVRejectsTable>(key, name);
+}
+
+void CSVRejectsTable::InitializeTable(ClientContext &context, const ReadCSVData &data) {
+	// (Re)Create the temporary rejects table
+	auto &catalog = Catalog::GetCatalog(context, TEMP_CATALOG);
+	auto info = make_uniq<CreateTableInfo>(TEMP_CATALOG, DEFAULT_SCHEMA, name);
+	info->temporary = true;
+	info->on_conflict = OnCreateConflict::ERROR_ON_CONFLICT;
+	info->columns.AddColumn(ColumnDefinition("file", LogicalType::VARCHAR));
+	info->columns.AddColumn(ColumnDefinition("line", LogicalType::BIGINT));
+	info->columns.AddColumn(ColumnDefinition("column", LogicalType::BIGINT));
+	info->columns.AddColumn(ColumnDefinition("column_name", LogicalType::VARCHAR));
+	info->columns.AddColumn(ColumnDefinition("parsed_value", LogicalType::VARCHAR));
+
+	if (!data.options.rejects_recovery_columns.empty()) {
+		child_list_t<LogicalType> recovery_key_components;
+		for (auto &col_name : data.options.rejects_recovery_columns) {
+			recovery_key_components.emplace_back(col_name, LogicalType::VARCHAR);
+		}
+		info->columns.AddColumn(ColumnDefinition("recovery_columns", LogicalType::STRUCT(recovery_key_components)));
+	}
+
+	info->columns.AddColumn(ColumnDefinition("error", LogicalType::VARCHAR));
+
+	catalog.CreateTable(context, std::move(info));
+
+	count = 0;
+}
+
+} // namespace duckdb

package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp

@@ -635,12 +635,11 @@ void ParallelCSVReader::ParseCSV(DataChunk &insert_chunk) {
 	}
 }
 
-idx_t ParallelCSVReader::GetLineError(idx_t line_error, idx_t buffer_idx) {
+idx_t ParallelCSVReader::GetLineError(idx_t line_error, idx_t buffer_idx, bool stop_at_first) {
 	while (true) {
 		if (buffer->line_info->CanItGetLine(file_idx, buffer_idx)) {
 			auto cur_start = verification_positions.beginning_of_first_line + buffer->buffer->GetCSVGlobalStart();
-			// line errors are 1-indexed
-			return buffer->line_info->GetLine(buffer_idx, line_error, file_idx, cur_start, false);
+			return buffer->line_info->GetLine(buffer_idx, line_error, file_idx, cur_start, false, stop_at_first);
 		}
 	}
 }