duckdb 0.8.1-dev96.0 → 0.8.2-dev1.0

package/src/duckdb/extension/json/json_scan.cpp

@@ -244,14 +244,13 @@ unique_ptr<GlobalTableFunctionState> JSONGlobalTableFunctionState::Init(ClientCo
 
 idx_t JSONGlobalTableFunctionState::MaxThreads() const {
 	auto &bind_data = state.bind_data;
-	if (bind_data.options.format == JSONFormat::NEWLINE_DELIMITED &&
-	    bind_data.options.compression == FileCompressionType::UNCOMPRESSED) {
+	if (bind_data.options.format == JSONFormat::NEWLINE_DELIMITED) {
 		return state.system_threads;
 	}
 
 	if (!state.json_readers.empty() && state.json_readers[0]->IsOpen()) {
 		auto &reader = *state.json_readers[0];
-		if (reader.IsParallel()) { // Auto-detected parallel scan
+		if (reader.GetFormat() == JSONFormat::NEWLINE_DELIMITED) { // Auto-detected NDJSON
 			return state.system_threads;
 		}
 	}
@@ -298,6 +297,7 @@ idx_t JSONScanLocalState::ReadNext(JSONScanGlobalState &gstate) {
 		if (!ReadNextBuffer(gstate)) {
 			return scan_count;
 		}
+		D_ASSERT(buffer_size != 0);
 		if (current_buffer_handle->buffer_index != 0 && current_reader->GetFormat() == JSONFormat::NEWLINE_DELIMITED) {
 			ReconstructFirstObject(gstate);
 			scan_count++;
@@ -308,8 +308,8 @@ idx_t JSONScanLocalState::ReadNext(JSONScanGlobalState &gstate) {
 	return scan_count;
 }
 
-static inline const char *NextNewline(const char *ptr, idx_t size) {
-	return (const char *)memchr(ptr, '\n', size);
+static inline const char *NextNewline(char *ptr, idx_t size) {
+	return char_ptr_cast(memchr(ptr, '\n', size));
 }
 
 static inline const char *PreviousNewline(const char *ptr) {
@@ -455,7 +455,21 @@ void JSONScanLocalState::ThrowInvalidAtEndError() {
 	throw InvalidInputException("Invalid JSON detected at the end of file \"%s\".", current_reader->GetFileName());
 }
 
-static pair<JSONFormat, JSONRecordType> DetectFormatAndRecordType(const char *const buffer_ptr, const idx_t buffer_size,
+bool JSONScanLocalState::IsParallel(JSONScanGlobalState &gstate) const {
+	if (bind_data.files.size() >= gstate.system_threads) {
+		// More files than threads, just parallelize over the files
+		return false;
+	}
+
+	if (current_reader->GetFormat() == JSONFormat::NEWLINE_DELIMITED) {
+		// NDJSON can be read in parallel
+		return true;
+	}
+
+	return false;
+}
+
+static pair<JSONFormat, JSONRecordType> DetectFormatAndRecordType(char *const buffer_ptr, const idx_t buffer_size,
                                                                   yyjson_alc *alc) {
 	// First we do the easy check whether it's NEWLINE_DELIMITED
 	auto line_end = NextNewline(buffer_ptr, buffer_size);
@@ -464,7 +478,7 @@ static pair<JSONFormat, JSONRecordType> DetectFormatAndRecordType(const char *co
 		SkipWhitespace(buffer_ptr, line_size, buffer_size);
 
 		yyjson_read_err error;
-		auto doc = JSONCommon::ReadDocumentUnsafe((char *)buffer_ptr, line_size, JSONCommon::READ_FLAG, alc, &error);
+		auto doc = JSONCommon::ReadDocumentUnsafe(buffer_ptr, line_size, JSONCommon::READ_FLAG, alc, &error);
 		if (error.code == YYJSON_READ_SUCCESS) { // We successfully read the line
 			if (yyjson_is_arr(doc->root) && line_size == buffer_size) {
 				// It's just one array, let's actually assume ARRAY, not NEWLINE_DELIMITED
@@ -500,8 +514,8 @@ static pair<JSONFormat, JSONRecordType> DetectFormatAndRecordType(const char *co
 
 	// It's definitely an ARRAY, but now we have to figure out if there's more than one top-level array
 	yyjson_read_err error;
-	auto doc = JSONCommon::ReadDocumentUnsafe((char *)buffer_ptr + buffer_offset, remaining, JSONCommon::READ_STOP_FLAG,
-	                                          alc, &error);
+	auto doc =
+	    JSONCommon::ReadDocumentUnsafe(buffer_ptr + buffer_offset, remaining, JSONCommon::READ_STOP_FLAG, alc, &error);
 	if (error.code == YYJSON_READ_SUCCESS) {
 		D_ASSERT(yyjson_is_arr(doc->root));
 
@@ -563,7 +577,7 @@ bool JSONScanLocalState::ReadNextBuffer(JSONScanGlobalState &gstate) {
 	} else {
 		buffer = gstate.allocator.Allocate(gstate.buffer_capacity);
 	}
-	buffer_ptr = (const char *)buffer.get();
+	buffer_ptr = char_ptr_cast(buffer.get());
 
 	idx_t buffer_index;
 	while (true) {
@@ -573,7 +587,7 @@ bool JSONScanLocalState::ReadNextBuffer(JSONScanGlobalState &gstate) {
 				if (is_last && gstate.bind_data.type != JSONScanType::SAMPLE) {
 					current_reader->CloseJSONFile();
 				}
-				if (current_reader->IsParallel()) {
+				if (IsParallel(gstate)) {
 					// If this threads' current reader is still the one at gstate.file_index,
 					// this thread can end the parallel scan
 					lock_guard<mutex> guard(gstate.lock);
@@ -599,7 +613,7 @@ bool JSONScanLocalState::ReadNextBuffer(JSONScanGlobalState &gstate) {
 			current_reader = gstate.json_readers[gstate.file_index].get();
 			if (current_reader->IsOpen()) {
 				// Can only be open from auto detection, so these should be known
-				if (!current_reader->IsParallel()) {
+				if (!IsParallel(gstate)) {
 					batch_index = gstate.batch_index++;
 					gstate.file_index++;
 				}
@@ -609,15 +623,15 @@ bool JSONScanLocalState::ReadNextBuffer(JSONScanGlobalState &gstate) {
 			current_reader->OpenJSONFile();
 			batch_index = gstate.batch_index++;
 			if (current_reader->GetFormat() != JSONFormat::AUTO_DETECT) {
-				if (!current_reader->IsParallel()) {
+				if (!IsParallel(gstate)) {
 					gstate.file_index++;
 				}
 				continue;
 			}
 
-			// If we have a low amount of files, we auto-detect within the lock,
+			// If we have less files than threads, we auto-detect within the lock,
 			// so other threads may join a parallel NDJSON scan
-			if (gstate.json_readers.size() < 100) {
+			if (gstate.json_readers.size() < gstate.system_threads) {
 				if (ReadAndAutoDetect(gstate, buffer_index, false)) {
 					continue;
 				}
@@ -637,7 +651,7 @@ bool JSONScanLocalState::ReadNextBuffer(JSONScanGlobalState &gstate) {
 	D_ASSERT(buffer_size != 0); // We should have read something if we got here
 
 	idx_t readers = 1;
-	if (current_reader->IsParallel()) {
+	if (current_reader->GetFormat() == JSONFormat::NEWLINE_DELIMITED) {
 		readers = is_last ? 1 : 2;
 	}
 
@@ -650,7 +664,7 @@ bool JSONScanLocalState::ReadNextBuffer(JSONScanGlobalState &gstate) {
 	lines_or_objects_in_buffer = 0;
 
 	// YYJSON needs this
-	memset((void *)(buffer_ptr + buffer_size), 0, YYJSON_PADDING_SIZE);
+	memset(buffer_ptr + buffer_size, 0, YYJSON_PADDING_SIZE);
 
 	return true;
 }
@@ -680,7 +694,7 @@ bool JSONScanLocalState::ReadAndAutoDetect(JSONScanGlobalState &gstate, idx_t &b
 		throw InvalidInputException("Expected file \"%s\" to contain records, detected non-record JSON instead.",
 		                            current_reader->GetFileName());
 	}
-	if (!already_incremented_file_idx && !current_reader->IsParallel()) {
+	if (!already_incremented_file_idx && !IsParallel(gstate)) {
 		gstate.file_index++;
 	}
 	return false;
@@ -739,13 +753,14 @@ void JSONScanLocalState::ReadNextBufferNoSeek(JSONScanGlobalState &gstate, idx_t
 		lock_guard<mutex> reader_guard(current_reader->lock);
 		buffer_index = current_reader->GetBufferIndex();
 
-		if (current_reader->IsOpen()) {
+		if (current_reader->IsOpen() && !current_reader->IsDone()) {
 			read_size = current_reader->GetFileHandle().Read(buffer_ptr + prev_buffer_remainder, request_size,
 			                                                 gstate.bind_data.type == JSONScanType::SAMPLE);
+			is_last = read_size < request_size;
 		} else {
 			read_size = 0;
+			is_last = false;
 		}
-		is_last = read_size < request_size;
 
 		if (!gstate.bind_data.ignore_errors && read_size == 0 && prev_buffer_remainder != 0) {
 			ThrowInvalidAtEndError();
@@ -796,13 +811,13 @@ void JSONScanLocalState::ReconstructFirstObject(JSONScanGlobalState &gstate) {
 	D_ASSERT(current_reader->GetFormat() == JSONFormat::NEWLINE_DELIMITED);
 
 	// Spinlock until the previous batch index has also read its buffer
-	JSONBufferHandle *previous_buffer_handle = nullptr;
+	optional_ptr<JSONBufferHandle> previous_buffer_handle;
 	while (!previous_buffer_handle) {
 		previous_buffer_handle = current_reader->GetBuffer(current_buffer_handle->buffer_index - 1);
 	}
 
 	// First we find the newline in the previous block
-	auto prev_buffer_ptr = (const char *)previous_buffer_handle->buffer.get() + previous_buffer_handle->buffer_size;
+	auto prev_buffer_ptr = char_ptr_cast(previous_buffer_handle->buffer.get()) + previous_buffer_handle->buffer_size;
 	auto part1_ptr = PreviousNewline(prev_buffer_ptr);
 	auto part1_size = prev_buffer_ptr - part1_ptr;
 
@@ -825,7 +840,7 @@ void JSONScanLocalState::ReconstructFirstObject(JSONScanGlobalState &gstate) {
 
 	// And copy the remainder of the line to the reconstruct buffer
 	memcpy(reconstruct_ptr + part1_size, buffer_ptr, part2_size);
-	memset((void *)(reconstruct_ptr + line_size), 0, YYJSON_PADDING_SIZE);
+	memset(reconstruct_ptr + line_size, 0, YYJSON_PADDING_SIZE);
 	buffer_offset += part2_size;
 
 	// We copied the object, so we are no longer reading the previous buffer
@@ -833,7 +848,7 @@ void JSONScanLocalState::ReconstructFirstObject(JSONScanGlobalState &gstate) {
 		current_reader->RemoveBuffer(current_buffer_handle->buffer_index - 1);
 	}
 
-	ParseJSON((char *)reconstruct_ptr, line_size, line_size);
+	ParseJSON(char_ptr_cast(reconstruct_ptr), line_size, line_size);
 }
 
 void JSONScanLocalState::ParseNextChunk() {
@@ -867,7 +882,7 @@ void JSONScanLocalState::ParseNextChunk() {
 		}
 
 		idx_t json_size = json_end - json_start;
-		ParseJSON((char *)json_start, json_size, remaining);
+		ParseJSON(json_start, json_size, remaining);
 		buffer_offset += json_size;
 
 		if (format == JSONFormat::ARRAY) {
@@ -947,9 +962,10 @@ void JSONScan::Serialize(FieldWriter &writer, const FunctionData *bind_data_p, c
 	bind_data.Serialize(writer);
 }
 
-unique_ptr<FunctionData> JSONScan::Deserialize(ClientContext &context, FieldReader &reader, TableFunction &function) {
+unique_ptr<FunctionData> JSONScan::Deserialize(PlanDeserializationState &state, FieldReader &reader,
+                                               TableFunction &function) {
 	auto result = make_uniq<JSONScanData>();
-	result->Deserialize(context, reader);
+	result->Deserialize(state.context, reader);
 	return std::move(result);
 }
 

package/src/duckdb/extension/parquet/column_reader.cpp

@@ -95,7 +95,11 @@ const uint64_t ParquetDecodeUtils::BITPACK_MASKS[] = {0,
                                                       576460752303423487,
                                                       1152921504606846975,
                                                       2305843009213693951,
-                                                      4611686018427387903};
+                                                      4611686018427387903,
+                                                      9223372036854775807,
+                                                      18446744073709551615ULL};
+
+const uint64_t ParquetDecodeUtils::BITPACK_MASKS_SIZE = sizeof(ParquetDecodeUtils::BITPACK_MASKS) / sizeof(uint64_t);
 
 const uint8_t ParquetDecodeUtils::BITPACK_DLEN = 8;
 

package/src/duckdb/extension/parquet/include/decode_utils.hpp

@@ -12,10 +12,16 @@ public:
 	}
 
 	static const uint64_t BITPACK_MASKS[];
+	static const uint64_t BITPACK_MASKS_SIZE;
 	static const uint8_t BITPACK_DLEN;
 
 	template <typename T>
 	static uint32_t BitUnpack(ByteBuffer &buffer, uint8_t &bitpack_pos, T *dest, uint32_t count, uint8_t width) {
+		if (width >= ParquetDecodeUtils::BITPACK_MASKS_SIZE) {
+			throw InvalidInputException("The width (%d) of the bitpacked data exceeds the supported max width (%d), "
+			                            "the file might be corrupted.",
+			                            width, ParquetDecodeUtils::BITPACK_MASKS_SIZE);
+		}
 		auto mask = BITPACK_MASKS[width];
 
 		for (uint32_t i = 0; i < count; i++) {

package/src/duckdb/extension/parquet/parquet-extension.cpp

@@ -411,8 +411,9 @@ public:
 		bind_data.parquet_options.Serialize(writer);
 	}
 
-	static unique_ptr<FunctionData> ParquetScanDeserialize(ClientContext &context, FieldReader &reader,
+	static unique_ptr<FunctionData> ParquetScanDeserialize(PlanDeserializationState &state, FieldReader &reader,
 	                                                       TableFunction &function) {
+		auto &context = state.context;
 		auto files = reader.ReadRequiredList<string>();
 		auto types = reader.ReadRequiredSerializableList<LogicalType, LogicalType>();
 		auto names = reader.ReadRequiredList<string>();
@@ -665,6 +666,28 @@ unique_ptr<LocalFunctionData> ParquetWriteInitializeLocal(ExecutionContext &cont
 	return make_uniq<ParquetWriteLocalState>(context.client, bind_data.sql_types);
 }
 
+// LCOV_EXCL_START
+static void ParquetCopySerialize(FieldWriter &writer, const FunctionData &bind_data_p, const CopyFunction &function) {
+	auto &bind_data = bind_data_p.Cast<ParquetWriteBindData>();
+	writer.WriteRegularSerializableList<LogicalType>(bind_data.sql_types);
+	writer.WriteList<string>(bind_data.column_names);
+	writer.WriteField<duckdb_parquet::format::CompressionCodec::type>(bind_data.codec);
+	writer.WriteField<idx_t>(bind_data.row_group_size);
+}
+
+static unique_ptr<FunctionData> ParquetCopyDeserialize(ClientContext &context, FieldReader &reader,
+                                                       CopyFunction &function) {
+	unique_ptr<ParquetWriteBindData> data = make_uniq<ParquetWriteBindData>();
+
+	data->sql_types = reader.ReadRequiredSerializableList<LogicalType, LogicalType>();
+	data->column_names = reader.ReadRequiredList<string>();
+	data->codec = reader.ReadRequired<duckdb_parquet::format::CompressionCodec::type>();
+	data->row_group_size = reader.ReadRequired<idx_t>();
+
+	return std::move(data);
+}
+// LCOV_EXCL_STOP
+
 //===--------------------------------------------------------------------===//
 // Execution Mode
 //===--------------------------------------------------------------------===//
@@ -764,6 +787,8 @@ void ParquetExtension::Load(DuckDB &db) {
 	function.prepare_batch = ParquetWritePrepareBatch;
 	function.flush_batch = ParquetWriteFlushBatch;
 	function.desired_batch_size = ParquetWriteDesiredBatchSize;
+	function.serialize = ParquetCopySerialize;
+	function.deserialize = ParquetCopyDeserialize;
 
 	function.extension = "parquet";
 	ExtensionUtil::RegisterFunction(db_instance, function);

package/src/duckdb/src/catalog/catalog.cpp

@@ -674,31 +674,19 @@ vector<reference<SchemaCatalogEntry>> Catalog::GetSchemas(ClientContext &context
 	return schemas;
 }
 
-bool Catalog::TypeExists(ClientContext &context, const string &catalog_name, const string &schema, const string &name) {
-	optional_ptr<CatalogEntry> entry;
-	entry = GetEntry(context, CatalogType::TYPE_ENTRY, catalog_name, schema, name, OnEntryNotFound::RETURN_NULL);
-	if (!entry) {
-		// look in the system catalog
-		entry = GetEntry(context, CatalogType::TYPE_ENTRY, SYSTEM_CATALOG, schema, name, OnEntryNotFound::RETURN_NULL);
-		if (!entry) {
-			return false;
-		}
-	}
-	return true;
-}
-
 vector<reference<SchemaCatalogEntry>> Catalog::GetSchemas(ClientContext &context, const string &catalog_name) {
 	vector<reference<Catalog>> catalogs;
 	if (IsInvalidCatalog(catalog_name)) {
-		unordered_set<string> name;
+		reference_set_t<Catalog> inserted_catalogs;
 
 		auto &search_path = *context.client_data->catalog_search_path;
 		for (auto &entry : search_path.Get()) {
-			if (name.find(entry.catalog) != name.end()) {
+			auto &catalog = Catalog::GetCatalog(context, entry.catalog);
+			if (inserted_catalogs.find(catalog) != inserted_catalogs.end()) {
 				continue;
 			}
-			name.insert(entry.catalog);
-			catalogs.push_back(Catalog::GetCatalog(context, entry.catalog));
+			inserted_catalogs.insert(catalog);
+			catalogs.push_back(catalog);
 		}
 	} else {
 		catalogs.push_back(Catalog::GetCatalog(context, catalog_name));

package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp

@@ -16,6 +16,9 @@
 #include "duckdb/parser/constraints/list.hpp"
 #include "duckdb/function/table/table_scan.hpp"
 #include "duckdb/storage/table_storage_info.hpp"
+#include "duckdb/planner/operator/logical_get.hpp"
+#include "duckdb/planner/operator/logical_projection.hpp"
+#include "duckdb/planner/operator/logical_update.hpp"
 
 namespace duckdb {
 
@@ -714,10 +717,13 @@ TableFunction DuckTableEntry::GetScanFunction(ClientContext &context, unique_ptr
 	return TableScanFunction::GetFunction();
 }
 
+vector<ColumnSegmentInfo> DuckTableEntry::GetColumnSegmentInfo() {
+	return storage->GetColumnSegmentInfo();
+}
+
 TableStorageInfo DuckTableEntry::GetStorageInfo(ClientContext &context) {
 	TableStorageInfo result;
 	result.cardinality = storage->info->cardinality.load();
-	storage->GetStorageInfo(result);
 	storage->info->indexes.Scan([&](Index &index) {
 		IndexInfo info;
 		info.is_primary = index.IsPrimary();

package/src/duckdb/src/catalog/catalog_entry/table_catalog_entry.cpp

@@ -9,6 +9,11 @@
 #include "duckdb/main/database.hpp"
 #include "duckdb/parser/constraints/list.hpp"
 #include "duckdb/parser/parsed_data/create_table_info.hpp"
+#include "duckdb/storage/table_storage_info.hpp"
+#include "duckdb/planner/operator/logical_update.hpp"
+#include "duckdb/planner/operator/logical_get.hpp"
+#include "duckdb/planner/constraints/bound_check_constraint.hpp"
+#include "duckdb/planner/operator/logical_projection.hpp"
 
 #include <sstream>
 
@@ -214,6 +219,122 @@ DataTable &TableCatalogEntry::GetStorage() {
 const vector<unique_ptr<BoundConstraint>> &TableCatalogEntry::GetBoundConstraints() {
 	throw InternalException("Calling GetBoundConstraints on a TableCatalogEntry that is not a DuckTableEntry");
 }
+
 // LCOV_EXCL_STOP
 
+static void BindExtraColumns(TableCatalogEntry &table, LogicalGet &get, LogicalProjection &proj, LogicalUpdate &update,
+                             physical_index_set_t &bound_columns) {
+	if (bound_columns.size() <= 1) {
+		return;
+	}
+	idx_t found_column_count = 0;
+	physical_index_set_t found_columns;
+	for (idx_t i = 0; i < update.columns.size(); i++) {
+		if (bound_columns.find(update.columns[i]) != bound_columns.end()) {
+			// this column is referenced in the CHECK constraint
+			found_column_count++;
+			found_columns.insert(update.columns[i]);
+		}
+	}
+	if (found_column_count > 0 && found_column_count != bound_columns.size()) {
+		// columns in this CHECK constraint were referenced, but not all were part of the UPDATE
+		// add them to the scan and update set
+		for (auto &check_column_id : bound_columns) {
+			if (found_columns.find(check_column_id) != found_columns.end()) {
+				// column is already projected
+				continue;
+			}
+			// column is not projected yet: project it by adding the clause "i=i" to the set of updated columns
+			auto &column = table.GetColumns().GetColumn(check_column_id);
+			update.expressions.push_back(make_uniq<BoundColumnRefExpression>(
+			    column.Type(), ColumnBinding(proj.table_index, proj.expressions.size())));
+			proj.expressions.push_back(make_uniq<BoundColumnRefExpression>(
+			    column.Type(), ColumnBinding(get.table_index, get.column_ids.size())));
+			get.column_ids.push_back(check_column_id.index);
+			update.columns.push_back(check_column_id);
+		}
+	}
+}
+
+static bool TypeSupportsRegularUpdate(const LogicalType &type) {
+	switch (type.id()) {
+	case LogicalTypeId::LIST:
+	case LogicalTypeId::MAP:
+	case LogicalTypeId::UNION:
+		// lists and maps and unions don't support updates directly
+		return false;
+	case LogicalTypeId::STRUCT: {
+		auto &child_types = StructType::GetChildTypes(type);
+		for (auto &entry : child_types) {
+			if (!TypeSupportsRegularUpdate(entry.second)) {
+				return false;
+			}
+		}
+		return true;
+	}
+	default:
+		return true;
+	}
+}
+
+vector<ColumnSegmentInfo> TableCatalogEntry::GetColumnSegmentInfo() {
+	return {};
+}
+
+void TableCatalogEntry::BindUpdateConstraints(LogicalGet &get, LogicalProjection &proj, LogicalUpdate &update,
+                                              ClientContext &context) {
+	// check the constraints and indexes of the table to see if we need to project any additional columns
+	// we do this for indexes with multiple columns and CHECK constraints in the UPDATE clause
+	// suppose we have a constraint CHECK(i + j < 10); now we need both i and j to check the constraint
+	// if we are only updating one of the two columns we add the other one to the UPDATE set
+	// with a "useless" update (i.e. i=i) so we can verify that the CHECK constraint is not violated
+	for (auto &constraint : GetBoundConstraints()) {
+		if (constraint->type == ConstraintType::CHECK) {
+			auto &check = constraint->Cast<BoundCheckConstraint>();
+			// check constraint! check if we need to add any extra columns to the UPDATE clause
+			BindExtraColumns(*this, get, proj, update, check.bound_columns);
+		}
+	}
+	if (update.return_chunk) {
+		physical_index_set_t all_columns;
+		for (auto &column : GetColumns().Physical()) {
+			all_columns.insert(column.Physical());
+		}
+		BindExtraColumns(*this, get, proj, update, all_columns);
+	}
+	// for index updates we always turn any update into an insert and a delete
+	// we thus need all the columns to be available, hence we check if the update touches any index columns
+	// If the returning keyword is used, we need access to the whole row in case the user requests it.
+	// Therefore switch the update to a delete and insert.
+	update.update_is_del_and_insert = false;
+	TableStorageInfo table_storage_info = GetStorageInfo(context);
+	for (auto index : table_storage_info.index_info) {
+		for (auto &column : update.columns) {
+			if (index.column_set.find(column.index) != index.column_set.end()) {
+				update.update_is_del_and_insert = true;
+				break;
+			}
+		}
+	};
+
+	// we also convert any updates on LIST columns into delete + insert
+	for (auto &col_index : update.columns) {
+		auto &column = GetColumns().GetColumn(col_index);
+		if (!TypeSupportsRegularUpdate(column.Type())) {
+			update.update_is_del_and_insert = true;
+			break;
+		}
+	}
+
+	if (update.update_is_del_and_insert) {
+		// the update updates a column required by an index or requires returning the updated rows,
+		// push projections for all columns
+		physical_index_set_t all_columns;
+		for (auto &column : GetColumns().Physical()) {
+			all_columns.insert(column.Physical());
+		}
+		BindExtraColumns(*this, get, proj, update, all_columns);
+	}
+}
+
 } // namespace duckdb

package/src/duckdb/src/catalog/catalog_search_path.cpp

@@ -5,6 +5,7 @@
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/catalog/catalog.hpp"
+#include "duckdb/main/database_manager.hpp"
 
 namespace duckdb {
 
@@ -126,37 +127,57 @@ void CatalogSearchPath::Reset() {
 	SetPaths(empty);
 }
 
-void CatalogSearchPath::Set(vector<CatalogSearchEntry> new_paths, bool is_set_schema) {
-	if (is_set_schema && new_paths.size() != 1) {
-		throw CatalogException("SET schema can set only 1 schema. This has %d", new_paths.size());
+string CatalogSearchPath::GetSetName(CatalogSetPathType set_type) {
+	switch (set_type) {
+	case CatalogSetPathType::SET_SCHEMA:
+		return "SET schema";
+	case CatalogSetPathType::SET_SCHEMAS:
+		return "SET search_path";
+	default:
+		throw InternalException("Unrecognized CatalogSetPathType");
+	}
+}
+
+void CatalogSearchPath::Set(vector<CatalogSearchEntry> new_paths, CatalogSetPathType set_type) {
+	if (set_type != CatalogSetPathType::SET_SCHEMAS && new_paths.size() != 1) {
+		throw CatalogException("%s can set only 1 schema. This has %d", GetSetName(set_type), new_paths.size());
 	}
 	for (auto &path : new_paths) {
-		if (!Catalog::GetSchema(context, path.catalog, path.schema, OnEntryNotFound::RETURN_NULL)) {
+		auto schema_entry = Catalog::GetSchema(context, path.catalog, path.schema, OnEntryNotFound::RETURN_NULL);
+		if (schema_entry) {
+			// we are setting a schema - update the catalog and schema
 			if (path.catalog.empty()) {
-				// only schema supplied - check if this is a database instead
-				auto schema = Catalog::GetSchema(context, path.schema, DEFAULT_SCHEMA, OnEntryNotFound::RETURN_NULL);
+				path.catalog = GetDefault().catalog;
+			}
+			continue;
+		}
+		// only schema supplied - check if this is a catalog instead
+		if (path.catalog.empty()) {
+			auto catalog = Catalog::GetCatalogEntry(context, path.schema);
+			if (catalog) {
+				auto schema = catalog->GetSchema(context, DEFAULT_SCHEMA, OnEntryNotFound::RETURN_NULL);
 				if (schema) {
 					path.catalog = std::move(path.schema);
 					path.schema = schema->name;
 					continue;
 				}
 			}
-			throw CatalogException("SET %s: No catalog + schema named %s found.",
-			                       is_set_schema ? "schema" : "search_path", path.ToString());
 		}
+		throw CatalogException("%s: No catalog + schema named \"%s\" found.", GetSetName(set_type), path.ToString());
 	}
-	if (is_set_schema) {
+	if (set_type == CatalogSetPathType::SET_SCHEMA) {
 		if (new_paths[0].catalog == TEMP_CATALOG || new_paths[0].catalog == SYSTEM_CATALOG) {
-			throw CatalogException("SET schema cannot be set to internal schema \"%s\"", new_paths[0].catalog);
+			throw CatalogException("%s cannot be set to internal schema \"%s\"", GetSetName(set_type),
+			                       new_paths[0].catalog);
 		}
 	}
 	this->set_paths = std::move(new_paths);
 	SetPaths(set_paths);
 }
 
-void CatalogSearchPath::Set(CatalogSearchEntry new_value, bool is_set_schema) {
+void CatalogSearchPath::Set(CatalogSearchEntry new_value, CatalogSetPathType set_type) {
 	vector<CatalogSearchEntry> new_paths {std::move(new_value)};
-	Set(std::move(new_paths), is_set_schema);
+	Set(std::move(new_paths), set_type);
 }
 
 const vector<CatalogSearchEntry> &CatalogSearchPath::Get() {
@@ -225,4 +246,20 @@ void CatalogSearchPath::SetPaths(vector<CatalogSearchEntry> new_paths) {
 	paths.emplace_back(SYSTEM_CATALOG, "pg_catalog");
 }
 
+bool CatalogSearchPath::SchemaInSearchPath(ClientContext &context, const string &catalog_name,
+                                           const string &schema_name) {
+	for (auto &path : paths) {
+		if (path.schema != schema_name) {
+			continue;
+		}
+		if (path.catalog == catalog_name) {
+			return true;
+		}
+		if (IsInvalidCatalog(path.catalog) && catalog_name == DatabaseManager::GetDefaultDatabase(context)) {
+			return true;
+		}
+	}
+	return false;
+}
+
 } // namespace duckdb