duckdb 1.4.2-dev4.0 → 1.4.3-dev0.0

package/src/duckdb/src/common/enums/compression_type.cpp

@@ -17,25 +17,60 @@ vector<string> ListCompressionTypes(void) {
 	return compression_types;
 }
 
-bool CompressionTypeIsDeprecated(CompressionType compression_type, optional_ptr<StorageManager> storage_manager) {
-	vector<CompressionType> types({CompressionType::COMPRESSION_PATAS, CompressionType::COMPRESSION_CHIMP});
-	if (storage_manager) {
-		if (storage_manager->GetStorageVersion() >= 5) {
-			//! NOTE: storage_manager is an optional_ptr because it's called from ForceCompressionSetting, which doesn't
-			//! have guaranteed access to a StorageManager The introduction of DICT_FSST deprecates Dictionary and FSST
-			//! compression methods
-			types.emplace_back(CompressionType::COMPRESSION_DICTIONARY);
-			types.emplace_back(CompressionType::COMPRESSION_FSST);
-		} else {
-			types.emplace_back(CompressionType::COMPRESSION_DICT_FSST);
-		}
+namespace {
+struct CompressionMethodRequirements {
+	CompressionType type;
+	optional_idx minimum_storage_version;
+	optional_idx maximum_storage_version;
+};
+} // namespace
+
+CompressionAvailabilityResult CompressionTypeIsAvailable(CompressionType compression_type,
+                                                         optional_ptr<StorageManager> storage_manager) {
+	//! Max storage compatibility
+	vector<CompressionMethodRequirements> candidates({{CompressionType::COMPRESSION_PATAS, optional_idx(), 0},
+	                                                  {CompressionType::COMPRESSION_CHIMP, optional_idx(), 0},
+	                                                  {CompressionType::COMPRESSION_DICTIONARY, 0, 4},
+	                                                  {CompressionType::COMPRESSION_FSST, 0, 4},
+	                                                  {CompressionType::COMPRESSION_DICT_FSST, 5, optional_idx()}});
+
+	optional_idx current_storage_version;
+	if (storage_manager && storage_manager->HasStorageVersion()) {
+		current_storage_version = storage_manager->GetStorageVersion();
 	}
-	for (auto &type : types) {
-		if (type == compression_type) {
-			return true;
+	for (auto &candidate : candidates) {
+		auto &type = candidate.type;
+		if (type != compression_type) {
+			continue;
+		}
+		auto &min = candidate.minimum_storage_version;
+		auto &max = candidate.maximum_storage_version;
+
+		if (!min.IsValid()) {
+			//! Used to signal: always deprecated
+			return CompressionAvailabilityResult::Deprecated();
+		}
+
+		if (!current_storage_version.IsValid()) {
+			//! Can't determine in this call whether it's available or not, default to available
+			return CompressionAvailabilityResult();
+		}
+
+		auto current_version = current_storage_version.GetIndex();
+		D_ASSERT(min.IsValid());
+		if (min.GetIndex() > current_version) {
+			//! Minimum required storage version is higher than the current storage version, this method isn't available
+			//! yet
+			return CompressionAvailabilityResult::NotAvailableYet();
+		}
+		if (max.IsValid() && max.GetIndex() < current_version) {
+			//! Maximum supported storage version is lower than the current storage version, this method is no longer
+			//! available
+			return CompressionAvailabilityResult::Deprecated();
 		}
+		return CompressionAvailabilityResult();
 	}
-	return false;
+	return CompressionAvailabilityResult();
 }
 
 CompressionType CompressionTypeFromString(const string &str) {

package/src/duckdb/src/common/exception/binder_exception.cpp

@@ -18,9 +18,14 @@ BinderException BinderException::ColumnNotFound(const string &name, const vector
 	extra_info["name"] = name;
 	if (!similar_bindings.empty()) {
 		extra_info["candidates"] = StringUtil::Join(similar_bindings, ",");
+		return BinderException(
+		    StringUtil::Format("Referenced column \"%s\" not found in FROM clause!%s", name, candidate_str),
+		    extra_info);
+	} else {
+		return BinderException(
+		    StringUtil::Format("Referenced column \"%s\" was not found because the FROM clause is missing", name),
+		    extra_info);
 	}
-	return BinderException(
-	    StringUtil::Format("Referenced column \"%s\" not found in FROM clause!%s", name, candidate_str), extra_info);
 }
 
 BinderException BinderException::NoMatchingFunction(const string &catalog_name, const string &schema_name,

package/src/duckdb/src/common/progress_bar/unscented_kalman_filter.cpp

@@ -254,11 +254,11 @@ void UnscentedKalmanFilter::UpdateInternal(double measured_progress) {
 	}
 
 	// Ensure progress stays in bounds
-	x[0] = std::max(0.0, std::min(1.0, x[0]));
+	x[0] = std::max(0.0, std::min(scale_factor, x[0]));
 }
 
 double UnscentedKalmanFilter::GetProgress() const {
-	return x[0];
+	return x[0] / scale_factor;
 }
 
 double UnscentedKalmanFilter::GetVelocity() const {

package/src/duckdb/src/common/random_engine.cpp

@@ -82,4 +82,14 @@ void RandomEngine::SetSeed(uint64_t seed) {
 	random_state->pcg.seed(seed);
 }
 
+void RandomEngine::RandomData(duckdb::data_ptr_t data, duckdb::idx_t len) {
+	while (len) {
+		const auto random_integer = NextRandomInteger();
+		const auto next = duckdb::MinValue<duckdb::idx_t>(len, sizeof(random_integer));
+		memcpy(data, duckdb::const_data_ptr_cast(&random_integer), next);
+		data += next;
+		len -= next;
+	}
+}
+
 } // namespace duckdb

package/src/duckdb/src/execution/expression_executor/execute_comparison.cpp

@@ -138,8 +138,19 @@ static idx_t TemplatedSelectOperation(Vector &left, Vector &right, optional_ptr<
 		                                                      false_sel.get());
 	case PhysicalType::LIST:
 	case PhysicalType::STRUCT:
-	case PhysicalType::ARRAY:
-		return NestedSelectOperation<OP>(left, right, sel, count, true_sel, false_sel, null_mask);
+	case PhysicalType::ARRAY: {
+		auto result_count = NestedSelectOperation<OP>(left, right, sel, count, true_sel, false_sel, null_mask);
+		if (true_sel && result_count > 0) {
+			std::sort(true_sel->data(), true_sel->data() + result_count);
+		}
+		if (false_sel) {
+			idx_t false_count = count - result_count;
+			if (false_count > 0) {
+				std::sort(false_sel->data(), false_sel->data() + false_count);
+			}
+		}
+		return result_count;
+	}
 	default:
 		throw InternalException("Invalid type for comparison");
 	}

package/src/duckdb/src/execution/index/art/art.cpp

@@ -522,7 +522,9 @@ ErrorData ART::Insert(IndexLock &l, DataChunk &chunk, Vector &row_ids, IndexAppe
 		if (keys[i].Empty()) {
 			continue;
 		}
-		D_ASSERT(ARTOperator::Lookup(*this, tree, keys[i], 0));
+		auto leaf = ARTOperator::Lookup(*this, tree, keys[i], 0);
+		D_ASSERT(leaf);
+		D_ASSERT(ARTOperator::LookupInLeaf(*this, *leaf, row_id_keys[i]));
 	}
 #endif
 	return ErrorData();
@@ -602,8 +604,9 @@ void ART::Delete(IndexLock &state, DataChunk &input, Vector &row_ids) {
 			continue;
 		}
 		auto leaf = ARTOperator::Lookup(*this, tree, keys[i], 0);
-		if (leaf && leaf->GetType() == NType::LEAF_INLINED) {
-			D_ASSERT(leaf->GetRowId() != row_id_keys[i].GetRowId());
+		if (leaf) {
+			auto contains_row_id = ARTOperator::LookupInLeaf(*this, *leaf, row_id_keys[i]);
+			D_ASSERT(!contains_row_id);
 		}
 	}
 #endif

package/src/duckdb/src/execution/index/bound_index.cpp

@@ -154,28 +154,39 @@ string BoundIndex::AppendRowError(DataChunk &input, idx_t index) {
 	return error;
 }
 
-void BoundIndex::ApplyBufferedAppends(const vector<LogicalType> &table_types, ColumnDataCollection &buffered_appends,
+void BoundIndex::ApplyBufferedReplays(const vector<LogicalType> &table_types,
+                                      vector<BufferedIndexData> &buffered_replays,
                                       const vector<StorageIndex> &mapped_column_ids) {
-	IndexAppendInfo index_append_info(IndexAppendMode::INSERT_DUPLICATES, nullptr);
-
-	ColumnDataScanState state;
-	buffered_appends.InitializeScan(state);
-
-	DataChunk scan_chunk;
-	buffered_appends.InitializeScanChunk(scan_chunk);
-	DataChunk table_chunk;
-	table_chunk.InitializeEmpty(table_types);
-
-	while (buffered_appends.Scan(state, scan_chunk)) {
-		for (idx_t i = 0; i < scan_chunk.ColumnCount() - 1; i++) {
-			auto col_id = mapped_column_ids[i].GetPrimaryIndex();
-			table_chunk.data[col_id].Reference(scan_chunk.data[i]);
-		}
-		table_chunk.SetCardinality(scan_chunk.size());
-
-		auto error = Append(table_chunk, scan_chunk.data.back(), index_append_info);
-		if (error.HasError()) {
-			throw InternalException("error while applying buffered appends: " + error.Message());
+	for (auto &replay : buffered_replays) {
+		ColumnDataScanState state;
+		auto &buffered_data = *replay.data;
+		buffered_data.InitializeScan(state);
+
+		DataChunk scan_chunk;
+		buffered_data.InitializeScanChunk(scan_chunk);
+		DataChunk table_chunk;
+		table_chunk.InitializeEmpty(table_types);
+
+		while (buffered_data.Scan(state, scan_chunk)) {
+			for (idx_t i = 0; i < scan_chunk.ColumnCount() - 1; i++) {
+				auto col_id = mapped_column_ids[i].GetPrimaryIndex();
+				table_chunk.data[col_id].Reference(scan_chunk.data[i]);
+			}
+			table_chunk.SetCardinality(scan_chunk.size());
+
+			switch (replay.type) {
+			case BufferedIndexReplay::INSERT_ENTRY: {
+				IndexAppendInfo index_append_info(IndexAppendMode::INSERT_DUPLICATES, nullptr);
+				auto error = Append(table_chunk, scan_chunk.data.back(), index_append_info);
+				if (error.HasError()) {
+					throw InternalException("error while applying buffered appends: " + error.Message());
+				}
+				continue;
+			}
+			case BufferedIndexReplay::DEL_ENTRY: {
+				Delete(table_chunk, scan_chunk.data.back());
+			}
+			}
 		}
 	}
 }

package/src/duckdb/src/execution/index/unbound_index.cpp

@@ -8,6 +8,10 @@
 
 namespace duckdb {
 
+BufferedIndexData::BufferedIndexData(BufferedIndexReplay replay_type, unique_ptr<ColumnDataCollection> data_p)
+    : type(replay_type), data(std::move(data_p)) {
+}
+
 UnboundIndex::UnboundIndex(unique_ptr<CreateInfo> create_info, IndexStorageInfo storage_info_p,
                            TableIOManager &table_io_manager, AttachedDatabase &db)
     : Index(create_info->Cast<CreateIndexInfo>().column_ids, table_io_manager, db), create_info(std::move(create_info)),
@@ -35,26 +39,33 @@ void UnboundIndex::CommitDrop() {
 	}
 }
 
-void UnboundIndex::BufferChunk(DataChunk &chunk, Vector &row_ids, const vector<StorageIndex> &mapped_column_ids_p) {
+void UnboundIndex::BufferChunk(DataChunk &index_column_chunk, Vector &row_ids,
+                               const vector<StorageIndex> &mapped_column_ids_p, BufferedIndexReplay replay_type) {
 	D_ASSERT(!column_ids.empty());
-	auto types = chunk.GetTypes();
+	auto types = index_column_chunk.GetTypes(); // column types
 	types.push_back(LogicalType::ROW_TYPE);
 
-	if (!buffered_appends) {
-		auto &allocator = Allocator::Get(db);
-		buffered_appends = make_uniq<ColumnDataCollection>(allocator, types);
+	auto &allocator = Allocator::Get(db);
+
+	BufferedIndexData buffered_data(replay_type, make_uniq<ColumnDataCollection>(allocator, types));
+
+	//! First time we are buffering data, canonical column_id mapping is stored.
+	//! This should be a sorted list of all the physical offsets of Indexed columns on this table.
+	if (mapped_column_ids.empty()) {
 		mapped_column_ids = mapped_column_ids_p;
 	}
 	D_ASSERT(mapped_column_ids == mapped_column_ids_p);
 
+	// Combined chunk has all the indexed columns and rowids.
 	DataChunk combined_chunk;
 	combined_chunk.InitializeEmpty(types);
-	for (idx_t i = 0; i < chunk.ColumnCount(); i++) {
-		combined_chunk.data[i].Reference(chunk.data[i]);
+	for (idx_t i = 0; i < index_column_chunk.ColumnCount(); i++) {
+		combined_chunk.data[i].Reference(index_column_chunk.data[i]);
 	}
 	combined_chunk.data.back().Reference(row_ids);
-	combined_chunk.SetCardinality(chunk.size());
-	buffered_appends->Append(combined_chunk);
+	combined_chunk.SetCardinality(index_column_chunk.size());
+	buffered_data.data->Append(combined_chunk);
+	buffered_replays.emplace_back(std::move(buffered_data));
 }
 
 } // namespace duckdb

package/src/duckdb/src/execution/join_hashtable.cpp

@@ -888,6 +888,7 @@ idx_t ScanStructure::ResolvePredicates(DataChunk &keys, SelectionVector &match_s
 	}
 
 	// If there is a matcher for the probing side because of non-equality predicates, use it
+	idx_t result_count;
 	if (ht.needs_chain_matcher) {
 		idx_t no_match_count = 0;
 		auto &matcher = no_match_sel ? ht.row_matcher_probe_no_match_sel : ht.row_matcher_probe;
@@ -895,12 +896,17 @@ idx_t ScanStructure::ResolvePredicates(DataChunk &keys, SelectionVector &match_s
 
 		// we need to only use the vectors with the indices of the columns that are used in the probe phase, namely
 		// the non-equality columns
-		return matcher->Match(keys, key_state.vector_data, match_sel, this->count, pointers, no_match_sel,
-		                      no_match_count);
+		result_count =
+		    matcher->Match(keys, key_state.vector_data, match_sel, this->count, pointers, no_match_sel, no_match_count);
 	} else {
 		// no match sel is the opposite of match sel
-		return this->count;
+		result_count = this->count;
 	}
+
+	// Update total probe match count
+	ht.total_probe_matches.fetch_add(result_count, std::memory_order_relaxed);
+
+	return result_count;
 }
 
 idx_t ScanStructure::ScanInnerJoin(DataChunk &keys, SelectionVector &result_vector) {

package/src/duckdb/src/execution/operator/helper/physical_buffered_batch_collector.cpp

@@ -94,7 +94,7 @@ unique_ptr<LocalSinkState> PhysicalBufferedBatchCollector::GetLocalSinkState(Exe
 unique_ptr<GlobalSinkState> PhysicalBufferedBatchCollector::GetGlobalSinkState(ClientContext &context) const {
 	auto state = make_uniq<BufferedBatchCollectorGlobalState>();
 	state->context = context.shared_from_this();
-	state->buffered_data = make_shared_ptr<BatchedBufferedData>(state->context);
+	state->buffered_data = make_shared_ptr<BatchedBufferedData>(context);
 	return std::move(state);
 }
 

package/src/duckdb/src/execution/operator/helper/physical_buffered_collector.cpp

@@ -48,7 +48,7 @@ SinkCombineResultType PhysicalBufferedCollector::Combine(ExecutionContext &conte
 unique_ptr<GlobalSinkState> PhysicalBufferedCollector::GetGlobalSinkState(ClientContext &context) const {
 	auto state = make_uniq<BufferedCollectorGlobalState>();
 	state->context = context.shared_from_this();
-	state->buffered_data = make_shared_ptr<SimpleBufferedData>(state->context);
+	state->buffered_data = make_shared_ptr<SimpleBufferedData>(context);
 	return std::move(state);
 }
 

package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp

@@ -164,6 +164,11 @@ public:
 		}
 	}
 
+	~HashJoinGlobalSinkState() override {
+		DUCKDB_LOG(context, PhysicalOperatorLogType, op, "PhysicalHashJoin", "GetData",
+		           {{"total_probe_matches", to_string(hash_table->total_probe_matches)}});
+	}
+
 	void ScheduleFinalize(Pipeline &pipeline, Event &event);
 	void InitializeProbeSpill();
 

package/src/duckdb/src/function/cast/cast_function_set.cpp

@@ -184,7 +184,9 @@ int64_t CastFunctionSet::ImplicitCastCost(optional_ptr<ClientContext> context, c
 			old_implicit_casting = DBConfig::GetSetting<OldImplicitCastingSetting>(*config);
 		}
 		if (old_implicit_casting) {
-			score = 149;
+			// very high cost to avoid choosing this cast if any other option is available
+			// (it should be more costly than casting to TEMPLATE if that is available)
+			score = 10000000000;
 		}
 	}
 	return score;

package/src/duckdb/src/function/macro_function.cpp

@@ -47,7 +47,7 @@ MacroBindResult MacroFunction::BindMacroFunction(
     InsertionOrderPreservingMap<unique_ptr<ParsedExpression>> &named_arguments, idx_t depth) {
 
 	ExpressionBinder expr_binder(binder, binder.context);
-
+	expr_binder.lambda_bindings = binder.lambda_bindings;
 	// Find argument types and separate positional and default arguments
 	vector<LogicalType> positional_arg_types;
 	InsertionOrderPreservingMap<LogicalType> named_arg_types;

package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp

@@ -198,7 +198,7 @@ scalar_function_t GetStringDecompressFunctionSwitch(const LogicalType &input_typ
 	case LogicalTypeId::UHUGEINT:
 		return GetStringDecompressFunction<uhugeint_t>(input_type);
 	case LogicalTypeId::HUGEINT:
-		return GetStringCompressFunction<hugeint_t>(input_type);
+		return GetStringDecompressFunction<hugeint_t>(input_type);
 	default:
 		throw InternalException("Unexpected type in GetStringDecompressFunctionSwitch");
 	}

package/src/duckdb/src/function/scalar/create_sort_key.cpp

@@ -696,13 +696,15 @@ void PrepareSortData(Vector &result, idx_t size, SortKeyLengthInfo &key_lengths,
 	}
 }
 
-void FinalizeSortData(Vector &result, idx_t size) {
+void FinalizeSortData(Vector &result, idx_t size, const SortKeyLengthInfo &key_lengths,
+                      const unsafe_vector<idx_t> &offsets) {
 	switch (result.GetType().id()) {
 	case LogicalTypeId::BLOB: {
 		auto result_data = FlatVector::GetData<string_t>(result);
 		// call Finalize on the result
 		for (idx_t r = 0; r < size; r++) {
-			result_data[r].Finalize();
+			result_data[r].SetSizeAndFinalize(offsets[r],
+			                                  key_lengths.variable_lengths[r] + key_lengths.constant_length);
 		}
 		break;
 	}
@@ -739,7 +741,7 @@ void CreateSortKeyInternal(vector<unique_ptr<SortKeyVectorData>> &sort_key_data,
 		SortKeyConstructInfo info(modifiers[c], offsets, data_pointers.get());
 		ConstructSortKey(*sort_key_data[c], info);
 	}
-	FinalizeSortData(result, row_count);
+	FinalizeSortData(result, row_count, key_lengths, offsets);
 }
 
 } // namespace

package/src/duckdb/src/function/scalar/operator/arithmetic.cpp

@@ -1220,7 +1220,7 @@ hugeint_t InterpolateOperator::Operation(const hugeint_t &lo, const double d, co
 
 template <>
 uhugeint_t InterpolateOperator::Operation(const uhugeint_t &lo, const double d, const uhugeint_t &hi) {
-	return Hugeint::Convert(Operation(Uhugeint::Cast<double>(lo), d, Uhugeint::Cast<double>(hi)));
+	return Uhugeint::Convert(Operation(Uhugeint::Cast<double>(lo), d, Uhugeint::Cast<double>(hi)));
 }
 
 static interval_t MultiplyByDouble(const interval_t &i, const double &d) { // NOLINT

package/src/duckdb/src/function/scalar/system/parse_log_message.cpp

@@ -77,8 +77,10 @@ void ParseLogMessageFunction(DataChunk &args, ExpressionState &state, Vector &re
 } // namespace
 
 ScalarFunction ParseLogMessage::GetFunction() {
-	return ScalarFunction({LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::ANY, ParseLogMessageFunction,
-	                      ParseLogMessageBind, nullptr, nullptr, nullptr, LogicalType(LogicalTypeId::INVALID));
+	auto fun = ScalarFunction({LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::ANY, ParseLogMessageFunction,
+	                          ParseLogMessageBind, nullptr, nullptr, nullptr, LogicalType(LogicalTypeId::INVALID));
+	fun.errors = FunctionErrors::CAN_THROW_RUNTIME_ERROR;
+	return fun;
 }
 
 } // namespace duckdb

package/src/duckdb/src/function/table/copy_csv.cpp

@@ -280,7 +280,31 @@ struct GlobalWriteCSVData : public GlobalFunctionData {
 		return writer.FileSize();
 	}
 
+	unique_ptr<CSVWriterState> GetLocalState(ClientContext &context, const idx_t flush_size) {
+		{
+			lock_guard<mutex> guard(local_state_lock);
+			if (!local_states.empty()) {
+				auto result = std::move(local_states.back());
+				local_states.pop_back();
+				return result;
+			}
+		}
+		auto result = make_uniq<CSVWriterState>(context, flush_size);
+		result->require_manual_flush = true;
+		return result;
+	}
+
+	void StoreLocalState(unique_ptr<CSVWriterState> lstate) {
+		lock_guard<mutex> guard(local_state_lock);
+		lstate->Reset();
+		local_states.push_back(std::move(lstate));
+	}
+
 	CSVWriter writer;
+
+private:
+	mutex local_state_lock;
+	vector<unique_ptr<CSVWriterState>> local_states;
 };
 
 static unique_ptr<LocalFunctionData> WriteCSVInitializeLocal(ExecutionContext &context, FunctionData &bind_data) {
@@ -371,9 +395,7 @@ CopyFunctionExecutionMode WriteCSVExecutionMode(bool preserve_insertion_order, b
 // Prepare Batch
 //===--------------------------------------------------------------------===//
 struct WriteCSVBatchData : public PreparedBatchData {
-	explicit WriteCSVBatchData(ClientContext &context, const idx_t flush_size)
-	    : writer_local_state(make_uniq<CSVWriterState>(context, flush_size)) {
-		writer_local_state->require_manual_flush = true;
+	explicit WriteCSVBatchData(unique_ptr<CSVWriterState> writer_state) : writer_local_state(std::move(writer_state)) {
 	}
 
 	//! The thread-local buffer to write data into
@@ -397,7 +419,8 @@ unique_ptr<PreparedBatchData> WriteCSVPrepareBatch(ClientContext &context, Funct
 	auto &global_state = gstate.Cast<GlobalWriteCSVData>();
 
 	// write CSV chunks to the batch data
-	auto batch = make_uniq<WriteCSVBatchData>(context, NextPowerOfTwo(collection->SizeInBytes()));
+	auto local_writer_state = global_state.GetLocalState(context, NextPowerOfTwo(collection->SizeInBytes()));
+	auto batch = make_uniq<WriteCSVBatchData>(std::move(local_writer_state));
 	for (auto &chunk : collection->Chunks()) {
 		WriteCSVChunkInternal(global_state.writer, *batch->writer_local_state, cast_chunk, chunk, executor);
 	}
@@ -412,6 +435,7 @@ void WriteCSVFlushBatch(ClientContext &context, FunctionData &bind_data, GlobalF
 	auto &csv_batch = batch.Cast<WriteCSVBatchData>();
 	auto &global_state = gstate.Cast<GlobalWriteCSVData>();
 	global_state.writer.Flush(*csv_batch.writer_local_state);
+	global_state.StoreLocalState(std::move(csv_batch.writer_local_state));
 }
 
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/function/table/direct_file_reader.cpp

@@ -52,6 +52,8 @@ void DirectFileReader::Scan(ClientContext &context, GlobalTableFunctionState &gl
 	}
 
 	auto files = state.file_list;
+
+	auto &regular_fs = FileSystem::GetFileSystem(context);
 	auto fs = CachingFileSystem::Get(context);
 	idx_t out_idx = 0;
 
@@ -65,6 +67,14 @@ void DirectFileReader::Scan(ClientContext &context, GlobalTableFunctionState &gl
 			flags |= FileFlags::FILE_FLAGS_DIRECT_IO;
 		}
 		file_handle = fs.OpenFile(QueryContext(context), file, flags);
+	} else {
+		// At least verify that the file exist
+		// The globbing behavior in remote filesystems can lead to files being listed that do not actually exist
+		if (FileSystem::IsRemoteFile(file.path) && !regular_fs.FileExists(file.path)) {
+			output.SetCardinality(0);
+			done = true;
+			return;
+		}
 	}
 
 	for (idx_t col_idx = 0; col_idx < state.column_ids.size(); col_idx++) {

package/src/duckdb/src/function/table/read_file.cpp

@@ -10,10 +10,43 @@
 
 namespace duckdb {
 
+namespace {
+
 //------------------------------------------------------------------------------
 // DirectMultiFileInfo
 //------------------------------------------------------------------------------
 
+template <class OP>
+struct DirectMultiFileInfo : MultiFileReaderInterface {
+	static unique_ptr<MultiFileReaderInterface> CreateInterface(ClientContext &context);
+	unique_ptr<BaseFileReaderOptions> InitializeOptions(ClientContext &context,
+	                                                    optional_ptr<TableFunctionInfo> info) override;
+	bool ParseCopyOption(ClientContext &context, const string &key, const vector<Value> &values,
+	                     BaseFileReaderOptions &options, vector<string> &expected_names,
+	                     vector<LogicalType> &expected_types) override;
+	bool ParseOption(ClientContext &context, const string &key, const Value &val, MultiFileOptions &file_options,
+	                 BaseFileReaderOptions &options) override;
+	unique_ptr<TableFunctionData> InitializeBindData(MultiFileBindData &multi_file_data,
+	                                                 unique_ptr<BaseFileReaderOptions> options) override;
+	void BindReader(ClientContext &context, vector<LogicalType> &return_types, vector<string> &names,
+	                MultiFileBindData &bind_data) override;
+	optional_idx MaxThreads(const MultiFileBindData &bind_data_p, const MultiFileGlobalState &global_state,
+	                        FileExpandResult expand_result) override;
+	unique_ptr<GlobalTableFunctionState> InitializeGlobalState(ClientContext &context, MultiFileBindData &bind_data,
+	                                                           MultiFileGlobalState &global_state) override;
+	unique_ptr<LocalTableFunctionState> InitializeLocalState(ExecutionContext &, GlobalTableFunctionState &) override;
+	shared_ptr<BaseFileReader> CreateReader(ClientContext &context, GlobalTableFunctionState &gstate,
+	                                        BaseUnionData &union_data, const MultiFileBindData &bind_data_p) override;
+	shared_ptr<BaseFileReader> CreateReader(ClientContext &context, GlobalTableFunctionState &gstate,
+	                                        const OpenFileInfo &file, idx_t file_idx,
+	                                        const MultiFileBindData &bind_data) override;
+	shared_ptr<BaseFileReader> CreateReader(ClientContext &context, const OpenFileInfo &file,
+	                                        BaseFileReaderOptions &options,
+	                                        const MultiFileOptions &file_options) override;
+	unique_ptr<NodeStatistics> GetCardinality(const MultiFileBindData &bind_data, idx_t file_count) override;
+	FileGlobInput GetGlobInput() override;
+};
+
 template <class OP>
 unique_ptr<MultiFileReaderInterface> DirectMultiFileInfo<OP>::CreateInterface(ClientContext &context) {
 	return make_uniq<DirectMultiFileInfo>();
@@ -132,14 +165,45 @@ FileGlobInput DirectMultiFileInfo<OP>::GetGlobInput() {
 }
 
 //------------------------------------------------------------------------------
-// Register
+// Operations
 //------------------------------------------------------------------------------
+
+struct ReadBlobOperation {
+	static constexpr const char *NAME = "read_blob";
+	static constexpr const char *FILE_TYPE = "blob";
+
+	static inline LogicalType TYPE() {
+		return LogicalType::BLOB;
+	}
+};
+
+struct ReadTextOperation {
+	static constexpr const char *NAME = "read_text";
+	static constexpr const char *FILE_TYPE = "text";
+
+	static inline LogicalType TYPE() {
+		return LogicalType::VARCHAR;
+	}
+};
+
 template <class OP>
 static TableFunction GetFunction() {
 	MultiFileFunction<DirectMultiFileInfo<OP>> table_function(OP::NAME);
+	// Erase extra multi file reader options
+	table_function.named_parameters.erase("filename");
+	table_function.named_parameters.erase("hive_partitioning");
+	table_function.named_parameters.erase("union_by_name");
+	table_function.named_parameters.erase("hive_types");
+	table_function.named_parameters.erase("hive_types_autocast");
 	return table_function;
 }
 
+} // namespace
+
+//------------------------------------------------------------------------------
+// Register
+//------------------------------------------------------------------------------
+
 void ReadBlobFunction::RegisterFunction(BuiltinFunctions &set) {
 	auto scan_fun = GetFunction<ReadBlobOperation>();
 	set.AddFunction(MultiFileReader::CreateFunctionSet(scan_fun));

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,5 +1,5 @@
 #ifndef DUCKDB_PATCH_VERSION
-#define DUCKDB_PATCH_VERSION "1"
+#define DUCKDB_PATCH_VERSION "2"
 #endif
 #ifndef DUCKDB_MINOR_VERSION
 #define DUCKDB_MINOR_VERSION 4
@@ -8,10 +8,10 @@
 #define DUCKDB_MAJOR_VERSION 1
 #endif
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "v1.4.1"
+#define DUCKDB_VERSION "v1.4.2"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "b390a7c376"
+#define DUCKDB_SOURCE_ID "68d7555f68"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/common/csv_writer.hpp

@@ -90,9 +90,6 @@ public:
 	//! Closes the writer, optionally writes a postfix
 	void Close();
 
-	unique_ptr<CSVWriterState> InitializeLocalWriteState(ClientContext &context, idx_t flush_size);
-	unique_ptr<CSVWriterState> InitializeLocalWriteState(DatabaseInstance &db, idx_t flush_size);
-
 	vector<unique_ptr<Expression>> string_casts;
 
 	idx_t BytesWritten();