duckdb 0.7.2-dev2820.0 → 0.7.2-dev2995.0

package/src/duckdb/src/execution/operator/schema/physical_create_type.cpp

@@ -3,6 +3,7 @@
 #include "duckdb/catalog/catalog.hpp"
 #include "duckdb/common/types/column/column_data_collection.hpp"
 #include "duckdb/catalog/catalog_entry/type_catalog_entry.hpp"
+#include "duckdb/common/string_map_set.hpp"
 
 namespace duckdb {
 
@@ -21,22 +22,22 @@ public:
 	Vector result;
 	idx_t size = 0;
 	idx_t capacity = STANDARD_VECTOR_SIZE;
+	string_set_t found_strings;
 };
 
 unique_ptr<GlobalSinkState> PhysicalCreateType::GetGlobalSinkState(ClientContext &context) const {
 	return make_uniq<CreateTypeGlobalState>(context);
 }
 
-SinkResultType PhysicalCreateType::Sink(ExecutionContext &context, GlobalSinkState &gstate_p, LocalSinkState &lstate_p,
-                                        DataChunk &input) const {
-	auto &gstate = gstate_p.Cast<CreateTypeGlobalState>();
-	idx_t total_row_count = gstate.size + input.size();
+SinkResultType PhysicalCreateType::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const {
+	auto &gstate = input.global_state.Cast<CreateTypeGlobalState>();
+	idx_t total_row_count = gstate.size + chunk.size();
 	if (total_row_count > NumericLimits<uint32_t>::Maximum()) {
 		throw InvalidInputException("Attempted to create ENUM of size %llu, which exceeds the maximum size of %llu",
 		                            total_row_count, NumericLimits<uint32_t>::Maximum());
 	}
 	UnifiedVectorFormat sdata;
-	input.data[0].ToUnifiedFormat(input.size(), sdata);
+	chunk.data[0].ToUnifiedFormat(chunk.size(), sdata);
 
 	if (total_row_count > gstate.capacity) {
 		// We must resize our result vector
@@ -47,13 +48,20 @@ SinkResultType PhysicalCreateType::Sink(ExecutionContext &context, GlobalSinkSta
 	auto src_ptr = (string_t *)sdata.data;
 	auto result_ptr = FlatVector::GetData<string_t>(gstate.result);
 	// Input vector has NULL value, we just throw an exception
-	for (idx_t i = 0; i < input.size(); i++) {
+	for (idx_t i = 0; i < chunk.size(); i++) {
 		idx_t idx = sdata.sel->get_index(i);
 		if (!sdata.validity.RowIsValid(idx)) {
 			throw InvalidInputException("Attempted to create ENUM type with NULL value!");
 		}
-		result_ptr[gstate.size++] =
-		    StringVector::AddStringOrBlob(gstate.result, src_ptr[idx].GetData(), src_ptr[idx].GetSize());
+		auto str = src_ptr[idx];
+		auto entry = gstate.found_strings.find(src_ptr[idx]);
+		if (entry != gstate.found_strings.end()) {
+			// entry was already found - skip
+			continue;
+		}
+		auto owned_string = StringVector::AddStringOrBlob(gstate.result, str.GetData(), str.GetSize());
+		gstate.found_strings.insert(owned_string);
+		result_ptr[gstate.size++] = owned_string;
 	}
 	return SinkResultType::NEED_MORE_INPUT;
 }
@@ -61,25 +69,8 @@ SinkResultType PhysicalCreateType::Sink(ExecutionContext &context, GlobalSinkSta
 //===--------------------------------------------------------------------===//
 // Source
 //===--------------------------------------------------------------------===//
-class CreateTypeSourceState : public GlobalSourceState {
-public:
-	CreateTypeSourceState() : finished(false) {
-	}
-
-	bool finished;
-};
-
-unique_ptr<GlobalSourceState> PhysicalCreateType::GetGlobalSourceState(ClientContext &context) const {
-	return make_uniq<CreateTypeSourceState>();
-}
-
-void PhysicalCreateType::GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
-                                 LocalSourceState &lstate) const {
-	auto &state = gstate.Cast<CreateTypeSourceState>();
-	if (state.finished) {
-		return;
-	}
-
+SourceResultType PhysicalCreateType::GetData(ExecutionContext &context, DataChunk &chunk,
+                                             OperatorSourceInput &input) const {
 	if (IsSink()) {
 		D_ASSERT(info->type == LogicalType::INVALID);
 		auto &g_sink_state = sink_state->Cast<CreateTypeGlobalState>();
@@ -91,7 +82,8 @@ void PhysicalCreateType::GetData(ExecutionContext &context, DataChunk &chunk, Gl
 	D_ASSERT(catalog_entry->type == CatalogType::TYPE_ENTRY);
 	auto &catalog_type = catalog_entry->Cast<TypeCatalogEntry>();
 	EnumType::SetCatalog(info->type, &catalog_type);
-	state.finished = true;
+
+	return SourceResultType::FINISHED;
 }
 
 } // namespace duckdb

package/src/duckdb/src/execution/operator/schema/physical_create_view.cpp

@@ -6,27 +6,12 @@ namespace duckdb {
 //===--------------------------------------------------------------------===//
 // Source
 //===--------------------------------------------------------------------===//
-class CreateViewSourceState : public GlobalSourceState {
-public:
-	CreateViewSourceState() : finished(false) {
-	}
-
-	bool finished;
-};
-
-unique_ptr<GlobalSourceState> PhysicalCreateView::GetGlobalSourceState(ClientContext &context) const {
-	return make_uniq<CreateViewSourceState>();
-}
-
-void PhysicalCreateView::GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
-                                 LocalSourceState &lstate) const {
-	auto &state = gstate.Cast<CreateViewSourceState>();
-	if (state.finished) {
-		return;
-	}
+SourceResultType PhysicalCreateView::GetData(ExecutionContext &context, DataChunk &chunk,
+                                             OperatorSourceInput &input) const {
 	auto &catalog = Catalog::GetCatalog(context.client, info->catalog);
 	catalog.CreateView(context.client, *info);
-	state.finished = true;
+
+	return SourceResultType::FINISHED;
 }
 
 } // namespace duckdb

package/src/duckdb/src/execution/operator/schema/physical_detach.cpp

@@ -11,27 +11,12 @@ namespace duckdb {
 //===--------------------------------------------------------------------===//
 // Source
 //===--------------------------------------------------------------------===//
-class DetachSourceState : public GlobalSourceState {
-public:
-	DetachSourceState() : finished(false) {
-	}
-
-	bool finished;
-};
-
-unique_ptr<GlobalSourceState> PhysicalDetach::GetGlobalSourceState(ClientContext &context) const {
-	return make_uniq<DetachSourceState>();
-}
-
-void PhysicalDetach::GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
-                             LocalSourceState &lstate) const {
-	auto &state = gstate.Cast<DetachSourceState>();
-	if (state.finished) {
-		return;
-	}
+SourceResultType PhysicalDetach::GetData(ExecutionContext &context, DataChunk &chunk,
+                                         OperatorSourceInput &input) const {
 	auto &db_manager = DatabaseManager::Get(context.client);
 	db_manager.DetachDatabase(context.client, info->name, info->if_not_found);
-	state.finished = true;
+
+	return SourceResultType::FINISHED;
 }
 
 } // namespace duckdb

package/src/duckdb/src/execution/operator/schema/physical_drop.cpp

@@ -11,24 +11,7 @@ namespace duckdb {
 //===--------------------------------------------------------------------===//
 // Source
 //===--------------------------------------------------------------------===//
-class DropSourceState : public GlobalSourceState {
-public:
-	DropSourceState() : finished(false) {
-	}
-
-	bool finished;
-};
-
-unique_ptr<GlobalSourceState> PhysicalDrop::GetGlobalSourceState(ClientContext &context) const {
-	return make_uniq<DropSourceState>();
-}
-
-void PhysicalDrop::GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
-                           LocalSourceState &lstate) const {
-	auto &state = gstate.Cast<DropSourceState>();
-	if (state.finished) {
-		return;
-	}
+SourceResultType PhysicalDrop::GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const {
 	switch (info->type) {
 	case CatalogType::PREPARED_STATEMENT: {
 		// DEALLOCATE silently ignores errors
@@ -62,7 +45,8 @@ void PhysicalDrop::GetData(ExecutionContext &context, DataChunk &chunk, GlobalSo
 		break;
 	}
 	}
-	state.finished = true;
+
+	return SourceResultType::FINISHED;
 }
 
 } // namespace duckdb

package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp

@@ -61,16 +61,15 @@ idx_t PhysicalRecursiveCTE::ProbeHT(DataChunk &chunk, RecursiveCTEState &state)
 	return new_group_count;
 }
 
-SinkResultType PhysicalRecursiveCTE::Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,
-                                          DataChunk &input) const {
-	auto &gstate = state.Cast<RecursiveCTEState>();
+SinkResultType PhysicalRecursiveCTE::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const {
+	auto &gstate = input.global_state.Cast<RecursiveCTEState>();
 	if (!union_all) {
-		idx_t match_count = ProbeHT(input, gstate);
+		idx_t match_count = ProbeHT(chunk, gstate);
 		if (match_count > 0) {
-			gstate.intermediate_table.Append(input);
+			gstate.intermediate_table.Append(chunk);
 		}
 	} else {
-		gstate.intermediate_table.Append(input);
+		gstate.intermediate_table.Append(chunk);
 	}
 	return SinkResultType::NEED_MORE_INPUT;
 }
@@ -78,8 +77,8 @@ SinkResultType PhysicalRecursiveCTE::Sink(ExecutionContext &context, GlobalSinkS
 //===--------------------------------------------------------------------===//
 // Source
 //===--------------------------------------------------------------------===//
-void PhysicalRecursiveCTE::GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate_p,
-                                   LocalSourceState &lstate) const {
+SourceResultType PhysicalRecursiveCTE::GetData(ExecutionContext &context, DataChunk &chunk,
+                                               OperatorSourceInput &input) const {
 	auto &gstate = sink_state->Cast<RecursiveCTEState>();
 	if (!gstate.initialized) {
 		gstate.intermediate_table.InitializeScan(gstate.scan_state);
@@ -117,6 +116,8 @@ void PhysicalRecursiveCTE::GetData(ExecutionContext &context, DataChunk &chunk,
 			gstate.intermediate_table.InitializeScan(gstate.scan_state);
 		}
 	}
+
+	return chunk.size() == 0 ? SourceResultType::FINISHED : SourceResultType::HAVE_MORE_OUTPUT;
 }
 
 void PhysicalRecursiveCTE::ExecuteRecursivePipelines(ExecutionContext &context) const {

package/src/duckdb/src/execution/operator/set/physical_union.cpp

@@ -28,7 +28,7 @@ void PhysicalUnion::BuildPipelines(Pipeline &current, MetaPipeline &meta_pipelin
 		order_matters = true;
 	}
 	if (sink) {
-		if (sink->SinkOrderDependent() && !sink->RequiresBatchIndex()) {
+		if (sink->SinkOrderDependent() || sink->RequiresBatchIndex()) {
 			order_matters = true;
 		}
 		if (!sink->ParallelSink()) {

package/src/duckdb/src/execution/physical_operator.cpp

@@ -72,8 +72,8 @@ unique_ptr<GlobalSourceState> PhysicalOperator::GetGlobalSourceState(ClientConte
 }
 
 // LCOV_EXCL_START
-void PhysicalOperator::GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
-                               LocalSourceState &lstate) const {
+SourceResultType PhysicalOperator::GetData(ExecutionContext &context, DataChunk &chunk,
+                                           OperatorSourceInput &input) const {
 	throw InternalException("Calling GetData on a node that is not a source!");
 }
 
@@ -91,10 +91,10 @@ double PhysicalOperator::GetProgress(ClientContext &context, GlobalSourceState &
 // Sink
 //===--------------------------------------------------------------------===//
 // LCOV_EXCL_START
-SinkResultType PhysicalOperator::Sink(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate,
-                                      DataChunk &input) const {
+SinkResultType PhysicalOperator::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const {
 	throw InternalException("Calling Sink on a node that is not a sink!");
 }
+
 // LCOV_EXCL_STOP
 
 void PhysicalOperator::Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const {
@@ -105,6 +105,9 @@ SinkFinalizeType PhysicalOperator::Finalize(Pipeline &pipeline, Event &event, Cl
 	return SinkFinalizeType::READY;
 }
 
+void PhysicalOperator::NextBatch(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate_p) const {
+}
+
 unique_ptr<LocalSinkState> PhysicalOperator::GetLocalSinkState(ExecutionContext &context) const {
 	return make_uniq<LocalSinkState>();
 }
@@ -238,7 +241,9 @@ OperatorResultType CachingPhysicalOperator::Execute(ExecutionContext &context, D
 		state.initialized = true;
 		state.can_cache_chunk = true;
 
-		if (!context.pipeline || !caching_supported) {
+		if (!context.client.config.enable_caching_operators) {
+			state.can_cache_chunk = false;
+		} else if (!context.pipeline || !caching_supported) {
 			state.can_cache_chunk = false;
 		} else if (!context.pipeline->GetSink()) {
 			// Disabling for pipelines without Sink, i.e. when pulling
@@ -252,6 +257,7 @@ OperatorResultType CachingPhysicalOperator::Execute(ExecutionContext &context, D
 	if (!state.can_cache_chunk) {
 		return child_result;
 	}
+	// TODO chunk size of 0 should not result in a cache being created!
 	if (chunk.size() < CACHE_THRESHOLD) {
 		// we have filtered out a significant amount of tuples
 		// add this chunk to the cache and continue

package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp

@@ -130,19 +130,18 @@ void RadixPartitionedHashTable::PopulateGroupChunk(DataChunk &group_chunk, DataC
 	group_chunk.Verify();
 }
 
-void RadixPartitionedHashTable::Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,
-                                     DataChunk &groups_input, DataChunk &payload_input,
-                                     const vector<idx_t> &filter) const {
-	auto &llstate = lstate.Cast<RadixHTLocalState>();
-	auto &gstate = state.Cast<RadixHTGlobalState>();
+void RadixPartitionedHashTable::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input,
+                                     DataChunk &payload_input, const vector<idx_t> &filter) const {
+	auto &llstate = input.local_state.Cast<RadixHTLocalState>();
+	auto &gstate = input.global_state.Cast<RadixHTGlobalState>();
 	D_ASSERT(!gstate.is_finalized);
 
 	DataChunk &group_chunk = llstate.group_chunk;
-	PopulateGroupChunk(group_chunk, groups_input);
+	PopulateGroupChunk(group_chunk, chunk);
 
 	// if we have non-combinable aggregates (e.g. string_agg) we cannot keep parallel hash
 	// tables
-	if (ForceSingleHT(state)) {
+	if (ForceSingleHT(input.global_state)) {
 		lock_guard<mutex> glock(gstate.lock);
 		gstate.is_empty = gstate.is_empty && group_chunk.size() == 0;
 		if (gstate.finalized_hts.empty()) {
@@ -304,7 +303,7 @@ private:
 };
 
 void RadixPartitionedHashTable::ScheduleTasks(Executor &executor, const shared_ptr<Event> &event,
-                                              GlobalSinkState &state, vector<unique_ptr<Task>> &tasks) const {
+                                              GlobalSinkState &state, vector<shared_ptr<Task>> &tasks) const {
 	auto &gstate = state.Cast<RadixHTGlobalState>();
 	if (!gstate.is_partitioned) {
 		return;
@@ -382,14 +381,14 @@ idx_t RadixPartitionedHashTable::Size(GlobalSinkState &sink_state) const {
 	return count;
 }
 
-void RadixPartitionedHashTable::GetData(ExecutionContext &context, DataChunk &chunk, GlobalSinkState &sink_state,
-                                        GlobalSourceState &gsstate, LocalSourceState &lsstate) const {
+SourceResultType RadixPartitionedHashTable::GetData(ExecutionContext &context, DataChunk &chunk,
+                                                    GlobalSinkState &sink_state, OperatorSourceInput &input) const {
 	auto &gstate = sink_state.Cast<RadixHTGlobalState>();
-	auto &state = gsstate.Cast<RadixHTGlobalSourceState>();
-	auto &lstate = lsstate.Cast<RadixHTLocalSourceState>();
+	auto &state = input.global_state.Cast<RadixHTGlobalSourceState>();
+	auto &lstate = input.local_state.Cast<RadixHTLocalSourceState>();
 	D_ASSERT(gstate.is_finalized);
 	if (state.finished) {
-		return;
+		return SourceResultType::FINISHED;
 	}
 
 	// special case hack to sort out aggregating from empty intermediates
@@ -421,11 +420,11 @@ void RadixPartitionedHashTable::GetData(ExecutionContext &context, DataChunk &ch
 			chunk.data[null_groups.size() + op.aggregates.size() + i].Reference(grouping_values[i]);
 		}
 		state.finished = true;
-		return;
+		return chunk.size() == 0 ? SourceResultType::FINISHED : SourceResultType::HAVE_MORE_OUTPUT;
 	}
 	if (gstate.is_empty) {
 		state.finished = true;
-		return;
+		return chunk.size() == 0 ? SourceResultType::FINISHED : SourceResultType::HAVE_MORE_OUTPUT;
 	}
 	idx_t elements_found = 0;
 
@@ -461,7 +460,7 @@ void RadixPartitionedHashTable::GetData(ExecutionContext &context, DataChunk &ch
 			ht_index = state.ht_index;
 			if (ht_index >= gstate.finalized_hts.size()) {
 				state.finished = true;
-				return;
+				return chunk.size() == 0 ? SourceResultType::FINISHED : SourceResultType::HAVE_MORE_OUTPUT;
 			}
 		}
 		D_ASSERT(ht_index < gstate.finalized_hts.size());
@@ -511,6 +510,7 @@ void RadixPartitionedHashTable::GetData(ExecutionContext &context, DataChunk &ch
 	for (idx_t i = 0; i < op.grouping_functions.size(); i++) {
 		chunk.data[op.GroupCount() + op.aggregates.size() + i].Reference(grouping_values[i]);
 	}
+	return chunk.size() == 0 ? SourceResultType::FINISHED : SourceResultType::HAVE_MORE_OUTPUT;
 }
 
 } // namespace duckdb

package/src/duckdb/src/function/table/arrow_conversion.cpp

@@ -253,6 +253,9 @@ static void SetVectorString(Vector &vector, idx_t size, char *cdata, T *offsets)
 		}
 		auto cptr = cdata + offsets[row_idx];
 		auto str_len = offsets[row_idx + 1] - offsets[row_idx];
+		if (str_len > NumericLimits<uint32_t>::Maximum()) { // LCOV_EXCL_START
+			throw ConversionException("DuckDB does not support Strings over 4GB");
+		} // LCOV_EXCL_STOP
 		strings[row_idx] = string_t(cptr, str_len);
 	}
 }
@@ -406,9 +409,6 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowScanLoca
 		auto original_type = arrow_convert_data[col_idx]->variable_sz_type[arrow_convert_idx.variable_sized_index++];
 		auto cdata = (char *)array.buffers[2];
 		if (original_type.first == ArrowVariableSizeType::SUPER_SIZE) {
-			if (((uint64_t *)array.buffers[1])[array.length] > NumericLimits<uint32_t>::Maximum()) { // LCOV_EXCL_START
-				throw ConversionException("DuckDB does not support Strings over 4GB");
-			} // LCOV_EXCL_STOP
 			auto offsets = (uint64_t *)array.buffers[1] + array.offset + scan_state.chunk_offset;
 			if (nested_offset != -1) {
 				offsets = (uint64_t *)array.buffers[1] + array.offset + nested_offset;

package/src/duckdb/src/function/table/read_csv.cpp

@@ -497,7 +497,8 @@ bool ParallelCSVGlobalState::Next(ClientContext &context, const ReadCSVData &bin
 		}
 		reader->options.file_path = current_file_path;
 		MultiFileReader::InitializeReader(*reader, bind_data.options.file_options, bind_data.reader_bind,
-		                                  bind_data.return_types, bind_data.return_names, column_ids, nullptr);
+		                                  bind_data.return_types, bind_data.return_names, column_ids, nullptr,
+		                                  bind_data.files.front());
 	} else {
 		// update the current reader
 		reader->SetBufferRead(std::move(result));
@@ -660,7 +661,8 @@ struct SingleThreadedCSVState : public GlobalTableFunctionState {
 				result->names = csv_names;
 			}
 			MultiFileReader::InitializeReader(*result, bind_data.options.file_options, bind_data.reader_bind,
-			                                  bind_data.return_types, bind_data.return_names, column_ids, nullptr);
+			                                  bind_data.return_types, bind_data.return_names, column_ids, nullptr,
+			                                  bind_data.files.front());
 		}
 		total_size = result->file_handle->FileSize();
 		return result;
@@ -707,14 +709,15 @@ static unique_ptr<GlobalTableFunctionState> SingleThreadedCSVInit(ClientContext
 		}
 	}
 	MultiFileReader::InitializeReader(*result->initial_reader, bind_data.options.file_options, bind_data.reader_bind,
-	                                  bind_data.return_types, bind_data.return_names, input.column_ids, input.filters);
+	                                  bind_data.return_types, bind_data.return_names, input.column_ids, input.filters,
+	                                  bind_data.files.front());
 	for (auto &reader : bind_data.union_readers) {
 		if (!reader) {
 			continue;
 		}
 		MultiFileReader::InitializeReader(*reader, bind_data.options.file_options, bind_data.reader_bind,
 		                                  bind_data.return_types, bind_data.return_names, input.column_ids,
-		                                  input.filters);
+		                                  input.filters, bind_data.files.front());
 	}
 	result->column_ids = input.column_ids;
 

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.7.2-dev2820"
+#define DUCKDB_VERSION "0.7.2-dev2995"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "fc797c18cf"
+#define DUCKDB_SOURCE_ID "9b1d80a9ee"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp

@@ -80,7 +80,7 @@ public:
 	virtual unique_ptr<BaseStatistics> GetStatistics(ClientContext &context, column_t column_id) = 0;
 
 	//! Serialize the meta information of the TableCatalogEntry a serializer
-	virtual void Serialize(Serializer &serializer) const;
+	void Serialize(Serializer &serializer) const;
 	//! Deserializes to a CreateTableInfo
 	static unique_ptr<CreateTableInfo> Deserialize(Deserializer &source, ClientContext &context);
 
@@ -103,6 +103,12 @@ public:
 	virtual TableStorageInfo GetStorageInfo(ClientContext &context) = 0;
 
 protected:
+	// This is used to serialize the entry by #Serialize(Serializer& ). It is virtual to allow
+	// Custom catalog implementations to override the default implementation. We can not make
+	// The Serialize method itself virtual as the logic is tightly coupled to the static
+	// Deserialize method.
+	virtual CreateTableInfo GetTableInfoForSerialization() const;
+
 	//! A list of columns that are part of this table
 	ColumnList columns;
 	//! A list of constraints that are part of this table

package/src/duckdb/src/include/duckdb/common/enums/operator_result_type.hpp

@@ -14,25 +14,37 @@ namespace duckdb {
 
 //! The OperatorResultType is used to indicate how data should flow around a regular (i.e. non-sink and non-source)
 //! physical operator
-//! There are three possible results:
+//! There are four possible results:
 //! NEED_MORE_INPUT means the operator is done with the current input and can consume more input if available
 //! If there is more input the operator will be called with more input, otherwise the operator will not be called again.
 //! HAVE_MORE_OUTPUT means the operator is not finished yet with the current input.
 //! The operator will be called again with the same input.
 //! FINISHED means the operator has finished the entire pipeline and no more processing is necessary.
 //! The operator will not be called again, and neither will any other operators in this pipeline.
-enum class OperatorResultType : uint8_t { NEED_MORE_INPUT, HAVE_MORE_OUTPUT, FINISHED };
+//! BLOCKED means the operator does not want to be called right now. e.g. because its currently doing async I/O. The
+//! operator has set the interrupt state and the caller is expected to handle it. Note that intermediate operators
+//! should currently not emit this state.
+enum class OperatorResultType : uint8_t { NEED_MORE_INPUT, HAVE_MORE_OUTPUT, FINISHED, BLOCKED };
 
 //! OperatorFinalizeResultType is used to indicate whether operators have finished flushing their cached results.
 //! FINISHED means the operator has flushed all cached data.
 //! HAVE_MORE_OUTPUT means the operator contains more results.
 enum class OperatorFinalizeResultType : uint8_t { HAVE_MORE_OUTPUT, FINISHED };
 
+//! SourceResultType is used to indicate the result of data being pulled out of a source.
+//! There are three possible results:
+//! HAVE_MORE_OUTPUT means the source has more output, this flag should only be set when data is returned, empty results
+//! should only occur for the FINISHED and BLOCKED flags
+//! FINISHED means the source is exhausted
+//! BLOCKED means the source is currently blocked, e.g. by some async I/O
+enum class SourceResultType : uint8_t { HAVE_MORE_OUTPUT, FINISHED, BLOCKED };
+
 //! The SinkResultType is used to indicate the result of data flowing into a sink
-//! There are two possible results:
+//! There are three possible results:
 //! NEED_MORE_INPUT means the sink needs more input
 //! FINISHED means the sink is finished executing, and more input will not change the result any further
-enum class SinkResultType : uint8_t { NEED_MORE_INPUT, FINISHED };
+//! BLOCKED means the sink is currently blocked, e.g. by some async I/O.
+enum class SinkResultType : uint8_t { NEED_MORE_INPUT, FINISHED, BLOCKED };
 
 //! The SinkFinalizeType is used to indicate the result of a Finalize call on a sink
 //! There are two possible results:

package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp

@@ -107,7 +107,8 @@ struct MultiFileReader {
 	DUCKDB_API static void CreateMapping(const string &file_name, const vector<LogicalType> &local_types,
 	                                     const vector<string> &local_names, const vector<LogicalType> &global_types,
 	                                     const vector<string> &global_names, const vector<column_t> &global_column_ids,
-	                                     optional_ptr<TableFilterSet> filters, MultiFileReaderData &reader_data);
+	                                     optional_ptr<TableFilterSet> filters, MultiFileReaderData &reader_data,
+	                                     const string &initial_file);
 	//! Finalize the reading of a chunk - applying any constants that are required
 	DUCKDB_API static void FinalizeChunk(const MultiFileReaderBindData &bind_data,
 	                                     const MultiFileReaderData &reader_data, DataChunk &chunk);
@@ -156,11 +157,11 @@ struct MultiFileReader {
 	static void InitializeReader(READER_CLASS &reader, const MultiFileReaderOptions &options,
 	                             const MultiFileReaderBindData &bind_data, const vector<LogicalType> &global_types,
 	                             const vector<string> &global_names, const vector<column_t> &global_column_ids,
-	                             optional_ptr<TableFilterSet> table_filters) {
+	                             optional_ptr<TableFilterSet> table_filters, const string &initial_file) {
 		FinalizeBind(options, bind_data, reader.GetFileName(), reader.GetNames(), global_types, global_names,
 		             global_column_ids, reader.reader_data);
 		CreateMapping(reader.GetFileName(), reader.GetTypes(), reader.GetNames(), global_types, global_names,
-		              global_column_ids, table_filters, reader.reader_data);
+		              global_column_ids, table_filters, reader.reader_data, initial_file);
 		reader.reader_data.filters = table_filters;
 	}
 
@@ -193,7 +194,7 @@ private:
 	static void CreateNameMapping(const string &file_name, const vector<LogicalType> &local_types,
 	                              const vector<string> &local_names, const vector<LogicalType> &global_types,
 	                              const vector<string> &global_names, const vector<column_t> &global_column_ids,
-	                              MultiFileReaderData &reader_data);
+	                              MultiFileReaderData &reader_data, const string &initial_file);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/optional_idx.hpp

@@ -0,0 +1,45 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/optional_idx.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/exception.hpp"
+
+namespace duckdb {
+
+class optional_idx {
+	static constexpr const idx_t INVALID_INDEX = idx_t(-1);
+
+public:
+	optional_idx() : index(INVALID_INDEX) {
+	}
+	optional_idx(idx_t index) : index(index) { // NOLINT: allow implicit conversion from idx_t
+		if (index == INVALID_INDEX) {
+			throw InternalException("optional_idx cannot be initialized with an invalid index");
+		}
+	}
+
+	static optional_idx Invalid() {
+		return INVALID_INDEX;
+	}
+
+	bool IsValid() const {
+		return index != DConstants::INVALID_INDEX;
+	}
+	idx_t GetIndex() {
+		if (index == INVALID_INDEX) {
+			throw InternalException("Attempting to get the index of an optional_idx that is not set");
+		}
+		return index;
+	}
+
+private:
+	idx_t index;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/set.hpp

@@ -11,5 +11,6 @@
 #include <set>
 
 namespace duckdb {
+using std::multiset;
 using std::set;
-}
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp

@@ -88,6 +88,21 @@ struct FormatFun {
 	static ScalarFunction GetFunction();
 };
 
+struct FormatBytesFun {
+	static constexpr const char *Name = "format_bytes";
+	static constexpr const char *Parameters = "bytes";
+	static constexpr const char *Description = "Converts bytes to a human-readable presentation (e.g. 16000 -> 16KB)";
+	static constexpr const char *Example = "format_bytes(1000 * 16)";
+
+	static ScalarFunction GetFunction();
+};
+
+struct FormatreadabledecimalsizeFun {
+	using ALIAS = FormatBytesFun;
+
+	static constexpr const char *Name = "formatReadableDecimalSize";
+};
+
 struct HammingFun {
 	static constexpr const char *Name = "hamming";
 	static constexpr const char *Parameters = "str1,str2";