duckdb 0.7.1-dev341.0 → 0.7.1-dev407.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.7.1-dev341.0",
+  "version": "0.7.1-dev407.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/extension/json/include/json_scan.hpp

@@ -296,6 +296,21 @@ public:
 		return lstate.GetBatchIndex();
 	}
 
+	static unique_ptr<NodeStatistics> JSONScanCardinality(ClientContext &context, const FunctionData *bind_data) {
+		auto &data = (JSONScanData &)*bind_data;
+		idx_t per_file_cardinality;
+		if (data.stored_readers.empty()) {
+			// The cardinality of an unknown JSON file is the almighty number 42 except when it's not
+			per_file_cardinality = 42;
+		} else {
+			// If we multiply the almighty number 42 by 10, we get the exact average size of a JSON
+			// Not really, but the average size of a lineitem row in JSON is around 360 bytes
+			per_file_cardinality = data.stored_readers[0]->GetFileHandle().FileSize() / 420;
+		}
+		// Obviously this can be improved but this is better than defaulting to 0
+		return make_unique<NodeStatistics>(per_file_cardinality * data.file_paths.size());
+	}
+
 	static void JSONScanSerialize(FieldWriter &writer, const FunctionData *bind_data_p, const TableFunction &function) {
 		auto &bind_data = (JSONScanData &)*bind_data_p;
 		bind_data.Serialize(writer);
@@ -316,6 +331,7 @@ public:
 
 		table_function.table_scan_progress = JSONScanProgress;
 		table_function.get_batch_index = JSONScanGetBatchIndex;
+		table_function.cardinality = JSONScanCardinality;
 
 		table_function.serialize = JSONScanSerialize;
 		table_function.deserialize = JSONScanDeserialize;

package/src/duckdb/extension/json/json_functions/json_structure.cpp

@@ -214,9 +214,6 @@ void JSONStructureNode::RefineCandidateTypesObject(yyjson_val *vals[], idx_t cou
 		}
 	}
 
-	if (count > STANDARD_VECTOR_SIZE) {
-		string_vector.Initialize(false, count);
-	}
 	for (idx_t child_idx = 0; child_idx < child_count; child_idx++) {
 		desc.children[child_idx].RefineCandidateTypes(child_vals[child_idx], count, string_vector, allocator,
 		                                              date_format_map);
@@ -431,6 +428,10 @@ static inline yyjson_mut_val *ConvertStructureArray(const JSONStructureNode &nod
 static inline yyjson_mut_val *ConvertStructureObject(const JSONStructureNode &node, yyjson_mut_doc *doc) {
 	D_ASSERT(node.descriptions.size() == 1 && node.descriptions[0].type == LogicalTypeId::STRUCT);
 	auto &desc = node.descriptions[0];
+	if (desc.children.empty()) {
+		// Empty struct - let's do JSON instead
+		return yyjson_mut_str(doc, JSONCommon::JSON_TYPE_NAME);
+	}
 
 	auto obj = yyjson_mut_obj(doc);
 	for (auto &child : desc.children) {
@@ -495,6 +496,10 @@ static LogicalType StructureToTypeObject(ClientContext &context, const JSONStruc
                                          idx_t depth) {
 	D_ASSERT(node.descriptions.size() == 1 && node.descriptions[0].type == LogicalTypeId::STRUCT);
 	auto &desc = node.descriptions[0];
+	if (desc.children.empty()) {
+		// Empty struct - let's do JSON instead
+		return JSONCommon::JSONType();
+	}
 
 	child_list_t<LogicalType> child_types;
 	child_types.reserve(desc.children.size());

package/src/duckdb/extension/json/json_functions/json_transform.cpp

@@ -58,6 +58,9 @@ static LogicalType StructureToTypeObject(yyjson_val *obj, ClientContext &context
 		child_types.emplace_back(key_str, StructureStringToType(val, context));
 	}
 	D_ASSERT(yyjson_obj_size(obj) == names.size());
+	if (child_types.empty()) {
+		throw InvalidInputException("Empty object in JSON structure");
+	}
 	return LogicalType::STRUCT(child_types);
 }
 
@@ -87,7 +90,7 @@ static unique_ptr<FunctionData> JSONTransformBind(ClientContext &context, Scalar
 	} else {
 		auto structure_val = ExpressionExecutor::EvaluateScalar(context, *arguments[1]);
 		if (!structure_val.DefaultTryCastAs(JSONCommon::JSONType())) {
-			throw InvalidInputException("cannot cast JSON structure to string");
+			throw InvalidInputException("Cannot cast JSON structure to string");
 		}
 		auto structure_string = structure_val.GetValueUnsafe<string_t>();
 		JSONAllocator json_allocator(Allocator::DefaultAllocator());
@@ -251,7 +254,10 @@ static bool TransformDecimal(yyjson_val *vals[], Vector &result, const idx_t cou
 
 bool JSONTransform::GetStringVector(yyjson_val *vals[], const idx_t count, const LogicalType &target,
                                     Vector &string_vector, JSONTransformOptions &options) {
-	auto data = (string_t *)FlatVector::GetData(string_vector);
+	if (count > STANDARD_VECTOR_SIZE) {
+		string_vector.Initialize(false, count);
+	}
+	auto data = FlatVector::GetData<string_t>(string_vector);
 	auto &validity = FlatVector::Validity(string_vector);
 	validity.SetAllValid(count);
 

package/src/duckdb/extension/json/json_functions.cpp

@@ -173,6 +173,7 @@ unique_ptr<TableRef> JSONFunctions::ReadJSONReplacement(ClientContext &context,
 		lower_name = lower_name.substr(0, lower_name.size() - 4);
 	}
 	if (!StringUtil::EndsWith(lower_name, ".json") && !StringUtil::Contains(lower_name, ".json?") &&
+	    !StringUtil::EndsWith(lower_name, ".jsonl") && !StringUtil::Contains(lower_name, ".jsonl?") &&
 	    !StringUtil::EndsWith(lower_name, ".ndjson") && !StringUtil::Contains(lower_name, ".ndjson?")) {
 		return nullptr;
 	}

package/src/duckdb/src/common/bind_helpers.cpp

@@ -4,6 +4,7 @@
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/types/value.hpp"
 #include "duckdb/common/case_insensitive_map.hpp"
+#include <numeric>
 
 namespace duckdb {
 
@@ -64,4 +65,58 @@ vector<bool> ParseColumnList(const Value &value, vector<string> &names, const st
 	return ParseColumnList(children, names, loption);
 }
 
+vector<idx_t> ParseColumnsOrdered(const vector<Value> &set, vector<string> &names, const string &loption) {
+	vector<idx_t> result;
+
+	if (set.empty()) {
+		throw BinderException("\"%s\" expects a column list or * as parameter", loption);
+	}
+
+	// Maps option to bool indicating if its found and the index in the original set
+	case_insensitive_map_t<std::pair<bool, idx_t>> option_map;
+	for (idx_t i = 0; i < set.size(); i++) {
+		option_map[set[i].ToString()] = {false, i};
+	}
+	result.resize(option_map.size());
+
+	for (idx_t i = 0; i < names.size(); i++) {
+		auto entry = option_map.find(names[i]);
+		if (entry != option_map.end()) {
+			result[entry->second.second] = i;
+			entry->second.first = true;
+		}
+	}
+	for (auto &entry : option_map) {
+		if (!entry.second.first) {
+			throw BinderException("\"%s\" expected to find %s, but it was not found in the table", loption,
+			                      entry.first.c_str());
+		}
+	}
+	return result;
+}
+
+vector<idx_t> ParseColumnsOrdered(const Value &value, vector<string> &names, const string &loption) {
+	vector<idx_t> result;
+
+	// Only accept a list of arguments
+	if (value.type().id() != LogicalTypeId::LIST) {
+		// Support a single argument if it's '*'
+		if (value.type().id() == LogicalTypeId::VARCHAR && value.GetValue<string>() == "*") {
+			result.resize(names.size(), 0);
+			std::iota(std::begin(result), std::end(result), 0);
+			return result;
+		}
+		throw BinderException("\"%s\" expects a column list or * as parameter", loption);
+	}
+	auto &children = ListValue::GetChildren(value);
+	// accept '*' as single argument
+	if (children.size() == 1 && children[0].type().id() == LogicalTypeId::VARCHAR &&
+	    children[0].GetValue<string>() == "*") {
+		result.resize(names.size(), 0);
+		std::iota(std::begin(result), std::end(result), 0);
+		return result;
+	}
+	return ParseColumnsOrdered(children, names, loption);
+}
+
 } // namespace duckdb

package/src/duckdb/src/common/types.cpp

@@ -424,7 +424,7 @@ string LogicalType::ToString() const {
 		auto &child_types = StructType::GetChildTypes(*this);
 		string ret = "STRUCT(";
 		for (size_t i = 0; i < child_types.size(); i++) {
-			ret += child_types[i].first + " " + child_types[i].second.ToString();
+			ret += KeywordHelper::WriteOptionallyQuoted(child_types[i].first) + " " + child_types[i].second.ToString();
 			if (i < child_types.size() - 1) {
 				ret += ", ";
 			}

package/src/duckdb/src/execution/physical_operator.cpp

@@ -252,15 +252,15 @@ OperatorResultType CachingPhysicalOperator::Execute(ExecutionContext &context, D
 	if (!state.initialized) {
 		state.initialized = true;
 		state.can_cache_chunk = true;
+
 		if (!context.pipeline || !caching_supported) {
 			state.can_cache_chunk = false;
-		}
-
-		if (context.pipeline->GetSink() && context.pipeline->GetSink()->RequiresBatchIndex()) {
+		} else if (!context.pipeline->GetSink()) {
+			// Disabling for pipelines without Sink, i.e. when pulling
 			state.can_cache_chunk = false;
-		}
-
-		if (context.pipeline->IsOrderDependent()) {
+		} else if (context.pipeline->GetSink()->RequiresBatchIndex()) {
+			state.can_cache_chunk = false;
+		} else if (context.pipeline->IsOrderDependent()) {
 			state.can_cache_chunk = false;
 		}
 	}

package/src/duckdb/src/function/table/system/duckdb_temporary_files.cpp

@@ -0,0 +1,59 @@
+#include "duckdb/function/table/system_functions.hpp"
+#include "duckdb/storage/buffer_manager.hpp"
+
+namespace duckdb {
+
+struct DuckDBTemporaryFilesData : public GlobalTableFunctionState {
+	DuckDBTemporaryFilesData() : offset(0) {
+	}
+
+	vector<TemporaryFileInformation> entries;
+	idx_t offset;
+};
+
+static unique_ptr<FunctionData> DuckDBTemporaryFilesBind(ClientContext &context, TableFunctionBindInput &input,
+                                                         vector<LogicalType> &return_types, vector<string> &names) {
+	names.emplace_back("path");
+	return_types.emplace_back(LogicalType::VARCHAR);
+
+	names.emplace_back("size");
+	return_types.emplace_back(LogicalType::BIGINT);
+
+	return nullptr;
+}
+
+unique_ptr<GlobalTableFunctionState> DuckDBTemporaryFilesInit(ClientContext &context, TableFunctionInitInput &input) {
+	auto result = make_unique<DuckDBTemporaryFilesData>();
+
+	result->entries = BufferManager::GetBufferManager(context).GetTemporaryFiles();
+	return std::move(result);
+}
+
+void DuckDBTemporaryFilesFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) {
+	auto &data = (DuckDBTemporaryFilesData &)*data_p.global_state;
+	if (data.offset >= data.entries.size()) {
+		// finished returning values
+		return;
+	}
+	// start returning values
+	// either fill up the chunk or return all the remaining columns
+	idx_t count = 0;
+	while (data.offset < data.entries.size() && count < STANDARD_VECTOR_SIZE) {
+		auto &entry = data.entries[data.offset++];
+		// return values:
+		idx_t col = 0;
+		// database_name, VARCHAR
+		output.SetValue(col++, count, entry.path);
+		// database_oid, BIGINT
+		output.SetValue(col++, count, Value::BIGINT(entry.size));
+		count++;
+	}
+	output.SetCardinality(count);
+}
+
+void DuckDBTemporaryFilesFun::RegisterFunction(BuiltinFunctions &set) {
+	set.AddFunction(TableFunction("duckdb_temporary_files", {}, DuckDBTemporaryFilesFunction, DuckDBTemporaryFilesBind,
+	                              DuckDBTemporaryFilesInit));
+}
+
+} // namespace duckdb

package/src/duckdb/src/function/table/system_functions.cpp

@@ -29,6 +29,7 @@ void BuiltinFunctions::RegisterSQLiteFunctions() {
 	DuckDBSequencesFun::RegisterFunction(*this);
 	DuckDBSettingsFun::RegisterFunction(*this);
 	DuckDBTablesFun::RegisterFunction(*this);
+	DuckDBTemporaryFilesFun::RegisterFunction(*this);
 	DuckDBTypesFun::RegisterFunction(*this);
 	DuckDBViewsFun::RegisterFunction(*this);
 	TestAllTypesFun::RegisterFunction(*this);

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.7.1-dev341"
+#define DUCKDB_VERSION "0.7.1-dev407"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "d58ab188ff"
+#define DUCKDB_SOURCE_ID "66ba97b5f9"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/common/bind_helpers.hpp

@@ -17,5 +17,7 @@ class Value;
 Value ConvertVectorToValue(vector<Value> set);
 vector<bool> ParseColumnList(const vector<Value> &set, vector<string> &names, const string &option_name);
 vector<bool> ParseColumnList(const Value &value, vector<string> &names, const string &option_name);
+vector<idx_t> ParseColumnsOrdered(const vector<Value> &set, vector<string> &names, const string &loption);
+vector<idx_t> ParseColumnsOrdered(const Value &value, vector<string> &names, const string &loption);
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp

@@ -89,6 +89,10 @@ struct DuckDBTablesFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
 
+struct DuckDBTemporaryFilesFun {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
 struct DuckDBTypesFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };

package/src/duckdb/src/include/duckdb/main/config.hpp

@@ -136,6 +136,8 @@ struct DBConfigOptions {
 	case_insensitive_map_t<Value> set_variables;
 	//! Database configuration variable default values;
 	case_insensitive_map_t<Value> set_variable_defaults;
+	//! Directory to store extension binaries in
+	string extension_directory;
 	//! Whether unsigned extensions should be loaded
 	bool allow_unsigned_extensions = false;
 	//! Enable emitting FSST Vectors

package/src/duckdb/src/include/duckdb/main/settings.hpp

@@ -216,6 +216,15 @@ struct ExplainOutputSetting {
 	static Value GetSetting(ClientContext &context);
 };
 
+struct ExtensionDirectorySetting {
+	static constexpr const char *Name = "extension_directory";
+	static constexpr const char *Description = "Set the directory to store extensions in";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::VARCHAR;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(ClientContext &context);
+};
+
 struct ExternalThreadsSetting {
 	static constexpr const char *Name = "external_threads";
 	static constexpr const char *Description = "The number of external threads that work on DuckDB tasks.";

package/src/duckdb/src/include/duckdb/parallel/pipeline_executor.hpp

@@ -69,13 +69,6 @@ private:
 	//! The final chunk used for moving data into the sink
 	DataChunk final_chunk;
 
-	//! Indicates that the first non-finished operator in the pipeline with RequireFinalExecute has some pending result
-	bool pending_final_execute = false;
-	//! The OperatorFinalizeResultType corresponding to the currently pending final_execute result
-	OperatorFinalizeResultType cached_final_execute_result;
-	//! Source has been exhausted
-	bool source_empty = false;
-
 	//! The operators that are not yet finished executing and have data remaining
 	//! If the stack of in_process_operators is empty, we fetch from the source instead
 	stack<idx_t> in_process_operators;

package/src/duckdb/src/include/duckdb/storage/buffer_manager.hpp

@@ -23,6 +23,11 @@ class DatabaseInstance;
 class TemporaryDirectoryHandle;
 struct EvictionQueue;
 
+struct TemporaryFileInformation {
+	string path;
+	idx_t size;
+};
+
 //! The buffer manager is in charge of handling memory management for the database. It hands out memory buffers that can
 //! be used by the database internally.
 //
@@ -98,6 +103,9 @@ public:
 	DUCKDB_API void ReserveMemory(idx_t size);
 	DUCKDB_API void FreeReservedMemory(idx_t size);
 
+	//! Returns a list of all temporary files
+	vector<TemporaryFileInformation> GetTemporaryFiles();
+
 private:
 	//! Register an in-memory buffer of arbitrary size, as long as it is >= BLOCK_SIZE. can_destroy signifies whether or
 	//! not the buffer can be destroyed when unpinned, or whether or not it needs to be written to a temporary file so

package/src/duckdb/src/main/config.cpp

@@ -67,6 +67,7 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_LOCAL(EnableProgressBarPrintSetting),
                                                  DUCKDB_GLOBAL(ExperimentalParallelCSVSetting),
                                                  DUCKDB_LOCAL(ExplainOutputSetting),
+                                                 DUCKDB_GLOBAL(ExtensionDirectorySetting),
                                                  DUCKDB_GLOBAL(ExternalThreadsSetting),
                                                  DUCKDB_LOCAL(FileSearchPathSetting),
                                                  DUCKDB_GLOBAL(ForceCompressionSetting),

package/src/duckdb/src/main/database.cpp

@@ -144,9 +144,15 @@ unique_ptr<AttachedDatabase> DatabaseInstance::CreateAttachedDatabase(AttachInfo
 		if (entry == config.storage_extensions.end()) {
 			throw BinderException("Unrecognized storage type \"%s\"", type);
 		}
-		// use storage extension to create the initial database
-		attached_database = make_unique<AttachedDatabase>(*this, Catalog::GetSystemCatalog(*this), *entry->second,
-		                                                  info.name, info, access_mode);
+
+		if (entry->second->attach != nullptr && entry->second->create_transaction_manager != nullptr) {
+			// use storage extension to create the initial database
+			attached_database = make_unique<AttachedDatabase>(*this, Catalog::GetSystemCatalog(*this), *entry->second,
+			                                                  info.name, info, access_mode);
+		} else {
+			attached_database = make_unique<AttachedDatabase>(*this, Catalog::GetSystemCatalog(*this), info.name,
+			                                                  info.path, access_mode);
+		}
 	} else {
 		// check if this is an in-memory database or not
 		attached_database =
@@ -200,6 +206,7 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf
 	AttachInfo info;
 	info.name = AttachedDatabase::ExtractDatabaseName(config.options.database_path);
 	info.path = config.options.database_path;
+
 	auto attached_database = CreateAttachedDatabase(info, database_type, config.options.access_mode);
 	auto initial_database = attached_database.get();
 	{

package/src/duckdb/src/main/extension/extension_install.cpp

@@ -40,20 +40,54 @@ const vector<string> ExtensionHelper::PathComponents() {
 
 string ExtensionHelper::ExtensionDirectory(ClientContext &context) {
 	auto &fs = FileSystem::GetFileSystem(context);
-	string local_path = fs.GetHomeDirectory(FileSystem::GetFileOpener(context));
-	if (!fs.DirectoryExists(local_path)) {
-		throw IOException("Can't find the home directory at '%s'\nSpecify a home directory using the SET "
-		                  "home_directory='/path/to/dir' option.",
-		                  local_path);
+	auto opener = FileSystem::GetFileOpener(context);
+	Value extension_directory_value;
+	string extension_directory;
+
+	if (context.TryGetCurrentSetting("extension_directory", extension_directory_value) &&
+	    !extension_directory_value.IsNull() &&
+	    !extension_directory_value.ToString().empty()) { // create the extension directory if not present
+		extension_directory = extension_directory_value.ToString();
+		// TODO this should probably live in the FileSystem
+		// convert random separators to platform-canonic
+		extension_directory = fs.ConvertSeparators(extension_directory);
+		// expand ~ in extension directory
+		extension_directory = fs.ExpandPath(extension_directory, opener);
+		if (!fs.DirectoryExists(extension_directory)) {
+			auto sep = fs.PathSeparator();
+			auto splits = StringUtil::Split(extension_directory, sep);
+			D_ASSERT(!splits.empty());
+			string extension_directory_prefix;
+			if (StringUtil::StartsWith(extension_directory, sep)) {
+				extension_directory_prefix = sep; // this is swallowed by Split otherwise
+			}
+			for (auto &split : splits) {
+				extension_directory_prefix = extension_directory_prefix + split + sep;
+				if (!fs.DirectoryExists(extension_directory_prefix)) {
+					fs.CreateDirectory(extension_directory_prefix);
+				}
+			}
+		}
+	} else { // otherwise default to home
+		string home_directory = fs.GetHomeDirectory(opener);
+		// exception if the home directory does not exist, don't create whatever we think is home
+		if (!fs.DirectoryExists(home_directory)) {
+			throw IOException("Can't find the home directory at '%s'\nSpecify a home directory using the SET "
+			                  "home_directory='/path/to/dir' option.",
+			                  home_directory);
+		}
+		extension_directory = home_directory;
 	}
+	D_ASSERT(fs.DirectoryExists(extension_directory));
+
 	auto path_components = PathComponents();
 	for (auto &path_ele : path_components) {
-		local_path = fs.JoinPath(local_path, path_ele);
-		if (!fs.DirectoryExists(local_path)) {
-			fs.CreateDirectory(local_path);
+		extension_directory = fs.JoinPath(extension_directory, path_ele);
+		if (!fs.DirectoryExists(extension_directory)) {
+			fs.CreateDirectory(extension_directory);
 		}
 	}
-	return local_path;
+	return extension_directory;
 }
 
 bool ExtensionHelper::CreateSuggestions(const string &extension_name, string &message) {

package/src/duckdb/src/main/extension/extension_load.cpp

@@ -32,7 +32,13 @@ ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileOpener *o
 
 	// shorthand case
 	if (!ExtensionHelper::IsFullPath(extension)) {
-		string local_path = fs.GetHomeDirectory(opener);
+		string local_path = !config.options.extension_directory.empty() ? config.options.extension_directory
+		                                                                : fs.GetHomeDirectory(opener);
+
+		// convert random separators to platform-canonic
+		local_path = fs.ConvertSeparators(local_path);
+		// expand ~ in extension directory
+		local_path = fs.ExpandPath(local_path, opener);
 		auto path_components = PathComponents();
 		for (auto &path_ele : path_components) {
 			local_path = fs.JoinPath(local_path, path_ele);
@@ -40,7 +46,6 @@ ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileOpener *o
 		string extension_name = ApplyExtensionAlias(extension);
 		filename = fs.JoinPath(local_path, extension_name + ".duckdb_extension");
 	}
-
 	if (!fs.FileExists(filename)) {
 		string message;
 		bool exact_match = ExtensionHelper::CreateSuggestions(extension, message);

package/src/duckdb/src/main/settings/settings.cpp

@@ -531,6 +531,22 @@ Value ExplainOutputSetting::GetSetting(ClientContext &context) {
 	}
 }
 
+//===--------------------------------------------------------------------===//
+// Extension Directory Setting
+//===--------------------------------------------------------------------===//
+void ExtensionDirectorySetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
+	auto new_directory = input.ToString();
+	config.options.extension_directory = input.ToString();
+}
+
+void ExtensionDirectorySetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) {
+	config.options.extension_directory = DBConfig().options.extension_directory;
+}
+
+Value ExtensionDirectorySetting::GetSetting(ClientContext &context) {
+	return Value(DBConfig::GetConfig(context).options.extension_directory);
+}
+
 //===--------------------------------------------------------------------===//
 // External Threads Setting
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/parallel/pipeline_executor.cpp

@@ -114,52 +114,6 @@ OperatorResultType PipelineExecutor::ExecutePushInternal(DataChunk &input, idx_t
 	}
 }
 
-// Pull a single DataChunk from the pipeline by flushing any operators holding cached output
-void PipelineExecutor::FlushCachingOperatorsPull(DataChunk &result) {
-	idx_t start_idx = IsFinished() ? idx_t(finished_processing_idx) : 0;
-	idx_t op_idx = start_idx;
-	while (op_idx < pipeline.operators.size()) {
-		if (!pipeline.operators[op_idx]->RequiresFinalExecute()) {
-			op_idx++;
-			continue;
-		}
-
-		OperatorFinalizeResultType finalize_result;
-		DataChunk &curr_chunk =
-		    op_idx + 1 >= intermediate_chunks.size() ? final_chunk : *intermediate_chunks[op_idx + 1];
-
-		if (pending_final_execute) {
-			// Still have a cached chunk from a last pull, reuse chunk
-			finalize_result = cached_final_execute_result;
-		} else {
-			// Flush the current operator
-			auto current_operator = pipeline.operators[op_idx];
-			StartOperator(current_operator);
-			finalize_result = current_operator->FinalExecute(context, curr_chunk, *current_operator->op_state,
-			                                                 *intermediate_states[op_idx]);
-			EndOperator(current_operator, &curr_chunk);
-		}
-
-		auto execute_result = Execute(curr_chunk, result, op_idx + 1);
-
-		if (execute_result == OperatorResultType::HAVE_MORE_OUTPUT) {
-			pending_final_execute = true;
-			cached_final_execute_result = finalize_result;
-		} else {
-			pending_final_execute = false;
-			if (finalize_result == OperatorFinalizeResultType::FINISHED) {
-				FinishProcessing(op_idx);
-				op_idx++;
-			}
-		}
-
-		// Some non-empty result was pulled from some caching operator, we're done for this pull
-		if (result.size() > 0) {
-			break;
-		}
-	}
-}
-
 // Push all remaining cached operator output through the pipeline
 void PipelineExecutor::FlushCachingOperatorsPush() {
 	idx_t start_idx = IsFinished() ? idx_t(finished_processing_idx) : 0;
@@ -223,21 +177,13 @@ void PipelineExecutor::ExecutePull(DataChunk &result) {
 		D_ASSERT(!pipeline.sink);
 		auto &source_chunk = pipeline.operators.empty() ? result : *intermediate_chunks[0];
 		while (result.size() == 0) {
-			if (source_empty) {
-				FlushCachingOperatorsPull(result);
-				break;
-			}
-
 			if (in_process_operators.empty()) {
 				source_chunk.Reset();
 				FetchFromSource(source_chunk);
-
 				if (source_chunk.size() == 0) {
-					source_empty = true;
-					continue;
+					break;
 				}
 			}
-
 			if (!pipeline.operators.empty()) {
 				auto state = Execute(source_chunk, result);
 				if (state == OperatorResultType::FINISHED) {

package/src/duckdb/src/planner/binder/statement/bind_copy.cpp

@@ -23,16 +23,6 @@
 
 namespace duckdb {
 
-static vector<idx_t> ColumnListToIndices(const vector<bool> &vec) {
-	vector<idx_t> ret;
-	for (idx_t i = 0; i < vec.size(); i++) {
-		if (vec[i]) {
-			ret.push_back(i);
-		}
-	}
-	return ret;
-}
-
 vector<string> GetUniqueNames(const vector<string> &original_names) {
 	unordered_set<string> name_set;
 	vector<string> unique_names;
@@ -115,7 +105,7 @@ BoundStatement Binder::BindCopyTo(CopyStatement &stmt) {
 		}
 		if (loption == "partition_by") {
 			auto converted = ConvertVectorToValue(std::move(option.second));
-			partition_cols = ColumnListToIndices(ParseColumnList(converted, select_node.names, loption));
+			partition_cols = ParseColumnsOrdered(converted, select_node.names, loption);
 			continue;
 		}
 		stmt.info->options[option.first] = option.second;

package/src/duckdb/src/storage/buffer_manager.cpp

@@ -62,7 +62,7 @@ BlockHandle::BlockHandle(BlockManager &block_manager, block_id_t block_id_p, uni
 	memory_charge = std::move(reservation);
 }
 
-BlockHandle::~BlockHandle() {
+BlockHandle::~BlockHandle() { // NOLINT: allow internal exceptions
 	// being destroyed, so any unswizzled pointers are just binary junk now.
 	unswizzled = nullptr;
 	auto &buffer_manager = block_manager.buffer_manager;
@@ -522,11 +522,8 @@ void BufferManager::PurgeQueue() {
 
 void BlockManager::UnregisterBlock(block_id_t block_id, bool can_destroy) {
 	if (block_id >= MAXIMUM_BLOCK) {
-		// in-memory buffer: destroy the buffer
-		if (!can_destroy) {
-			// buffer could have been offloaded to disk: remove the file
-			buffer_manager.DeleteTemporaryFile(block_id);
-		}
+		// in-memory buffer: buffer could have been offloaded to disk: remove the file
+		buffer_manager.DeleteTemporaryFile(block_id);
 	} else {
 		lock_guard<mutex> lock(blocks_lock);
 		// on-disk block: erase from list of blocks in manager
@@ -608,7 +605,11 @@ public:
 	//! Returns true if the max_index has been altered
 	bool RemoveIndex(idx_t index) {
 		// remove this block from the set of blocks
-		indexes_in_use.erase(index);
+		auto entry = indexes_in_use.find(index);
+		if (entry == indexes_in_use.end()) {
+			throw InternalException("RemoveIndex - index %llu not found in indexes_in_use", index);
+		}
+		indexes_in_use.erase(entry);
 		free_indexes.insert(index);
 		// check if we can truncate the file
 
@@ -617,7 +618,7 @@ public:
 		if (max_index_in_use < max_index) {
 			// max index in use is lower than the max_index
 			// reduce the max_index
-			max_index = max_index_in_use + 1;
+			max_index = indexes_in_use.empty() ? 0 : max_index_in_use + 1;
 			// we can remove any free_indexes that are larger than the current max_index
 			while (!free_indexes.empty()) {
 				auto max_entry = *free_indexes.rbegin();
@@ -693,16 +694,15 @@ public:
 
 	unique_ptr<FileBuffer> ReadTemporaryBuffer(block_id_t id, idx_t block_index,
 	                                           unique_ptr<FileBuffer> reusable_buffer) {
-		auto buffer =
-		    ReadTemporaryBufferInternal(BufferManager::GetBufferManager(db), *handle, GetPositionInFile(block_index),
-		                                Storage::BLOCK_SIZE, id, std::move(reusable_buffer));
-		{
-			// remove the block (and potentially truncate the temp file)
-			TemporaryFileLock lock(file_lock);
-			D_ASSERT(handle);
-			RemoveTempBlockIndex(lock, block_index);
-		}
-		return buffer;
+		return ReadTemporaryBufferInternal(BufferManager::GetBufferManager(db), *handle, GetPositionInFile(block_index),
+		                                   Storage::BLOCK_SIZE, id, std::move(reusable_buffer));
+	}
+
+	void EraseBlockIndex(block_id_t block_index) {
+		// remove the block (and potentially truncate the temp file)
+		TemporaryFileLock lock(file_lock);
+		D_ASSERT(handle);
+		RemoveTempBlockIndex(lock, block_index);
 	}
 
 	bool DeleteIfEmpty() {
@@ -718,6 +718,14 @@ public:
 		return true;
 	}
 
+	TemporaryFileInformation GetTemporaryFile() {
+		TemporaryFileLock lock(file_lock);
+		TemporaryFileInformation info;
+		info.path = path;
+		info.size = GetPositionInFile(index_manager.GetMaxIndex());
+		return info;
+	}
+
 private:
 	void CreateFileIfNotExists(TemporaryFileLock &) {
 		if (handle) {
@@ -818,7 +826,7 @@ public:
 		{
 			// remove the block (and potentially erase the temp file)
 			TemporaryManagerLock lock(manager_lock);
-			EraseUsedBlock(lock, id, handle, index.file_index);
+			EraseUsedBlock(lock, id, handle, index);
 		}
 		return buffer;
 	}
@@ -827,14 +835,29 @@ public:
 		TemporaryManagerLock lock(manager_lock);
 		auto index = GetTempBlockIndex(lock, id);
 		auto handle = GetFileHandle(lock, index.file_index);
-		EraseUsedBlock(lock, id, handle, index.file_index);
+		EraseUsedBlock(lock, id, handle, index);
+	}
+
+	vector<TemporaryFileInformation> GetTemporaryFiles() {
+		lock_guard<mutex> lock(manager_lock);
+		vector<TemporaryFileInformation> result;
+		for (auto &file : files) {
+			result.push_back(file.second->GetTemporaryFile());
+		}
+		return result;
 	}
 
 private:
-	void EraseUsedBlock(TemporaryManagerLock &lock, block_id_t id, TemporaryFileHandle *handle, idx_t file_index) {
-		used_blocks.erase(id);
+	void EraseUsedBlock(TemporaryManagerLock &lock, block_id_t id, TemporaryFileHandle *handle,
+	                    TemporaryFileIndex index) {
+		auto entry = used_blocks.find(id);
+		if (entry == used_blocks.end()) {
+			throw InternalException("EraseUsedBlock - Block %llu not found in used blocks", id);
+		}
+		used_blocks.erase(entry);
+		handle->EraseBlockIndex(index.block_index);
 		if (handle->DeleteIfEmpty()) {
-			EraseFileHandle(lock, file_index);
+			EraseFileHandle(lock, index.file_index);
 		}
 	}
 
@@ -992,6 +1015,35 @@ void BufferManager::DeleteTemporaryFile(block_id_t id) {
 	}
 }
 
+vector<TemporaryFileInformation> BufferManager::GetTemporaryFiles() {
+	vector<TemporaryFileInformation> result;
+	if (temp_directory.empty()) {
+		return result;
+	}
+	{
+		lock_guard<mutex> temp_handle_guard(temp_handle_lock);
+		if (temp_directory_handle) {
+			result = temp_directory_handle->GetTempFile().GetTemporaryFiles();
+		}
+	}
+	auto &fs = FileSystem::GetFileSystem(db);
+	fs.ListFiles(temp_directory, [&](const string &name, bool is_dir) {
+		if (is_dir) {
+			return;
+		}
+		if (!StringUtil::EndsWith(name, ".block")) {
+			return;
+		}
+		TemporaryFileInformation info;
+		info.path = name;
+		auto handle = fs.OpenFile(name, FileFlags::FILE_FLAGS_READ);
+		info.size = fs.GetFileSize(*handle);
+		handle.reset();
+		result.push_back(info);
+	});
+	return result;
+}
+
 string BufferManager::InMemoryWarning() {
 	if (!temp_directory.empty()) {
 		return "";

package/src/duckdb/ub_src_function_table_system.cpp

@@ -22,6 +22,8 @@
 
 #include "src/function/table/system/duckdb_tables.cpp"
 
+#include "src/function/table/system/duckdb_temporary_files.cpp"
+
 #include "src/function/table/system/duckdb_types.cpp"
 
 #include "src/function/table/system/duckdb_views.cpp"