duckdb 0.7.2-dev2820.0 → 0.7.2-dev2995.0

package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp

@@ -93,7 +93,7 @@ public:
 
 	//! Convert a transient in-memory segment into a persistent segment blocked by an on-disk block.
 	//! Only used during checkpointing.
-	void ConvertToPersistent(BlockManager *block_manager, block_id_t block_id);
+	void ConvertToPersistent(optional_ptr<BlockManager> block_manager, block_id_t block_id);
 	//! Updates pointers to refer to the given block and offset. This is only used
 	//! when sharing a block among segments. This is invoked only AFTER the block is written.
 	void MarkAsPersistent(shared_ptr<BlockHandle> block, uint32_t offset_in_block);

package/src/duckdb/src/include/duckdb/storage/table/list_column_data.hpp

@@ -17,8 +17,7 @@ namespace duckdb {
 class ListColumnData : public ColumnData {
 public:
 	ListColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
-	               LogicalType type, ColumnData *parent = nullptr);
-	ListColumnData(ColumnData &original, idx_t start_row, ColumnData *parent = nullptr);
+	               LogicalType type, optional_ptr<ColumnData> parent = nullptr);
 
 	//! The child-column of the list
 	unique_ptr<ColumnData> child_column;
@@ -26,6 +25,7 @@ public:
 	ValidityColumnData validity;
 
 public:
+	void SetStart(idx_t new_start) override;
 	bool CheckZonemap(ColumnScanState &state, TableFilter &filter) override;
 
 	void InitializeScan(ColumnScanState &state) override;

package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -59,20 +59,20 @@ public:
 public:
 	RowGroup(RowGroupCollection &collection, idx_t start, idx_t count);
 	RowGroup(RowGroupCollection &collection, RowGroupPointer &&pointer);
-	RowGroup(RowGroup &row_group, RowGroupCollection &collection, idx_t start);
 	~RowGroup();
 
 private:
 	//! The RowGroupCollection this row-group is a part of
-	RowGroupCollection &collection;
+	reference<RowGroupCollection> collection;
 	//! The version info of the row_group (inserted and deleted tuple info)
 	shared_ptr<VersionNode> version_info;
 	//! The column data of the row_group
 	vector<shared_ptr<ColumnData>> columns;
 
 public:
+	void MoveToCollection(RowGroupCollection &collection, idx_t new_start);
 	RowGroupCollection &GetCollection() {
-		return collection;
+		return collection.get();
 	}
 	DatabaseInstance &GetDatabase();
 	BlockManager &GetBlockManager();

package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp

@@ -44,6 +44,7 @@ public:
 	void AppendRowGroup(SegmentLock &l, idx_t start_row);
 	//! Get the nth row-group, negative numbers start from the back (so -1 is the last row group, etc)
 	RowGroup *GetRowGroup(int64_t index);
+	idx_t RowGroupCount();
 	void Verify();
 
 	void InitializeScan(CollectionScanState &state, const vector<column_t> &column_ids, TableFilterSet *table_filters);

package/src/duckdb/src/include/duckdb/storage/table/segment_base.hpp

@@ -27,7 +27,7 @@ public:
 	}
 
 	//! The start row id of this chunk
-	const idx_t start;
+	idx_t start;
 	//! The amount of entries in this storage chunk
 	atomic<idx_t> count;
 	//! The next segment after this one

package/src/duckdb/src/include/duckdb/storage/table/segment_tree.hpp

@@ -63,6 +63,14 @@ public:
 		LoadAllSegments(l);
 		return std::move(nodes);
 	}
+	vector<SegmentNode<T>> MoveSegments() {
+		auto l = Lock();
+		return MoveSegments(l);
+	}
+	idx_t GetSegmentCount() {
+		auto l = Lock();
+		return nodes.size();
+	}
 	//! Gets a pointer to the nth segment. Negative numbers start from the back.
 	T *GetSegmentByIndex(int64_t index) {
 		auto l = Lock();
@@ -243,6 +251,20 @@ public:
 		return SegmentIterationHelper(*this);
 	}
 
+	void Reinitialize() {
+		if (nodes.empty()) {
+			return;
+		}
+		idx_t offset = nodes[0].node->start;
+		for (auto &entry : nodes) {
+			if (entry.node->start != offset) {
+				throw InternalException("In SegmentTree::Reinitialize - gap found between nodes!");
+			}
+			entry.row_start = offset;
+			offset += entry.node->count;
+		}
+	}
+
 protected:
 	atomic<bool> finished_loading;
 

package/src/duckdb/src/include/duckdb/storage/table/standard_column_data.hpp

@@ -17,13 +17,13 @@ namespace duckdb {
 class StandardColumnData : public ColumnData {
 public:
 	StandardColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
-	                   LogicalType type, ColumnData *parent = nullptr);
-	StandardColumnData(ColumnData &original, idx_t start_row, ColumnData *parent = nullptr);
+	                   LogicalType type, optional_ptr<ColumnData> parent = nullptr);
 
 	//! The validity column data
 	ValidityColumnData validity;
 
 public:
+	void SetStart(idx_t new_start) override;
 	bool CheckZonemap(ColumnScanState &state, TableFilter &filter) override;
 
 	void InitializeScan(ColumnScanState &state) override;
@@ -51,7 +51,7 @@ public:
 	                                                        PartialBlockManager &partial_block_manager) override;
 	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, PartialBlockManager &partial_block_manager,
 	                                             ColumnCheckpointInfo &checkpoint_info) override;
-	void CheckpointScan(ColumnSegment *segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
+	void CheckpointScan(ColumnSegment &segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
 	                    Vector &scan_vector) override;
 
 	void DeserializeColumn(Deserializer &source) override;

package/src/duckdb/src/include/duckdb/storage/table/struct_column_data.hpp

@@ -17,8 +17,7 @@ namespace duckdb {
 class StructColumnData : public ColumnData {
 public:
 	StructColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
-	                 LogicalType type, ColumnData *parent = nullptr);
-	StructColumnData(ColumnData &original, idx_t start_row, ColumnData *parent = nullptr);
+	                 LogicalType type, optional_ptr<ColumnData> parent = nullptr);
 
 	//! The sub-columns of the struct
 	vector<unique_ptr<ColumnData>> sub_columns;
@@ -26,6 +25,7 @@ public:
 	ValidityColumnData validity;
 
 public:
+	void SetStart(idx_t new_start) override;
 	bool CheckZonemap(ColumnScanState &state, TableFilter &filter) override;
 	idx_t GetMaxEntry() override;
 

package/src/duckdb/src/include/duckdb/storage/table/update_segment.hpp

@@ -23,8 +23,6 @@ struct UpdateNode;
 class UpdateSegment {
 public:
 	UpdateSegment(ColumnData &column_data);
-	// Construct a duplicate of 'other' with 'new_owner' as it's column data
-	UpdateSegment(UpdateSegment &other, ColumnData &new_owner);
 	~UpdateSegment();
 
 	ColumnData &column_data;

package/src/duckdb/src/include/duckdb/storage/table/validity_column_data.hpp

@@ -16,8 +16,7 @@ namespace duckdb {
 class ValidityColumnData : public ColumnData {
 public:
 	ValidityColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
-	                   ColumnData *parent);
-	ValidityColumnData(ColumnData &original, idx_t start_row, ColumnData *parent = nullptr);
+	                   ColumnData &parent);
 
 public:
 	bool CheckZonemap(ColumnScanState &state, TableFilter &filter) override;

package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp

@@ -11,6 +11,7 @@
 #include "duckdb/storage/table/row_group_collection.hpp"
 #include "duckdb/storage/table/table_index_list.hpp"
 #include "duckdb/storage/table/table_statistics.hpp"
+#include "duckdb/storage/optimistic_data_writer.hpp"
 
 namespace duckdb {
 class AttachedDatabase;
@@ -20,35 +21,6 @@ class WriteAheadLog;
 struct LocalAppendState;
 struct TableAppendState;
 
-class OptimisticDataWriter {
-public:
-	OptimisticDataWriter(DataTable &table);
-	OptimisticDataWriter(DataTable &table, OptimisticDataWriter &parent);
-	~OptimisticDataWriter();
-
-	void CheckFlushToDisk(RowGroupCollection &row_groups);
-	//! Flushes a specific row group to disk
-	void FlushToDisk(RowGroup *row_group);
-	//! Flushes the final row group to disk (if any)
-	void FlushToDisk(RowGroupCollection &row_groups, bool force = false);
-	//! Final flush: flush the partial block manager to disk
-	void FinalFlush();
-
-	void Rollback();
-
-private:
-	//! Prepare a write to disk
-	bool PrepareWrite();
-
-private:
-	//! The table
-	DataTable &table;
-	//! The partial block manager (if we created one yet)
-	unique_ptr<PartialBlockManager> partial_manager;
-	//! The set of blocks that have been pre-emptively written to disk
-	unordered_set<block_id_t> written_blocks;
-};
-
 class LocalTableStorage : public std::enable_shared_from_this<LocalTableStorage> {
 public:
 	// Create a new LocalTableStorage
@@ -76,13 +48,14 @@ public:
 	OptimisticDataWriter optimistic_writer;
 	//! The set of all optimistic data writers associated with this table
 	vector<unique_ptr<OptimisticDataWriter>> optimistic_writers;
+	//! Whether or not storage was merged
+	bool merged_storage = false;
 
 public:
 	void InitializeScan(CollectionScanState &state, optional_ptr<TableFilterSet> table_filters = nullptr);
-	//! Check if we should flush the previously written row-group to disk
-	void CheckFlushToDisk();
-	//! Flushes the final row group to disk (if any)
-	void FlushToDisk();
+	//! Write a new row group to disk (if possible)
+	void WriteNewRowGroup();
+	void FlushBlocks();
 	void Rollback();
 	idx_t EstimatedSize();
 
@@ -93,6 +66,7 @@ public:
 
 	//! Creates an optimistic writer for this table
 	OptimisticDataWriter &CreateOptimisticWriter();
+	void FinalizeOptimisticWriter(OptimisticDataWriter &writer);
 };
 
 class LocalTableManager {
@@ -114,7 +88,7 @@ private:
 class LocalStorage {
 public:
 	// Threshold to merge row groups instead of appending
-	static constexpr const idx_t MERGE_THRESHOLD = RowGroup::ROW_GROUP_SIZE / 2;
+	static constexpr const idx_t MERGE_THRESHOLD = RowGroup::ROW_GROUP_SIZE;
 
 public:
 	struct CommitState {
@@ -150,6 +124,7 @@ public:
 	void LocalMerge(DataTable &table, RowGroupCollection &collection);
 	//! Create an optimistic writer for the specified table
 	OptimisticDataWriter &CreateOptimisticWriter(DataTable &table);
+	void FinalizeOptimisticWriter(DataTable &table, OptimisticDataWriter &writer);
 
 	//! Delete a set of rows from the local storage
 	idx_t Delete(DataTable &table, Vector &row_ids, idx_t count);

package/src/duckdb/src/include/duckdb/verification/no_operator_caching_verifier.hpp

@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/verification/unoptimized_statement_verifier.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/verification/statement_verifier.hpp"
+
+namespace duckdb {
+
+class NoOperatorCachingVerifier : public StatementVerifier {
+public:
+	explicit NoOperatorCachingVerifier(unique_ptr<SQLStatement> statement_p);
+	static unique_ptr<StatementVerifier> Create(const SQLStatement &statement_p);
+
+	bool DisableOperatorCaching() const override {
+		return true;
+	}
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/verification/statement_verifier.hpp

@@ -20,6 +20,7 @@ enum class VerificationType : uint8_t {
 	DESERIALIZED,
 	PARSED,
 	UNOPTIMIZED,
+	NO_OPERATOR_CACHING,
 	PREPARED,
 	EXTERNAL,
 
@@ -59,6 +60,10 @@ public:
 		return false;
 	}
 
+	virtual bool DisableOperatorCaching() const {
+		return false;
+	}
+
 	virtual bool ForceExternal() const {
 		return false;
 	}

package/src/duckdb/src/main/attached_database.cpp

@@ -1,9 +1,11 @@
 #include "duckdb/main/attached_database.hpp"
-#include "duckdb/storage/storage_manager.hpp"
-#include "duckdb/transaction/duck_transaction_manager.hpp"
-#include "duckdb/common/file_system.hpp"
+
 #include "duckdb/catalog/duck_catalog.hpp"
+#include "duckdb/common/file_system.hpp"
+#include "duckdb/parser/parsed_data/attach_info.hpp"
 #include "duckdb/storage/storage_extension.hpp"
+#include "duckdb/storage/storage_manager.hpp"
+#include "duckdb/transaction/duck_transaction_manager.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/main/client_verify.cpp

@@ -28,6 +28,10 @@ PreservedError ClientContext::VerifyQuery(ClientContextLock &lock, const string
 		statement_verifiers.emplace_back(StatementVerifier::Create(VerificationType::DESERIALIZED, stmt));
 		statement_verifiers.emplace_back(StatementVerifier::Create(VerificationType::UNOPTIMIZED, stmt));
 		prepared_statement_verifier = StatementVerifier::Create(VerificationType::PREPARED, stmt);
+#ifdef DUCKDB_DEBUG_ASYNC_SINK_SOURCE
+		// This verification is quite slow, so we only run it for the async sink/source debug mode
+		statement_verifiers.emplace_back(StatementVerifier::Create(VerificationType::NO_OPERATOR_CACHING, stmt));
+#endif
 	}
 	if (config.verify_external) {
 		statement_verifiers.emplace_back(StatementVerifier::Create(VerificationType::EXTERNAL, stmt));

package/src/duckdb/src/main/config.cpp

@@ -14,6 +14,10 @@
 
 namespace duckdb {
 
+#ifdef DEBUG
+bool DBConfigOptions::debug_print_bindings = false;
+#endif
+
 #define DUCKDB_GLOBAL(_PARAM)                                                                                          \
 	{                                                                                                                  \
 		_PARAM::Name, _PARAM::Description, _PARAM::InputType, _PARAM::SetGlobal, nullptr, _PARAM::ResetGlobal,         \

package/src/duckdb/src/main/database.cpp

@@ -1,24 +1,24 @@
 #include "duckdb/main/database.hpp"
 
 #include "duckdb/catalog/catalog.hpp"
-#include "duckdb/main/database_manager.hpp"
 #include "duckdb/common/virtual_file_system.hpp"
+#include "duckdb/execution/operator/helper/physical_set.hpp"
+#include "duckdb/function/cast/cast_function_set.hpp"
+#include "duckdb/function/compression_function.hpp"
+#include "duckdb/main/attached_database.hpp"
 #include "duckdb/main/client_context.hpp"
-#include "duckdb/parallel/task_scheduler.hpp"
-#include "duckdb/storage/storage_manager.hpp"
-#include "duckdb/storage/object_cache.hpp"
-#include "duckdb/transaction/transaction_manager.hpp"
 #include "duckdb/main/connection_manager.hpp"
-#include "duckdb/function/compression_function.hpp"
-#include "duckdb/main/extension_helper.hpp"
-#include "duckdb/function/cast/cast_function_set.hpp"
+#include "duckdb/main/database_manager.hpp"
 #include "duckdb/main/error_manager.hpp"
-#include "duckdb/main/attached_database.hpp"
+#include "duckdb/main/extension_helper.hpp"
+#include "duckdb/parallel/task_scheduler.hpp"
 #include "duckdb/parser/parsed_data/attach_info.hpp"
 #include "duckdb/storage/magic_bytes.hpp"
-#include "duckdb/storage/storage_extension.hpp"
-#include "duckdb/execution/operator/helper/physical_set.hpp"
+#include "duckdb/storage/object_cache.hpp"
 #include "duckdb/storage/standard_buffer_manager.hpp"
+#include "duckdb/storage/storage_extension.hpp"
+#include "duckdb/storage/storage_manager.hpp"
+#include "duckdb/transaction/transaction_manager.hpp"
 
 #ifndef DUCKDB_NO_THREADS
 #include "duckdb/common/thread.hpp"
@@ -149,8 +149,9 @@ duckdb::unique_ptr<AttachedDatabase> DatabaseInstance::CreateAttachedDatabase(At
                                                                               AccessMode access_mode) {
 	duckdb::unique_ptr<AttachedDatabase> attached_database;
 	if (!type.empty()) {
-		// find the storage extensionon database
-		auto entry = config.storage_extensions.find(type);
+		// find the storage extension
+		auto extension_name = ExtensionHelper::ApplyExtensionAlias(type);
+		auto entry = config.storage_extensions.find(extension_name);
 		if (entry == config.storage_extensions.end()) {
 			throw BinderException("Unrecognized storage type \"%s\"", type);
 		}
@@ -171,6 +172,33 @@ duckdb::unique_ptr<AttachedDatabase> DatabaseInstance::CreateAttachedDatabase(At
 	return attached_database;
 }
 
+void DatabaseInstance::CreateDatabase(const string &database_type) {
+	AttachInfo info;
+	info.name = AttachedDatabase::ExtractDatabaseName(config.options.database_path);
+	info.path = config.options.database_path;
+
+	auto attached_database = CreateAttachedDatabase(info, database_type, config.options.access_mode);
+	auto initial_database = attached_database.get();
+	{
+		Connection con(*this);
+		con.BeginTransaction();
+		db_manager->AddDatabase(*con.context, std::move(attached_database));
+		con.Commit();
+	}
+
+	// initialize the database
+	initial_database->Initialize();
+}
+
+void ThrowExtensionSetUnrecognizedOptions(const unordered_map<string, Value> &unrecognized_options) {
+	auto unrecognized_options_iter = unrecognized_options.begin();
+	string unrecognized_option_keys = unrecognized_options_iter->first;
+	for (; unrecognized_options_iter == unrecognized_options.end(); ++unrecognized_options_iter) {
+		unrecognized_option_keys = "," + unrecognized_options_iter->first;
+	}
+	throw InvalidInputException("Unrecognized configuration property \"%s\"", unrecognized_option_keys);
+}
+
 void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_config) {
 	DBConfig default_config;
 	DBConfig *config_ptr = &default_config;
@@ -208,27 +236,9 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf
 
 	// check if we are opening a standard DuckDB database or an extension database
 	auto database_type = ExtractDatabaseType(config.options.database_path);
-	if (!database_type.empty()) {
-		// we are opening an extension database, run storage_init
-		ExtensionHelper::StorageInit(database_type, config);
-	}
-	AttachInfo info;
-	info.name = AttachedDatabase::ExtractDatabaseName(config.options.database_path);
-	info.path = config.options.database_path;
-
-	auto attached_database = CreateAttachedDatabase(info, database_type, config.options.access_mode);
-	auto initial_database = attached_database.get();
-	{
-		Connection con(*this);
-		con.BeginTransaction();
-		db_manager->AddDatabase(*con.context, std::move(attached_database));
-		con.Commit();
-	}
 
 	// initialize the system catalog
 	db_manager->InitializeSystemCatalog();
-	// initialize the database
-	initial_database->Initialize();
 
 	if (!database_type.empty()) {
 		// if we are opening an extension database - load the extension
@@ -236,24 +246,11 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf
 	}
 
 	if (!config.options.unrecognized_options.empty()) {
-		// check if all unrecognized options can be handled by the loaded extension(s)
-		for (auto &unrecognized_option : config.options.unrecognized_options) {
-			auto entry = config.extension_parameters.find(unrecognized_option.first);
-			if (entry == config.extension_parameters.end()) {
-				throw InvalidInputException("Unrecognized configuration property \"%s\"", unrecognized_option.first);
-			}
-		}
+		ThrowExtensionSetUnrecognizedOptions(config.options.unrecognized_options);
+	}
 
-		// if so - set the options
-		Connection con(*this);
-		con.BeginTransaction();
-		for (auto &unrecognized_option : config.options.unrecognized_options) {
-			auto entry = config.extension_parameters.find(unrecognized_option.first);
-			D_ASSERT(entry != config.extension_parameters.end());
-			PhysicalSet::SetExtensionVariable(*con.context, entry->second, unrecognized_option.first, SetScope::GLOBAL,
-			                                  unrecognized_option.second);
-		}
-		con.Commit();
+	if (!db_manager->HasDefaultDatabase()) {
+		CreateDatabase(database_type);
 	}
 
 	// only increase thread count after storage init because we get races on catalog otherwise

package/src/duckdb/src/main/extension/extension_load.cpp

@@ -19,7 +19,7 @@ namespace duckdb {
 //===--------------------------------------------------------------------===//
 typedef void (*ext_init_fun_t)(DatabaseInstance &);
 typedef const char *(*ext_version_fun_t)(void);
-typedef void (*ext_storage_init_t)(DBConfig &);
+typedef bool (*ext_is_storage_t)(void);
 
 template <class T>
 static T LoadFunctionFromDLL(void *dll, const string &function_name, const string &filename) {
@@ -30,14 +30,18 @@ static T LoadFunctionFromDLL(void *dll, const string &function_name, const strin
 	return (T)function;
 }
 
-void ComputeSHA256(FileHandle *handle, const idx_t start, const idx_t end, std::string *res) {
+static void ComputeSHA256String(const std::string &to_hash, std::string *res) {
+	// Invoke MbedTls function to actually compute sha256
+	*res = duckdb_mbedtls::MbedTlsWrapper::ComputeSha256Hash(to_hash);
+}
+
+static void ComputeSHA256FileSegment(FileHandle *handle, const idx_t start, const idx_t end, std::string *res) {
 	const idx_t len = end - start;
 	string file_content;
 	file_content.resize(len);
 	handle->Read((void *)file_content.data(), len, start);
 
-	// Invoke MbedTls function to actually compute sha256
-	*res = duckdb_mbedtls::MbedTlsWrapper::ComputeSha256Hash(file_content);
+	ComputeSHA256String(file_content, res);
 }
 
 bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileOpener *opener, const string &extension,
@@ -86,19 +90,19 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileOpener *opener, const
 
 		const idx_t maxLenChunks = 1024 * 1024;
 		const idx_t numChunks = (signature_offset + maxLenChunks - 1) / maxLenChunks;
-		std::vector<std::string> chunks(numChunks);
+		std::vector<std::string> hash_chunks(numChunks);
 		std::vector<idx_t> splits(numChunks + 1);
 
-		splits.back() = signature_offset;
-		for (idx_t i = 0; i < chunks.size(); i++) {
+		for (idx_t i = 0; i < numChunks; i++) {
 			splits[i] = maxLenChunks * i;
 		}
+		splits.back() = signature_offset;
 
 #ifndef DUCKDB_NO_THREADS
 		std::vector<std::thread> threads;
 		threads.reserve(numChunks);
 		for (idx_t i = 0; i < numChunks; i++) {
-			threads.emplace_back(ComputeSHA256, handle.get(), splits[i], splits[i + 1], &chunks[i]);
+			threads.emplace_back(ComputeSHA256FileSegment, handle.get(), splits[i], splits[i + 1], &hash_chunks[i]);
 		}
 
 		for (auto &thread : threads) {
@@ -106,26 +110,26 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileOpener *opener, const
 		}
 #else
 		for (idx_t i = 0; i < numChunks; i++) {
-			ComputeSHA256(handle.get(), splits[i], splits[i + 1], &chunks[i]);
+			ComputeSHA256FileSegment(handle.get(), splits[i], splits[i + 1], &hash_chunks[i]);
 		}
 #endif // DUCKDB_NO_THREADS
 
-		string file_content;
-		file_content.reserve(256 * numChunks);
+		string hash_concatenation;
+		hash_concatenation.reserve(32 * numChunks); // 256 bits -> 32 bytes per chunk
 
-		for (auto &chunk : chunks) {
-			file_content += chunk;
+		for (auto &hash_chunk : hash_chunks) {
+			hash_concatenation += hash_chunk;
 		}
 
-		string hash;
-		ComputeSHA256(handle.get(), 0, file_content.size(), &hash);
+		string two_level_hash;
+		ComputeSHA256String(hash_concatenation, &two_level_hash);
 
 		// TODO maybe we should do a stream read / hash update here
 		handle->Read((void *)signature.data(), signature.size(), signature_offset);
 
 		bool any_valid = false;
 		for (auto &key : ExtensionHelper::GetPublicKeys()) {
-			if (duckdb_mbedtls::MbedTlsWrapper::IsValidSha256Signature(key, signature, hash)) {
+			if (duckdb_mbedtls::MbedTlsWrapper::IsValidSha256Signature(key, signature, two_level_hash)) {
 				any_valid = true;
 				break;
 			}
@@ -201,7 +205,18 @@ ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileOpener *o
 	string error;
 	ExtensionInitResult result;
 	if (!TryInitialLoad(config, opener, extension, result, error)) {
-		throw IOException(error);
+		if (!ExtensionHelper::AllowAutoInstall(extension)) {
+			throw IOException(error);
+		}
+		// the extension load failed - try installing the extension
+		if (!config.file_system) {
+			throw InternalException("Attempting to install an extension without a file system");
+		}
+		ExtensionHelper::InstallExtension(config, *config.file_system, extension, false);
+		// try loading again
+		if (!TryInitialLoad(config, nullptr, extension, result, error)) {
+			throw IOException(error);
+		}
 	}
 	return result;
 }
@@ -251,38 +266,6 @@ void ExtensionHelper::LoadExternalExtension(ClientContext &context, const string
 	LoadExternalExtension(DatabaseInstance::GetDatabase(context), FileSystem::GetFileOpener(context), extension);
 }
 
-void ExtensionHelper::StorageInit(string &extension, DBConfig &config) {
-	extension = ExtensionHelper::ApplyExtensionAlias(extension);
-	ExtensionInitResult res;
-	string error;
-	if (!TryInitialLoad(config, nullptr, extension, res, error)) {
-		if (!ExtensionHelper::AllowAutoInstall(extension)) {
-			throw IOException(error);
-		}
-		// the extension load failed - try installing the extension
-		if (!config.file_system) {
-			throw InternalException("Attempting to install an extension without a file system");
-		}
-		ExtensionHelper::InstallExtension(config, *config.file_system, extension, false);
-		// try loading again
-		if (!TryInitialLoad(config, nullptr, extension, res, error)) {
-			throw IOException(error);
-		}
-	}
-	auto storage_fun_name = res.basename + "_storage_init";
-
-	ext_storage_init_t storage_init_fun;
-	storage_init_fun = LoadFunctionFromDLL<ext_storage_init_t>(res.lib_hdl, storage_fun_name, res.filename);
-
-	try {
-		(*storage_init_fun)(config);
-	} catch (std::exception &e) {
-		throw InvalidInputException(
-		    "Storage initialization function \"%s\" from file \"%s\" threw an exception: \"%s\"", storage_fun_name,
-		    res.filename, e.what());
-	}
-}
-
 string ExtensionHelper::ExtractExtensionPrefixFromPath(const string &path) {
 	auto first_colon = path.find(':');
 	if (first_colon == string::npos || first_colon < 2) { // needs to be at least two characters because windows c: ...

package/src/duckdb/src/parallel/event.cpp

@@ -72,7 +72,7 @@ void Event::InsertEvent(shared_ptr<Event> replacement_event) {
 	executor.AddEvent(std::move(replacement_event));
 }
 
-void Event::SetTasks(vector<unique_ptr<Task>> tasks) {
+void Event::SetTasks(vector<shared_ptr<Task>> tasks) {
 	auto &ts = TaskScheduler::GetScheduler(executor.context);
 	D_ASSERT(total_tasks == 0);
 	D_ASSERT(!tasks.empty());