duckdb 0.7.1-dev240.0 → 0.7.1-dev320.0

package/src/duckdb/src/include/duckdb/execution/index/art/art_key.hpp

@@ -28,24 +28,24 @@ public:
 
 public:
 	template <class T>
-	static inline Key CreateKey(ArenaAllocator &allocator, T element) {
+	static inline Key CreateKey(ArenaAllocator &allocator, const LogicalType &type, T element) {
 		auto data = Key::CreateData<T>(allocator, element);
 		return Key(data, sizeof(element));
 	}
 
 	template <class T>
-	static inline Key CreateKey(ArenaAllocator &allocator, const Value &element) {
-		return CreateKey(allocator, element.GetValueUnsafe<T>());
+	static inline Key CreateKey(ArenaAllocator &allocator, const LogicalType &type, const Value &element) {
+		return CreateKey(allocator, type, element.GetValueUnsafe<T>());
 	}
 
 	template <class T>
-	static inline void CreateKey(ArenaAllocator &allocator, Key &key, T element) {
+	static inline void CreateKey(ArenaAllocator &allocator, const LogicalType &type, Key &key, T element) {
 		key.data = Key::CreateData<T>(allocator, element);
 		key.len = sizeof(element);
 	}
 
 	template <class T>
-	static inline void CreateKey(ArenaAllocator &allocator, Key &key, const Value element) {
+	static inline void CreateKey(ArenaAllocator &allocator, const LogicalType &type, Key &key, const Value element) {
 		key.data = Key::CreateData<T>(allocator, element.GetValueUnsafe<T>());
 		key.len = sizeof(element);
 	}
@@ -76,12 +76,9 @@ private:
 };
 
 template <>
-Key Key::CreateKey(ArenaAllocator &allocator, string_t value);
+Key Key::CreateKey(ArenaAllocator &allocator, const LogicalType &type, string_t value);
 template <>
-Key Key::CreateKey(ArenaAllocator &allocator, const char *value);
+Key Key::CreateKey(ArenaAllocator &allocator, const LogicalType &type, const char *value);
 template <>
-void Key::CreateKey(ArenaAllocator &allocator, Key &key, string_t value);
-template <>
-void Key::CreateKey(ArenaAllocator &allocator, Key &key, const char *value);
-
+void Key::CreateKey(ArenaAllocator &allocator, const LogicalType &type, Key &key, string_t value);
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/{extension_functions.hpp → extension_entries.hpp}

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/main/extension_functions.hpp
+// duckdb/main/extension_entries.hpp
 //
 //
 //===----------------------------------------------------------------------===//
@@ -12,12 +12,12 @@
 
 namespace duckdb {
 
-struct ExtensionFunction {
-	char function[48];
+struct ExtensionEntry {
+	char name[48];
 	char extension[48];
 };
 
-static constexpr ExtensionFunction EXTENSION_FUNCTIONS[] = {
+static constexpr ExtensionEntry EXTENSION_FUNCTIONS[] = {
     {"->>", "json"},
     {"array_to_json", "json"},
     {"create_fts_index", "fts"},
@@ -30,9 +30,9 @@ static constexpr ExtensionFunction EXTENSION_FUNCTIONS[] = {
     {"from_json", "json"},
     {"from_json_strict", "json"},
     {"from_substrait", "substrait"},
+    {"from_substrait_json", "substrait"},
     {"get_substrait", "substrait"},
     {"get_substrait_json", "substrait"},
-    {"from_substrait_json", "substrait"},
     {"icu_calendar_names", "icu"},
     {"icu_sort_key", "icu"},
     {"json", "json"},
@@ -88,4 +88,25 @@ static constexpr ExtensionFunction EXTENSION_FUNCTIONS[] = {
     {"visualize_json_profiling_output", "visualizer"},
     {"visualize_last_profiling_output", "visualizer"},
 };
+
+static constexpr ExtensionEntry EXTENSION_SETTINGS[] = {
+    {"binary_as_string", "parquet"},
+    {"calendar", "icu"},
+    {"http_retries", "httpfs"},
+    {"http_retry_backoff", "httpfs"},
+    {"http_retry_wait_ms", "httpfs"},
+    {"http_timeout", "httpfs"},
+    {"s3_access_key_id", "httpfs"},
+    {"s3_endpoint", "httpfs"},
+    {"s3_region", "httpfs"},
+    {"s3_secret_access_key", "httpfs"},
+    {"s3_session_token", "httpfs"},
+    {"s3_uploader_max_filesize", "httpfs"},
+    {"s3_uploader_max_parts_per_file", "httpfs"},
+    {"s3_uploader_thread_limit", "httpfs"},
+    {"s3_url_style", "httpfs"},
+    {"s3_use_ssl", "httpfs"},
+    {"sqlite_all_varchar", "sqlite_scanner"},
+    {"timezone", "icu"},
+};
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/extension_helper.hpp

@@ -62,6 +62,9 @@ public:
 	//! Apply any known extension aliases
 	static string ApplyExtensionAlias(string extension_name);
 
+	static string GetExtensionName(const string &extension);
+	static bool IsFullPath(const string &extension);
+
 private:
 	static const vector<string> PathComponents();
 	static ExtensionInitResult InitialLoad(DBConfig &context, FileOpener *opener, const string &extension);

package/src/duckdb/src/include/duckdb/planner/binder.hpp

@@ -108,6 +108,9 @@ public:
 
 	unique_ptr<BoundCreateTableInfo> BindCreateTableInfo(unique_ptr<CreateInfo> info);
 	unique_ptr<BoundCreateTableInfo> BindCreateTableInfo(unique_ptr<CreateInfo> info, SchemaCatalogEntry *schema);
+
+	vector<unique_ptr<Expression>> BindCreateIndexExpressions(TableCatalogEntry *table, CreateIndexInfo *info);
+
 	void BindCreateViewInfo(CreateViewInfo &base);
 	SchemaCatalogEntry *BindSchema(CreateInfo &info);
 	SchemaCatalogEntry *BindCreateFunctionInfo(CreateInfo &info);

package/src/duckdb/src/include/duckdb/planner/expression_binder/index_binder.hpp

@@ -10,20 +10,27 @@
 
 #include "duckdb/planner/expression_binder.hpp"
 #include "duckdb/common/unordered_map.hpp"
+#include "duckdb/parser/parsed_data/create_index_info.hpp"
+#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 
 namespace duckdb {
 class BoundColumnRefExpression;
 
-//! The INDEX binder is responsible for binding an expression within an Index statement
+//! The IndexBinder is responsible for binding an expression within an index statement
 class IndexBinder : public ExpressionBinder {
 public:
-	IndexBinder(Binder &binder, ClientContext &context);
+	IndexBinder(Binder &binder, ClientContext &context, TableCatalogEntry *table = nullptr,
+	            CreateIndexInfo *info = nullptr);
 
 protected:
 	BindResult BindExpression(unique_ptr<ParsedExpression> *expr_ptr, idx_t depth,
 	                          bool root_expression = false) override;
-
 	string UnsupportedAggregateMessage() override;
+
+private:
+	// only for WAL replay
+	TableCatalogEntry *table;
+	CreateIndexInfo *info;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/data_table.hpp

@@ -122,6 +122,12 @@ public:
 	void UpdateColumn(TableCatalogEntry &table, ClientContext &context, Vector &row_ids,
 	                  const vector<column_t> &column_path, DataChunk &updates);
 
+	//! Add an index to the DataTable. NOTE: for CREATE (UNIQUE) INDEX statements, we use the PhysicalCreateIndex
+	//! operator. This function is only used during the WAL replay, and is a much less performant index creation
+	//! approach.
+	void WALAddIndex(ClientContext &context, unique_ptr<Index> index,
+	                 const vector<unique_ptr<Expression>> &expressions);
+
 	//! Fetches an append lock
 	void AppendLock(TableAppendState &state);
 	//! Begin appending structs to this table, obtaining necessary locks, etc
@@ -176,7 +182,7 @@ public:
 	static bool IsForeignKeyIndex(const vector<PhysicalIndex> &fk_keys, Index &index, ForeignKeyType fk_type);
 
 	//! Initializes a special scan that is used to create an index on the table, it keeps locks on the table
-	void InitializeCreateIndexScan(CreateIndexScanState &state, const vector<column_t> &column_ids);
+	void InitializeWALCreateIndexScan(CreateIndexScanState &state, const vector<column_t> &column_ids);
 	//! Scans the next chunk for the CREATE INDEX operator
 	bool CreateIndexScan(TableScanState &state, DataChunk &result, TableScanType type);
 

package/src/duckdb/src/include/duckdb/storage/index.hpp

@@ -40,11 +40,11 @@ public:
 	IndexType type;
 	//! Associated table io manager
 	TableIOManager &table_io_manager;
-	//! Column identifiers to extract from the base table
+	//! Column identifiers to extract key columns from the base table
 	vector<column_t> column_ids;
-	//! Unordered_set of column_ids used by the index
+	//! Unordered set of column_ids used by the index
 	unordered_set<column_t> column_id_set;
-	//! Unbound expressions used by the index
+	//! Unbound expressions used by the index during optimizations
 	vector<unique_ptr<Expression>> unbound_expressions;
 	//! The physical types stored in the index
 	vector<PhysicalType> types;
@@ -64,94 +64,103 @@ public:
 	bool track_memory;
 
 public:
-	//! Initialize a scan on the index with the given expression and column ids
-	//! to fetch from the base table when we only have one query predicate
+	//! Initialize a single predicate scan on the index with the given expression and column IDs
 	virtual unique_ptr<IndexScanState> InitializeScanSinglePredicate(const Transaction &transaction, const Value &value,
 	                                                                 ExpressionType expressionType) = 0;
-	//! Initialize a scan on the index with the given expression and column ids
-	//! to fetch from the base table for two query predicates
+	//! Initialize a two predicate scan on the index with the given expression and column IDs
 	virtual unique_ptr<IndexScanState> InitializeScanTwoPredicates(Transaction &transaction, const Value &low_value,
 	                                                               ExpressionType low_expression_type,
 	                                                               const Value &high_value,
 	                                                               ExpressionType high_expression_type) = 0;
-	//! Perform a lookup on the index, fetching up to max_count result ids. Returns true if all row ids were fetched,
-	//! and false otherwise.
+	//! Performs a lookup on the index, fetching up to max_count result IDs. Returns true if all row IDs were fetched,
+	//! and false otherwise
 	virtual bool Scan(Transaction &transaction, DataTable &table, IndexScanState &state, idx_t max_count,
 	                  vector<row_t> &result_ids) = 0;
 
 	//! Obtain a lock on the index
 	virtual void InitializeLock(IndexLock &state);
-	//! Called when data is appended to the index. The lock obtained from InitializeAppend must be held
+	//! Called when data is appended to the index. The lock obtained from InitializeLock must be held
 	virtual bool Append(IndexLock &state, DataChunk &entries, Vector &row_identifiers) = 0;
+	//! Obtains a lock and calls Append while holding that lock
 	bool Append(DataChunk &entries, Vector &row_identifiers);
-	//! Verify that data can be appended to the index
+	//! Verify that data can be appended to the index without a constraint violation
 	virtual void VerifyAppend(DataChunk &chunk) = 0;
-	//! Verify that data can be appended to the index
+	//! Verify that data can be appended to the index without a constraint violation using the conflict manager
 	virtual void VerifyAppend(DataChunk &chunk, ConflictManager &conflict_manager) = 0;
-	//! Verify that data can be appended to the index for foreign key constraint
-	virtual void VerifyAppendForeignKey(DataChunk &chunk) = 0;
-	//! Verify that data can be delete from the index for foreign key constraint
-	virtual void VerifyDeleteForeignKey(DataChunk &chunk) = 0;
+	//! Performs constraint checking for a chunk of input data
+	virtual void CheckConstraintsForChunk(DataChunk &input, ConflictManager &conflict_manager) = 0;
 
-	//! Called when data inside the index is Deleted
+	//! Delete a chunk of entries from the index. The lock obtained from InitializeLock must be held
 	virtual void Delete(IndexLock &state, DataChunk &entries, Vector &row_identifiers) = 0;
+	//! Obtains a lock and calls Delete while holding that lock
 	void Delete(DataChunk &entries, Vector &row_identifiers);
 
-	//! Insert data into the index. Does not lock the index.
+	//! Insert a chunk of entries into the index
 	virtual bool Insert(IndexLock &lock, DataChunk &input, Vector &row_identifiers) = 0;
 
-	//! Merge other_index into this index.
+	//! Merge another index into this index. The lock obtained from InitializeLock must be held, and the other
+	//! index must also be locked during the merge
 	virtual bool MergeIndexes(IndexLock &state, Index *other_index) = 0;
+	//! Obtains a lock and calls MergeIndexes while holding that lock
 	bool MergeIndexes(Index *other_index);
 
 	//! Returns the string representation of an index
 	virtual string ToString() = 0;
-	//! Verifies that the memory_size value of the index matches its actual size
+	//! Verifies that the in-memory size value of the index matches its actual size
 	virtual void Verify() = 0;
-
-	//! Returns true if the index is affected by updates on the specified column ids, and false otherwise
+	//! Increases the memory size by the difference between the old size and the current size
+	//! and performs verifications
+	virtual void IncreaseAndVerifyMemorySize(idx_t old_memory_size) = 0;
+
+	//! Increases the in-memory size value
+	inline void IncreaseMemorySize(idx_t size) {
+		memory_size += size;
+	};
+	//! Decreases the in-memory size value
+	inline void DecreaseMemorySize(idx_t size) {
+		D_ASSERT(memory_size >= size);
+		memory_size -= size;
+	};
+
+	//! Returns true if the index is affected by updates on the specified column IDs, and false otherwise
 	bool IndexIsUpdated(const vector<PhysicalIndex> &column_ids) const;
 
-	//! Returns how many of the input values were found in the 'input' chunk, with the option to also record what those
-	//! matches were. For this purpose, nulls count as a match, and are returned in 'null_count'
-	virtual void LookupValues(DataChunk &input, ConflictManager &conflict_manager) = 0;
-
 	//! Returns unique flag
 	bool IsUnique() {
 		return (constraint_type == IndexConstraintType::UNIQUE || constraint_type == IndexConstraintType::PRIMARY);
 	}
-	//! Returns primary flag
+	//! Returns primary key flag
 	bool IsPrimary() {
 		return (constraint_type == IndexConstraintType::PRIMARY);
 	}
-	//! Returns foreign flag
+	//! Returns foreign key flag
 	bool IsForeign() {
 		return (constraint_type == IndexConstraintType::FOREIGN);
 	}
-	//! Serializes the index and returns the pair of block_id offset positions
-	virtual BlockPointer Serialize(duckdb::MetaBlockWriter &writer);
-	BlockPointer GetBlockPointer();
 
-	//! Returns block/offset of where index was most recently serialized.
+	//! Serializes the index and returns the pair of block_id offset positions
+	virtual BlockPointer Serialize(MetaBlockWriter &writer);
+	//! Returns the serialized data pointer to the block and offset of the serialized index
 	BlockPointer GetSerializedDataPointer() const {
 		return serialized_data_pointer;
 	}
 
-protected:
+	//! Execute the index expressions on an input chunk
 	void ExecuteExpressions(DataChunk &input, DataChunk &result);
 
-	//! Lock used for updating the index
+protected:
+	//! Lock used for any changes to the index
 	mutex lock;
-
-	//! Pointer to most recently checkpointed index data.
+	//! Pointer to serialized index data
 	BlockPointer serialized_data_pointer;
 
 private:
-	//! Bound expressions used by the index
+	//! Bound expressions used during expression execution
 	vector<unique_ptr<Expression>> bound_expressions;
-	//! Expression executor for the index expressions
+	//! Expression executor to execute the index expressions
 	ExpressionExecutor executor;
 
+	//! Bind the unbound expressions of the index
 	unique_ptr<Expression> BindExpression(unique_ptr<Expression> expr);
 };
 

package/src/duckdb/src/include/duckdb/storage/write_ahead_log.hpp

@@ -15,6 +15,7 @@
 #include "duckdb/catalog/catalog_entry/scalar_macro_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/sequence_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/table_macro_catalog_entry.hpp"
+#include "duckdb/catalog/catalog_entry/index_catalog_entry.hpp"
 #include "duckdb/main/attached_database.hpp"
 #include "duckdb/storage/storage_info.hpp"
 
@@ -77,6 +78,9 @@ protected:
 	void ReplayCreateTableMacro();
 	void ReplayDropTableMacro();
 
+	void ReplayCreateIndex();
+	void ReplayDropIndex();
+
 	void ReplayUseTable();
 	void ReplayInsert();
 	void ReplayDelete();
@@ -125,6 +129,9 @@ public:
 	void WriteCreateTableMacro(TableMacroCatalogEntry *entry);
 	void WriteDropTableMacro(TableMacroCatalogEntry *entry);
 
+	void WriteCreateIndex(IndexCatalogEntry *entry);
+	void WriteDropIndex(IndexCatalogEntry *entry);
+
 	void WriteCreateType(TypeCatalogEntry *entry);
 	void WriteDropType(TypeCatalogEntry *entry);
 	//! Sets the table used for subsequent insert/delete/update commands

package/src/duckdb/src/main/database.cpp

@@ -356,7 +356,8 @@ idx_t DuckDB::NumberOfThreads() {
 }
 
 bool DatabaseInstance::ExtensionIsLoaded(const std::string &name) {
-	return loaded_extensions.find(name) != loaded_extensions.end();
+	auto extension_name = ExtensionHelper::GetExtensionName(name);
+	return loaded_extensions.find(extension_name) != loaded_extensions.end();
 }
 
 bool DuckDB::ExtensionIsLoaded(const std::string &name) {
@@ -364,7 +365,8 @@ bool DuckDB::ExtensionIsLoaded(const std::string &name) {
 }
 
 void DatabaseInstance::SetExtensionLoaded(const std::string &name) {
-	loaded_extensions.insert(name);
+	auto extension_name = ExtensionHelper::GetExtensionName(name);
+	loaded_extensions.insert(extension_name);
 }
 
 bool DatabaseInstance::TryGetCurrentSetting(const std::string &key, Value &result) {

package/src/duckdb/src/main/extension/extension_load.cpp

@@ -31,7 +31,7 @@ ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileOpener *o
 	auto filename = fs.ConvertSeparators(extension);
 
 	// shorthand case
-	if (!StringUtil::Contains(extension, ".") && !StringUtil::Contains(extension, fs.PathSeparator())) {
+	if (!ExtensionHelper::IsFullPath(extension)) {
 		string local_path = fs.GetHomeDirectory(opener);
 		auto path_components = PathComponents();
 		for (auto &path_ele : path_components) {
@@ -121,9 +121,28 @@ ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileOpener *o
 	return res;
 }
 
+bool ExtensionHelper::IsFullPath(const string &extension) {
+	return StringUtil::Contains(extension, ".") || StringUtil::Contains(extension, "/") ||
+	       StringUtil::Contains(extension, "\\");
+}
+
+string ExtensionHelper::GetExtensionName(const string &extension) {
+	if (!IsFullPath(extension)) {
+		return extension;
+	}
+	auto splits = StringUtil::Split(StringUtil::Replace(extension, "\\", "/"), '/');
+	if (splits.empty()) {
+		return extension;
+	}
+	splits = StringUtil::Split(splits.back(), '.');
+	if (splits.empty()) {
+		return extension;
+	}
+	return StringUtil::Lower(splits.front());
+}
+
 void ExtensionHelper::LoadExternalExtension(DatabaseInstance &db, FileOpener *opener, const string &extension) {
-	auto &loaded_extensions = db.LoadedExtensions();
-	if (loaded_extensions.find(extension) != loaded_extensions.end()) {
+	if (db.ExtensionIsLoaded(extension)) {
 		return;
 	}
 

package/src/duckdb/src/parser/parsed_data/create_index_info.cpp

@@ -16,6 +16,9 @@ unique_ptr<CreateInfo> CreateIndexInfo::Copy() const {
 	for (auto &expr : expressions) {
 		result->expressions.push_back(expr->Copy());
 	}
+	for (auto &expr : parsed_expressions) {
+		result->parsed_expressions.push_back(expr->Copy());
+	}
 
 	result->scan_types = scan_types;
 	result->names = names;

package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp

@@ -16,6 +16,8 @@
 #include "duckdb/parser/expression/list.hpp"
 #include "duckdb/common/index_map.hpp"
 #include "duckdb/planner/expression_iterator.hpp"
+#include "duckdb/planner/expression_binder/index_binder.hpp"
+#include "duckdb/parser/parsed_data/create_index_info.hpp"
 
 #include <algorithm>
 
@@ -300,4 +302,15 @@ unique_ptr<BoundCreateTableInfo> Binder::BindCreateTableInfo(unique_ptr<CreateIn
 	return BindCreateTableInfo(std::move(info), schema);
 }
 
+vector<unique_ptr<Expression>> Binder::BindCreateIndexExpressions(TableCatalogEntry *table, CreateIndexInfo *info) {
+	vector<unique_ptr<Expression>> expressions;
+
+	auto index_binder = IndexBinder(*this, this->context, table, info);
+	for (auto &expr : info->expressions) {
+		expressions.push_back(index_binder.Bind(expr));
+	}
+
+	return expressions;
+}
+
 } // namespace duckdb

package/src/duckdb/src/planner/expression_binder/index_binder.cpp

@@ -1,8 +1,14 @@
 #include "duckdb/planner/expression_binder/index_binder.hpp"
 
+#include "duckdb/parser/parsed_data/create_index_info.hpp"
+#include "duckdb/parser/expression/columnref_expression.hpp"
+#include "duckdb/planner/expression/bound_columnref_expression.hpp"
+#include "duckdb/planner/column_binding.hpp"
+
 namespace duckdb {
 
-IndexBinder::IndexBinder(Binder &binder, ClientContext &context) : ExpressionBinder(binder, context) {
+IndexBinder::IndexBinder(Binder &binder, ClientContext &context, TableCatalogEntry *table, CreateIndexInfo *info)
+    : ExpressionBinder(binder, context), table(table), info(info) {
 }
 
 BindResult IndexBinder::BindExpression(unique_ptr<ParsedExpression> *expr_ptr, idx_t depth, bool root_expression) {
@@ -12,6 +18,31 @@ BindResult IndexBinder::BindExpression(unique_ptr<ParsedExpression> *expr_ptr, i
 		return BindResult("window functions are not allowed in index expressions");
 	case ExpressionClass::SUBQUERY:
 		return BindResult("cannot use subquery in index expressions");
+	case ExpressionClass::COLUMN_REF: {
+		if (table) {
+			// WAL replay
+			// we assume that the parsed expressions have qualified column names
+			// and that the columns exist in the table
+			auto &col_ref = (ColumnRefExpression &)expr;
+			auto col_idx = table->GetColumnIndex(col_ref.column_names.back());
+			auto col_type = table->GetColumn(col_idx).GetType();
+
+			// find the col_idx in the index.column_ids
+			auto col_id_idx = DConstants::INVALID_INDEX;
+			for (idx_t i = 0; i < info->column_ids.size(); i++) {
+				if (col_idx.index == info->column_ids[i]) {
+					col_id_idx = i;
+				}
+			}
+
+			if (col_id_idx == DConstants::INVALID_INDEX) {
+				throw InternalException("failed to replay CREATE INDEX statement - column id not found");
+			}
+			return BindResult(
+			    make_unique<BoundColumnRefExpression>(col_ref.alias, col_type, ColumnBinding(0, col_id_idx)));
+		}
+		return ExpressionBinder::BindExpression(expr_ptr, depth);
+	}
 	default:
 		return ExpressionBinder::BindExpression(expr_ptr, depth);
 	}

package/src/duckdb/src/storage/buffer_manager.cpp

@@ -230,6 +230,7 @@ public:
 private:
 	DatabaseInstance &db;
 	string temp_directory;
+	bool created_directory = false;
 	unique_ptr<TemporaryFileManager> temp_file;
 };
 
@@ -868,7 +869,10 @@ TemporaryDirectoryHandle::TemporaryDirectoryHandle(DatabaseInstance &db, string
     : db(db), temp_directory(std::move(path_p)), temp_file(make_unique<TemporaryFileManager>(db, temp_directory)) {
 	auto &fs = FileSystem::GetFileSystem(db);
 	if (!temp_directory.empty()) {
-		fs.CreateDirectory(temp_directory);
+		if (!fs.DirectoryExists(temp_directory)) {
+			fs.CreateDirectory(temp_directory);
+			created_directory = true;
+		}
 	}
 }
 TemporaryDirectoryHandle::~TemporaryDirectoryHandle() {
@@ -877,7 +881,30 @@ TemporaryDirectoryHandle::~TemporaryDirectoryHandle() {
 	// then delete the temporary file directory
 	auto &fs = FileSystem::GetFileSystem(db);
 	if (!temp_directory.empty()) {
-		fs.RemoveDirectory(temp_directory);
+		bool delete_directory = created_directory;
+		vector<string> files_to_delete;
+		if (!created_directory) {
+			bool deleted_everything = true;
+			fs.ListFiles(temp_directory, [&](const string &path, bool isdir) {
+				if (isdir) {
+					deleted_everything = false;
+					return;
+				}
+				if (!StringUtil::StartsWith(path, "duckdb_temp_")) {
+					deleted_everything = false;
+					return;
+				}
+				files_to_delete.push_back(path);
+			});
+		}
+		if (delete_directory) {
+			// we want to remove all files in the directory
+			fs.RemoveDirectory(temp_directory);
+		} else {
+			for (auto &file : files_to_delete) {
+				fs.RemoveFile(fs.JoinPath(temp_directory, file));
+			}
+		}
 	}
 }
 
@@ -887,7 +914,7 @@ TemporaryFileManager &TemporaryDirectoryHandle::GetTempFile() {
 
 string BufferManager::GetTemporaryPath(block_id_t id) {
 	auto &fs = FileSystem::GetFileSystem(db);
-	return fs.JoinPath(temp_directory, to_string(id) + ".block");
+	return fs.JoinPath(temp_directory, "duckdb_temp_block-" + to_string(id) + ".block");
 }
 
 void BufferManager::RequireTemporaryDirectory() {

package/src/duckdb/src/storage/compression/bitpacking.cpp

@@ -447,8 +447,8 @@ public:
 
 		static void ReserveSpace(BitpackingCompressState<T> *state, idx_t data_bytes) {
 			idx_t meta_bytes = sizeof(bitpacking_metadata_encoded_t);
-			state->FlushAndCreateSegmentIfFull(data_bytes + meta_bytes);
-			D_ASSERT(data_bytes + meta_bytes <= state->RemainingSize());
+			state->FlushAndCreateSegmentIfFull(data_bytes, meta_bytes);
+			D_ASSERT(state->CanStore(data_bytes, meta_bytes));
 		}
 
 		static void UpdateStats(BitpackingCompressState<T> *state, idx_t count) {
@@ -461,9 +461,12 @@ public:
 		}
 	};
 
-	// Space remaining between the metadata_ptr growing down and data ptr growing up
-	idx_t RemainingSize() {
-		return metadata_ptr - data_ptr;
+	bool CanStore(idx_t data_bytes, idx_t meta_bytes) {
+		auto required_data_bytes = AlignValue<idx_t>((data_ptr + data_bytes) - data_ptr);
+		auto required_meta_bytes = Storage::BLOCK_SIZE - (metadata_ptr - data_ptr) + meta_bytes;
+
+		return required_data_bytes + required_meta_bytes <=
+		       Storage::BLOCK_SIZE - BitpackingPrimitives::BITPACKING_HEADER_SIZE;
 	}
 
 	void CreateEmptySegment(idx_t row_start) {
@@ -489,8 +492,8 @@ public:
 		}
 	}
 
-	void FlushAndCreateSegmentIfFull(idx_t required_space) {
-		if (RemainingSize() < required_space) {
+	void FlushAndCreateSegmentIfFull(idx_t required_data_bytes, idx_t required_meta_bytes) {
+		if (!CanStore(required_data_bytes, required_meta_bytes)) {
 			auto row_start = current_segment->start + current_segment->count;
 			FlushSegment();
 			CreateEmptySegment(row_start);
@@ -505,6 +508,12 @@ public:
 		idx_t metadata_offset = AlignValue(data_ptr - base_ptr);
 		idx_t metadata_size = base_ptr + Storage::BLOCK_SIZE - metadata_ptr;
 		idx_t total_segment_size = metadata_offset + metadata_size;
+
+		// Asserting things are still sane here
+		if (!CanStore(0, 0)) {
+			throw InternalException("Error in bitpacking size calculation");
+		}
+
 		memmove(base_ptr + metadata_offset, metadata_ptr, metadata_size);
 
 		// Store the offset of the metadata of the first group (which is at the highest address).