duckdb 0.6.2-dev971.0 → 0.7.1-dev2.0

package/src/duckdb/src/include/duckdb/planner/parsed_data/bound_create_function_info.hpp

@@ -14,7 +14,7 @@ namespace duckdb {
 class CatalogEntry;
 
 struct BoundCreateFunctionInfo {
-	explicit BoundCreateFunctionInfo(unique_ptr<CreateInfo> base) : base(move(base)) {
+	explicit BoundCreateFunctionInfo(unique_ptr<CreateInfo> base) : base(std::move(base)) {
 	}
 
 	//! The schema to create the table in

package/src/duckdb/src/include/duckdb/planner/parsed_data/bound_create_table_info.hpp

@@ -23,7 +23,7 @@ namespace duckdb {
 class CatalogEntry;
 
 struct BoundCreateTableInfo {
-	explicit BoundCreateTableInfo(unique_ptr<CreateInfo> base_p) : base(move(base_p)) {
+	explicit BoundCreateTableInfo(unique_ptr<CreateInfo> base_p) : base(std::move(base_p)) {
 		D_ASSERT(base);
 	}
 

package/src/duckdb/src/include/duckdb/planner/table_binding.hpp

@@ -25,6 +25,7 @@ class TableCatalogEntry;
 class TableFunctionCatalogEntry;
 class BoundTableFunction;
 class StandardEntry;
+struct ColumnBinding;
 
 enum class BindingType { BASE, TABLE, DUMMY, CATALOG_ENTRY };
 
@@ -40,6 +41,7 @@ struct Binding {
 	string alias;
 	//! The table index of the binding
 	idx_t index;
+	//! The types of the bound columns
 	vector<LogicalType> types;
 	//! Column names of the subquery
 	vector<string> names;
@@ -81,6 +83,11 @@ public:
 	BindResult Bind(ColumnRefExpression &colref, idx_t depth) override;
 	StandardEntry *GetStandardEntry() override;
 	string ColumnNotFoundError(const string &column_name) const override;
+	// These are columns that are present in the name_map, appearing in the order that they're bound
+	const vector<column_t> &GetBoundColumnIds() const;
+
+protected:
+	ColumnBinding GetColumnBinding(column_t column_index);
 };
 
 //! DummyBinding is like the Binding, except the alias and index are set by default. Used for binding lambdas and macro
@@ -99,6 +106,7 @@ public:
 
 public:
 	BindResult Bind(ColumnRefExpression &colref, idx_t depth) override;
+	BindResult Bind(ColumnRefExpression &colref, idx_t lambda_index, idx_t depth);
 
 	//! Given the parameter colref, returns a copy of the argument that was supplied for this parameter
 	unique_ptr<ParsedExpression> ParamToArg(ColumnRefExpression &colref);

package/src/duckdb/src/include/duckdb/planner/tableref/bound_basetableref.hpp

@@ -18,7 +18,7 @@ class TableCatalogEntry;
 class BoundBaseTableRef : public BoundTableRef {
 public:
 	BoundBaseTableRef(TableCatalogEntry *table, unique_ptr<LogicalOperator> get)
-	    : BoundTableRef(TableReferenceType::BASE_TABLE), table(table), get(move(get)) {
+	    : BoundTableRef(TableReferenceType::BASE_TABLE), table(table), get(std::move(get)) {
 	}
 
 	TableCatalogEntry *table;

package/src/duckdb/src/include/duckdb/planner/tableref/bound_joinref.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/planner/binder.hpp"
 #include "duckdb/common/enums/join_type.hpp"
+#include "duckdb/common/enums/joinref_type.hpp"
 #include "duckdb/planner/bound_tableref.hpp"
 #include "duckdb/planner/expression.hpp"
 
@@ -18,7 +19,8 @@ namespace duckdb {
 //! Represents a join
 class BoundJoinRef : public BoundTableRef {
 public:
-	BoundJoinRef() : BoundTableRef(TableReferenceType::JOIN), lateral(false) {
+	explicit BoundJoinRef(JoinRefType ref_type)
+	    : BoundTableRef(TableReferenceType::JOIN), type(JoinType::INNER), ref_type(ref_type), lateral(false) {
 	}
 
 	//! The binder used to bind the LHS of the join
@@ -33,6 +35,8 @@ public:
 	unique_ptr<Expression> condition;
 	//! The join type
 	JoinType type;
+	//! Join condition type
+	JoinRefType ref_type;
 	//! Whether or not this is a lateral join
 	bool lateral;
 	//! The correlated columns of the right-side with the left-side

package/src/duckdb/src/include/duckdb/planner/tableref/{bound_crossproductref.hpp → bound_pos_join_ref.hpp}

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/planner/tableref/bound_crossproductref.hpp
+// duckdb/planner/tableref/bound_pos_join_ref.hpp
 //
 //
 //===----------------------------------------------------------------------===//
@@ -13,21 +13,21 @@
 
 namespace duckdb {
 
-//! Represents a cross product
-class BoundCrossProductRef : public BoundTableRef {
+//! Represents a positional join
+class BoundPositionalJoinRef : public BoundTableRef {
 public:
-	BoundCrossProductRef() : BoundTableRef(TableReferenceType::CROSS_PRODUCT), lateral(false) {
+	BoundPositionalJoinRef() : BoundTableRef(TableReferenceType::POSITIONAL_JOIN), lateral(false) {
 	}
 
-	//! The binder used to bind the LHS of the cross product
+	//! The binder used to bind the LHS of the positional join
 	shared_ptr<Binder> left_binder;
-	//! The binder used to bind the RHS of the cross product
+	//! The binder used to bind the RHS of the positional join
 	shared_ptr<Binder> right_binder;
-	//! The left hand side of the cross product
+	//! The left hand side of the positional join
 	unique_ptr<BoundTableRef> left;
-	//! The right hand side of the cross product
+	//! The right hand side of the positional join
 	unique_ptr<BoundTableRef> right;
-	//! Whether or not this is a lateral cross product
+	//! Whether or not this is a lateral positional join
 	bool lateral;
 	//! The correlated columns of the right-side with the left-side
 	vector<CorrelatedColumnInfo> correlated_columns;

package/src/duckdb/src/include/duckdb/planner/tableref/bound_subqueryref.hpp

@@ -18,7 +18,7 @@ namespace duckdb {
 class BoundSubqueryRef : public BoundTableRef {
 public:
 	BoundSubqueryRef(shared_ptr<Binder> binder_p, unique_ptr<BoundQueryNode> subquery)
-	    : BoundTableRef(TableReferenceType::SUBQUERY), binder(move(binder_p)), subquery(move(subquery)) {
+	    : BoundTableRef(TableReferenceType::SUBQUERY), binder(std::move(binder_p)), subquery(std::move(subquery)) {
 	}
 
 	//! The binder used to bind the subquery

package/src/duckdb/src/include/duckdb/planner/tableref/bound_table_function.hpp

@@ -17,7 +17,7 @@ namespace duckdb {
 class BoundTableFunction : public BoundTableRef {
 public:
 	explicit BoundTableFunction(unique_ptr<LogicalOperator> get)
-	    : BoundTableRef(TableReferenceType::TABLE_FUNCTION), get(move(get)) {
+	    : BoundTableRef(TableReferenceType::TABLE_FUNCTION), get(std::move(get)) {
 	}
 
 	unique_ptr<LogicalOperator> get;

package/src/duckdb/src/include/duckdb/planner/tableref/list.hpp

@@ -1,5 +1,4 @@
 #include "duckdb/planner/tableref/bound_basetableref.hpp"
-#include "duckdb/planner/tableref/bound_crossproductref.hpp"
 #include "duckdb/planner/tableref/bound_cteref.hpp"
 #include "duckdb/planner/tableref/bound_dummytableref.hpp"
 #include "duckdb/planner/tableref/bound_expressionlistref.hpp"

package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp

@@ -31,16 +31,21 @@ public:
 	DUCKDB_API ArenaAllocator(Allocator &allocator, idx_t initial_capacity = ARENA_ALLOCATOR_INITIAL_CAPACITY);
 	DUCKDB_API ~ArenaAllocator();
 
-	data_ptr_t Allocate(idx_t size);
+	DUCKDB_API data_ptr_t Allocate(idx_t size);
+	DUCKDB_API data_ptr_t Reallocate(data_ptr_t pointer, idx_t old_size, idx_t size);
+
+	DUCKDB_API data_ptr_t AllocateAligned(idx_t size);
+	DUCKDB_API data_ptr_t ReallocateAligned(data_ptr_t pointer, idx_t old_size, idx_t size);
+
 	//! Resets the current head and destroys all previous arena chunks
-	void Reset();
-	void Destroy();
-	void Move(ArenaAllocator &allocator);
+	DUCKDB_API void Reset();
+	DUCKDB_API void Destroy();
+	DUCKDB_API void Move(ArenaAllocator &allocator);
 
-	ArenaChunk *GetHead();
-	ArenaChunk *GetTail();
+	DUCKDB_API ArenaChunk *GetHead();
+	DUCKDB_API ArenaChunk *GetTail();
 
-	bool IsEmpty();
+	DUCKDB_API bool IsEmpty();
 
 private:
 	//! Internal allocator that is used by the arena allocator

package/src/duckdb/src/include/duckdb/storage/buffer_manager.hpp

@@ -57,17 +57,24 @@ public:
 	//! blocks can be evicted
 	void SetLimit(idx_t limit = (idx_t)-1);
 
-	static BufferManager &GetBufferManager(ClientContext &context);
+	DUCKDB_API static BufferManager &GetBufferManager(ClientContext &context);
 	DUCKDB_API static BufferManager &GetBufferManager(DatabaseInstance &db);
 	DUCKDB_API static BufferManager &GetBufferManager(AttachedDatabase &db);
 
+	//! Returns the currently allocated memory
 	idx_t GetUsedMemory() {
 		return current_memory;
 	}
+	//! Returns the maximum available memory
 	idx_t GetMaxMemory() {
 		return maximum_memory;
 	}
 
+	//! Increases the currently allocated memory, but the actual allocation does not go through the buffer manager
+	void IncreaseUsedMemory(idx_t size);
+	//! Decrease the currently allocated memory, but the actual deallocation does not go through the buffer manager
+	void DecreaseUsedMemory(idx_t size);
+
 	const string &GetTemporaryDirectory() {
 		return temp_directory;
 	}
@@ -85,10 +92,8 @@ public:
 	}
 
 	//! Construct a managed buffer.
-	//! The block_id is just used for internal tracking. It doesn't map to any actual
-	//! BlockManager.
-	virtual unique_ptr<FileBuffer> ConstructManagedBuffer(idx_t size, unique_ptr<FileBuffer> &&source,
-	                                                      FileBufferType type = FileBufferType::MANAGED_BUFFER);
+	unique_ptr<FileBuffer> ConstructManagedBuffer(idx_t size, unique_ptr<FileBuffer> &&source,
+	                                              FileBufferType type = FileBufferType::MANAGED_BUFFER);
 
 	DUCKDB_API void ReserveMemory(idx_t size);
 	DUCKDB_API void FreeReservedMemory(idx_t size);

package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp

@@ -11,6 +11,7 @@
 #include "duckdb/storage/checkpoint/row_group_writer.hpp"
 
 namespace duckdb {
+class DuckTableEntry;
 
 //! The table data writer is responsible for writing the data of a table to
 //! storage.
@@ -35,7 +36,7 @@ public:
 	virtual void AddRowGroup(RowGroupPointer &&row_group_pointer, unique_ptr<RowGroupWriter> &&writer);
 
 protected:
-	TableCatalogEntry &table;
+	DuckTableEntry &table;
 	// Pointers to the start of each row group.
 	vector<RowGroupPointer> row_group_pointers;
 };

package/src/duckdb/src/include/duckdb/storage/checkpoint_manager.hpp

@@ -25,15 +25,13 @@ class TypeCatalogEntry;
 
 class CheckpointWriter {
 public:
-	explicit CheckpointWriter(AttachedDatabase &db) : db(db), catalog(Catalog::GetCatalog(db)) {
+	explicit CheckpointWriter(AttachedDatabase &db) : db(db) {
 	}
 	virtual ~CheckpointWriter() {
 	}
 
 	//! The database
 	AttachedDatabase &db;
-	//! The catalog
-	Catalog &catalog;
 
 	virtual MetaBlockWriter &GetMetaBlockWriter() = 0;
 	virtual unique_ptr<TableDataWriter> GetTableDataWriter(TableCatalogEntry &table) = 0;

package/src/duckdb/src/include/duckdb/storage/compression/chimp/chimp_compress.hpp

@@ -118,7 +118,7 @@ public:
 		auto &type = checkpointer.GetType();
 		auto compressed_segment = ColumnSegment::CreateTransientSegment(db, type, row_start);
 		compressed_segment->function = function;
-		current_segment = move(compressed_segment);
+		current_segment = std::move(compressed_segment);
 		next_group_byte_index_start = ChimpPrimitives::HEADER_SIZE;
 
 		auto &buffer_manager = BufferManager::GetBufferManager(db);
@@ -149,7 +149,7 @@ public:
 		current_segment->count++;
 
 		if (is_valid) {
-			T floating_point_value = *(T *)(&value);
+			T floating_point_value = Load<T>((const_data_ptr_t)&value);
 			NumericStatistics::Update<T>(current_segment->stats, floating_point_value);
 		} else {
 			//! FIXME: find a cheaper alternative to storing a NULL
@@ -251,7 +251,7 @@ public:
 		//  Store the offset of the metadata of the first group (which is at the highest address).
 		Store<uint32_t>(metadata_offset + metadata_size, dataptr);
 		handle.Destroy();
-		checkpoint_state.FlushSegment(move(current_segment), total_segment_size);
+		checkpoint_state.FlushSegment(std::move(current_segment), total_segment_size);
 	}
 
 	void Finalize() {

package/src/duckdb/src/include/duckdb/storage/compression/patas/patas_analyze.hpp

@@ -102,7 +102,7 @@ struct EmptyPatasWriter {
 		using EXACT_TYPE = typename FloatingToExact<VALUE_TYPE>::type;
 
 		auto state_wrapper = (PatasAnalyzeState<VALUE_TYPE> *)state_p;
-		state_wrapper->WriteValue(*(EXACT_TYPE *)(&uncompressed_value), is_valid);
+		state_wrapper->WriteValue(Load<EXACT_TYPE>((const_data_ptr_t)&uncompressed_value), is_valid);
 	}
 };
 

package/src/duckdb/src/include/duckdb/storage/compression/patas/patas_compress.hpp

@@ -52,7 +52,7 @@ public:
 				NumericStatistics::Update<VALUE_TYPE>(state_wrapper->current_segment->stats, value);
 			}
 
-			state_wrapper->WriteValue(*(EXACT_TYPE *)(&value));
+			state_wrapper->WriteValue(Load<EXACT_TYPE>((const_data_ptr_t)&value));
 		}
 	};
 
@@ -130,7 +130,7 @@ public:
 		auto &type = checkpointer.GetType();
 		auto compressed_segment = ColumnSegment::CreateTransientSegment(db, type, row_start);
 		compressed_segment->function = function;
-		current_segment = move(compressed_segment);
+		current_segment = std::move(compressed_segment);
 
 		auto &buffer_manager = BufferManager::GetBufferManager(db);
 		handle = buffer_manager.Pin(current_segment->block);
@@ -203,7 +203,7 @@ public:
 		// Store the offset to the metadata
 		Store<uint32_t>(metadata_offset + metadata_size, dataptr);
 		handle.Destroy();
-		checkpoint_state.FlushSegment(move(current_segment), total_segment_size);
+		checkpoint_state.FlushSegment(std::move(current_segment), total_segment_size);
 	}
 
 	void Finalize() {

package/src/duckdb/src/include/duckdb/storage/data_table.hpp

@@ -29,6 +29,7 @@ class ClientContext;
 class ColumnDataCollection;
 class ColumnDefinition;
 class DataTable;
+class DuckTransaction;
 class OptimisticDataWriter;
 class RowGroup;
 class StorageManager;
@@ -37,6 +38,8 @@ class TableIOManager;
 class Transaction;
 class WriteAheadLog;
 class TableDataWriter;
+class ConflictManager;
+enum class VerifyExistenceType : uint8_t;
 
 //! DataTable represents a physical table on disk
 class DataTable {
@@ -68,7 +71,7 @@ public:
 
 	void InitializeScan(TableScanState &state, const vector<column_t> &column_ids,
 	                    TableFilterSet *table_filter = nullptr);
-	void InitializeScan(Transaction &transaction, TableScanState &state, const vector<column_t> &column_ids,
+	void InitializeScan(DuckTransaction &transaction, TableScanState &state, const vector<column_t> &column_ids,
 	                    TableFilterSet *table_filters = nullptr);
 
 	//! Returns the maximum amount of threads that should be assigned to scan this data table
@@ -80,16 +83,17 @@ public:
 	//! from offset and store them in result. Offset is incremented with how many
 	//! elements were returned.
 	//! Returns true if all pushed down filters were executed during data fetching
-	void Scan(Transaction &transaction, DataChunk &result, TableScanState &state);
+	void Scan(DuckTransaction &transaction, DataChunk &result, TableScanState &state);
 
 	//! Fetch data from the specific row identifiers from the base table
-	void Fetch(Transaction &transaction, DataChunk &result, const vector<column_t> &column_ids, Vector &row_ids,
-	           idx_t fetch_count, ColumnFetchState &state);
+	void Fetch(DuckTransaction &transaction, DataChunk &result, const vector<column_t> &column_ids,
+	           const Vector &row_ids, idx_t fetch_count, ColumnFetchState &state);
 
 	//! Initializes an append to transaction-local storage
 	void InitializeLocalAppend(LocalAppendState &state, ClientContext &context);
 	//! Append a DataChunk to the transaction-local storage of the table.
-	void LocalAppend(LocalAppendState &state, TableCatalogEntry &table, ClientContext &context, DataChunk &chunk);
+	void LocalAppend(LocalAppendState &state, TableCatalogEntry &table, ClientContext &context, DataChunk &chunk,
+	                 bool unsafe = false);
 	//! Finalizes a transaction-local append
 	void FinalizeLocalAppend(LocalAppendState &state);
 	//! Append a chunk to the transaction-local storage of this table
@@ -121,7 +125,7 @@ public:
 	//! Fetches an append lock
 	void AppendLock(TableAppendState &state);
 	//! Begin appending structs to this table, obtaining necessary locks, etc
-	void InitializeAppend(Transaction &transaction, TableAppendState &state, idx_t append_count);
+	void InitializeAppend(DuckTransaction &transaction, TableAppendState &state, idx_t append_count);
 	//! Append a chunk to the table using the AppendState obtained from InitializeAppend
 	void Append(DataChunk &chunk, TableAppendState &state);
 	//! Commit the append
@@ -168,7 +172,7 @@ public:
 
 	idx_t GetTotalRows();
 
-	vector<vector<Value>> GetStorageInfo();
+	void GetStorageInfo(TableStorageInfo &result);
 	static bool IsForeignKeyIndex(const vector<PhysicalIndex> &fk_keys, Index &index, ForeignKeyType fk_type);
 
 	//! Initializes a special scan that is used to create an index on the table, it keeps locks on the table
@@ -177,7 +181,8 @@ public:
 	bool CreateIndexScan(TableScanState &state, DataChunk &result, TableScanType type);
 
 	//! Verify constraints with a chunk from the Append containing all columns of the table
-	void VerifyAppendConstraints(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk);
+	void VerifyAppendConstraints(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk,
+	                             ConflictManager *conflict_manager = nullptr);
 
 private:
 	//! Verify the new added constraints against current persistent&local data
@@ -192,7 +197,7 @@ private:
 	                              idx_t end_row);
 
 	void VerifyForeignKeyConstraint(const BoundForeignKeyConstraint &bfk, ClientContext &context, DataChunk &chunk,
-	                                bool is_append);
+	                                VerifyExistenceType verify_type);
 	void VerifyAppendForeignKeyConstraint(const BoundForeignKeyConstraint &bfk, ClientContext &context,
 	                                      DataChunk &chunk);
 	void VerifyDeleteForeignKeyConstraint(const BoundForeignKeyConstraint &bfk, ClientContext &context,

package/src/duckdb/src/include/duckdb/storage/database_size.hpp

@@ -0,0 +1,24 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/database_size.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+
+namespace duckdb {
+
+struct DatabaseSize {
+	idx_t total_blocks = 0;
+	idx_t block_size = 0;
+	idx_t free_blocks = 0;
+	idx_t used_blocks = 0;
+	idx_t bytes = 0;
+	idx_t wal_size = 0;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/index.hpp

@@ -17,20 +17,23 @@
 #include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/meta_block_writer.hpp"
 #include "duckdb/execution/expression_executor.hpp"
+#include "duckdb/common/types/constraint_conflict_info.hpp"
 
 namespace duckdb {
 
 class ClientContext;
 class TableIOManager;
 class Transaction;
+class ConflictManager;
 
 struct IndexLock;
 
 //! The index is an abstract base class that serves as the basis for indexes
 class Index {
 public:
-	Index(IndexType type, TableIOManager &table_io_manager, const vector<column_t> &column_ids,
-	      const vector<unique_ptr<Expression>> &unbound_expressions, IndexConstraintType constraint_type);
+	Index(AttachedDatabase &db, IndexType type, TableIOManager &table_io_manager, const vector<column_t> &column_ids,
+	      const vector<unique_ptr<Expression>> &unbound_expressions, IndexConstraintType constraint_type,
+	      bool track_memory);
 	virtual ~Index() = default;
 
 	//! The type of the index
@@ -39,7 +42,7 @@ public:
 	TableIOManager &table_io_manager;
 	//! Column identifiers to extract from the base table
 	vector<column_t> column_ids;
-	//! unordered_set of column_ids used by the index
+	//! Unordered_set of column_ids used by the index
 	unordered_set<column_t> column_id_set;
 	//! Unbound expressions used by the index
 	vector<unique_ptr<Expression>> unbound_expressions;
@@ -47,18 +50,29 @@ public:
 	vector<PhysicalType> types;
 	//! The logical types of the expressions
 	vector<LogicalType> logical_types;
-	//! constraint type
+	//! Index constraint type (primary key, foreign key, ...)
 	IndexConstraintType constraint_type;
 
+	//! Attached database instance
+	AttachedDatabase &db;
+	//! Buffer manager of the database instance
+	BufferManager &buffer_manager;
+	//! The size of the index in memory
+	//! This does not track the size of the index meta information, but only allocated nodes and leaves
+	idx_t memory_size;
+	//! Flag determining if this index's size is tracked by the buffer manager
+	bool track_memory;
+
 public:
 	//! Initialize a scan on the index with the given expression and column ids
 	//! to fetch from the base table when we only have one query predicate
-	virtual unique_ptr<IndexScanState> InitializeScanSinglePredicate(Transaction &transaction, Value value,
+	virtual unique_ptr<IndexScanState> InitializeScanSinglePredicate(const Transaction &transaction, const Value &value,
 	                                                                 ExpressionType expressionType) = 0;
 	//! Initialize a scan on the index with the given expression and column ids
 	//! to fetch from the base table for two query predicates
-	virtual unique_ptr<IndexScanState> InitializeScanTwoPredicates(Transaction &transaction, Value low_value,
-	                                                               ExpressionType low_expression_type, Value high_value,
+	virtual unique_ptr<IndexScanState> InitializeScanTwoPredicates(Transaction &transaction, const Value &low_value,
+	                                                               ExpressionType low_expression_type,
+	                                                               const Value &high_value,
 	                                                               ExpressionType high_expression_type) = 0;
 	//! Perform a lookup on the index, fetching up to max_count result ids. Returns true if all row ids were fetched,
 	//! and false otherwise.
@@ -72,10 +86,12 @@ public:
 	bool Append(DataChunk &entries, Vector &row_identifiers);
 	//! Verify that data can be appended to the index
 	virtual void VerifyAppend(DataChunk &chunk) = 0;
+	//! Verify that data can be appended to the index
+	virtual void VerifyAppend(DataChunk &chunk, ConflictManager &conflict_manager) = 0;
 	//! Verify that data can be appended to the index for foreign key constraint
-	virtual void VerifyAppendForeignKey(DataChunk &chunk, string *err_msg_ptr) = 0;
+	virtual void VerifyAppendForeignKey(DataChunk &chunk) = 0;
 	//! Verify that data can be delete from the index for foreign key constraint
-	virtual void VerifyDeleteForeignKey(DataChunk &chunk, string *err_msg_ptr) = 0;
+	virtual void VerifyDeleteForeignKey(DataChunk &chunk) = 0;
 
 	//! Called when data inside the index is Deleted
 	virtual void Delete(IndexLock &state, DataChunk &entries, Vector &row_identifiers) = 0;
@@ -83,8 +99,6 @@ public:
 
 	//! Insert data into the index. Does not lock the index.
 	virtual bool Insert(IndexLock &lock, DataChunk &input, Vector &row_identifiers) = 0;
-	//! Construct an index from sorted chunks of keys.
-	virtual void ConstructAndMerge(IndexLock &lock, PayloadScanner &scanner, Allocator &allocator) = 0;
 
 	//! Merge other_index into this index.
 	virtual bool MergeIndexes(IndexLock &state, Index *other_index) = 0;
@@ -92,10 +106,16 @@ public:
 
 	//! Returns the string representation of an index
 	virtual string ToString() = 0;
+	//! Verifies that the memory_size value of the index matches its actual size
+	virtual void Verify() = 0;
 
 	//! Returns true if the index is affected by updates on the specified column ids, and false otherwise
 	bool IndexIsUpdated(const vector<PhysicalIndex> &column_ids) const;
 
+	//! Returns how many of the input values were found in the 'input' chunk, with the option to also record what those
+	//! matches were. For this purpose, nulls count as a match, and are returned in 'null_count'
+	virtual void LookupValues(DataChunk &input, ConflictManager &conflict_manager) = 0;
+
 	//! Returns unique flag
 	bool IsUnique() {
 		return (constraint_type == IndexConstraintType::UNIQUE || constraint_type == IndexConstraintType::PRIMARY);
@@ -110,6 +130,7 @@ public:
 	}
 	//! Serializes the index and returns the pair of block_id offset positions
 	virtual BlockPointer Serialize(duckdb::MetaBlockWriter &writer);
+	BlockPointer GetBlockPointer();
 
 	//! Returns block/offset of where index was most recently serialized.
 	BlockPointer GetSerializedDataPointer() const {

package/src/duckdb/src/include/duckdb/storage/magic_bytes.hpp

@@ -0,0 +1,28 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/magic_bytes.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+
+namespace duckdb {
+class FileSystem;
+
+enum class DataFileType : uint8_t {
+	FILE_DOES_NOT_EXIST, // file does not exist
+	DUCKDB_FILE,         // duckdb database file
+	SQLITE_FILE,         // sqlite database file
+	PARQUET_FILE         // parquet file
+};
+
+class MagicBytes {
+public:
+	static DataFileType CheckMagicBytes(FileSystem *fs, const string &path);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/object_cache.hpp

@@ -49,7 +49,7 @@ public:
 
 	void Put(string key, shared_ptr<ObjectCacheEntry> value) {
 		lock_guard<mutex> glock(lock);
-		cache[key] = move(value);
+		cache[key] = std::move(value);
 	}
 
 	DUCKDB_API static ObjectCache &GetObjectCache(ClientContext &context);

package/src/duckdb/src/include/duckdb/storage/partial_block_manager.hpp

@@ -36,7 +36,7 @@ struct PartialBlockState {
 };
 
 struct PartialBlock {
-	explicit PartialBlock(PartialBlockState state) : state(move(state)) {
+	explicit PartialBlock(PartialBlockState state) : state(std::move(state)) {
 	}
 	virtual ~PartialBlock() {
 	}
@@ -68,10 +68,10 @@ public:
 	// 20% free / 80% utilization
 	static constexpr const idx_t DEFAULT_MAX_PARTIAL_BLOCK_SIZE = Storage::BLOCK_SIZE / 5 * 4;
 	// Max number of shared references to a block. No effective limit by default.
-	static constexpr const idx_t DEFAULT_MAX_USE_COUNT = 1 << 20;
+	static constexpr const idx_t DEFAULT_MAX_USE_COUNT = 1u << 20;
 	// No point letting map size grow unbounded. We'll drop blocks with the
 	// least free space first.
-	static constexpr const idx_t MAX_BLOCK_MAP_SIZE = 1 << 31;
+	static constexpr const idx_t MAX_BLOCK_MAP_SIZE = 1u << 31;
 
 public:
 	PartialBlockManager(BlockManager &block_manager, uint32_t max_partial_block_size = DEFAULT_MAX_PARTIAL_BLOCK_SIZE,

package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp

@@ -17,6 +17,7 @@
 #include "duckdb/common/vector.hpp"
 
 namespace duckdb {
+
 class DatabaseInstance;
 
 //! SingleFileBlockManager is an implementation for a BlockManager which manages blocks in a single file
@@ -25,7 +26,11 @@ class SingleFileBlockManager : public BlockManager {
 	static constexpr uint64_t BLOCK_START = Storage::FILE_HEADER_SIZE * 3;
 
 public:
-	SingleFileBlockManager(AttachedDatabase &db, string path, bool read_only, bool create_new, bool use_direct_io);
+	SingleFileBlockManager(AttachedDatabase &db, string path, bool read_only, bool use_direct_io);
+
+	void GetFileFlags(uint8_t &flags, FileLockType &lock, bool create_new);
+	void CreateNewDatabase();
+	void LoadExistingDatabase();
 
 	//! Creates a new Block using the specified block_id and returns a pointer
 	unique_ptr<Block> CreateBlock(block_id_t block_id, FileBuffer *source_buffer) override;
@@ -59,6 +64,9 @@ private:
 
 	void Initialize(DatabaseHeader &header);
 
+	void ReadAndChecksum(FileBuffer &handle, uint64_t location) const;
+	void ChecksumAndWrite(FileBuffer &handle, uint64_t location) const;
+
 	//! Return the blocks to which we will write the free list and modified blocks
 	vector<block_id_t> GetFreeListBlocks();