duckdb 0.9.3-dev2.0 → 0.10.0

package/src/duckdb/src/include/duckdb/storage/table/scan_state.hpp

@@ -31,6 +31,7 @@ class TableFilterSet;
 class ColumnData;
 class DuckTransaction;
 class RowGroupSegmentTree;
+struct TableScanOptions;
 
 struct SegmentScanState {
 	virtual ~SegmentScanState() {
@@ -92,9 +93,11 @@ struct ColumnScanState {
 	vector<unique_ptr<SegmentScanState>> previous_states;
 	//! The last read offset in the child state (used for LIST columns only)
 	idx_t last_offset = 0;
+	//! Contains TableScan level config for scanning
+	optional_ptr<TableScanOptions> scan_options;
 
 public:
-	void Initialize(const LogicalType &type);
+	void Initialize(const LogicalType &type, optional_ptr<TableScanOptions> options);
 	//! Move the scan state forward by "count" rows (including all child states)
 	void Next(idx_t count);
 	//! Move ONLY this state forward by "count" rows (i.e. not the child states)
@@ -134,6 +137,7 @@ public:
 	const vector<storage_t> &GetColumnIds();
 	TableFilterSet *GetFilters();
 	AdaptiveFilter *GetAdaptiveFilter();
+	TableScanOptions &GetOptions();
 	bool Scan(DuckTransaction &transaction, DataChunk &result);
 	bool ScanCommitted(DataChunk &result, TableScanType type);
 	bool ScanCommitted(DataChunk &result, SegmentLock &l, TableScanType type);
@@ -142,6 +146,11 @@ private:
 	TableScanState &parent;
 };
 
+struct TableScanOptions {
+	//! Test config that forces fetching rows one by one instead of regular scans
+	bool force_fetch_row = false;
+};
+
 class TableScanState {
 public:
 	TableScanState() : table_state(*this), local_state(*this), table_filters(nullptr) {};
@@ -150,6 +159,8 @@ public:
 	CollectionScanState table_state;
 	//! Transaction-local scan state
 	CollectionScanState local_state;
+	//! Options for scanning
+	TableScanOptions options;
 
 public:
 	void Initialize(vector<storage_t> column_ids, TableFilterSet *table_filters = nullptr);

package/src/duckdb/src/include/duckdb/storage/table/standard_column_data.hpp

@@ -57,7 +57,7 @@ public:
 	void GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector<duckdb::idx_t> col_path,
 	                          vector<duckdb::ColumnSegmentInfo> &result) override;
 
-	void DeserializeColumn(Deserializer &deserializer) override;
+	void DeserializeColumn(Deserializer &deserializer, BaseStatistics &target_stats) override;
 
 	void Verify(RowGroup &parent) override;
 };

package/src/duckdb/src/include/duckdb/storage/table/struct_column_data.hpp

@@ -57,7 +57,7 @@ public:
 	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, PartialBlockManager &partial_block_manager,
 	                                             ColumnCheckpointInfo &checkpoint_info) override;
 
-	void DeserializeColumn(Deserializer &source) override;
+	void DeserializeColumn(Deserializer &source, BaseStatistics &target_stats) override;
 
 	void GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector<duckdb::idx_t> col_path,
 	                          vector<duckdb::ColumnSegmentInfo> &result) override;

package/src/duckdb/src/include/duckdb/storage/table/table_index_list.hpp

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/storage/table_index_list.hpp
+// duckdb/storage/table/table_index_list.hpp
 //
 //
 //===----------------------------------------------------------------------===//
@@ -14,13 +14,14 @@
 namespace duckdb {
 
 class ConflictManager;
+struct IndexStorageInfo;
+struct DataTableInfo;
 
 class TableIndexList {
 public:
-	//! Scan the catalog set, invoking the callback method for every entry
+	//! Scan the indexes, invoking the callback method for every entry
 	template <class T>
 	void Scan(T &&callback) {
-		// lock the catalog set
 		lock_guard<mutex> lock(indexes_lock);
 		for (auto &index : indexes) {
 			if (callback(*index)) {
@@ -28,26 +29,29 @@ public:
 			}
 		}
 	}
-
+	//! Returns a reference to the indexes of this table
 	const vector<unique_ptr<Index>> &Indexes() const {
 		return indexes;
 	}
-
+	//! Adds an index to the list of indexes of this table
 	void AddIndex(unique_ptr<Index> index);
-
-	void RemoveIndex(Index &index);
-
+	//! Removes an index from the list of indexes of this table
+	void RemoveIndex(const string &name);
+	//! Completely removes all remaining memory of an index after dropping the catalog entry
+	void CommitDrop(const string &name);
+	//! Returns true, if the index name does not exist
+	bool NameIsUnique(const string &name);
+	//! Initializes unknown indexes that might now be present after an extension load
+	void InitializeIndexes(ClientContext &context, DataTableInfo &table_info);
 	bool Empty();
-
 	idx_t Count();
-
 	void Move(TableIndexList &other);
 
 	Index *FindForeignKeyIndex(const vector<PhysicalIndex> &fk_keys, ForeignKeyType fk_type);
 	void VerifyForeignKey(const vector<PhysicalIndex> &fk_keys, DataChunk &chunk, ConflictManager &conflict_manager);
 
-	//! Serialize all indexes owned by this table, returns a vector of block info of all indexes
-	vector<BlockPointer> SerializeIndexes(duckdb::MetadataWriter &writer);
+	//! Serialize all indexes of this table
+	vector<IndexStorageInfo> GetStorageInfos();
 
 	vector<column_t> GetRequiredColumns();
 

package/src/duckdb/src/include/duckdb/storage/table/table_statistics.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/types/data_chunk.hpp"
+#include "duckdb/execution/reservoir_sample.hpp"
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/storage/statistics/column_statistics.hpp"
 
@@ -57,6 +58,9 @@ private:
 	mutex stats_lock;
 	//! Column statistics
 	vector<shared_ptr<ColumnStatistics>> column_stats;
+	//! The table sample
+	//! Sample for table
+	unique_ptr<BlockingSample> table_sample;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table_storage_info.hpp

@@ -8,12 +8,15 @@
 
 #pragma once
 
-#include "duckdb/storage/storage_info.hpp"
 #include "duckdb/common/types/value.hpp"
-#include "duckdb/common/unordered_set.hpp"
+#include "duckdb/common/unordered_map.hpp"
+#include "duckdb/storage/block.hpp"
+#include "duckdb/storage/index_storage_info.hpp"
+#include "duckdb/storage/storage_info.hpp"
 
 namespace duckdb {
 
+//! Column segment information
 struct ColumnSegmentInfo {
 	idx_t row_group_index;
 	idx_t column_id;
@@ -31,13 +34,7 @@ struct ColumnSegmentInfo {
 	string segment_info;
 };
 
-struct IndexInfo {
-	bool is_unique;
-	bool is_primary;
-	bool is_foreign;
-	unordered_set<column_t> column_set;
-};
-
+//! Table storage information
 class TableStorageInfo {
 public:
 	//! The (estimated) cardinality of the table

package/src/duckdb/src/include/duckdb/storage/temporary_memory_manager.hpp

@@ -0,0 +1,119 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/temporary_memory_manager.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/atomic.hpp"
+#include "duckdb/common/mutex.hpp"
+#include "duckdb/common/reference_map.hpp"
+#include "duckdb/storage/storage_info.hpp"
+
+namespace duckdb {
+
+class ClientContext;
+class TemporaryMemoryManager;
+
+//! State of the temporary memory to be managed concurrently with other states
+//! As long as this is within scope, it is active
+class TemporaryMemoryState {
+	friend class TemporaryMemoryManager;
+
+private:
+	TemporaryMemoryState(TemporaryMemoryManager &temporary_memory_manager, idx_t minimum_reservation);
+
+public:
+	~TemporaryMemoryState();
+
+public:
+	//! Set the remaining size needed for this state, and updates the reservation
+	void SetRemainingSize(ClientContext &context, idx_t new_remaining_size);
+	//! Get the remaining size that was set for this state
+	idx_t GetRemainingSize() const;
+	//! Set the minimum reservation for this state
+	void SetMinimumReservation(idx_t new_minimum_reservation);
+	//! Get the reservation of this state
+	idx_t GetReservation() const;
+
+private:
+	//! The TemporaryMemoryManager that owns this state
+	TemporaryMemoryManager &temporary_memory_manager;
+
+	//! The remaining size needed if it could fit fully in memory
+	atomic<idx_t> remaining_size;
+	//! The minimum reservation for this state
+	atomic<idx_t> minimum_reservation;
+	//! How much memory this operator has reserved
+	atomic<idx_t> reservation;
+};
+
+//! TemporaryMemoryManager is a one-of class owned by the buffer pool that tries to dynamically assign memory
+//! to concurrent states, such that their combined memory usage does not exceed the limit
+class TemporaryMemoryManager {
+	//! TemporaryMemoryState is a friend class so it can access the private methods of this class,
+	//! but it should not access the private fields!
+	friend class TemporaryMemoryState;
+
+public:
+	TemporaryMemoryManager();
+
+private:
+	//! TemporaryMemoryState is initialized with a minimum reservation guarantee, which is either
+	//! 512 blocks per state per thread, which is 0.125GB per thread for Storage::BLOCK_ALLOC_SIZE = 262144
+	static constexpr const idx_t MINIMUM_RESERVATION_PER_STATE_PER_THREAD = idx_t(512) * Storage::BLOCK_ALLOC_SIZE;
+	//! Or 1/16th of main memory, if that is lower
+	static constexpr const idx_t MINIMUM_RESERVATION_MEMORY_LIMIT_DIVISOR = 16;
+
+	//! The maximum ratio of the memory limit that we reserve using the TemporaryMemoryManager
+	static constexpr const double MAXIMUM_MEMORY_LIMIT_RATIO = 0.8;
+	//! The maximum ratio of the remaining memory that we reserve per TemporaryMemoryState
+	static constexpr const double MAXIMUM_FREE_MEMORY_RATIO = double(2) / double(3);
+
+public:
+	//! Get the TemporaryMemoryManager
+	static TemporaryMemoryManager &Get(ClientContext &context);
+	//! Register a TemporaryMemoryState
+	unique_ptr<TemporaryMemoryState> Register(ClientContext &context);
+
+private:
+	//! Locks the TemporaryMemoryManager
+	unique_lock<mutex> Lock();
+	//! Update memory_limit, has_temporary_directory, and num_threads (must hold the lock)
+	void UpdateConfiguration(ClientContext &context);
+	//! Update the TemporaryMemoryState to the new remaining size, and updates the reservation (must hold the lock)
+	void UpdateState(ClientContext &context, TemporaryMemoryState &temporary_memory_state);
+	//! Set the remaining size of a TemporaryMemoryState (must hold the lock)
+	void SetRemainingSize(TemporaryMemoryState &temporary_memory_state, idx_t new_remaining_size);
+	//! Set the reservation of a TemporaryMemoryState (must hold the lock)
+	void SetReservation(TemporaryMemoryState &temporary_memory_state, idx_t new_reservation);
+	//! Unregister a TemporaryMemoryState (called by the destructor of TemporaryMemoryState)
+	void Unregister(TemporaryMemoryState &temporary_memory_state);
+	//! Verify internal counts (must hold the lock)
+	void Verify() const;
+
+private:
+	//! Lock because TemporaryMemoryManager is used concurrently
+	mutex lock;
+
+	//! Memory limit of the buffer pool
+	idx_t memory_limit;
+	//! Whether there is a temporary directory that we can offload blocks to
+	bool has_temporary_directory;
+	//! Number of threads
+	idx_t num_threads;
+	//! Max memory per query
+	idx_t query_max_memory;
+
+	//! Currently active states
+	reference_set_t<TemporaryMemoryState> active_states;
+	//! The sum of reservations of all active states
+	idx_t reservation;
+	//! The sum of the remaining size of all active states
+	idx_t remaining_size;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/write_ahead_log.hpp

@@ -8,15 +8,16 @@
 
 #pragma once
 
-#include "duckdb/common/helper.hpp"
-#include "duckdb/common/types/data_chunk.hpp"
-#include "duckdb/common/enums/wal_type.hpp"
-#include "duckdb/common/serializer/buffered_file_writer.hpp"
+#include "duckdb/catalog/catalog_entry/index_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/scalar_macro_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/sequence_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/table_macro_catalog_entry.hpp"
-#include "duckdb/catalog/catalog_entry/index_catalog_entry.hpp"
+#include "duckdb/common/enums/wal_type.hpp"
+#include "duckdb/common/helper.hpp"
+#include "duckdb/common/serializer/buffered_file_writer.hpp"
+#include "duckdb/common/types/data_chunk.hpp"
 #include "duckdb/main/attached_database.hpp"
+#include "duckdb/storage/block.hpp"
 #include "duckdb/storage/storage_info.hpp"
 
 namespace duckdb {
@@ -34,56 +35,7 @@ class TypeCatalogEntry;
 class TableCatalogEntry;
 class Transaction;
 class TransactionManager;
-
-class ReplayState {
-public:
-	ReplayState(AttachedDatabase &db, ClientContext &context)
-	    : db(db), context(context), catalog(db.GetCatalog()), deserialize_only(false) {
-	}
-
-	AttachedDatabase &db;
-	ClientContext &context;
-	Catalog &catalog;
-	optional_ptr<TableCatalogEntry> current_table;
-	bool deserialize_only;
-	MetaBlockPointer checkpoint_id;
-
-public:
-	void ReplayEntry(WALType entry_type, BinaryDeserializer &deserializer);
-
-protected:
-	virtual void ReplayCreateTable(BinaryDeserializer &deserializer);
-	void ReplayDropTable(BinaryDeserializer &deserializer);
-	void ReplayAlter(BinaryDeserializer &deserializer);
-
-	void ReplayCreateView(BinaryDeserializer &deserializer);
-	void ReplayDropView(BinaryDeserializer &deserializer);
-
-	void ReplayCreateSchema(BinaryDeserializer &deserializer);
-	void ReplayDropSchema(BinaryDeserializer &deserializer);
-
-	void ReplayCreateType(BinaryDeserializer &deserializer);
-	void ReplayDropType(BinaryDeserializer &deserializer);
-
-	void ReplayCreateSequence(BinaryDeserializer &deserializer);
-	void ReplayDropSequence(BinaryDeserializer &deserializer);
-	void ReplaySequenceValue(BinaryDeserializer &deserializer);
-
-	void ReplayCreateMacro(BinaryDeserializer &deserializer);
-	void ReplayDropMacro(BinaryDeserializer &deserializer);
-
-	void ReplayCreateTableMacro(BinaryDeserializer &deserializer);
-	void ReplayDropTableMacro(BinaryDeserializer &deserializer);
-
-	void ReplayCreateIndex(BinaryDeserializer &deserializer);
-	void ReplayDropIndex(BinaryDeserializer &deserializer);
-
-	void ReplayUseTable(BinaryDeserializer &deserializer);
-	void ReplayInsert(BinaryDeserializer &deserializer);
-	void ReplayDelete(BinaryDeserializer &deserializer);
-	void ReplayUpdate(BinaryDeserializer &deserializer);
-	void ReplayCheckpoint(BinaryDeserializer &deserializer);
-};
+class WriteAheadLogDeserializer;
 
 //! The WriteAheadLog (WAL) is a log that is used to provide durability. Prior
 //! to committing a transaction it writes the changes the transaction made to
@@ -107,6 +59,12 @@ public:
 	//! Gets the total bytes written to the WAL since startup
 	idx_t GetTotalWritten();
 
+	BufferedFileWriter &GetWriter() {
+		return *writer;
+	}
+
+	void WriteVersion();
+
 	virtual void WriteCreateTable(const TableCatalogEntry &entry);
 	void WriteDropTable(const TableCatalogEntry &entry);
 

package/src/duckdb/src/include/duckdb/transaction/duck_transaction.hpp

@@ -39,7 +39,7 @@ public:
 
 	//! Commit the current transaction with the given commit identifier. Returns an error message if the transaction
 	//! commit failed, or an empty string if the commit was sucessful
-	string Commit(AttachedDatabase &db, transaction_t commit_id, bool checkpoint) noexcept;
+	ErrorData Commit(AttachedDatabase &db, transaction_t commit_id, bool checkpoint) noexcept;
 	//! Returns whether or not a commit of this transaction should trigger an automatic checkpoint
 	bool AutomaticCheckpoint(AttachedDatabase &db);
 

package/src/duckdb/src/include/duckdb/transaction/duck_transaction_manager.hpp

@@ -20,17 +20,17 @@ class DuckTransactionManager : public TransactionManager {
 
 public:
 	explicit DuckTransactionManager(AttachedDatabase &db);
-	~DuckTransactionManager();
+	~DuckTransactionManager() override;
 
 public:
 	static DuckTransactionManager &Get(AttachedDatabase &db);
 
 	//! Start a new transaction
-	Transaction *StartTransaction(ClientContext &context) override;
+	Transaction &StartTransaction(ClientContext &context) override;
 	//! Commit the given transaction
-	string CommitTransaction(ClientContext &context, Transaction *transaction) override;
+	ErrorData CommitTransaction(ClientContext &context, Transaction &transaction) override;
 	//! Rollback the given transaction
-	void RollbackTransaction(Transaction *transaction) override;
+	void RollbackTransaction(Transaction &transaction) override;
 
 	void Checkpoint(ClientContext &context, bool force = false) override;
 
@@ -49,7 +49,6 @@ private:
 	bool CanCheckpoint(optional_ptr<DuckTransaction> current = nullptr);
 	//! Remove the given transaction from the list of active transactions
 	void RemoveTransaction(DuckTransaction &transaction) noexcept;
-	void LockClients(vector<ClientLockWrapper> &client_locks, ClientContext &context);
 
 private:
 	//! The current start timestamp used by transactions

package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp

@@ -61,8 +61,8 @@ public:
 
 	void AppendToIndexes(DuckTransaction &transaction, TableAppendState &append_state, idx_t append_count,
 	                     bool append_to_table);
-	PreservedError AppendToIndexes(DuckTransaction &transaction, RowGroupCollection &source, TableIndexList &index_list,
-	                               const vector<LogicalType> &table_types, row_t &start_row);
+	ErrorData AppendToIndexes(DuckTransaction &transaction, RowGroupCollection &source, TableIndexList &index_list,
+	                          const vector<LogicalType> &table_types, row_t &start_row);
 
 	//! Creates an optimistic writer for this table
 	OptimisticDataWriter &CreateOptimisticWriter();

package/src/duckdb/src/include/duckdb/transaction/meta_transaction.hpp

@@ -14,6 +14,8 @@
 #include "duckdb/common/types/timestamp.hpp"
 #include "duckdb/common/unordered_map.hpp"
 #include "duckdb/common/optional_ptr.hpp"
+#include "duckdb/common/reference_map.hpp"
+#include "duckdb/common/error_data.hpp"
 
 namespace duckdb {
 class AttachedDatabase;
@@ -44,8 +46,9 @@ public:
 	}
 
 	Transaction &GetTransaction(AttachedDatabase &db);
+	void RemoveTransaction(AttachedDatabase &db);
 
-	string Commit();
+	ErrorData Commit();
 	void Rollback();
 
 	idx_t GetActiveQuery();
@@ -58,9 +61,9 @@ public:
 
 private:
 	//! The set of active transactions for each database
-	unordered_map<AttachedDatabase *, Transaction *> transactions;
+	reference_map_t<AttachedDatabase, reference<Transaction>> transactions;
 	//! The set of transactions in order of when they were started
-	vector<optional_ptr<AttachedDatabase>> all_transactions;
+	vector<reference<AttachedDatabase>> all_transactions;
 	//! The database we are modifying - we can only modify one database per transaction
 	optional_ptr<AttachedDatabase> modified_database;
 };

package/src/duckdb/src/include/duckdb/transaction/transaction_context.hpp

@@ -23,7 +23,7 @@ class TransactionManager;
 //! current transaction
 class TransactionContext {
 public:
-	TransactionContext(ClientContext &context);
+	explicit TransactionContext(ClientContext &context);
 	~TransactionContext();
 
 	MetaTransaction &ActiveTransaction() {
@@ -33,8 +33,8 @@ public:
 		return *current_transaction;
 	}
 
-	bool HasActiveTransaction() {
-		return !!current_transaction;
+	bool HasActiveTransaction() const {
+		return current_transaction.get();
 	}
 
 	void BeginTransaction();
@@ -43,7 +43,7 @@ public:
 	void ClearTransaction();
 
 	void SetAutoCommit(bool value);
-	bool IsAutoCommit() {
+	bool IsAutoCommit() const {
 		return auto_commit;
 	}
 

package/src/duckdb/src/include/duckdb/transaction/transaction_manager.hpp

@@ -32,11 +32,11 @@ public:
 	virtual ~TransactionManager();
 
 	//! Start a new transaction
-	virtual Transaction *StartTransaction(ClientContext &context) = 0;
+	virtual Transaction &StartTransaction(ClientContext &context) = 0;
 	//! Commit the given transaction. Returns a non-empty error message on failure.
-	virtual string CommitTransaction(ClientContext &context, Transaction *transaction) = 0;
+	virtual ErrorData CommitTransaction(ClientContext &context, Transaction &transaction) = 0;
 	//! Rollback the given transaction
-	virtual void RollbackTransaction(Transaction *transaction) = 0;
+	virtual void RollbackTransaction(Transaction &transaction) = 0;
 
 	virtual void Checkpoint(ClientContext &context, bool force = false) = 0;
 

package/src/duckdb/src/include/duckdb/verification/fetch_row_verifier.hpp

@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/verification/unoptimized_statement_verifier.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/verification/statement_verifier.hpp"
+
+namespace duckdb {
+
+class FetchRowVerifier : public StatementVerifier {
+public:
+	explicit FetchRowVerifier(unique_ptr<SQLStatement> statement_p);
+	static unique_ptr<StatementVerifier> Create(const SQLStatement &statement_p);
+
+	bool ForceFetchRow() const override {
+		return true;
+	}
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/verification/statement_verifier.hpp

@@ -23,6 +23,7 @@ enum class VerificationType : uint8_t {
 	NO_OPERATOR_CACHING,
 	PREPARED,
 	EXTERNAL,
+	FETCH_ROW_AS_SCAN,
 
 	INVALID
 };
@@ -67,6 +68,10 @@ public:
 	virtual bool ForceExternal() const {
 		return false;
 	}
+
+	virtual bool ForceFetchRow() const {
+		return false;
+	}
 };
 
 } // namespace duckdb