duckdb 0.5.2-dev733.0 → 0.5.2-dev746.0

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "37960df94"
-#define DUCKDB_VERSION "v0.5.2-dev733"
+#define DUCKDB_SOURCE_ID "90356fc58"
+#define DUCKDB_VERSION "v0.5.2-dev746"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -3681,8 +3681,16 @@ struct string_t {
 	friend class StringSegment;
 
 public:
-	static constexpr idx_t PREFIX_LENGTH = 4 * sizeof(char);
-	static constexpr idx_t INLINE_LENGTH = 12;
+	static constexpr idx_t PREFIX_BYTES = 4 * sizeof(char);
+	static constexpr idx_t INLINE_BYTES = 12 * sizeof(char);
+	static constexpr idx_t HEADER_SIZE = sizeof(uint32_t) + PREFIX_BYTES;
+#ifndef DUCKDB_DEBUG_NO_INLINE
+	static constexpr idx_t PREFIX_LENGTH = PREFIX_BYTES;
+	static constexpr idx_t INLINE_LENGTH = INLINE_BYTES;
+#else
+	static constexpr idx_t PREFIX_LENGTH = 0;
+	static constexpr idx_t INLINE_LENGTH = 0;
+#endif
 
 	string_t() = default;
 	explicit string_t(uint32_t len) {
@@ -3694,7 +3702,7 @@ public:
 		if (IsInlined()) {
 			// zero initialize the prefix first
 			// this makes sure that strings with length smaller than 4 still have an equal prefix
-			memset(value.inlined.inlined, 0, INLINE_LENGTH);
+			memset(value.inlined.inlined, 0, INLINE_BYTES);
 			if (GetSize() == 0) {
 				return;
 			}
@@ -3702,7 +3710,11 @@ public:
 			memcpy(value.inlined.inlined, data, GetSize());
 		} else {
 			// large string: store pointer
+#ifndef DUCKDB_DEBUG_NO_INLINE
 			memcpy(value.pointer.prefix, data, PREFIX_LENGTH);
+#else
+			memset(value.pointer.prefix, 0, PREFIX_BYTES);
+#endif
 			value.pointer.ptr = (char *)data;
 		}
 	}
@@ -3743,15 +3755,19 @@ public:
 
 	void Finalize() {
 		// set trailing NULL byte
-		auto dataptr = (char *)GetDataUnsafe();
 		if (GetSize() <= INLINE_LENGTH) {
 			// fill prefix with zeros if the length is smaller than the prefix length
-			for (idx_t i = GetSize(); i < INLINE_LENGTH; i++) {
+			for (idx_t i = GetSize(); i < INLINE_BYTES; i++) {
 				value.inlined.inlined[i] = '\0';
 			}
 		} else {
 			// copy the data into the prefix
+#ifndef DUCKDB_DEBUG_NO_INLINE
+			auto dataptr = (char *)GetDataUnsafe();
 			memcpy(value.pointer.prefix, dataptr, PREFIX_LENGTH);
+#else
+			memset(value.pointer.prefix, 0, PREFIX_BYTES);
+#endif
 		}
 	}
 
@@ -7865,7 +7881,7 @@ struct StringComparisonOperators {
 			}
 		} else {
 			// large string: first check prefix and length
-			if (memcmp(&a, &b, sizeof(uint32_t) + string_t::PREFIX_LENGTH) == 0) {
+			if (memcmp(&a, &b, string_t::HEADER_SIZE) == 0) {
 				// prefix and length are equal: check main string
 				if (memcmp(a.value.pointer.ptr, b.value.pointer.ptr, a.GetSize()) == 0) {
 					// entire string is equal
@@ -14740,13 +14756,235 @@ private:
 
 
 
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/function/compression_function.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/map.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+#include <map>
+
+namespace duckdb {
+using std::map;
+using std::multimap;
+} // namespace duckdb
+
+
+
+
+namespace duckdb {
+class DatabaseInstance;
+class ColumnData;
+class ColumnDataCheckpointer;
+class ColumnSegment;
+class SegmentStatistics;
+
+struct ColumnFetchState;
+struct ColumnScanState;
+struct SegmentScanState;
+
+struct AnalyzeState {
+	virtual ~AnalyzeState() {
+	}
+};
+
+struct CompressionState {
+	virtual ~CompressionState() {
+	}
+};
+
+struct CompressedSegmentState {
+	virtual ~CompressedSegmentState() {
+	}
+};
+
+struct CompressionAppendState {
+	CompressionAppendState(BufferHandle handle_p) : handle(move(handle_p)) {
+	}
+	virtual ~CompressionAppendState() {
+	}
+
+	BufferHandle handle;
+};
+
+//===--------------------------------------------------------------------===//
+// Analyze
+//===--------------------------------------------------------------------===//
+//! The analyze functions are used to determine whether or not to use this compression method
+//! The system first determines the potential compression methods to use based on the physical type of the column
+//! After that the following steps are taken:
+//! 1. The init_analyze is called to initialize the analyze state of every candidate compression method
+//! 2. The analyze method is called with all of the input data in the order in which it must be stored.
+//!    analyze can return "false". In that case, the compression method is taken out of consideration early.
+//! 3. The final_analyze method is called, which should return a score for the compression method
+
+//! The system then decides which compression function to use based on the analyzed score (returned from final_analyze)
+typedef unique_ptr<AnalyzeState> (*compression_init_analyze_t)(ColumnData &col_data, PhysicalType type);
+typedef bool (*compression_analyze_t)(AnalyzeState &state, Vector &input, idx_t count);
+typedef idx_t (*compression_final_analyze_t)(AnalyzeState &state);
+
+//===--------------------------------------------------------------------===//
+// Compress
+//===--------------------------------------------------------------------===//
+typedef unique_ptr<CompressionState> (*compression_init_compression_t)(ColumnDataCheckpointer &checkpointer,
+                                                                       unique_ptr<AnalyzeState> state);
+typedef void (*compression_compress_data_t)(CompressionState &state, Vector &scan_vector, idx_t count);
+typedef void (*compression_compress_finalize_t)(CompressionState &state);
+
+//===--------------------------------------------------------------------===//
+// Uncompress / Scan
+//===--------------------------------------------------------------------===//
+typedef unique_ptr<SegmentScanState> (*compression_init_segment_scan_t)(ColumnSegment &segment);
+typedef void (*compression_scan_vector_t)(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count,
+                                          Vector &result);
+typedef void (*compression_scan_partial_t)(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count,
+                                           Vector &result, idx_t result_offset);
+typedef void (*compression_fetch_row_t)(ColumnSegment &segment, ColumnFetchState &state, row_t row_id, Vector &result,
+                                        idx_t result_idx);
+typedef void (*compression_skip_t)(ColumnSegment &segment, ColumnScanState &state, idx_t skip_count);
+
+//===--------------------------------------------------------------------===//
+// Append (optional)
+//===--------------------------------------------------------------------===//
+typedef unique_ptr<CompressedSegmentState> (*compression_init_segment_t)(ColumnSegment &segment, block_id_t block_id);
+typedef unique_ptr<CompressionAppendState> (*compression_init_append_t)(ColumnSegment &segment);
+typedef idx_t (*compression_append_t)(CompressionAppendState &append_state, ColumnSegment &segment,
+                                      SegmentStatistics &stats, UnifiedVectorFormat &data, idx_t offset, idx_t count);
+typedef idx_t (*compression_finalize_append_t)(ColumnSegment &segment, SegmentStatistics &stats);
+typedef void (*compression_revert_append_t)(ColumnSegment &segment, idx_t start_row);
+
+class CompressionFunction {
+public:
+	CompressionFunction(CompressionType type, PhysicalType data_type, compression_init_analyze_t init_analyze,
+	                    compression_analyze_t analyze, compression_final_analyze_t final_analyze,
+	                    compression_init_compression_t init_compression, compression_compress_data_t compress,
+	                    compression_compress_finalize_t compress_finalize, compression_init_segment_scan_t init_scan,
+	                    compression_scan_vector_t scan_vector, compression_scan_partial_t scan_partial,
+	                    compression_fetch_row_t fetch_row, compression_skip_t skip,
+	                    compression_init_segment_t init_segment = nullptr,
+	                    compression_init_append_t init_append = nullptr, compression_append_t append = nullptr,
+	                    compression_finalize_append_t finalize_append = nullptr,
+	                    compression_revert_append_t revert_append = nullptr)
+	    : type(type), data_type(data_type), init_analyze(init_analyze), analyze(analyze), final_analyze(final_analyze),
+	      init_compression(init_compression), compress(compress), compress_finalize(compress_finalize),
+	      init_scan(init_scan), scan_vector(scan_vector), scan_partial(scan_partial), fetch_row(fetch_row), skip(skip),
+	      init_segment(init_segment), init_append(init_append), append(append), finalize_append(finalize_append),
+	      revert_append(revert_append) {
+	}
+
+	//! Compression type
+	CompressionType type;
+	//! The data type this function can compress
+	PhysicalType data_type;
+
+	//! Analyze step: determine which compression function is the most effective
+	//! init_analyze is called once to set up the analyze state
+	compression_init_analyze_t init_analyze;
+	//! analyze is called several times (once per vector in the row group)
+	//! analyze should return true, unless compression is no longer possible with this compression method
+	//! in that case false should be returned
+	compression_analyze_t analyze;
+	//! final_analyze should return the score of the compression function
+	//! ideally this is the exact number of bytes required to store the data
+	//! this is not required/enforced: it can be an estimate as well
+	//! also this function can return DConstants::INVALID_INDEX to skip this compression method
+	compression_final_analyze_t final_analyze;
+
+	//! Compression step: actually compress the data
+	//! init_compression is called once to set up the comperssion state
+	compression_init_compression_t init_compression;
+	//! compress is called several times (once per vector in the row group)
+	compression_compress_data_t compress;
+	//! compress_finalize is called after
+	compression_compress_finalize_t compress_finalize;
+
+	//! init_scan is called to set up the scan state
+	compression_init_segment_scan_t init_scan;
+	//! scan_vector scans an entire vector using the scan state
+	compression_scan_vector_t scan_vector;
+	//! scan_partial scans a subset of a vector
+	//! this can request > vector_size as well
+	//! this is used if a vector crosses segment boundaries, or for child columns of lists
+	compression_scan_partial_t scan_partial;
+	//! fetch an individual row from the compressed vector
+	//! used for index lookups
+	compression_fetch_row_t fetch_row;
+	//! Skip forward in the compressed segment
+	compression_skip_t skip;
+
+	// Append functions
+	//! This only really needs to be defined for uncompressed segments
+
+	//! Initialize a compressed segment (optional)
+	compression_init_segment_t init_segment;
+	//! Initialize the append state (optional)
+	compression_init_append_t init_append;
+	//! Append to the compressed segment (optional)
+	compression_append_t append;
+	//! Finalize an append to the segment
+	compression_finalize_append_t finalize_append;
+	//! Revert append (optional)
+	compression_revert_append_t revert_append;
+};
+
+//! The set of compression functions
+struct CompressionFunctionSet {
+	mutex lock;
+	map<CompressionType, map<PhysicalType, CompressionFunction>> functions;
+};
+
+} // namespace duckdb
+
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/transaction/transaction_data.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+namespace duckdb {
+class Transaction;
+
+struct TransactionData {
+	TransactionData(Transaction &transaction_p);
+	TransactionData(transaction_t transaction_id_p, transaction_t start_time_p);
+
+	Transaction *transaction;
+	transaction_t transaction_id;
+	transaction_t start_time;
+};
+
+} // namespace duckdb
+
 
 namespace duckdb {
 class ColumnSegment;
 class DataTable;
+class LocalTableStorage;
 class RowGroup;
 class UpdateSegment;
-class ValiditySegment;
 
 struct TableAppendState;
 
@@ -14757,6 +14995,8 @@ struct ColumnAppendState {
 	vector<ColumnAppendState> child_appends;
 	//! The write lock that is held by the append
 	unique_ptr<StorageLockKey> lock;
+	//! The compression append state
+	unique_ptr<CompressionAppendState> append_state;
 };
 
 struct RowGroupAppendState {
@@ -14778,14 +15018,26 @@ struct IndexLock {
 };
 
 struct TableAppendState {
-	TableAppendState() : row_group_append_state(*this) {
-	}
+	TableAppendState();
+	~TableAppendState();
 
 	RowGroupAppendState row_group_append_state;
 	unique_lock<mutex> append_lock;
 	row_t row_start;
 	row_t current_row;
-	idx_t remaining_append_count;
+	//! The total number of rows appended by the append operation
+	idx_t total_append_count;
+	//! The first row-group that has been appended to
+	RowGroup *start_row_group;
+	//! The transaction data
+	TransactionData transaction;
+	//! The remaining append count, only if the append count is known beforehand
+	idx_t remaining;
+};
+
+struct LocalAppendState {
+	TableAppendState append_state;
+	LocalTableStorage *storage;
 };
 
 } // namespace duckdb
@@ -17959,7 +18211,7 @@ public:
 	              row_t row_id, DataChunk &result, idx_t result_idx);
 
 	//! Append count rows to the version info
-	void AppendVersionInfo(TransactionData transaction, idx_t start, idx_t count, transaction_t commit_id);
+	void AppendVersionInfo(TransactionData transaction, idx_t count);
 	//! Commit a previous append made by RowGroup::AppendVersionInfo
 	void CommitAppend(transaction_t commit_id, idx_t start, idx_t count);
 	//! Revert a previous append made by RowGroup::AppendVersionInfo
@@ -17972,7 +18224,7 @@ public:
 	static void Serialize(RowGroupPointer &pointer, Serializer &serializer);
 	static RowGroupPointer Deserialize(Deserializer &source, const vector<ColumnDefinition> &columns);
 
-	void InitializeAppend(TransactionData transaction, RowGroupAppendState &append_state, idx_t remaining_append_count);
+	void InitializeAppend(RowGroupAppendState &append_state);
 	void Append(RowGroupAppendState &append_state, DataChunk &chunk, idx_t append_count);
 
 	void Update(TransactionData transaction, DataChunk &updates, row_t *ids, idx_t offset, idx_t count,
@@ -18130,8 +18382,14 @@ public:
 	void Fetch(TransactionData transaction, DataChunk &result, const vector<column_t> &column_ids,
 	           Vector &row_identifiers, idx_t fetch_count, ColumnFetchState &state);
 
+	//! Initialize an append of a variable number of rows. FinalizeAppend must be called after appending is done.
+	void InitializeAppend(TableAppendState &state);
+	//! Initialize an append with a known number of rows. FinalizeAppend should not be called after appending is done.
 	void InitializeAppend(TransactionData transaction, TableAppendState &state, idx_t append_count);
-	void Append(TransactionData transaction, DataChunk &chunk, TableAppendState &state, TableStatistics &stats);
+	//! Append a chunk to a table.
+	void Append(DataChunk &chunk, TableAppendState &state, TableStatistics &stats);
+	//! FinalizeAppend flushes an append with a variable number of rows.
+	void FinalizeAppend(TransactionData transaction, TableAppendState &state);
 	void CommitAppend(transaction_t commit_id, idx_t row_start, idx_t count);
 	void RevertAppendInternal(idx_t start_row, idx_t count);
 
@@ -20525,16 +20783,19 @@ public:
 	bool NextParallelScan(ClientContext &context, DataTable *table, ParallelCollectionScanState &state,
 	                      CollectionScanState &scan_state);
 
+	//! Begin appending to the local storage
+	void InitializeAppend(LocalAppendState &state, DataTable *table);
 	//! Append a chunk to the local storage
-	void Append(DataTable *table, DataChunk &chunk);
+	static void Append(LocalAppendState &state, DataChunk &chunk);
+	//! Finish appending to the local storage
+	static void FinalizeAppend(LocalAppendState &state);
 	//! Delete a set of rows from the local storage
 	idx_t Delete(DataTable *table, Vector &row_ids, idx_t count);
 	//! Update a set of rows in the local storage
 	void Update(DataTable *table, Vector &row_ids, const vector<column_t> &column_ids, DataChunk &data);
 
 	//! Commits the local storage, writing it to the WAL and completing the commit
-	void Commit(LocalStorage::CommitState &commit_state, Transaction &transaction, WriteAheadLog *log,
-	            transaction_t commit_id);
+	void Commit(LocalStorage::CommitState &commit_state, Transaction &transaction);
 
 	bool ChangesMade() noexcept {
 		return table_storage.size() > 0;
@@ -20575,6 +20836,7 @@ private:
 
 
 
+
 namespace duckdb {
 class SequenceCatalogEntry;
 
@@ -20665,20 +20927,6 @@ private:
 	Transaction(const Transaction &) = delete;
 };
 
-struct TransactionData {
-	TransactionData(Transaction &transaction_p) // NOLINT
-	    : transaction(&transaction_p), transaction_id(transaction_p.transaction_id),
-	      start_time(transaction_p.start_time) {
-	}
-	TransactionData(transaction_t transaction_id_p, transaction_t start_time_p)
-	    : transaction(nullptr), transaction_id(transaction_id_p), start_time(start_time_p) {
-	}
-
-	Transaction *transaction;
-	transaction_t transaction_id;
-	transaction_t start_time;
-};
-
 } // namespace duckdb
 
 #include <functional>
@@ -24599,7 +24847,7 @@ private:
 	unique_ptr<ConnectionManager> connection_manager;
 	unordered_set<std::string> loaded_extensions;
 	//! Set to true if a fatal exception has occurred
-	bool is_invalidated = false;
+	atomic<bool> is_invalidated;
 };
 
 //! The database object. This object holds the catalog and all the
@@ -25827,198 +26075,6 @@ public:
 
 
 
-//===----------------------------------------------------------------------===//
-//                         DuckDB
-//
-// duckdb/function/compression_function.hpp
-//
-//
-//===----------------------------------------------------------------------===//
-
-
-
-
-
-
-//===----------------------------------------------------------------------===//
-//                         DuckDB
-//
-// duckdb/common/map.hpp
-//
-//
-//===----------------------------------------------------------------------===//
-
-
-
-#include <map>
-
-namespace duckdb {
-using std::map;
-using std::multimap;
-} // namespace duckdb
-
-
-
-
-namespace duckdb {
-class DatabaseInstance;
-class ColumnData;
-class ColumnDataCheckpointer;
-class ColumnSegment;
-class SegmentStatistics;
-
-struct ColumnFetchState;
-struct ColumnScanState;
-struct SegmentScanState;
-
-struct AnalyzeState {
-	virtual ~AnalyzeState() {
-	}
-};
-
-struct CompressionState {
-	virtual ~CompressionState() {
-	}
-};
-
-struct CompressedSegmentState {
-	virtual ~CompressedSegmentState() {
-	}
-};
-
-struct UncompressedCompressState : public CompressionState {
-	explicit UncompressedCompressState(ColumnDataCheckpointer &checkpointer);
-
-	ColumnDataCheckpointer &checkpointer;
-	unique_ptr<ColumnSegment> current_segment;
-
-	virtual void CreateEmptySegment(idx_t row_start);
-	void FlushSegment(idx_t segment_size);
-	void Finalize(idx_t segment_size);
-};
-
-//===--------------------------------------------------------------------===//
-// Analyze
-//===--------------------------------------------------------------------===//
-//! The analyze functions are used to determine whether or not to use this compression method
-//! The system first determines the potential compression methods to use based on the physical type of the column
-//! After that the following steps are taken:
-//! 1. The init_analyze is called to initialize the analyze state of every candidate compression method
-//! 2. The analyze method is called with all of the input data in the order in which it must be stored.
-//!    analyze can return "false". In that case, the compression method is taken out of consideration early.
-//! 3. The final_analyze method is called, which should return a score for the compression method
-
-//! The system then decides which compression function to use based on the analyzed score (returned from final_analyze)
-typedef unique_ptr<AnalyzeState> (*compression_init_analyze_t)(ColumnData &col_data, PhysicalType type);
-typedef bool (*compression_analyze_t)(AnalyzeState &state, Vector &input, idx_t count);
-typedef idx_t (*compression_final_analyze_t)(AnalyzeState &state);
-
-//===--------------------------------------------------------------------===//
-// Compress
-//===--------------------------------------------------------------------===//
-typedef unique_ptr<CompressionState> (*compression_init_compression_t)(ColumnDataCheckpointer &checkpointer,
-                                                                       unique_ptr<AnalyzeState> state);
-typedef void (*compression_compress_data_t)(CompressionState &state, Vector &scan_vector, idx_t count);
-typedef void (*compression_compress_finalize_t)(CompressionState &state);
-
-//===--------------------------------------------------------------------===//
-// Uncompress / Scan
-//===--------------------------------------------------------------------===//
-typedef unique_ptr<SegmentScanState> (*compression_init_segment_scan_t)(ColumnSegment &segment);
-typedef void (*compression_scan_vector_t)(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count,
-                                          Vector &result);
-typedef void (*compression_scan_partial_t)(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count,
-                                           Vector &result, idx_t result_offset);
-typedef void (*compression_fetch_row_t)(ColumnSegment &segment, ColumnFetchState &state, row_t row_id, Vector &result,
-                                        idx_t result_idx);
-typedef void (*compression_skip_t)(ColumnSegment &segment, ColumnScanState &state, idx_t skip_count);
-
-//===--------------------------------------------------------------------===//
-// Append (optional)
-//===--------------------------------------------------------------------===//
-typedef unique_ptr<CompressedSegmentState> (*compression_init_segment_t)(ColumnSegment &segment, block_id_t block_id);
-typedef idx_t (*compression_append_t)(ColumnSegment &segment, SegmentStatistics &stats, UnifiedVectorFormat &data,
-                                      idx_t offset, idx_t count);
-typedef idx_t (*compression_finalize_append_t)(ColumnSegment &segment, SegmentStatistics &stats);
-typedef void (*compression_revert_append_t)(ColumnSegment &segment, idx_t start_row);
-
-class CompressionFunction {
-public:
-	CompressionFunction(CompressionType type, PhysicalType data_type, compression_init_analyze_t init_analyze,
-	                    compression_analyze_t analyze, compression_final_analyze_t final_analyze,
-	                    compression_init_compression_t init_compression, compression_compress_data_t compress,
-	                    compression_compress_finalize_t compress_finalize, compression_init_segment_scan_t init_scan,
-	                    compression_scan_vector_t scan_vector, compression_scan_partial_t scan_partial,
-	                    compression_fetch_row_t fetch_row, compression_skip_t skip,
-	                    compression_init_segment_t init_segment = nullptr, compression_append_t append = nullptr,
-	                    compression_finalize_append_t finalize_append = nullptr,
-	                    compression_revert_append_t revert_append = nullptr)
-	    : type(type), data_type(data_type), init_analyze(init_analyze), analyze(analyze), final_analyze(final_analyze),
-	      init_compression(init_compression), compress(compress), compress_finalize(compress_finalize),
-	      init_scan(init_scan), scan_vector(scan_vector), scan_partial(scan_partial), fetch_row(fetch_row), skip(skip),
-	      init_segment(init_segment), append(append), finalize_append(finalize_append), revert_append(revert_append) {
-	}
-
-	//! Compression type
-	CompressionType type;
-	//! The data type this function can compress
-	PhysicalType data_type;
-
-	//! Analyze step: determine which compression function is the most effective
-	//! init_analyze is called once to set up the analyze state
-	compression_init_analyze_t init_analyze;
-	//! analyze is called several times (once per vector in the row group)
-	//! analyze should return true, unless compression is no longer possible with this compression method
-	//! in that case false should be returned
-	compression_analyze_t analyze;
-	//! final_analyze should return the score of the compression function
-	//! ideally this is the exact number of bytes required to store the data
-	//! this is not required/enforced: it can be an estimate as well
-	//! also this function can return DConstants::INVALID_INDEX to skip this compression method
-	compression_final_analyze_t final_analyze;
-
-	//! Compression step: actually compress the data
-	//! init_compression is called once to set up the comperssion state
-	compression_init_compression_t init_compression;
-	//! compress is called several times (once per vector in the row group)
-	compression_compress_data_t compress;
-	//! compress_finalize is called after
-	compression_compress_finalize_t compress_finalize;
-
-	//! init_scan is called to set up the scan state
-	compression_init_segment_scan_t init_scan;
-	//! scan_vector scans an entire vector using the scan state
-	compression_scan_vector_t scan_vector;
-	//! scan_partial scans a subset of a vector
-	//! this can request > vector_size as well
-	//! this is used if a vector crosses segment boundaries, or for child columns of lists
-	compression_scan_partial_t scan_partial;
-	//! fetch an individual row from the compressed vector
-	//! used for index lookups
-	compression_fetch_row_t fetch_row;
-	//! Skip forward in the compressed segment
-	compression_skip_t skip;
-
-	// Append functions
-	//! This only really needs to be defined for uncompressed segments
-
-	//! Initialize a compressed segment (optional)
-	compression_init_segment_t init_segment;
-	//! Append to the compressed segment (optional)
-	compression_append_t append;
-	//! Finalize an append to the segment
-	compression_finalize_append_t finalize_append;
-	//! Revert append (optional)
-	compression_revert_append_t revert_append;
-};
-
-//! The set of compression functions
-struct CompressionFunctionSet {
-	mutex lock;
-	map<CompressionType, map<PhysicalType, CompressionFunction>> functions;
-};
-
-} // namespace duckdb
 
 
 namespace duckdb {
@@ -26084,7 +26140,7 @@ public:
 	//! Finalize the segment for appending - no more appends can follow on this segment
 	//! The segment should be compacted as much as possible
 	//! Returns the number of bytes occupied within the segment
-	idx_t FinalizeAppend();
+	idx_t FinalizeAppend(ColumnAppendState &state);
 	//! Revert an append made to this segment
 	void RevertAppend(idx_t start_row);
 
@@ -26191,6 +26247,7 @@ struct DataTableInfo {
 
 namespace duckdb {
 class ClientContext;
+class ColumnDataCollection;
 class ColumnDefinition;
 class DataTable;
 class RowGroup;
@@ -26249,8 +26306,17 @@ public:
 	void Fetch(Transaction &transaction, DataChunk &result, const vector<column_t> &column_ids, Vector &row_ids,
 	           idx_t fetch_count, ColumnFetchState &state);
 
-	//! Append a DataChunk to the table. Throws an exception if the columns don't match the tables' columns.
-	void Append(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk);
+	//! Initializes an append to transaction-local storage
+	void InitializeLocalAppend(LocalAppendState &state, ClientContext &context);
+	//! Append a DataChunk to the transaction-local storage of the table.
+	void LocalAppend(LocalAppendState &state, TableCatalogEntry &table, ClientContext &context, DataChunk &chunk);
+	//! Finalizes a transaction-local append
+	void FinalizeLocalAppend(LocalAppendState &state);
+	//! Append a chunk to the transaction-local storage of this table
+	void LocalAppend(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk);
+	//! Append a column data collection to the transaction-local storage of this table
+	void LocalAppend(TableCatalogEntry &table, ClientContext &context, ColumnDataCollection &collection);
+
 	//! Delete the entries with the specified row identifier from the table
 	idx_t Delete(TableCatalogEntry &table, ClientContext &context, Vector &row_ids, idx_t count);
 	//! Update the entries with the specified row identifier from the table
@@ -26272,8 +26338,8 @@ public:
 	void AppendLock(TableAppendState &state);
 	//! Begin appending structs to this table, obtaining necessary locks, etc
 	void InitializeAppend(Transaction &transaction, TableAppendState &state, idx_t append_count);
-	//! Append a chunk to the table using the AppendState obtained from BeginAppend
-	void Append(Transaction &transaction, DataChunk &chunk, TableAppendState &state);
+	//! Append a chunk to the table using the AppendState obtained from InitializeAppend
+	void Append(DataChunk &chunk, TableAppendState &state);
 	//! Commit the append
 	void CommitAppend(transaction_t commit_id, idx_t row_start, idx_t count);
 	//! Write a segment of the table to the WAL