duckdb 0.3.3-dev585.0 → 0.3.3-dev681.0

package/src/duckdb.cpp

@@ -34007,28 +34007,68 @@ hash_t Hash(char *val) {
 	return Hash<const char *>(val);
 }
 
-// Jenkins hash function: https://en.wikipedia.org/wiki/Jenkins_hash_function
-uint32_t JenkinsOneAtATimeHash(const char *key, size_t length) {
-	size_t i = 0;
-	uint32_t hash = 0;
-	while (i != length) {
-		hash += key[i++];
-		hash += hash << 10;
-		hash ^= hash >> 6;
-	}
-	hash += hash << 3;
-	hash ^= hash >> 11;
-	hash += hash << 15;
-	return hash;
+// MIT License
+// Copyright (c) 2018-2021 Martin Ankerl
+// https://github.com/martinus/robin-hood-hashing/blob/3.11.5/LICENSE
+hash_t HashBytes(void *ptr, size_t len) noexcept {
+	static constexpr uint64_t M = UINT64_C(0xc6a4a7935bd1e995);
+	static constexpr uint64_t SEED = UINT64_C(0xe17a1465);
+	static constexpr unsigned int R = 47;
+
+	auto const *const data64 = static_cast<uint64_t const *>(ptr);
+	uint64_t h = SEED ^ (len * M);
+
+	size_t const n_blocks = len / 8;
+	for (size_t i = 0; i < n_blocks; ++i) {
+		auto k = Load<uint64_t>(reinterpret_cast<const_data_ptr_t>(data64 + i));
+
+		k *= M;
+		k ^= k >> R;
+		k *= M;
+
+		h ^= k;
+		h *= M;
+	}
+
+	auto const *const data8 = reinterpret_cast<uint8_t const *>(data64 + n_blocks);
+	switch (len & 7U) {
+	case 7:
+		h ^= static_cast<uint64_t>(data8[6]) << 48U;
+		DUCKDB_EXPLICIT_FALLTHROUGH;
+	case 6:
+		h ^= static_cast<uint64_t>(data8[5]) << 40U;
+		DUCKDB_EXPLICIT_FALLTHROUGH;
+	case 5:
+		h ^= static_cast<uint64_t>(data8[4]) << 32U;
+		DUCKDB_EXPLICIT_FALLTHROUGH;
+	case 4:
+		h ^= static_cast<uint64_t>(data8[3]) << 24U;
+		DUCKDB_EXPLICIT_FALLTHROUGH;
+	case 3:
+		h ^= static_cast<uint64_t>(data8[2]) << 16U;
+		DUCKDB_EXPLICIT_FALLTHROUGH;
+	case 2:
+		h ^= static_cast<uint64_t>(data8[1]) << 8U;
+		DUCKDB_EXPLICIT_FALLTHROUGH;
+	case 1:
+		h ^= static_cast<uint64_t>(data8[0]);
+		h *= M;
+		DUCKDB_EXPLICIT_FALLTHROUGH;
+	default:
+		break;
+	}
+	h ^= h >> R;
+	h *= M;
+	h ^= h >> R;
+	return static_cast<hash_t>(h);
 }
 
 hash_t Hash(const char *val, size_t size) {
-	auto hash_val = JenkinsOneAtATimeHash(val, size);
-	return Hash<uint32_t>(hash_val);
+	return HashBytes((void *)val, size);
 }
 
 hash_t Hash(uint8_t *val, size_t size) {
-	return Hash((const char *)val, size);
+	return HashBytes((void *)val, size);
 }
 
 } // namespace duckdb
@@ -77369,6 +77409,11 @@ struct BitpackingFun {
 	static bool TypeIsSupported(PhysicalType type);
 };
 
+struct DictionaryCompressionFun {
+	static CompressionFunction GetFunction(PhysicalType type);
+	static bool TypeIsSupported(PhysicalType type);
+};
+
 } // namespace duckdb
 
 
@@ -77389,6 +77434,8 @@ static DefaultCompressionMethod internal_compression_methods[] = {
     {CompressionType::COMPRESSION_UNCOMPRESSED, UncompressedFun::GetFunction, UncompressedFun::TypeIsSupported},
     {CompressionType::COMPRESSION_RLE, RLEFun::GetFunction, RLEFun::TypeIsSupported},
     {CompressionType::COMPRESSION_BITPACKING, BitpackingFun::GetFunction, BitpackingFun::TypeIsSupported},
+    {CompressionType::COMPRESSION_DICTIONARY, DictionaryCompressionFun::GetFunction,
+     DictionaryCompressionFun::TypeIsSupported},
     {CompressionType::COMPRESSION_AUTO, nullptr, nullptr}};
 
 static CompressionFunction *FindCompressionFunction(CompressionFunctionSet &set, CompressionType type,
@@ -77438,6 +77485,7 @@ vector<CompressionFunction *> DBConfig::GetCompressionFunctions(PhysicalType dat
 	TryLoadCompression(*this, result, CompressionType::COMPRESSION_UNCOMPRESSED, data_type);
 	TryLoadCompression(*this, result, CompressionType::COMPRESSION_RLE, data_type);
 	TryLoadCompression(*this, result, CompressionType::COMPRESSION_BITPACKING, data_type);
+	TryLoadCompression(*this, result, CompressionType::COMPRESSION_DICTIONARY, data_type);
 	return result;
 }
 
@@ -160145,6 +160193,16 @@ void PartialBlock::FlushToDisk(DatabaseInstance &db) {
 }
 
 } // namespace duckdb
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/bitpacking.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
 
 
 // LICENSE_CHANGE_BEGIN
@@ -161316,262 +161374,55 @@ inline void fastpack(const uint64_t *__restrict in,
 
 
 
-
-
-//===----------------------------------------------------------------------===//
-//                         DuckDB
-//
-// duckdb/storage/table/column_data_checkpointer.hpp
-//
-//
-//===----------------------------------------------------------------------===//
-
-
-
-//===----------------------------------------------------------------------===//
-//                         DuckDB
-//
-// duckdb/storage/table/column_data.hpp
-//
-//
-//===----------------------------------------------------------------------===//
-
-
-
-
-
-
-
-
-
-
-//===----------------------------------------------------------------------===//
-//                         DuckDB
-//
-// duckdb/storage/table/column_checkpoint_state.hpp
-//
-//
-//===----------------------------------------------------------------------===//
-
-
-
-
-
-
-
-
-
 namespace duckdb {
-class ColumnData;
-class DatabaseInstance;
-class RowGroup;
-class TableDataWriter;
-
-struct ColumnCheckpointState {
-	ColumnCheckpointState(RowGroup &row_group, ColumnData &column_data, TableDataWriter &writer);
-	virtual ~ColumnCheckpointState();
-
-	RowGroup &row_group;
-	ColumnData &column_data;
-	TableDataWriter &writer;
-	SegmentTree new_tree;
-	vector<DataPointer> data_pointers;
-	unique_ptr<BaseStatistics> global_stats;
-
-public:
-	virtual unique_ptr<BaseStatistics> GetStatistics() {
-		return global_stats->Copy();
-	}
-
-	virtual void FlushSegment(unique_ptr<ColumnSegment> segment, idx_t segment_size);
-	virtual void FlushToDisk();
-};
-
-} // namespace duckdb
-
-
-
-namespace duckdb {
-class ColumnData;
-class ColumnSegment;
-class DatabaseInstance;
-class RowGroup;
-class TableDataWriter;
-class Transaction;
-
-struct DataTableInfo;
-
-struct ColumnCheckpointInfo {
-	ColumnCheckpointInfo(CompressionType compression_type_p) : compression_type(compression_type_p) {};
-	CompressionType compression_type;
-};
-
-class ColumnData {
-	friend class ColumnDataCheckpointer;
-
-public:
-	ColumnData(DataTableInfo &info, idx_t column_index, idx_t start_row, LogicalType type, ColumnData *parent);
-	virtual ~ColumnData();
-
-	//! Table info for the column
-	DataTableInfo &info;
-	//! The column index of the column, either within the parent table or within the parent
-	idx_t column_index;
-	//! The start row
-	idx_t start;
-	//! The type of the column
-	LogicalType type;
-	//! The parent column (if any)
-	ColumnData *parent;
-
-public:
-	virtual bool CheckZonemap(ColumnScanState &state, TableFilter &filter) = 0;
-
-	DatabaseInstance &GetDatabase() const;
-	DataTableInfo &GetTableInfo() const;
-	virtual idx_t GetMaxEntry();
-
-	//! The root type of the column
-	const LogicalType &RootType() const;
-
-	//! Initialize a scan of the column
-	virtual void InitializeScan(ColumnScanState &state);
-	//! Initialize a scan starting at the specified offset
-	virtual void InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx);
-	//! Scan the next vector from the column
-	virtual idx_t Scan(Transaction &transaction, idx_t vector_index, ColumnScanState &state, Vector &result);
-	virtual idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates);
-	virtual void ScanCommittedRange(idx_t row_group_start, idx_t offset_in_row_group, idx_t count, Vector &result);
-	virtual idx_t ScanCount(ColumnScanState &state, Vector &result, idx_t count);
-	//! Select
-	virtual void Select(Transaction &transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
-	                    SelectionVector &sel, idx_t &count, const TableFilter &filter);
-	virtual void FilterScan(Transaction &transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
-	                        SelectionVector &sel, idx_t count);
-	virtual void FilterScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, SelectionVector &sel,
-	                                 idx_t count, bool allow_updates);
-
-	//! Skip the scan forward by "count" rows
-	virtual void Skip(ColumnScanState &state, idx_t count = STANDARD_VECTOR_SIZE);
-
-	//! Initialize an appending phase for this column
-	virtual void InitializeAppend(ColumnAppendState &state);
-	//! Append a vector of type [type] to the end of the column
-	virtual void Append(BaseStatistics &stats, ColumnAppendState &state, Vector &vector, idx_t count);
-	virtual void AppendData(BaseStatistics &stats, ColumnAppendState &state, VectorData &vdata, idx_t count);
-	//! Revert a set of appends to the ColumnData
-	virtual void RevertAppend(row_t start_row);
-
-	//! Fetch the vector from the column data that belongs to this specific row
-	virtual idx_t Fetch(ColumnScanState &state, row_t row_id, Vector &result);
-	//! Fetch a specific row id and append it to the vector
-	virtual void FetchRow(Transaction &transaction, ColumnFetchState &state, row_t row_id, Vector &result,
-	                      idx_t result_idx);
-
-	virtual void Update(Transaction &transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
-	                    idx_t update_count);
-	virtual void UpdateColumn(Transaction &transaction, const vector<column_t> &column_path, Vector &update_vector,
-	                          row_t *row_ids, idx_t update_count, idx_t depth);
-	virtual unique_ptr<BaseStatistics> GetUpdateStatistics();
-
-	virtual void CommitDropColumn();
-
-	virtual unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group, TableDataWriter &writer);
-	virtual unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, TableDataWriter &writer,
-	                                                     ColumnCheckpointInfo &checkpoint_info);
-
-	virtual void CheckpointScan(ColumnSegment *segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
-	                            Vector &scan_vector);
-
-	virtual void DeserializeColumn(Deserializer &source);
-	static shared_ptr<ColumnData> Deserialize(DataTableInfo &info, idx_t column_index, idx_t start_row,
-	                                          Deserializer &source, const LogicalType &type, ColumnData *parent);
-
-	virtual void GetStorageInfo(idx_t row_group_index, vector<idx_t> col_path, vector<vector<Value>> &result);
-	virtual void Verify(RowGroup &parent);
-
-	static shared_ptr<ColumnData> CreateColumn(DataTableInfo &info, idx_t column_index, idx_t start_row,
-	                                           const LogicalType &type, ColumnData *parent = nullptr);
-	static unique_ptr<ColumnData> CreateColumnUnique(DataTableInfo &info, idx_t column_index, idx_t start_row,
-	                                                 const LogicalType &type, ColumnData *parent = nullptr);
-
-protected:
-	//! Append a transient segment
-	void AppendTransientSegment(idx_t start_row);
 
-	//! Scans a base vector from the column
-	idx_t ScanVector(ColumnScanState &state, Vector &result, idx_t remaining);
-	//! Scans a vector from the column merged with any potential updates
-	//! If ALLOW_UPDATES is set to false, the function will instead throw an exception if any updates are found
-	template <bool SCAN_COMMITTED, bool ALLOW_UPDATES>
-	idx_t ScanVector(Transaction *transaction, idx_t vector_index, ColumnScanState &state, Vector &result);
-
-protected:
-	//! The segments holding the data of this column segment
-	SegmentTree data;
-	//! The lock for the updates
-	mutex update_lock;
-	//! The updates for this column segment
-	unique_ptr<UpdateSegment> updates;
-};
-
-} // namespace duckdb
-
-
-
-namespace duckdb {
+using bitpacking_width_t = uint8_t;
 
-class ColumnDataCheckpointer {
-public:
-	ColumnDataCheckpointer(ColumnData &col_data_p, RowGroup &row_group_p, ColumnCheckpointState &state_p,
-	                       ColumnCheckpointInfo &checkpoint_info);
+class BitpackingPrimitives {
 
 public:
-	DatabaseInstance &GetDatabase();
-	const LogicalType &GetType() const;
-	ColumnData &GetColumnData();
-	RowGroup &GetRowGroup();
-	ColumnCheckpointState &GetCheckpointState();
-
-	void Checkpoint(unique_ptr<SegmentBase> segment);
-
-private:
-	void ScanSegments(const std::function<void(Vector &, idx_t)> &callback);
-	unique_ptr<AnalyzeState> DetectBestCompressionMethod(idx_t &compression_idx);
-	void WriteToDisk();
-	bool HasChanges();
-	void WritePersistentSegments();
-
-private:
-	ColumnData &col_data;
-	RowGroup &row_group;
-	ColumnCheckpointState &state;
-	bool is_validity;
-	Vector intermediate;
-	unique_ptr<SegmentBase> owned_segment;
-	vector<CompressionFunction *> compression_functions;
-	ColumnCheckpointInfo &checkpoint_info;
-};
-
-} // namespace duckdb
-
+	static constexpr const idx_t BITPACKING_ALGORITHM_GROUP_SIZE = 32;
+	static constexpr const idx_t BITPACKING_HEADER_SIZE = sizeof(uint64_t);
+	static constexpr const bool BYTE_ALIGNED = false;
 
+	// To ensure enough data is available, use GetRequiredSize() to determine the correct size for dst buffer
+	// Note: input should be aligned to BITPACKING_ALGORITHM_GROUP_SIZE for good performance.
+	template <class T, bool ASSUME_INPUT_ALIGNED = false>
+	inline static void PackBuffer(data_ptr_t dst, T *src, idx_t count, bitpacking_width_t width) {
+		if (ASSUME_INPUT_ALIGNED) {
+			for (idx_t i = 0; i < count; i += BITPACKING_ALGORITHM_GROUP_SIZE) {
+				PackGroup<T>(dst + (i * width) / 8, src + i, width);
+			}
+		} else {
+			idx_t misaligned_count = count % BITPACKING_ALGORITHM_GROUP_SIZE;
+			T tmp_buffer[BITPACKING_ALGORITHM_GROUP_SIZE]; // TODO maybe faster on the heap?
 
-#include <functional>
+			if (misaligned_count) {
+				count -= misaligned_count;
+			}
 
-namespace duckdb {
+			for (idx_t i = 0; i < count; i += BITPACKING_ALGORITHM_GROUP_SIZE) {
+				PackGroup<T>(dst + (i * width) / 8, src + i, width);
+			}
 
-using bitpacking_width_t = uint8_t;
+			// Input was not aligned to BITPACKING_ALGORITHM_GROUP_SIZE, we need a copy
+			if (misaligned_count) {
+				memcpy(tmp_buffer, src + count, misaligned_count * sizeof(T));
+				PackGroup<T>(dst + (count * width) / 8, tmp_buffer, width);
+			}
+		}
+	}
 
-// Note that optimizations in scanning only work if this value is equal to STANDARD_VECTOR_SIZE, however we keep them
-// separated to prevent the code from break on lower vector sizes
-static constexpr const idx_t BITPACKING_WIDTH_GROUP_SIZE = 1024;
+	// Unpacks a block of BITPACKING_ALGORITHM_GROUP_SIZE values
+	// Assumes both src and dst to be of the correct size
+	template <class T>
+	inline static void UnPackBuffer(data_ptr_t dst, data_ptr_t src, idx_t count, bitpacking_width_t width,
+	                                bool skip_sign_extension = false) {
 
-class BitpackingPrimitives {
-public:
-	static constexpr const idx_t BITPACKING_ALGORITHM_GROUP_SIZE = 32;
-	static constexpr const idx_t BITPACKING_HEADER_SIZE = sizeof(uint64_t);
-	static constexpr const bool BYTE_ALIGNED = false;
+		for (idx_t i = 0; i < count; i += BITPACKING_ALGORITHM_GROUP_SIZE) {
+			UnPackGroup<T>(dst + i * sizeof(T), src + (i * width) / 8, width, skip_sign_extension);
+		}
+	}
 
 	// Packs a block of BITPACKING_ALGORITHM_GROUP_SIZE values
 	template <class T>
@@ -161586,44 +161437,35 @@ public:
 		return UnPackGroup<T>(dst, src, width, skip_sign_extension);
 	}
 
+	// Calculates the minimum required number of bits per value that can store all values
+	template <class T>
+	inline static bitpacking_width_t MinimumBitWidth(T value) {
+		return FindMinimumBitWidth<T, BYTE_ALIGNED>(value, value);
+	}
+
 	// Calculates the minimum required number of bits per value that can store all values
 	template <class T>
 	inline static bitpacking_width_t MinimumBitWidth(T *values, idx_t count) {
 		return FindMinimumBitWidth<T, BYTE_ALIGNED>(values, count);
 	}
 
-private:
 	template <class T>
-	static bitpacking_width_t MinimumBitWidth(T min_value, T max_value) {
-		bitpacking_width_t required_bits;
-
-		if (std::is_signed<T>::value) {
-			if (min_value == NumericLimits<T>::Minimum()) {
-				// handle special case of the minimal value, as it cannot be negated like all other values.
-				return sizeof(T) * 8;
-			} else {
-				max_value = MaxValue((T)-min_value, max_value);
-			}
-		}
-
-		if (max_value == 0) {
-			return 0;
-		}
-
-		if (std::is_signed<T>::value) {
-			required_bits = 1;
-		} else {
-			required_bits = 0;
-		}
+	inline static idx_t GetRequiredSize(idx_t count, bitpacking_width_t width) {
+		count = RoundUpToAlgorithmGroupSize(count);
+		return ((count * width) / 8);
+	}
 
-		while (max_value) {
-			required_bits++;
-			max_value >>= 1;
+	template <class T>
+	inline static T RoundUpToAlgorithmGroupSize(T num_to_round) {
+		int remainder = num_to_round % BITPACKING_ALGORITHM_GROUP_SIZE;
+		if (remainder == 0) {
+			return num_to_round;
 		}
 
-		return GetEffectiveWidth<T>(required_bits);
+		return num_to_round + BITPACKING_ALGORITHM_GROUP_SIZE - remainder;
 	}
 
+private:
 	template <class T, bool round_to_next_byte = false>
 	static bitpacking_width_t FindMinimumBitWidth(T *values, idx_t count) {
 		T min_value = values[0];
@@ -161641,23 +161483,57 @@ private:
 			}
 		}
 
-		bitpacking_width_t calc_width = MinimumBitWidth<T>(std::is_signed<T>::value ? min_value : 0, max_value);
+		return FindMinimumBitWidth<T, round_to_next_byte>(min_value, max_value);
+	}
+
+	template <class T, bool round_to_next_byte = false>
+	static bitpacking_width_t FindMinimumBitWidth(T min_value, T max_value) {
+		bitpacking_width_t bitwidth;
+		T value;
+
+		if (std::is_signed<T>::value) {
+			if (min_value == NumericLimits<T>::Minimum()) {
+				// handle special case of the minimal value, as it cannot be negated like all other values.
+				return sizeof(T) * 8;
+			} else {
+				value = MaxValue((T)-min_value, max_value);
+			}
+		} else {
+			value = max_value;
+		}
+
+		if (value == 0) {
+			return 0;
+		}
+
+		if (std::is_signed<T>::value) {
+			bitwidth = 1;
+		} else {
+			bitwidth = 0;
+		}
+
+		while (value) {
+			bitwidth++;
+			value >>= 1;
+		}
+
+		bitwidth = GetEffectiveWidth<T>(bitwidth);
 
 		// Assert results are correct
 #ifdef DEBUG
-		if (calc_width < sizeof(T) * 8 && calc_width != 0) {
+		if (bitwidth < sizeof(T) * 8 && bitwidth != 0) {
 			if (std::is_signed<T>::value) {
-				D_ASSERT((int64_t)max_value <= (int64_t)(1L << (calc_width - 1)) - 1);
-				D_ASSERT((int64_t)min_value >= (int64_t)(-1 * ((1L << (calc_width - 1)) - 1) - 1));
+				D_ASSERT((int64_t)max_value <= (int64_t)(1L << (bitwidth - 1)) - 1);
+				D_ASSERT((int64_t)min_value >= (int64_t)(-1 * ((1L << (bitwidth - 1)) - 1) - 1));
 			} else {
-				D_ASSERT((uint64_t)max_value <= (uint64_t)(1L << (calc_width)) - 1);
+				D_ASSERT((uint64_t)max_value <= (uint64_t)(1L << (bitwidth)) - 1);
 			}
 		}
 #endif
 		if (round_to_next_byte) {
-			return (calc_width / 8 + (calc_width % 8 != 0)) * 8;
+			return (bitwidth / 8 + (bitwidth % 8 != 0)) * 8;
 		} else {
-			return calc_width;
+			return bitwidth;
 		}
 	}
 
@@ -161727,6 +161603,263 @@ private:
 	}
 };
 
+} // namespace duckdb
+
+
+
+
+
+
+
+
+
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/table/column_data_checkpointer.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/table/column_data.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+
+
+
+
+
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/table/column_checkpoint_state.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+
+
+
+
+namespace duckdb {
+class ColumnData;
+class DatabaseInstance;
+class RowGroup;
+class TableDataWriter;
+
+struct ColumnCheckpointState {
+	ColumnCheckpointState(RowGroup &row_group, ColumnData &column_data, TableDataWriter &writer);
+	virtual ~ColumnCheckpointState();
+
+	RowGroup &row_group;
+	ColumnData &column_data;
+	TableDataWriter &writer;
+	SegmentTree new_tree;
+	vector<DataPointer> data_pointers;
+	unique_ptr<BaseStatistics> global_stats;
+
+public:
+	virtual unique_ptr<BaseStatistics> GetStatistics() {
+		return global_stats->Copy();
+	}
+
+	virtual void FlushSegment(unique_ptr<ColumnSegment> segment, idx_t segment_size);
+	virtual void FlushToDisk();
+};
+
+} // namespace duckdb
+
+
+
+namespace duckdb {
+class ColumnData;
+class ColumnSegment;
+class DatabaseInstance;
+class RowGroup;
+class TableDataWriter;
+class Transaction;
+
+struct DataTableInfo;
+
+struct ColumnCheckpointInfo {
+	ColumnCheckpointInfo(CompressionType compression_type_p) : compression_type(compression_type_p) {};
+	CompressionType compression_type;
+};
+
+class ColumnData {
+	friend class ColumnDataCheckpointer;
+
+public:
+	ColumnData(DataTableInfo &info, idx_t column_index, idx_t start_row, LogicalType type, ColumnData *parent);
+	virtual ~ColumnData();
+
+	//! Table info for the column
+	DataTableInfo &info;
+	//! The column index of the column, either within the parent table or within the parent
+	idx_t column_index;
+	//! The start row
+	idx_t start;
+	//! The type of the column
+	LogicalType type;
+	//! The parent column (if any)
+	ColumnData *parent;
+
+public:
+	virtual bool CheckZonemap(ColumnScanState &state, TableFilter &filter) = 0;
+
+	DatabaseInstance &GetDatabase() const;
+	DataTableInfo &GetTableInfo() const;
+	virtual idx_t GetMaxEntry();
+
+	//! The root type of the column
+	const LogicalType &RootType() const;
+
+	//! Initialize a scan of the column
+	virtual void InitializeScan(ColumnScanState &state);
+	//! Initialize a scan starting at the specified offset
+	virtual void InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx);
+	//! Scan the next vector from the column
+	virtual idx_t Scan(Transaction &transaction, idx_t vector_index, ColumnScanState &state, Vector &result);
+	virtual idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates);
+	virtual void ScanCommittedRange(idx_t row_group_start, idx_t offset_in_row_group, idx_t count, Vector &result);
+	virtual idx_t ScanCount(ColumnScanState &state, Vector &result, idx_t count);
+	//! Select
+	virtual void Select(Transaction &transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+	                    SelectionVector &sel, idx_t &count, const TableFilter &filter);
+	virtual void FilterScan(Transaction &transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+	                        SelectionVector &sel, idx_t count);
+	virtual void FilterScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, SelectionVector &sel,
+	                                 idx_t count, bool allow_updates);
+
+	//! Skip the scan forward by "count" rows
+	virtual void Skip(ColumnScanState &state, idx_t count = STANDARD_VECTOR_SIZE);
+
+	//! Initialize an appending phase for this column
+	virtual void InitializeAppend(ColumnAppendState &state);
+	//! Append a vector of type [type] to the end of the column
+	virtual void Append(BaseStatistics &stats, ColumnAppendState &state, Vector &vector, idx_t count);
+	virtual void AppendData(BaseStatistics &stats, ColumnAppendState &state, VectorData &vdata, idx_t count);
+	//! Revert a set of appends to the ColumnData
+	virtual void RevertAppend(row_t start_row);
+
+	//! Fetch the vector from the column data that belongs to this specific row
+	virtual idx_t Fetch(ColumnScanState &state, row_t row_id, Vector &result);
+	//! Fetch a specific row id and append it to the vector
+	virtual void FetchRow(Transaction &transaction, ColumnFetchState &state, row_t row_id, Vector &result,
+	                      idx_t result_idx);
+
+	virtual void Update(Transaction &transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
+	                    idx_t update_count);
+	virtual void UpdateColumn(Transaction &transaction, const vector<column_t> &column_path, Vector &update_vector,
+	                          row_t *row_ids, idx_t update_count, idx_t depth);
+	virtual unique_ptr<BaseStatistics> GetUpdateStatistics();
+
+	virtual void CommitDropColumn();
+
+	virtual unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group, TableDataWriter &writer);
+	virtual unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, TableDataWriter &writer,
+	                                                     ColumnCheckpointInfo &checkpoint_info);
+
+	virtual void CheckpointScan(ColumnSegment *segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
+	                            Vector &scan_vector);
+
+	virtual void DeserializeColumn(Deserializer &source);
+	static shared_ptr<ColumnData> Deserialize(DataTableInfo &info, idx_t column_index, idx_t start_row,
+	                                          Deserializer &source, const LogicalType &type, ColumnData *parent);
+
+	virtual void GetStorageInfo(idx_t row_group_index, vector<idx_t> col_path, vector<vector<Value>> &result);
+	virtual void Verify(RowGroup &parent);
+
+	static shared_ptr<ColumnData> CreateColumn(DataTableInfo &info, idx_t column_index, idx_t start_row,
+	                                           const LogicalType &type, ColumnData *parent = nullptr);
+	static unique_ptr<ColumnData> CreateColumnUnique(DataTableInfo &info, idx_t column_index, idx_t start_row,
+	                                                 const LogicalType &type, ColumnData *parent = nullptr);
+
+protected:
+	//! Append a transient segment
+	void AppendTransientSegment(idx_t start_row);
+
+	//! Scans a base vector from the column
+	idx_t ScanVector(ColumnScanState &state, Vector &result, idx_t remaining);
+	//! Scans a vector from the column merged with any potential updates
+	//! If ALLOW_UPDATES is set to false, the function will instead throw an exception if any updates are found
+	template <bool SCAN_COMMITTED, bool ALLOW_UPDATES>
+	idx_t ScanVector(Transaction *transaction, idx_t vector_index, ColumnScanState &state, Vector &result);
+
+protected:
+	//! The segments holding the data of this column segment
+	SegmentTree data;
+	//! The lock for the updates
+	mutex update_lock;
+	//! The updates for this column segment
+	unique_ptr<UpdateSegment> updates;
+};
+
+} // namespace duckdb
+
+
+
+namespace duckdb {
+
+class ColumnDataCheckpointer {
+public:
+	ColumnDataCheckpointer(ColumnData &col_data_p, RowGroup &row_group_p, ColumnCheckpointState &state_p,
+	                       ColumnCheckpointInfo &checkpoint_info);
+
+public:
+	DatabaseInstance &GetDatabase();
+	const LogicalType &GetType() const;
+	ColumnData &GetColumnData();
+	RowGroup &GetRowGroup();
+	ColumnCheckpointState &GetCheckpointState();
+
+	void Checkpoint(unique_ptr<SegmentBase> segment);
+
+private:
+	void ScanSegments(const std::function<void(Vector &, idx_t)> &callback);
+	unique_ptr<AnalyzeState> DetectBestCompressionMethod(idx_t &compression_idx);
+	void WriteToDisk();
+	bool HasChanges();
+	void WritePersistentSegments();
+
+private:
+	ColumnData &col_data;
+	RowGroup &row_group;
+	ColumnCheckpointState &state;
+	bool is_validity;
+	Vector intermediate;
+	unique_ptr<SegmentBase> owned_segment;
+	vector<CompressionFunction *> compression_functions;
+	ColumnCheckpointInfo &checkpoint_info;
+};
+
+} // namespace duckdb
+
+
+
+#include <functional>
+
+namespace duckdb {
+
+// Note that optimizations in scanning only work if this value is equal to STANDARD_VECTOR_SIZE, however we keep them
+// separated to prevent the code from break on lower vector sizes
+static constexpr const idx_t BITPACKING_WIDTH_GROUP_SIZE = 1024;
+
 struct EmptyBitpackingWriter {
 	template <class T>
 	static void Operation(T *values, bool *validity, bitpacking_width_t width, idx_t count, void *data_ptr) {
@@ -161891,12 +162024,8 @@ public:
 
 	void WriteValues(T *values, bitpacking_width_t width, idx_t count) {
 		// TODO we can optimize this by stopping early if count < BITPACKING_WIDTH_GROUP_SIZE
-		idx_t compress_loops = BITPACKING_WIDTH_GROUP_SIZE / BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE;
-		for (idx_t i = 0; i < compress_loops; i++) {
-			BitpackingPrimitives::PackBlock<T>(
-			    data_ptr, &values[i * BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE], width);
-			data_ptr += (BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE * width) / 8;
-		}
+		BitpackingPrimitives::PackBuffer<T, false>(data_ptr, values, count, width);
+		data_ptr += (BITPACKING_WIDTH_GROUP_SIZE * width) / 8;
 
 		Store<bitpacking_width_t>(width, width_ptr);
 		width_ptr -= sizeof(bitpacking_width_t);
@@ -162181,6 +162310,11 @@ bool BitpackingFun::TypeIsSupported(PhysicalType type) {
 }
 
 } // namespace duckdb
+
+
+
+
+
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -162241,6 +162375,842 @@ public:
 
 
 
+
+
+
+
+
+namespace duckdb {
+struct StringDictionaryContainer {
+	//! The size of the dictionary
+	uint32_t size;
+	//! The end of the dictionary (typically Storage::BLOCK_SIZE)
+	uint32_t end;
+
+	void Verify() {
+		D_ASSERT(size <= Storage::BLOCK_SIZE);
+		D_ASSERT(end <= Storage::BLOCK_SIZE);
+		D_ASSERT(size <= end);
+	}
+};
+
+struct StringScanState : public SegmentScanState {
+	unique_ptr<BufferHandle> handle;
+};
+
+struct UncompressedStringStorage {
+public:
+	//! Dictionary header size at the beginning of the string segment (offset + length)
+	static constexpr uint16_t DICTIONARY_HEADER_SIZE = sizeof(uint32_t) + sizeof(uint32_t);
+	//! Marker used in length field to indicate the presence of a big string
+	static constexpr uint16_t BIG_STRING_MARKER = (uint16_t)-1;
+	//! Base size of big string marker (block id + offset)
+	static constexpr idx_t BIG_STRING_MARKER_BASE_SIZE = sizeof(block_id_t) + sizeof(int32_t);
+	//! The marker size of the big string
+	static constexpr idx_t BIG_STRING_MARKER_SIZE = BIG_STRING_MARKER_BASE_SIZE + sizeof(uint16_t);
+	//! The size below which the segment is compacted on flushing
+	static constexpr size_t COMPACTION_FLUSH_LIMIT = (size_t)Storage::BLOCK_SIZE / 5 * 4;
+
+public:
+	static unique_ptr<AnalyzeState> StringInitAnalyze(ColumnData &col_data, PhysicalType type);
+	static bool StringAnalyze(AnalyzeState &state_p, Vector &input, idx_t count);
+	static idx_t StringFinalAnalyze(AnalyzeState &state_p);
+	static unique_ptr<SegmentScanState> StringInitScan(ColumnSegment &segment);
+	static void StringScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result,
+	                              idx_t result_offset);
+	static void StringScan(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result);
+	static void StringFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id, Vector &result,
+	                           idx_t result_idx);
+	static unique_ptr<CompressedSegmentState> StringInitSegment(ColumnSegment &segment, block_id_t block_id);
+
+	static idx_t StringAppend(ColumnSegment &segment, SegmentStatistics &stats, VectorData &data, idx_t offset,
+	                          idx_t count) {
+		return StringAppendBase(segment, stats, data, offset, count);
+	}
+
+	template <bool DUPLICATE_ELIMINATE = false>
+	static idx_t StringAppendBase(ColumnSegment &segment, SegmentStatistics &stats, VectorData &data, idx_t offset,
+	                              idx_t count, std::unordered_map<string, int32_t> *seen_strings = nullptr) {
+		auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
+		auto handle = buffer_manager.Pin(segment.block);
+
+		D_ASSERT(segment.GetBlockOffset() == 0);
+		auto source_data = (string_t *)data.data;
+		auto result_data = (int32_t *)(handle->node->buffer + DICTIONARY_HEADER_SIZE);
+		for (idx_t i = 0; i < count; i++) {
+			auto source_idx = data.sel->get_index(offset + i);
+			auto target_idx = segment.count.load();
+			idx_t remaining_space = RemainingSpace(segment, *handle);
+			if (remaining_space < sizeof(int32_t)) {
+				// string index does not fit in the block at all
+				return i;
+			}
+			remaining_space -= sizeof(int32_t);
+			if (!data.validity.RowIsValid(source_idx)) {
+				// null value is stored as -1
+				result_data[target_idx] = 0;
+			} else {
+				auto dictionary = GetDictionary(segment, *handle);
+				auto end = handle->node->buffer + dictionary.end;
+
+				dictionary.Verify();
+
+				int32_t match;
+				bool found;
+				if (DUPLICATE_ELIMINATE) {
+					auto search = seen_strings->find(source_data[source_idx].GetString());
+					if (search != seen_strings->end()) {
+						match = search->second;
+						found = true;
+					} else {
+						found = false;
+					}
+				}
+
+				if (DUPLICATE_ELIMINATE && found) {
+					// We have seen this string
+					result_data[target_idx] = match;
+				} else {
+					// Unknown string, continue
+					// non-null value, check if we can fit it within the block
+					idx_t string_length = source_data[source_idx].GetSize();
+					idx_t dictionary_length = string_length + sizeof(uint16_t);
+
+					// determine whether or not we have space in the block for this string
+					bool use_overflow_block = false;
+					idx_t required_space = dictionary_length;
+					if (required_space >= StringUncompressed::STRING_BLOCK_LIMIT) {
+						// string exceeds block limit, store in overflow block and only write a marker here
+						required_space = BIG_STRING_MARKER_SIZE;
+						use_overflow_block = true;
+					}
+					if (required_space > remaining_space) {
+						// no space remaining: return how many tuples we ended up writing
+						return i;
+					}
+
+					// we have space: write the string
+					UpdateStringStats(stats, source_data[source_idx]);
+
+					if (use_overflow_block) {
+						// write to overflow blocks
+						block_id_t block;
+						int32_t offset;
+						// write the string into the current string block
+						WriteString(segment, source_data[source_idx], block, offset);
+						dictionary.size += BIG_STRING_MARKER_SIZE;
+						auto dict_pos = end - dictionary.size;
+
+						// write a big string marker into the dictionary
+						WriteStringMarker(dict_pos, block, offset);
+					} else {
+						// string fits in block, append to dictionary and increment dictionary position
+						D_ASSERT(string_length < NumericLimits<uint16_t>::Maximum());
+						dictionary.size += required_space;
+						auto dict_pos = end - dictionary.size; // first write the length as u16
+						Store<uint16_t>(string_length, dict_pos);
+						// now write the actual string data into the dictionary
+						memcpy(dict_pos + sizeof(uint16_t), source_data[source_idx].GetDataUnsafe(), string_length);
+					}
+					D_ASSERT(RemainingSpace(segment, *handle) <= Storage::BLOCK_SIZE);
+					// place the dictionary offset into the set of vectors
+					dictionary.Verify();
+
+					result_data[target_idx] = dictionary.size;
+					if (DUPLICATE_ELIMINATE) {
+						seen_strings->insert({source_data[source_idx].GetString(), dictionary.size});
+					}
+					SetDictionary(segment, *handle, dictionary);
+				}
+			}
+			segment.count++;
+		}
+		return count;
+	}
+
+	static idx_t FinalizeAppend(ColumnSegment &segment, SegmentStatistics &stats);
+
+public:
+	static inline void UpdateStringStats(SegmentStatistics &stats, const string_t &new_value) {
+		auto &sstats = (StringStatistics &)*stats.statistics;
+		sstats.Update(new_value);
+	}
+
+	static void SetDictionary(ColumnSegment &segment, BufferHandle &handle, StringDictionaryContainer dict);
+	static StringDictionaryContainer GetDictionary(ColumnSegment &segment, BufferHandle &handle);
+	static idx_t RemainingSpace(ColumnSegment &segment, BufferHandle &handle);
+	static void WriteString(ColumnSegment &segment, string_t string, block_id_t &result_block, int32_t &result_offset);
+	static void WriteStringMemory(ColumnSegment &segment, string_t string, block_id_t &result_block,
+	                              int32_t &result_offset);
+	static string_t ReadString(ColumnSegment &segment, Vector &result, block_id_t block, int32_t offset);
+	static string_t ReadString(data_ptr_t target, int32_t offset);
+	static void WriteStringMarker(data_ptr_t target, block_id_t block_id, int32_t offset);
+	static void ReadStringMarker(data_ptr_t target, block_id_t &block_id, int32_t &offset);
+
+	static string_location_t FetchStringLocation(StringDictionaryContainer dict, data_ptr_t baseptr,
+	                                             int32_t dict_offset);
+	static string_t FetchStringFromDict(ColumnSegment &segment, StringDictionaryContainer dict, Vector &result,
+	                                    data_ptr_t baseptr, int32_t dict_offset);
+	static string_t FetchString(ColumnSegment &segment, StringDictionaryContainer dict, Vector &result,
+	                            data_ptr_t baseptr, string_location_t location);
+};
+} // namespace duckdb
+
+
+
+namespace duckdb {
+
+struct StringHash {
+	std::size_t operator()(const string_t &k) const {
+		return Hash(k.GetDataUnsafe(), k.GetSize());
+	}
+};
+
+struct StringCompare {
+	bool operator()(const string_t &lhs, const string_t &rhs) const {
+		return StringComparisonOperators::EqualsOrNot<false>(lhs, rhs);
+	}
+};
+
+// Abstract class for keeping compression state either for compression or size analysis
+class DictionaryCompressionState : public CompressionState {
+public:
+	bool UpdateState(Vector &scan_vector, idx_t count) {
+		VectorData vdata;
+		scan_vector.Orrify(count, vdata);
+		auto data = (string_t *)vdata.data;
+		Verify();
+
+		for (idx_t i = 0; i < count; i++) {
+			auto idx = vdata.sel->get_index(i);
+			size_t string_size = 0;
+			bool new_string = false;
+			auto row_is_valid = vdata.validity.RowIsValid(idx);
+
+			if (row_is_valid) {
+				string_size = data[idx].GetSize();
+				if (string_size >= StringUncompressed::STRING_BLOCK_LIMIT) {
+					// Big strings not implemented for dictionary compression
+					return false;
+				}
+				new_string = !LookupString(data[idx]);
+			}
+
+			bool fits = HasEnoughSpace(new_string, string_size);
+			if (!fits) {
+				Flush();
+				new_string = true;
+				D_ASSERT(HasEnoughSpace(new_string, string_size));
+			}
+
+			if (!row_is_valid) {
+				AddNull();
+			} else if (new_string) {
+				AddNewString(data[idx]);
+			} else {
+				AddLastLookup();
+			}
+
+			Verify();
+		}
+
+		return true;
+	}
+
+protected:
+	// Should verify the State
+	virtual void Verify() = 0;
+	// Performs a lookup of str, storing the result internally
+	virtual bool LookupString(string_t str) = 0;
+	// Add the most recently looked up str to compression state
+	virtual void AddLastLookup() = 0;
+	// Add string to the state that is known to not be seen yet
+	virtual void AddNewString(string_t str) = 0;
+	// Add a null value to the compression state
+	virtual void AddNull() = 0;
+	// Check if we have enough space to add a string
+	virtual bool HasEnoughSpace(bool new_string, size_t string_size) = 0;
+	// Flush the segment to disk if compressing or reset the counters if analyzing
+	virtual void Flush(bool final = false) = 0;
+};
+
+typedef struct {
+	uint32_t dict_size;
+	uint32_t dict_end;
+	uint32_t index_buffer_offset;
+	uint32_t index_buffer_count;
+	uint32_t bitpacking_width;
+} dictionary_compression_header_t;
+
+struct DictionaryCompressionStorage {
+	static constexpr float MINIMUM_COMPRESSION_RATIO = 1.2;
+	static constexpr uint16_t DICTIONARY_HEADER_SIZE = sizeof(dictionary_compression_header_t);
+	static constexpr size_t COMPACTION_FLUSH_LIMIT = (size_t)Storage::BLOCK_SIZE / 5 * 4;
+
+	static unique_ptr<AnalyzeState> StringInitAnalyze(ColumnData &col_data, PhysicalType type);
+	static bool StringAnalyze(AnalyzeState &state_p, Vector &input, idx_t count);
+	static idx_t StringFinalAnalyze(AnalyzeState &state_p);
+
+	static unique_ptr<CompressionState> InitCompression(ColumnDataCheckpointer &checkpointer,
+	                                                    unique_ptr<AnalyzeState> state);
+	static void Compress(CompressionState &state_p, Vector &scan_vector, idx_t count);
+	static void FinalizeCompress(CompressionState &state_p);
+
+	static unique_ptr<SegmentScanState> StringInitScan(ColumnSegment &segment);
+	template <bool ALLOW_DICT_VECTORS>
+	static void StringScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result,
+	                              idx_t result_offset);
+	static void StringScan(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result);
+	static void StringFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id, Vector &result,
+	                           idx_t result_idx);
+
+	static bool HasEnoughSpace(idx_t current_count, idx_t index_count, idx_t dict_size,
+	                           bitpacking_width_t packing_width);
+	static idx_t RequiredSpace(idx_t current_count, idx_t index_count, idx_t dict_size,
+	                           bitpacking_width_t packing_width);
+
+	static StringDictionaryContainer GetDictionary(ColumnSegment &segment, BufferHandle &handle);
+	static void SetDictionary(ColumnSegment &segment, BufferHandle &handle, StringDictionaryContainer container);
+	static string_t FetchStringFromDict(ColumnSegment &segment, StringDictionaryContainer dict, data_ptr_t baseptr,
+	                                    int32_t dict_offset, uint16_t string_len);
+	static uint16_t GetStringLength(uint32_t *index_buffer_ptr, sel_t index);
+};
+
+// Dictionary compression uses a combination of bitpacking and a dictionary to compress string segments. The data is
+// stored across three buffers: the index buffer, the selection buffer and the dictionary. Firstly the Index buffer
+// contains the offsets into the dictionary which are also used to determine the string lengths. Each value in the
+// dictionary gets a single unique index in the index buffer. Secondly, the selection buffer maps the tuples to an index
+// in the index buffer. The selection buffer is compressed with bitpacking. Finally, the dictionary contains simply all
+// the unique strings without lenghts or null termination as we can deduce the lengths from the index buffer. The
+// addition of the selection buffer is done for two reasons: firstly, to allow the scan to emit dictionary vectors by
+// scanning the whole dictionary at once and then scanning the selection buffer for each emitted vector. Secondly, it
+// allows for efficient bitpacking compression as the selection values should remain relatively small.
+struct DictionaryCompressionCompressState : public DictionaryCompressionState {
+	explicit DictionaryCompressionCompressState(ColumnDataCheckpointer &checkpointer) : checkpointer(checkpointer) {
+		auto &db = checkpointer.GetDatabase();
+		auto &config = DBConfig::GetConfig(db);
+		function = config.GetCompressionFunction(CompressionType::COMPRESSION_DICTIONARY, PhysicalType::VARCHAR);
+		CreateEmptySegment(checkpointer.GetRowGroup().start);
+	}
+
+	void CreateEmptySegment(idx_t row_start) {
+		auto &db = checkpointer.GetDatabase();
+		auto &type = checkpointer.GetType();
+		auto compressed_segment = ColumnSegment::CreateTransientSegment(db, type, row_start);
+		current_segment = move(compressed_segment);
+
+		current_segment->function = function;
+
+		// Reset the buffers and string map
+		current_string_map.clear();
+		index_buffer.clear();
+		index_buffer.push_back(0); // Reserve index 0 for null strings
+		selection_buffer.clear();
+
+		current_width = 0;
+		next_width = 0;
+
+		// Reset the pointers into the current segment
+		auto &buffer_manager = BufferManager::GetBufferManager(current_segment->db);
+		current_handle = buffer_manager.Pin(current_segment->block);
+		current_dictionary = DictionaryCompressionStorage::GetDictionary(*current_segment, *current_handle);
+		current_end_ptr = current_handle->node->buffer + current_dictionary.end;
+	}
+
+	ColumnDataCheckpointer &checkpointer;
+	CompressionFunction *function;
+
+	// State regarding current segment
+	unique_ptr<ColumnSegment> current_segment;
+	unique_ptr<BufferHandle> current_handle;
+	StringDictionaryContainer current_dictionary;
+	data_ptr_t current_end_ptr;
+
+	// Buffers and map for current segment
+	std::unordered_map<string_t, uint32_t, StringHash, StringCompare> current_string_map;
+	std::vector<uint32_t> index_buffer;
+	std::vector<uint32_t> selection_buffer;
+
+	bitpacking_width_t current_width = 0;
+	bitpacking_width_t next_width = 0;
+
+	// Result of latest LookupString call
+	uint32_t latest_lookup_result;
+
+	void Verify() override {
+		current_dictionary.Verify();
+		D_ASSERT(current_segment->count == selection_buffer.size());
+		D_ASSERT(DictionaryCompressionStorage::HasEnoughSpace(current_segment->count.load(), index_buffer.size(),
+		                                                      current_dictionary.size, current_width));
+		D_ASSERT(current_dictionary.end == Storage::BLOCK_SIZE);
+		D_ASSERT(index_buffer.size() == current_string_map.size() + 1); // +1 is for null value
+	}
+
+	bool LookupString(string_t str) override {
+		auto search = current_string_map.find(str);
+		auto has_result = search != current_string_map.end();
+
+		if (has_result) {
+			latest_lookup_result = search->second;
+		}
+		return has_result;
+	}
+
+	void AddNewString(string_t str) override {
+		UncompressedStringStorage::UpdateStringStats(current_segment->stats, str);
+
+		// Copy string to dict
+		current_dictionary.size += str.GetSize();
+		auto dict_pos = current_end_ptr - current_dictionary.size;
+		memcpy(dict_pos, str.GetDataUnsafe(), str.GetSize());
+		current_dictionary.Verify();
+		D_ASSERT(current_dictionary.end == Storage::BLOCK_SIZE);
+
+		// Update buffers and map
+		index_buffer.push_back(current_dictionary.size);
+		selection_buffer.push_back(index_buffer.size() - 1);
+		current_string_map.insert({str, index_buffer.size() - 1});
+		DictionaryCompressionStorage::SetDictionary(*current_segment, *current_handle, current_dictionary);
+
+		current_width = next_width;
+		current_segment->count++;
+	}
+
+	void AddNull() override {
+		selection_buffer.push_back(0);
+		current_segment->count++;
+	}
+
+	void AddLastLookup() override {
+		selection_buffer.push_back(latest_lookup_result);
+		current_segment->count++;
+	}
+
+	bool HasEnoughSpace(bool new_string, size_t string_size) override {
+		if (new_string) {
+			next_width = BitpackingPrimitives::MinimumBitWidth(index_buffer.size() - 1 + new_string);
+			return DictionaryCompressionStorage::HasEnoughSpace(current_segment->count.load() + 1,
+			                                                    index_buffer.size() + 1,
+			                                                    current_dictionary.size + string_size, next_width);
+		} else {
+			return DictionaryCompressionStorage::HasEnoughSpace(current_segment->count.load() + 1, index_buffer.size(),
+			                                                    current_dictionary.size, current_width);
+		}
+	}
+
+	void Flush(bool final = false) override {
+		auto next_start = current_segment->start + current_segment->count;
+
+		auto segment_size = Finalize();
+		auto &state = checkpointer.GetCheckpointState();
+		state.FlushSegment(move(current_segment), segment_size);
+
+		if (!final) {
+			CreateEmptySegment(next_start);
+		}
+	}
+
+	idx_t Finalize() {
+		auto &buffer_manager = BufferManager::GetBufferManager(current_segment->db);
+		auto handle = buffer_manager.Pin(current_segment->block);
+		D_ASSERT(current_dictionary.end == Storage::BLOCK_SIZE);
+
+		// calculate sizes
+		auto compressed_selection_buffer_size =
+		    BitpackingPrimitives::GetRequiredSize<sel_t>(current_segment->count, current_width);
+		auto index_buffer_size = index_buffer.size() * sizeof(uint32_t);
+		auto total_size = DictionaryCompressionStorage::DICTIONARY_HEADER_SIZE + compressed_selection_buffer_size +
+		                  index_buffer_size + current_dictionary.size;
+
+		// calculate ptr and offsets
+		auto base_ptr = handle->node->buffer;
+		auto header_ptr = (dictionary_compression_header_t *)base_ptr;
+		auto compressed_selection_buffer_offset = DictionaryCompressionStorage::DICTIONARY_HEADER_SIZE;
+		auto index_buffer_offset = compressed_selection_buffer_offset + compressed_selection_buffer_size;
+
+		// Write compressed selection buffer
+		BitpackingPrimitives::PackBuffer<sel_t, false>(base_ptr + compressed_selection_buffer_offset,
+		                                               (sel_t *)(selection_buffer.data()), current_segment->count,
+		                                               current_width);
+
+		// Write the index buffer
+		memcpy(base_ptr + index_buffer_offset, index_buffer.data(), index_buffer_size);
+
+		// Store sizes and offsets in segment header
+		Store<uint32_t>(index_buffer_offset, (data_ptr_t)&header_ptr->index_buffer_offset);
+		Store<uint32_t>(index_buffer.size(), (data_ptr_t)&header_ptr->index_buffer_count);
+		Store<uint32_t>((uint32_t)current_width, (data_ptr_t)&header_ptr->bitpacking_width);
+
+		D_ASSERT(current_width == BitpackingPrimitives::MinimumBitWidth(index_buffer.size() - 1));
+		D_ASSERT(DictionaryCompressionStorage::HasEnoughSpace(current_segment->count, index_buffer.size(),
+		                                                      current_dictionary.size, current_width));
+		D_ASSERT((uint64_t)*max_element(std::begin(selection_buffer), std::end(selection_buffer)) ==
+		         index_buffer.size() - 1);
+
+		if (total_size >= DictionaryCompressionStorage::COMPACTION_FLUSH_LIMIT) {
+			// the block is full enough, don't bother moving around the dictionary
+			return Storage::BLOCK_SIZE;
+		}
+		// the block has space left: figure out how much space we can save
+		auto move_amount = Storage::BLOCK_SIZE - total_size;
+		// move the dictionary so it lines up exactly with the offsets
+		auto new_dictionary_offset = index_buffer_offset + index_buffer_size;
+		memmove(base_ptr + new_dictionary_offset, base_ptr + current_dictionary.end - current_dictionary.size,
+		        current_dictionary.size);
+		current_dictionary.end -= move_amount;
+		D_ASSERT(current_dictionary.end == total_size);
+		// write the new dictionary (with the updated "end")
+		DictionaryCompressionStorage::SetDictionary(*current_segment, *handle, current_dictionary);
+		return total_size;
+	}
+};
+
+//===--------------------------------------------------------------------===//
+// Analyze
+//===--------------------------------------------------------------------===//
+struct DictionaryCompressionAnalyzeState : public AnalyzeState, DictionaryCompressionState {
+	DictionaryCompressionAnalyzeState()
+	    : segment_count(0), current_tuple_count(0), current_unique_count(0), current_dict_size(0), current_width(0),
+	      next_width(0) {
+	}
+
+	size_t segment_count;
+	idx_t current_tuple_count;
+	idx_t current_unique_count;
+	size_t current_dict_size;
+	std::unordered_set<string_t, StringHash, StringCompare> current_set;
+	bitpacking_width_t current_width;
+	bitpacking_width_t next_width;
+
+	bool LookupString(string_t str) override {
+		return current_set.count(str);
+	}
+
+	void AddNewString(string_t str) override {
+		current_tuple_count++;
+		current_unique_count++;
+		current_dict_size += str.GetSize();
+		current_set.insert(str);
+		current_width = next_width;
+	}
+
+	void AddLastLookup() override {
+		current_tuple_count++;
+	}
+
+	void AddNull() override {
+		current_tuple_count++;
+	}
+
+	bool HasEnoughSpace(bool new_string, size_t string_size) override {
+		if (new_string) {
+			next_width =
+			    BitpackingPrimitives::MinimumBitWidth(current_unique_count + 2); // 1 for null, one for new string
+			return DictionaryCompressionStorage::HasEnoughSpace(current_tuple_count + 1, current_unique_count + 1,
+			                                                    current_dict_size + string_size, next_width);
+		} else {
+			return DictionaryCompressionStorage::HasEnoughSpace(current_tuple_count + 1, current_unique_count,
+			                                                    current_dict_size, current_width);
+		}
+	}
+
+	void Flush(bool final = false) override {
+		segment_count++;
+		current_tuple_count = 0;
+		current_unique_count = 0;
+		current_dict_size = 0;
+		current_set.clear();
+	}
+	void Verify() override {};
+};
+
+unique_ptr<AnalyzeState> DictionaryCompressionStorage::StringInitAnalyze(ColumnData &col_data, PhysicalType type) {
+	return make_unique<DictionaryCompressionAnalyzeState>();
+}
+
+bool DictionaryCompressionStorage::StringAnalyze(AnalyzeState &state_p, Vector &input, idx_t count) {
+	auto &state = (DictionaryCompressionAnalyzeState &)state_p;
+	return state.UpdateState(input, count);
+}
+
+idx_t DictionaryCompressionStorage::StringFinalAnalyze(AnalyzeState &state_p) {
+	auto &state = (DictionaryCompressionAnalyzeState &)state_p;
+
+	auto width = BitpackingPrimitives::MinimumBitWidth(state.current_unique_count + 1);
+	auto req_space =
+	    RequiredSpace(state.current_tuple_count, state.current_unique_count, state.current_dict_size, width);
+
+	return MINIMUM_COMPRESSION_RATIO * (state.segment_count * Storage::BLOCK_SIZE + req_space);
+}
+
+//===--------------------------------------------------------------------===//
+// Compress
+//===--------------------------------------------------------------------===//
+unique_ptr<CompressionState> DictionaryCompressionStorage::InitCompression(ColumnDataCheckpointer &checkpointer,
+                                                                           unique_ptr<AnalyzeState> state) {
+	return make_unique<DictionaryCompressionCompressState>(checkpointer);
+}
+
+void DictionaryCompressionStorage::Compress(CompressionState &state_p, Vector &scan_vector, idx_t count) {
+	auto &state = (DictionaryCompressionCompressState &)state_p;
+	state.UpdateState(scan_vector, count);
+}
+
+void DictionaryCompressionStorage::FinalizeCompress(CompressionState &state_p) {
+	auto &state = (DictionaryCompressionCompressState &)state_p;
+	state.Flush(true);
+}
+
+//===--------------------------------------------------------------------===//
+// Scan
+//===--------------------------------------------------------------------===//
+struct CompressedStringScanState : public StringScanState {
+	unique_ptr<BufferHandle> handle;
+	buffer_ptr<Vector> dictionary;
+	bitpacking_width_t current_width;
+	buffer_ptr<SelectionVector> sel_vec;
+	idx_t sel_vec_size = 0;
+};
+
+unique_ptr<SegmentScanState> DictionaryCompressionStorage::StringInitScan(ColumnSegment &segment) {
+	auto state = make_unique<CompressedStringScanState>();
+	auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
+	state->handle = buffer_manager.Pin(segment.block);
+
+	auto baseptr = state->handle->node->buffer + segment.GetBlockOffset();
+
+	// Load header values
+	auto dict = DictionaryCompressionStorage::GetDictionary(segment, *(state->handle));
+	auto header_ptr = (dictionary_compression_header_t *)baseptr;
+	auto index_buffer_offset = Load<uint32_t>((data_ptr_t)&header_ptr->index_buffer_offset);
+	auto index_buffer_count = Load<uint32_t>((data_ptr_t)&header_ptr->index_buffer_count);
+	state->current_width = (bitpacking_width_t)(Load<uint32_t>((data_ptr_t)&header_ptr->bitpacking_width));
+
+	auto index_buffer_ptr = (uint32_t *)(baseptr + index_buffer_offset);
+
+	state->dictionary = make_buffer<Vector>(segment.type, index_buffer_count);
+	auto dict_child_data = FlatVector::GetData<string_t>(*(state->dictionary));
+
+	for (uint32_t i = 0; i < index_buffer_count; i++) {
+		// NOTE: the passing of dict_child_vector, will not be used, its for big strings
+		uint16_t str_len = GetStringLength(index_buffer_ptr, i);
+		dict_child_data[i] = FetchStringFromDict(segment, dict, baseptr, index_buffer_ptr[i], str_len);
+	}
+
+	return move(state);
+}
+
+//===--------------------------------------------------------------------===//
+// Scan base data
+//===--------------------------------------------------------------------===//
+template <bool ALLOW_DICT_VECTORS>
+void DictionaryCompressionStorage::StringScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count,
+                                                     Vector &result, idx_t result_offset) {
+	// clear any previously locked buffers and get the primary buffer handle
+	auto &scan_state = (CompressedStringScanState &)*state.scan_state;
+	auto start = segment.GetRelativeIndex(state.row_index);
+
+	auto baseptr = scan_state.handle->node->buffer + segment.GetBlockOffset();
+	auto dict = DictionaryCompressionStorage::GetDictionary(segment, *scan_state.handle);
+
+	auto header_ptr = (dictionary_compression_header_t *)baseptr;
+	auto index_buffer_offset = Load<uint32_t>((data_ptr_t)&header_ptr->index_buffer_offset);
+	auto index_buffer_ptr = (uint32_t *)(baseptr + index_buffer_offset);
+
+	auto base_data = (data_ptr_t)(baseptr + DICTIONARY_HEADER_SIZE);
+	auto result_data = FlatVector::GetData<string_t>(result);
+
+	if (!ALLOW_DICT_VECTORS || scan_count != STANDARD_VECTOR_SIZE ||
+	    start % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE != 0) {
+		// Emit regular vector
+
+		// Handling non-bitpacking-group-aligned start values;
+		idx_t start_offset = start % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE;
+
+		// We will scan in blocks of BITPACKING_ALGORITHM_GROUP_SIZE, so we may scan some extra values.
+		idx_t decompress_count = BitpackingPrimitives::RoundUpToAlgorithmGroupSize(scan_count + start_offset);
+
+		// Create a decompression buffer of sufficient size if we don't already have one.
+		if (!scan_state.sel_vec || scan_state.sel_vec_size < decompress_count) {
+			scan_state.sel_vec_size = decompress_count;
+			scan_state.sel_vec = make_buffer<SelectionVector>(decompress_count);
+		}
+
+		data_ptr_t src = &base_data[((start - start_offset) * scan_state.current_width) / 8];
+		sel_t *sel_vec_ptr = scan_state.sel_vec->data();
+
+		BitpackingPrimitives::UnPackBuffer<sel_t>((data_ptr_t)sel_vec_ptr, src, decompress_count,
+		                                          scan_state.current_width);
+
+		for (idx_t i = 0; i < scan_count; i++) {
+			// Lookup dict offset in index buffer
+			auto string_number = scan_state.sel_vec->get_index(i + start_offset);
+			auto dict_offset = index_buffer_ptr[string_number];
+			uint16_t str_len = GetStringLength(index_buffer_ptr, string_number);
+			result_data[result_offset + i] = FetchStringFromDict(segment, dict, baseptr, dict_offset, str_len);
+		}
+
+	} else {
+		D_ASSERT(start % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE == 0);
+		D_ASSERT(scan_count == STANDARD_VECTOR_SIZE);
+		D_ASSERT(result_offset == 0);
+
+		idx_t decompress_count = BitpackingPrimitives::RoundUpToAlgorithmGroupSize(scan_count);
+
+		// Create a selection vector of sufficient size if we don't already have one.
+		if (!scan_state.sel_vec || scan_state.sel_vec_size < decompress_count) {
+			scan_state.sel_vec_size = decompress_count;
+			scan_state.sel_vec = make_buffer<SelectionVector>(decompress_count);
+		}
+
+		// Scanning 1024 values, emitting a dict vector
+		data_ptr_t dst = (data_ptr_t)(scan_state.sel_vec->data());
+		data_ptr_t src = (data_ptr_t)&base_data[(start * scan_state.current_width) / 8];
+
+		BitpackingPrimitives::UnPackBuffer<sel_t>(dst, src, scan_count, scan_state.current_width);
+
+		result.Slice(*(scan_state.dictionary), *scan_state.sel_vec, scan_count);
+	}
+}
+
+void DictionaryCompressionStorage::StringScan(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count,
+                                              Vector &result) {
+	StringScanPartial<true>(segment, state, scan_count, result, 0);
+}
+
+//===--------------------------------------------------------------------===//
+// Fetch
+//===--------------------------------------------------------------------===//
+void DictionaryCompressionStorage::StringFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id,
+                                                  Vector &result, idx_t result_idx) {
+	// fetch a single row from the string segment
+	// first pin the main buffer if it is not already pinned
+	auto primary_id = segment.block->BlockId();
+
+	BufferHandle *handle_ptr;
+	auto entry = state.handles.find(primary_id);
+	if (entry == state.handles.end()) {
+		// not pinned yet: pin it
+		auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
+		auto handle = buffer_manager.Pin(segment.block);
+		handle_ptr = handle.get();
+		state.handles[primary_id] = move(handle);
+	} else {
+		// already pinned: use the pinned handle
+		handle_ptr = entry->second.get();
+	}
+
+	auto baseptr = handle_ptr->node->buffer + segment.GetBlockOffset();
+	auto header_ptr = (dictionary_compression_header_t *)baseptr;
+	auto dict = DictionaryCompressionStorage::GetDictionary(segment, *handle_ptr);
+	auto index_buffer_offset = Load<uint32_t>((data_ptr_t)&header_ptr->index_buffer_offset);
+	auto width = (bitpacking_width_t)(Load<uint32_t>((data_ptr_t)&header_ptr->bitpacking_width));
+	auto index_buffer_ptr = (uint32_t *)(baseptr + index_buffer_offset);
+	auto base_data = (data_ptr_t)(baseptr + DICTIONARY_HEADER_SIZE);
+	auto result_data = FlatVector::GetData<string_t>(result);
+
+	// Handling non-bitpacking-group-aligned start values;
+	idx_t start_offset = row_id % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE;
+
+	// Decompress part of selection buffer we need for this value.
+	sel_t decompression_buffer[BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE];
+	data_ptr_t src = (data_ptr_t)&base_data[((row_id - start_offset) * width) / 8];
+	BitpackingPrimitives::UnPackBuffer<sel_t>((data_ptr_t)decompression_buffer, src,
+	                                          BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE, width);
+
+	auto selection_value = decompression_buffer[start_offset];
+	auto dict_offset = index_buffer_ptr[selection_value];
+	uint16_t str_len = GetStringLength(index_buffer_ptr, selection_value);
+
+	result_data[result_idx] = FetchStringFromDict(segment, dict, baseptr, dict_offset, str_len);
+}
+
+//===--------------------------------------------------------------------===//
+// Helper Functions
+//===--------------------------------------------------------------------===//
+bool DictionaryCompressionStorage::HasEnoughSpace(idx_t current_count, idx_t index_count, idx_t dict_size,
+                                                  bitpacking_width_t packing_width) {
+	return RequiredSpace(current_count, index_count, dict_size, packing_width) <= Storage::BLOCK_SIZE;
+}
+
+idx_t DictionaryCompressionStorage::RequiredSpace(idx_t current_count, idx_t index_count, idx_t dict_size,
+                                                  bitpacking_width_t packing_width) {
+	idx_t base_space = DICTIONARY_HEADER_SIZE + dict_size;
+	idx_t string_number_space = BitpackingPrimitives::GetRequiredSize<sel_t>(current_count, packing_width);
+	idx_t index_space = index_count * sizeof(uint32_t);
+
+	idx_t used_space = base_space + index_space + string_number_space;
+
+	return used_space;
+}
+
+StringDictionaryContainer DictionaryCompressionStorage::GetDictionary(ColumnSegment &segment, BufferHandle &handle) {
+	auto header_ptr = (dictionary_compression_header_t *)(handle.node->buffer + segment.GetBlockOffset());
+	StringDictionaryContainer container;
+	container.size = Load<uint32_t>((data_ptr_t)&header_ptr->dict_size);
+	container.end = Load<uint32_t>((data_ptr_t)&header_ptr->dict_end);
+	return container;
+}
+
+void DictionaryCompressionStorage::SetDictionary(ColumnSegment &segment, BufferHandle &handle,
+                                                 StringDictionaryContainer container) {
+	auto header_ptr = (dictionary_compression_header_t *)(handle.node->buffer + segment.GetBlockOffset());
+	Store<uint32_t>(container.size, (data_ptr_t)&header_ptr->dict_size);
+	Store<uint32_t>(container.end, (data_ptr_t)&header_ptr->dict_end);
+}
+
+string_t DictionaryCompressionStorage::FetchStringFromDict(ColumnSegment &segment, StringDictionaryContainer dict,
+                                                           data_ptr_t baseptr, int32_t dict_offset,
+                                                           uint16_t string_len) {
+	D_ASSERT(dict_offset >= 0 && dict_offset <= Storage::BLOCK_SIZE);
+
+	if (dict_offset == 0) {
+		return string_t(nullptr, 0);
+	}
+	// normal string: read string from this block
+	auto dict_end = baseptr + dict.end;
+	auto dict_pos = dict_end - dict_offset;
+
+	auto str_ptr = (char *)(dict_pos);
+	return string_t(str_ptr, string_len);
+}
+
+uint16_t DictionaryCompressionStorage::GetStringLength(uint32_t *index_buffer_ptr, sel_t index) {
+	if (index == 0) {
+		return 0;
+	} else {
+		return index_buffer_ptr[index] - index_buffer_ptr[index - 1];
+	}
+}
+
+//===--------------------------------------------------------------------===//
+// Get Function
+//===--------------------------------------------------------------------===//
+CompressionFunction DictionaryCompressionFun::GetFunction(PhysicalType data_type) {
+	return CompressionFunction(
+	    CompressionType::COMPRESSION_DICTIONARY, data_type, DictionaryCompressionStorage ::StringInitAnalyze,
+	    DictionaryCompressionStorage::StringAnalyze, DictionaryCompressionStorage::StringFinalAnalyze,
+	    DictionaryCompressionStorage::InitCompression, DictionaryCompressionStorage::Compress,
+	    DictionaryCompressionStorage::FinalizeCompress, DictionaryCompressionStorage::StringInitScan,
+	    DictionaryCompressionStorage::StringScan, DictionaryCompressionStorage::StringScanPartial<false>,
+	    DictionaryCompressionStorage::StringFetchRow, UncompressedFunctions::EmptySkip);
+}
+
+bool DictionaryCompressionFun::TypeIsSupported(PhysicalType type) {
+	return type == PhysicalType::VARCHAR;
+}
+} // namespace duckdb
+
+
+
+
+
+
+
+
+
+
+
 namespace duckdb {
 
 //===--------------------------------------------------------------------===//
@@ -162272,35 +163242,31 @@ idx_t FixedSizeFinalAnalyze(AnalyzeState &state_p) {
 //===--------------------------------------------------------------------===//
 // Compress
 //===--------------------------------------------------------------------===//
-struct UncompressedCompressState : public CompressionState {
-	explicit UncompressedCompressState(ColumnDataCheckpointer &checkpointer) : checkpointer(checkpointer) {
-		CreateEmptySegment(checkpointer.GetRowGroup().start);
-	}
-
-	void CreateEmptySegment(idx_t row_start) {
-		auto &db = checkpointer.GetDatabase();
-		auto &type = checkpointer.GetType();
-		auto compressed_segment = ColumnSegment::CreateTransientSegment(db, type, row_start);
-		if (type.InternalType() == PhysicalType::VARCHAR) {
-			auto &state = (UncompressedStringSegmentState &)*compressed_segment->GetSegmentState();
-			state.overflow_writer = make_unique<WriteOverflowStringsToDisk>(db);
-		}
-		current_segment = move(compressed_segment);
-	}
+UncompressedCompressState::UncompressedCompressState(ColumnDataCheckpointer &checkpointer)
+    : checkpointer(checkpointer) {
+	CreateEmptySegment(checkpointer.GetRowGroup().start);
+}
 
-	void FlushSegment(idx_t segment_size) {
-		auto &state = checkpointer.GetCheckpointState();
-		state.FlushSegment(move(current_segment), segment_size);
+void UncompressedCompressState::CreateEmptySegment(idx_t row_start) {
+	auto &db = checkpointer.GetDatabase();
+	auto &type = checkpointer.GetType();
+	auto compressed_segment = ColumnSegment::CreateTransientSegment(db, type, row_start);
+	if (type.InternalType() == PhysicalType::VARCHAR) {
+		auto &state = (UncompressedStringSegmentState &)*compressed_segment->GetSegmentState();
+		state.overflow_writer = make_unique<WriteOverflowStringsToDisk>(db);
 	}
+	current_segment = move(compressed_segment);
+}
 
-	void Finalize(idx_t segment_size) {
-		FlushSegment(segment_size);
-		current_segment.reset();
-	}
+void UncompressedCompressState::FlushSegment(idx_t segment_size) {
+	auto &state = checkpointer.GetCheckpointState();
+	state.FlushSegment(move(current_segment), segment_size);
+}
 
-	ColumnDataCheckpointer &checkpointer;
-	unique_ptr<ColumnSegment> current_segment;
-};
+void UncompressedCompressState::Finalize(idx_t segment_size) {
+	FlushSegment(segment_size);
+	current_segment.reset();
+}
 
 unique_ptr<CompressionState> UncompressedFunctions::InitCompression(ColumnDataCheckpointer &checkpointer,
                                                                     unique_ptr<AnalyzeState> state) {
@@ -163059,17 +164025,6 @@ bool RLEFun::TypeIsSupported(PhysicalType type) {
 } // namespace duckdb
 
 
-
-
-
-
-
-
-
-
-
-
-
 namespace duckdb {
 
 //===--------------------------------------------------------------------===//
@@ -163082,70 +164037,6 @@ UncompressedStringSegmentState::~UncompressedStringSegmentState() {
 	}
 }
 
-struct StringDictionaryContainer {
-	//! The size of the dictionary
-	uint32_t size;
-	//! The end of the dictionary (typically Storage::BLOCK_SIZE)
-	uint32_t end;
-
-	void Verify() {
-		D_ASSERT(size <= Storage::BLOCK_SIZE);
-		D_ASSERT(end <= Storage::BLOCK_SIZE);
-		D_ASSERT(size <= end);
-	}
-};
-
-struct UncompressedStringStorage {
-public:
-	//! Dictionary header size at the beginning of the string segment (offset + length)
-	static constexpr uint16_t DICTIONARY_HEADER_SIZE = sizeof(uint32_t) + sizeof(uint32_t);
-	//! Marker used in length field to indicate the presence of a big string
-	static constexpr uint16_t BIG_STRING_MARKER = (uint16_t)-1;
-	//! Base size of big string marker (block id + offset)
-	static constexpr idx_t BIG_STRING_MARKER_BASE_SIZE = sizeof(block_id_t) + sizeof(int32_t);
-	//! The marker size of the big string
-	static constexpr idx_t BIG_STRING_MARKER_SIZE = BIG_STRING_MARKER_BASE_SIZE + sizeof(uint16_t);
-
-public:
-	static unique_ptr<AnalyzeState> StringInitAnalyze(ColumnData &col_data, PhysicalType type);
-	static bool StringAnalyze(AnalyzeState &state_p, Vector &input, idx_t count);
-	static idx_t StringFinalAnalyze(AnalyzeState &state_p);
-	static unique_ptr<SegmentScanState> StringInitScan(ColumnSegment &segment);
-	static void StringScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result,
-	                              idx_t result_offset);
-	static void StringScan(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result);
-	static void StringFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id, Vector &result,
-	                           idx_t result_idx);
-	static unique_ptr<CompressedSegmentState> StringInitSegment(ColumnSegment &segment, block_id_t block_id);
-	static idx_t StringAppend(ColumnSegment &segment, SegmentStatistics &stats, VectorData &data, idx_t offset,
-	                          idx_t count);
-	static idx_t FinalizeAppend(ColumnSegment &segment, SegmentStatistics &stats);
-
-public:
-	static inline void UpdateStringStats(SegmentStatistics &stats, const string_t &new_value) {
-		auto &sstats = (StringStatistics &)*stats.statistics;
-		sstats.Update(new_value);
-	}
-
-	static void SetDictionary(ColumnSegment &segment, BufferHandle &handle, StringDictionaryContainer dict);
-	static StringDictionaryContainer GetDictionary(ColumnSegment &segment, BufferHandle &handle);
-	static idx_t RemainingSpace(ColumnSegment &segment, BufferHandle &handle);
-	static void WriteString(ColumnSegment &segment, string_t string, block_id_t &result_block, int32_t &result_offset);
-	static void WriteStringMemory(ColumnSegment &segment, string_t string, block_id_t &result_block,
-	                              int32_t &result_offset);
-	static string_t ReadString(ColumnSegment &segment, Vector &result, block_id_t block, int32_t offset);
-	static string_t ReadString(data_ptr_t target, int32_t offset);
-	static void WriteStringMarker(data_ptr_t target, block_id_t block_id, int32_t offset);
-	static void ReadStringMarker(data_ptr_t target, block_id_t &block_id, int32_t &offset);
-
-	static string_location_t FetchStringLocation(StringDictionaryContainer dict, data_ptr_t baseptr,
-	                                             int32_t dict_offset);
-	static string_t FetchStringFromDict(ColumnSegment &segment, StringDictionaryContainer dict, Vector &result,
-	                                    data_ptr_t baseptr, int32_t dict_offset);
-	static string_t FetchString(ColumnSegment &segment, StringDictionaryContainer dict, Vector &result,
-	                            data_ptr_t baseptr, string_location_t location);
-};
-
 //===--------------------------------------------------------------------===//
 // Analyze
 //===--------------------------------------------------------------------===//
@@ -163190,10 +164081,6 @@ idx_t UncompressedStringStorage::StringFinalAnalyze(AnalyzeState &state_p) {
 //===--------------------------------------------------------------------===//
 // Scan
 //===--------------------------------------------------------------------===//
-struct StringScanState : public SegmentScanState {
-	unique_ptr<BufferHandle> handle;
-};
-
 unique_ptr<SegmentScanState> UncompressedStringStorage::StringInitScan(ColumnSegment &segment) {
 	auto result = make_unique<StringScanState>();
 	auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
@@ -163270,82 +164157,6 @@ unique_ptr<CompressedSegmentState> UncompressedStringStorage::StringInitSegment(
 	return make_unique<UncompressedStringSegmentState>();
 }
 
-idx_t UncompressedStringStorage::StringAppend(ColumnSegment &segment, SegmentStatistics &stats, VectorData &data,
-                                              idx_t offset, idx_t count) {
-	auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
-	auto handle = buffer_manager.Pin(segment.block);
-
-	D_ASSERT(segment.GetBlockOffset() == 0);
-	auto source_data = (string_t *)data.data;
-	auto result_data = (int32_t *)(handle->node->buffer + DICTIONARY_HEADER_SIZE);
-	for (idx_t i = 0; i < count; i++) {
-		auto source_idx = data.sel->get_index(offset + i);
-		auto target_idx = segment.count.load();
-		idx_t remaining_space = RemainingSpace(segment, *handle);
-		if (remaining_space < sizeof(int32_t)) {
-			// string index does not fit in the block at all
-			return i;
-		}
-		remaining_space -= sizeof(int32_t);
-		if (!data.validity.RowIsValid(source_idx)) {
-			// null value is stored as -1
-			result_data[target_idx] = 0;
-		} else {
-			auto dictionary = GetDictionary(segment, *handle);
-			auto end = handle->node->buffer + dictionary.end;
-
-			dictionary.Verify();
-			// non-null value, check if we can fit it within the block
-			idx_t string_length = source_data[source_idx].GetSize();
-			idx_t dictionary_length = string_length + sizeof(uint16_t);
-
-			// determine whether or not we have space in the block for this string
-			bool use_overflow_block = false;
-			idx_t required_space = dictionary_length;
-			if (required_space >= StringUncompressed::STRING_BLOCK_LIMIT) {
-				// string exceeds block limit, store in overflow block and only write a marker here
-				required_space = BIG_STRING_MARKER_SIZE;
-				use_overflow_block = true;
-			}
-			if (required_space > remaining_space) {
-				// no space remaining: return how many tuples we ended up writing
-				return i;
-			}
-			// we have space: write the string
-			UpdateStringStats(stats, source_data[source_idx]);
-
-			if (use_overflow_block) {
-				// write to overflow blocks
-				block_id_t block;
-				int32_t offset;
-				// write the string into the current string block
-				WriteString(segment, source_data[source_idx], block, offset);
-				dictionary.size += BIG_STRING_MARKER_SIZE;
-				auto dict_pos = end - dictionary.size;
-
-				// write a big string marker into the dictionary
-				WriteStringMarker(dict_pos, block, offset);
-			} else {
-				// string fits in block, append to dictionary and increment dictionary position
-				D_ASSERT(string_length < NumericLimits<uint16_t>::Maximum());
-				dictionary.size += required_space;
-				auto dict_pos = end - dictionary.size; // first write the length as u16
-				Store<uint16_t>(string_length, dict_pos);
-				// now write the actual string data into the dictionary
-				memcpy(dict_pos + sizeof(uint16_t), source_data[source_idx].GetDataUnsafe(), string_length);
-			}
-			D_ASSERT(RemainingSpace(segment, *handle) <= Storage::BLOCK_SIZE);
-			// place the dictionary offset into the set of vectors
-			dictionary.Verify();
-
-			result_data[target_idx] = dictionary.size;
-			SetDictionary(segment, *handle, dictionary);
-		}
-		segment.count++;
-	}
-	return count;
-}
-
 idx_t UncompressedStringStorage::FinalizeAppend(ColumnSegment &segment, SegmentStatistics &stats) {
 	auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
 	auto handle = buffer_manager.Pin(segment.block);
@@ -163354,7 +164165,7 @@ idx_t UncompressedStringStorage::FinalizeAppend(ColumnSegment &segment, SegmentS
 	// compute the total size required to store this segment
 	auto offset_size = DICTIONARY_HEADER_SIZE + segment.count * sizeof(int32_t);
 	auto total_size = offset_size + dict.size;
-	if (total_size >= Storage::BLOCK_SIZE / 5 * 4) {
+	if (total_size >= COMPACTION_FLUSH_LIMIT) {
 		// the block is full enough, don't bother moving around the dictionary
 		return Storage::BLOCK_SIZE;
 	}