duckdb 0.4.1-dev1225.0 → 0.4.1-dev1254.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.4.1-dev1225.0",
+  "version": "0.4.1-dev1254.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -8230,10 +8230,10 @@ SequenceException::SequenceException(const string &msg) : Exception(ExceptionTyp
 InterruptException::InterruptException() : Exception(ExceptionType::INTERRUPT, "Interrupted!") {
 }
 
-FatalException::FatalException(const string &msg) : Exception(ExceptionType::FATAL, msg) {
+FatalException::FatalException(ExceptionType type, const string &msg) : Exception(type, msg) {
 }
 
-InternalException::InternalException(const string &msg) : Exception(ExceptionType::INTERNAL, msg) {
+InternalException::InternalException(const string &msg) : FatalException(ExceptionType::INTERNAL, msg) {
 }
 
 InvalidInputException::InvalidInputException(const string &msg) : Exception(ExceptionType::INVALID_INPUT, msg) {
@@ -118897,6 +118897,11 @@ unique_ptr<DataChunk> ClientContext::FetchInternal(ClientContextLock &lock, Exec
 		// standard exceptions do not invalidate the current transaction
 		result.error = ex.what();
 		invalidate_query = false;
+	} catch (FatalException &ex) {
+		// fatal exceptions invalidate the entire database
+		result.error = ex.what();
+		auto &db = DatabaseInstance::GetDatabase(*this);
+		db.Invalidate();
 	} catch (std::exception &ex) {
 		result.error = ex.what();
 	} catch (...) { // LCOV_EXCL_START
@@ -118910,6 +118915,10 @@ unique_ptr<DataChunk> ClientContext::FetchInternal(ClientContextLock &lock, Exec
 void ClientContext::BeginTransactionInternal(ClientContextLock &lock, bool requires_valid_transaction) {
 	// check if we are on AutoCommit. In this case we should start a transaction
 	D_ASSERT(!active_query);
+	auto &db = DatabaseInstance::GetDatabase(*this);
+	if (db.IsInvalidated()) {
+		throw FatalException("Failed: database has been invalidated!");
+	}
 	if (requires_valid_transaction && transaction.HasActiveTransaction() &&
 	    transaction.ActiveTransaction().IsInvalidated()) {
 		throw Exception("Failed: transaction has been invalidated!");
@@ -118958,6 +118967,10 @@ string ClientContext::EndQueryInternal(ClientContextLock &lock, bool success, bo
 				ActiveTransaction().Invalidate();
 			}
 		}
+	} catch (FatalException &ex) {
+		auto &db = DatabaseInstance::GetDatabase(*this);
+		db.Invalidate();
+		error = ex.what();
 	} catch (std::exception &ex) {
 		error = ex.what();
 	} catch (...) { // LCOV_EXCL_START
@@ -119401,7 +119414,17 @@ unique_ptr<PendingQueryResult> ClientContext::PendingStatementOrPreparedStatemen
     shared_ptr<PreparedStatementData> &prepared, PendingQueryParameters parameters) {
 	unique_ptr<PendingQueryResult> result;
 
-	BeginQueryInternal(lock, query);
+	try {
+		BeginQueryInternal(lock, query);
+	} catch (FatalException &ex) {
+		// fatal exceptions invalidate the entire database
+		auto &db = DatabaseInstance::GetDatabase(*this);
+		db.Invalidate();
+		result = make_unique<PendingQueryResult>(ex.what());
+		return result;
+	} catch (std::exception &ex) {
+		return make_unique<PendingQueryResult>(ex.what());
+	}
 	// start the profiler
 	auto &profiler = QueryProfiler::Get(*this);
 	profiler.StartQuery(query, IsExplainAnalyze(statement ? statement.get() : prepared->unbound_statement.get()));
@@ -119425,6 +119448,11 @@ unique_ptr<PendingQueryResult> ClientContext::PendingStatementOrPreparedStatemen
 		// standard exceptions do not invalidate the current transaction
 		result = make_unique<PendingQueryResult>(ex.what());
 		invalidate_query = false;
+	} catch (FatalException &ex) {
+		// fatal exceptions invalidate the entire database
+		auto &db = DatabaseInstance::GetDatabase(*this);
+		db.Invalidate();
+		result = make_unique<PendingQueryResult>(ex.what());
 	} catch (std::exception &ex) {
 		// other types of exceptions do invalidate the current transaction
 		result = make_unique<PendingQueryResult>(ex.what());
@@ -119902,6 +119930,10 @@ void ClientContext::RunFunctionInTransactionInternal(ClientContextLock &lock, co
 			transaction.Rollback();
 		}
 		throw;
+	} catch (FatalException &ex) {
+		auto &db = DatabaseInstance::GetDatabase(*this);
+		db.Invalidate();
+		throw;
 	} catch (std::exception &ex) {
 		if (require_new_transaction) {
 			transaction.Rollback();
@@ -121405,6 +121437,13 @@ string ClientConfig::ExtractTimezoneFromConfig(ClientConfig &config) {
 	}
 }
 
+void DatabaseInstance::Invalidate() {
+	this->is_invalidated = true;
+}
+bool DatabaseInstance::IsInvalidated() {
+	return this->is_invalidated;
+}
+
 } // namespace duckdb
 
 
@@ -183064,6 +183103,13 @@ public:
 		return FindMinimumBitWidth<T, BYTE_ALIGNED>(values, count);
 	}
 
+	// Calculates the minimum required number of bits per value that can store all values,
+	// given a predetermined minimum and maximum value of the buffer
+	template <class T>
+	inline static bitpacking_width_t MinimumBitWidth(T minimum, T maximum) {
+		return FindMinimumBitWidth<T, BYTE_ALIGNED>(minimum, maximum);
+	}
+
 	template <class T>
 	inline static idx_t GetRequiredSize(idx_t count, bitpacking_width_t width) {
 		count = RoundUpToAlgorithmGroupSize(count);
@@ -183152,6 +183198,18 @@ private:
 		}
 	}
 
+	// Sign bit extension
+	template <class T, class T_U = typename std::make_unsigned<T>::type>
+	static void SignExtend(data_ptr_t dst, bitpacking_width_t width) {
+		T const mask = ((T_U)1) << (width - 1);
+		for (idx_t i = 0; i < BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE; ++i) {
+			T value = Load<T>(dst + i * sizeof(T));
+			value = value & ((((T_U)1) << width) - ((T_U)1));
+			T result = (value ^ mask) - mask;
+			Store(result, dst + i * sizeof(T));
+		}
+	}
+
 	template <class T>
 	static void UnPackGroup(data_ptr_t dst, data_ptr_t src, bitpacking_width_t width,
 	                        bool skip_sign_extension = false) {
@@ -183175,33 +183233,14 @@ private:
 	// Prevent compression at widths that are ineffective
 	template <class T>
 	static bitpacking_width_t GetEffectiveWidth(bitpacking_width_t width) {
-		if (width > 56) {
-			return 64;
-		}
-
-		if (width > 28 && (std::is_same<T, uint32_t>::value || std::is_same<T, int32_t>::value)) {
-			return 32;
-		}
-
-		else if (width > 14 && (std::is_same<T, uint16_t>::value || std::is_same<T, int16_t>::value)) {
-			return 16;
+		auto bits_of_type = sizeof(T) * 8;
+		auto type_size = sizeof(T);
+		if (width + type_size > bits_of_type) {
+			return bits_of_type;
 		}
-
 		return width;
 	}
 
-	// Sign bit extension
-	template <class T, class T_U = typename std::make_unsigned<T>::type>
-	static void SignExtend(data_ptr_t dst, bitpacking_width_t width) {
-		T const mask = ((T_U)1) << (width - 1);
-		for (idx_t i = 0; i < BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE; ++i) {
-			T value = Load<T>(dst + i * sizeof(T));
-			value = value & ((((T_U)1) << width) - ((T_U)1));
-			T result = (value ^ mask) - mask;
-			Store(result, dst + i * sizeof(T));
-		}
-	}
-
 	template <class T>
 	static void PackGroup(data_ptr_t dst, T *values, bitpacking_width_t width) {
 		if (std::is_same<T, uint8_t>::value || std::is_same<T, int8_t>::value) {
@@ -183465,17 +183504,19 @@ private:
 
 
 
+
 #include <functional>
 
 namespace duckdb {
 
 // Note that optimizations in scanning only work if this value is equal to STANDARD_VECTOR_SIZE, however we keep them
 // separated to prevent the code from break on lower vector sizes
-static constexpr const idx_t BITPACKING_WIDTH_GROUP_SIZE = 1024;
+static constexpr const idx_t BITPACKING_METADATA_GROUP_SIZE = 1024;
 
 struct EmptyBitpackingWriter {
 	template <class T>
-	static void Operation(T *values, bool *validity, bitpacking_width_t width, idx_t count, void *data_ptr) {
+	static void Operation(T *values, bool *validity, bitpacking_width_t width, T frame_of_reference, idx_t count,
+	                      void *data_ptr) {
 	}
 };
 
@@ -183483,39 +183524,95 @@ template <class T>
 struct BitpackingState {
 public:
 	BitpackingState() : compression_buffer_idx(0), total_size(0), data_ptr(nullptr) {
+		ResetMinMax();
 	}
 
-	T compression_buffer[BITPACKING_WIDTH_GROUP_SIZE];
-	bool compression_buffer_validity[BITPACKING_WIDTH_GROUP_SIZE];
+	T compression_buffer[BITPACKING_METADATA_GROUP_SIZE];
+	bool compression_buffer_validity[BITPACKING_METADATA_GROUP_SIZE];
 	idx_t compression_buffer_idx;
 	idx_t total_size;
 	void *data_ptr;
 
+	bool min_max_set;
+	T minimum;
+	T maximum;
+
 public:
-	template <class OP>
+	void SubtractFrameOfReference(const T &frame_of_reference) {
+		for (idx_t i = 0; i < compression_buffer_idx; i++) {
+			compression_buffer[i] -= frame_of_reference;
+		}
+	}
+
+	void ResetMinMax() {
+		min_max_set = false;
+		//! We set these to 0, in case all values are NULL, in which case the min and max will never be set.
+		minimum = 0;
+		maximum = 0;
+	}
+
+	bool TryUpdateMinMax(T value) {
+		bool updated = false;
+		if (!min_max_set || value < minimum) {
+			minimum = value;
+			updated = true;
+		}
+		if (!min_max_set || value > maximum) {
+			maximum = value;
+			updated = true;
+		}
+		min_max_set = min_max_set || updated;
+		//! Only when either of the values are updated, do we need to test the overflow
+		if (updated) {
+			T ignore;
+			return TrySubtractOperator::Operation(maximum, minimum, ignore);
+		}
+		return true;
+	}
+
+	T GetFrameOfReference() {
+		return minimum;
+	}
+	T Maximum() {
+		return maximum;
+	}
+
+	template <class OP, class T_U = typename std::make_unsigned<T>::type>
 	void Flush() {
-		bitpacking_width_t width = BitpackingPrimitives::MinimumBitWidth<T>(compression_buffer, compression_buffer_idx);
-		OP::Operation(compression_buffer, compression_buffer_validity, width, compression_buffer_idx, data_ptr);
-		total_size += (BITPACKING_WIDTH_GROUP_SIZE * width) / 8 + sizeof(bitpacking_width_t);
+		T frame_of_reference = GetFrameOfReference();
+		SubtractFrameOfReference(frame_of_reference);
+
+		//! Because of FOR, we can guarantee that all values are positive
+		T_U adjusted_maximum = T_U(Maximum() - frame_of_reference);
+
+		bitpacking_width_t width = BitpackingPrimitives::MinimumBitWidth<T_U>((T_U)0, adjusted_maximum);
+		OP::template Operation<T>(compression_buffer, compression_buffer_validity, width, frame_of_reference,
+		                          compression_buffer_idx, data_ptr);
+		total_size += (BITPACKING_METADATA_GROUP_SIZE * width) / 8 + sizeof(bitpacking_width_t) + sizeof(T);
 		compression_buffer_idx = 0;
+		ResetMinMax();
 	}
 
 	template <class OP = EmptyBitpackingWriter>
-	void Update(T *data, ValidityMask &validity, idx_t idx) {
+	bool Update(T *data, ValidityMask &validity, idx_t idx) {
 
 		if (validity.RowIsValid(idx)) {
 			compression_buffer_validity[compression_buffer_idx] = true;
 			compression_buffer[compression_buffer_idx++] = data[idx];
+			if (!TryUpdateMinMax(data[idx])) {
+				return false;
+			}
 		} else {
 			// We write zero for easy bitwidth analysis of the compression buffer later
 			compression_buffer_validity[compression_buffer_idx] = false;
 			compression_buffer[compression_buffer_idx++] = 0;
 		}
 
-		if (compression_buffer_idx == BITPACKING_WIDTH_GROUP_SIZE) {
+		if (compression_buffer_idx == BITPACKING_METADATA_GROUP_SIZE) {
 			// Calculate bitpacking width;
 			Flush<OP>();
 		}
+		return true;
 	}
 };
 
@@ -183541,9 +183638,10 @@ bool BitpackingAnalyze(AnalyzeState &state, Vector &input, idx_t count) {
 	auto data = (T *)vdata.data;
 	for (idx_t i = 0; i < count; i++) {
 		auto idx = vdata.sel->get_index(i);
-		analyze_state.state.template Update<EmptyBitpackingWriter>(data, vdata.validity, idx);
+		if (!analyze_state.state.template Update<EmptyBitpackingWriter>(data, vdata.validity, idx)) {
+			return false;
+		}
 	}
-
 	return true;
 }
 
@@ -183577,19 +183675,25 @@ public:
 
 	// Ptr to next free spot in segment;
 	data_ptr_t data_ptr;
-	// Ptr to next free spot for storing bitwidths (growing downwards).
-	data_ptr_t width_ptr;
+	// Ptr to next free spot for storing bitwidths and frame-of-references (growing downwards).
+	data_ptr_t metadata_ptr;
 
 	BitpackingState<T> state;
 
 public:
 	struct BitpackingWriter {
+
 		template <class VALUE_TYPE>
-		static void Operation(VALUE_TYPE *values, bool *validity, bitpacking_width_t width, idx_t count,
-		                      void *data_ptr) {
+		static void Operation(VALUE_TYPE *values, bool *validity, bitpacking_width_t width,
+		                      VALUE_TYPE frame_of_reference, idx_t count, void *data_ptr) {
 			auto state = (BitpackingCompressState<T> *)data_ptr;
+			auto total_bits_needed = (width * BITPACKING_METADATA_GROUP_SIZE);
+			D_ASSERT(total_bits_needed % 8 == 0);
+			auto total_bytes_needed = total_bits_needed / 8;
+			total_bytes_needed += sizeof(bitpacking_width_t);
+			total_bytes_needed += sizeof(VALUE_TYPE);
 
-			if (state->RemainingSize() < (width * BITPACKING_WIDTH_GROUP_SIZE) / 8 + sizeof(bitpacking_width_t)) {
+			if (state->RemainingSize() < total_bytes_needed) {
 				// Segment is full
 				auto row_start = state->current_segment->start + state->current_segment->count;
 				state->FlushSegment();
@@ -183598,17 +183702,17 @@ public:
 
 			for (idx_t i = 0; i < count; i++) {
 				if (validity[i]) {
-					NumericStatistics::Update<T>(state->current_segment->stats, values[i]);
+					NumericStatistics::Update<T>(state->current_segment->stats, values[i] + frame_of_reference);
 				}
 			}
 
-			state->WriteValues(values, width, count);
+			state->WriteValues(values, width, frame_of_reference, count);
 		}
 	};
 
-	// Space remaining between the width_ptr growing down and data ptr growing up
+	// Space remaining between the metadata_ptr growing down and data ptr growing up
 	idx_t RemainingSize() {
-		return width_ptr - data_ptr;
+		return metadata_ptr - data_ptr;
 	}
 
 	void CreateEmptySegment(idx_t row_start) {
@@ -183621,7 +183725,8 @@ public:
 		handle = buffer_manager.Pin(current_segment->block);
 
 		data_ptr = handle.Ptr() + current_segment->GetBlockOffset() + BitpackingPrimitives::BITPACKING_HEADER_SIZE;
-		width_ptr = handle.Ptr() + current_segment->GetBlockOffset() + Storage::BLOCK_SIZE - sizeof(bitpacking_width_t);
+		metadata_ptr =
+		    handle.Ptr() + current_segment->GetBlockOffset() + Storage::BLOCK_SIZE - sizeof(bitpacking_width_t);
 	}
 
 	void Append(UnifiedVectorFormat &vdata, idx_t count) {
@@ -183634,13 +183739,15 @@ public:
 		}
 	}
 
-	void WriteValues(T *values, bitpacking_width_t width, idx_t count) {
-		// TODO we can optimize this by stopping early if count < BITPACKING_WIDTH_GROUP_SIZE
+	void WriteValues(T *values, bitpacking_width_t width, T frame_of_reference, idx_t count) {
+		// TODO we can optimize this by stopping early if count < BITPACKING_METADATA_GROUP_SIZE
 		BitpackingPrimitives::PackBuffer<T, false>(data_ptr, values, count, width);
-		data_ptr += (BITPACKING_WIDTH_GROUP_SIZE * width) / 8;
+		data_ptr += (BITPACKING_METADATA_GROUP_SIZE * width) / 8;
 
-		Store<bitpacking_width_t>(width, width_ptr);
-		width_ptr -= sizeof(bitpacking_width_t);
+		Store<bitpacking_width_t>(width, metadata_ptr);
+		metadata_ptr -= sizeof(T);
+		Store<T>(frame_of_reference, metadata_ptr);
+		metadata_ptr -= sizeof(bitpacking_width_t);
 
 		current_segment->count += count;
 	}
@@ -183649,14 +183756,14 @@ public:
 		auto &state = checkpointer.GetCheckpointState();
 		auto dataptr = handle.Ptr();
 
-		// Compact the segment by moving the widths next to the data.
-		idx_t minimal_widths_offset = AlignValue(data_ptr - dataptr);
-		idx_t widths_size = dataptr + Storage::BLOCK_SIZE - width_ptr - 1;
-		idx_t total_segment_size = minimal_widths_offset + widths_size;
-		memmove(dataptr + minimal_widths_offset, width_ptr + 1, widths_size);
+		// Compact the segment by moving the metadata next to the data.
+		idx_t metadata_offset = AlignValue(data_ptr - dataptr);
+		idx_t metadata_size = dataptr + Storage::BLOCK_SIZE - metadata_ptr - 1;
+		idx_t total_segment_size = metadata_offset + metadata_size;
+		memmove(dataptr + metadata_offset, metadata_ptr + 1, metadata_size);
 
-		// Store the offset of the first width (which is at the highest address).
-		Store<idx_t>(minimal_widths_offset + widths_size - 1, dataptr);
+		// Store the offset of the metadata of the first group (which is at the highest address).
+		Store<idx_t>(metadata_offset + metadata_size - 1, dataptr);
 		handle.Destroy();
 
 		state.FlushSegment(move(current_segment), total_segment_size);
@@ -183699,14 +183806,14 @@ public:
 		auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
 		handle = buffer_manager.Pin(segment.block);
 		auto dataptr = handle.Ptr();
-		current_width_group_ptr = dataptr + segment.GetBlockOffset() + BitpackingPrimitives::BITPACKING_HEADER_SIZE;
+		current_metadata_group_ptr = dataptr + segment.GetBlockOffset() + BitpackingPrimitives::BITPACKING_HEADER_SIZE;
 
 		// load offset to bitpacking widths pointer
-		auto bitpacking_widths_offset = Load<idx_t>(dataptr + segment.GetBlockOffset());
-		bitpacking_width_ptr = dataptr + segment.GetBlockOffset() + bitpacking_widths_offset;
+		auto bitpacking_metadata_offset = Load<idx_t>(dataptr + segment.GetBlockOffset());
+		bitpacking_metadata_ptr = dataptr + segment.GetBlockOffset() + bitpacking_metadata_offset;
 
-		// load the bitwidth of the first vector
-		LoadCurrentBitWidth();
+		// load the metadata of the first vector
+		LoadCurrentMetaData();
 	}
 
 	BufferHandle handle;
@@ -183715,32 +183822,37 @@ public:
 	T decompression_buffer[BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE];
 
 	idx_t position_in_group = 0;
-	data_ptr_t current_width_group_ptr;
-	data_ptr_t bitpacking_width_ptr;
+	data_ptr_t current_metadata_group_ptr;
+	data_ptr_t bitpacking_metadata_ptr;
 	bitpacking_width_t current_width;
+	T current_frame_of_reference;
 
 public:
-	void LoadCurrentBitWidth() {
-		D_ASSERT(bitpacking_width_ptr > handle.Ptr() && bitpacking_width_ptr < handle.Ptr() + Storage::BLOCK_SIZE);
-		current_width = Load<bitpacking_width_t>(bitpacking_width_ptr);
+	//! Loads the current group header, and sets pointer to next header
+	void LoadCurrentMetaData() {
+		D_ASSERT(bitpacking_metadata_ptr > handle.Ptr() &&
+		         bitpacking_metadata_ptr < handle.Ptr() + Storage::BLOCK_SIZE);
+		current_width = Load<bitpacking_width_t>(bitpacking_metadata_ptr);
+		bitpacking_metadata_ptr -= sizeof(T);
+		current_frame_of_reference = Load<T>(bitpacking_metadata_ptr);
+		bitpacking_metadata_ptr -= sizeof(bitpacking_width_t);
 		LoadDecompressFunction();
 	}
 
 	void Skip(ColumnSegment &segment, idx_t skip_count) {
 		while (skip_count > 0) {
-			if (position_in_group + skip_count < BITPACKING_WIDTH_GROUP_SIZE) {
+			if (position_in_group + skip_count < BITPACKING_METADATA_GROUP_SIZE) {
 				// We're not leaving this bitpacking group, we can perform all skips.
 				position_in_group += skip_count;
 				break;
 			} else {
 				// The skip crosses the current bitpacking group, we skip the remainder of this group.
-				auto skipping = BITPACKING_WIDTH_GROUP_SIZE - position_in_group;
+				auto skipping = BITPACKING_METADATA_GROUP_SIZE - position_in_group;
 				position_in_group = 0;
-				current_width_group_ptr += (current_width * BITPACKING_WIDTH_GROUP_SIZE) / 8;
+				current_metadata_group_ptr += (current_width * BITPACKING_METADATA_GROUP_SIZE) / 8;
 
-				// Update width pointer and load new width
-				bitpacking_width_ptr -= sizeof(bitpacking_width_t);
-				LoadCurrentBitWidth();
+				// Load new width
+				LoadCurrentMetaData();
 
 				skip_count -= skipping;
 			}
@@ -183758,6 +183870,16 @@ unique_ptr<SegmentScanState> BitpackingInitScan(ColumnSegment &segment) {
 	return move(result);
 }
 
+template <class T>
+static void ApplyFrameOfReference(T *dst, T frame_of_reference, idx_t size) {
+	if (!frame_of_reference) {
+		return;
+	}
+	for (idx_t i = 0; i < size; i++) {
+		dst[i] += frame_of_reference;
+	}
+}
+
 //===--------------------------------------------------------------------===//
 // Scan base data
 //===--------------------------------------------------------------------===//
@@ -183770,31 +183892,28 @@ void BitpackingScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t
 	result.SetVectorType(VectorType::FLAT_VECTOR);
 
 	// Fast path for when no compression was used, we can do a single memcopy
-	if (STANDARD_VECTOR_SIZE == BITPACKING_WIDTH_GROUP_SIZE) {
-		if (scan_state.current_width == sizeof(T) * 8 && scan_count <= BITPACKING_WIDTH_GROUP_SIZE &&
-		    scan_state.position_in_group == 0) {
-
-			memcpy(result_data + result_offset, scan_state.current_width_group_ptr, scan_count * sizeof(T));
-			scan_state.current_width_group_ptr += scan_count * sizeof(T);
-			scan_state.bitpacking_width_ptr -= sizeof(bitpacking_width_t);
-			scan_state.LoadCurrentBitWidth();
+	if (STANDARD_VECTOR_SIZE == BITPACKING_METADATA_GROUP_SIZE) {
+		if (scan_state.current_frame_of_reference == 0 && scan_state.current_width == sizeof(T) * 8 &&
+		    scan_count <= BITPACKING_METADATA_GROUP_SIZE && scan_state.position_in_group == 0) {
+
+			memcpy(result_data + result_offset, scan_state.current_metadata_group_ptr, scan_count * sizeof(T));
+			scan_state.current_metadata_group_ptr += scan_count * sizeof(T);
+			scan_state.LoadCurrentMetaData();
 			return;
 		}
 	}
 
-	// Determine if we can skip sign extension during compression
-	auto &nstats = (NumericStatistics &)*segment.stats.statistics;
-	bool skip_sign_extend = std::is_signed<T>::value && nstats.min >= 0;
+	//! Because FOR offsets all our values to be 0 or above, we can always skip sign extension here
+	bool skip_sign_extend = true;
 
 	idx_t scanned = 0;
 
 	while (scanned < scan_count) {
-		// Exhausted this width group, move pointers to next group and load bitwidth for next group.
-		if (scan_state.position_in_group >= BITPACKING_WIDTH_GROUP_SIZE) {
+		// Exhausted this metadata group, move pointers to next group and load metadata for next group.
+		if (scan_state.position_in_group >= BITPACKING_METADATA_GROUP_SIZE) {
 			scan_state.position_in_group = 0;
-			scan_state.bitpacking_width_ptr -= sizeof(bitpacking_width_t);
-			scan_state.current_width_group_ptr += (scan_state.current_width * BITPACKING_WIDTH_GROUP_SIZE) / 8;
-			scan_state.LoadCurrentBitWidth();
+			scan_state.current_metadata_group_ptr += (scan_state.current_width * BITPACKING_METADATA_GROUP_SIZE) / 8;
+			scan_state.LoadCurrentMetaData();
 		}
 
 		idx_t offset_in_compression_group =
@@ -183805,7 +183924,7 @@ void BitpackingScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t
 
 		// Calculate start of compression algorithm group
 		data_ptr_t current_position_ptr =
-		    scan_state.current_width_group_ptr + scan_state.position_in_group * scan_state.current_width / 8;
+		    scan_state.current_metadata_group_ptr + scan_state.position_in_group * scan_state.current_width / 8;
 		data_ptr_t decompression_group_start_pointer =
 		    current_position_ptr - offset_in_compression_group * scan_state.current_width / 8;
 
@@ -183824,7 +183943,7 @@ void BitpackingScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t
 			memcpy(current_result_ptr, scan_state.decompression_buffer + offset_in_compression_group,
 			       to_scan * sizeof(T));
 		}
-
+		ApplyFrameOfReference((T *)current_result_ptr, scan_state.current_frame_of_reference, to_scan);
 		scanned += to_scan;
 		scan_state.position_in_group += to_scan;
 	}
@@ -183851,16 +183970,18 @@ void BitpackingFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t r
 	    scan_state.position_in_group % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE;
 
 	data_ptr_t decompression_group_start_pointer =
-	    scan_state.current_width_group_ptr +
+	    scan_state.current_metadata_group_ptr +
 	    (scan_state.position_in_group - offset_in_compression_group) * scan_state.current_width / 8;
 
-	auto &nstats = (NumericStatistics &)*segment.stats.statistics;
-	bool skip_sign_extend = std::is_signed<T>::value && nstats.min >= 0;
+	//! Because FOR offsets all our values to be 0 or above, we can always skip sign extension here
+	bool skip_sign_extend = true;
 
 	scan_state.decompress_function((data_ptr_t)scan_state.decompression_buffer, decompression_group_start_pointer,
 	                               scan_state.current_width, skip_sign_extend);
 
 	*current_result_ptr = *(T *)(scan_state.decompression_buffer + offset_in_compression_group);
+	//! Apply FOR to result
+	*current_result_ptr += scan_state.current_frame_of_reference;
 }
 template <class T>
 void BitpackingSkip(ColumnSegment &segment, ColumnScanState &state, idx_t skip_count) {
@@ -190334,7 +190455,7 @@ string ValidityStatistics::ToString() const {
 
 namespace duckdb {
 
-const uint64_t VERSION_NUMBER = 37;
+const uint64_t VERSION_NUMBER = 38;
 
 } // namespace duckdb
 
@@ -191789,7 +191910,8 @@ void ColumnDataCheckpointer::ScanSegments(const std::function<void(Vector &, idx
 	}
 }
 
-void ForceCompression(vector<CompressionFunction *> &compression_functions, CompressionType compression_type) {
+CompressionType ForceCompression(vector<CompressionFunction *> &compression_functions,
+                                 CompressionType compression_type) {
 	// On of the force_compression flags has been set
 	// check if this compression method is available
 	bool found = false;
@@ -191802,25 +191924,31 @@ void ForceCompression(vector<CompressionFunction *> &compression_functions, Comp
 	if (found) {
 		// the force_compression method is available
 		// clear all other compression methods
+		// except the uncompressed method, so we can fall back on that
 		for (idx_t i = 0; i < compression_functions.size(); i++) {
+			if (compression_functions[i]->type == CompressionType::COMPRESSION_UNCOMPRESSED) {
+				continue;
+			}
 			if (compression_functions[i]->type != compression_type) {
 				compression_functions[i] = nullptr;
 			}
 		}
 	}
+	return found ? compression_type : CompressionType::COMPRESSION_AUTO;
 }
 
 unique_ptr<AnalyzeState> ColumnDataCheckpointer::DetectBestCompressionMethod(idx_t &compression_idx) {
 	D_ASSERT(!compression_functions.empty());
 	auto &config = DBConfig::GetConfig(GetDatabase());
+	CompressionType forced_method = CompressionType::COMPRESSION_AUTO;
 
 	auto compression_type = checkpoint_info.compression_type;
 	if (compression_type != CompressionType::COMPRESSION_AUTO) {
-		ForceCompression(compression_functions, compression_type);
+		forced_method = ForceCompression(compression_functions, compression_type);
 	}
 	if (compression_type == CompressionType::COMPRESSION_AUTO &&
 	    config.options.force_compression != CompressionType::COMPRESSION_AUTO) {
-		ForceCompression(compression_functions, config.options.force_compression);
+		forced_method = ForceCompression(compression_functions, config.options.force_compression);
 	}
 	// set up the analyze states for each compression method
 	vector<unique_ptr<AnalyzeState>> analyze_states;
@@ -191858,12 +191986,18 @@ unique_ptr<AnalyzeState> ColumnDataCheckpointer::DetectBestCompressionMethod(idx
 		if (!compression_functions[i]) {
 			continue;
 		}
+		//! Check if the method type is the forced method (if forced is used)
+		bool forced_method_found = compression_functions[i]->type == forced_method;
 		auto score = compression_functions[i]->final_analyze(*analyze_states[i]);
-		if (score < best_score) {
+		if (score < best_score || forced_method_found) {
 			compression_idx = i;
 			best_score = score;
 			state = move(analyze_states[i]);
 		}
+		//! If we have found the forced method, we're done
+		if (forced_method_found) {
+			break;
+		}
 	}
 	return state;
 }
@@ -191892,7 +192026,7 @@ void ColumnDataCheckpointer::WriteToDisk() {
 	auto analyze_state = DetectBestCompressionMethod(compression_idx);
 
 	if (!analyze_state) {
-		throw InternalException("No suitable compression/storage method found to store column");
+		throw FatalException("No suitable compression/storage method found to store column");
 	}
 
 	// now that we have analyzed the compression functions we can start writing to disk