duckdb 0.6.2-dev971.0 → 0.7.1-dev2.0

package/src/duckdb/src/storage/compression/fsst.cpp

@@ -212,17 +212,11 @@ public:
 		}
 	}
 
-	void CreateEmptySegment(idx_t row_start) {
-		auto &db = checkpointer.GetDatabase();
-		auto &type = checkpointer.GetType();
-		auto compressed_segment = ColumnSegment::CreateTransientSegment(db, type, row_start);
-		current_segment = move(compressed_segment);
-
-		current_segment->function = function;
-
-		// Reset the buffers and string map
+	void Reset() {
 		index_buffer.clear();
 		current_width = 0;
+		max_compressed_string_length = 0;
+		last_fitting_size = 0;
 
 		// Reset the pointers into the current segment
 		auto &buffer_manager = BufferManager::GetBufferManager(current_segment->db);
@@ -231,11 +225,21 @@ public:
 		current_end_ptr = current_handle.Ptr() + current_dictionary.end;
 	}
 
-	void UpdateState(string_t uncompressed_string, unsigned char *compressed_string, size_t compressed_string_len) {
+	void CreateEmptySegment(idx_t row_start) {
+		auto &db = checkpointer.GetDatabase();
+		auto &type = checkpointer.GetType();
+		auto compressed_segment = ColumnSegment::CreateTransientSegment(db, type, row_start);
+		current_segment = std::move(compressed_segment);
+		current_segment->function = function;
+		Reset();
+	}
 
+	void UpdateState(string_t uncompressed_string, unsigned char *compressed_string, size_t compressed_string_len) {
 		if (!HasEnoughSpace(compressed_string_len)) {
 			Flush();
-			D_ASSERT(HasEnoughSpace(compressed_string_len));
+			if (!HasEnoughSpace(compressed_string_len)) {
+				throw InternalException("FSST string compression failed due to insufficient space in empty block");
+			};
 		}
 
 		UncompressedStringStorage::UpdateStringStats(current_segment->stats, uncompressed_string);
@@ -258,7 +262,9 @@ public:
 	void AddNull() {
 		if (!HasEnoughSpace(0)) {
 			Flush();
-			D_ASSERT(HasEnoughSpace(0));
+			if (!HasEnoughSpace(0)) {
+				throw InternalException("FSST string compression failed due to insufficient space in empty block");
+			};
 		}
 		index_buffer.push_back(0);
 		current_segment->count++;
@@ -269,7 +275,7 @@ public:
 		UncompressedStringStorage::UpdateStringStats(current_segment->stats, "");
 	}
 
-	bool HasEnoughSpace(size_t string_len) {
+	size_t GetRequiredSize(size_t string_len) {
 		bitpacking_width_t required_minimum_width;
 		if (string_len > max_compressed_string_length) {
 			required_minimum_width = BitpackingPrimitives::MinimumBitWidth(string_len);
@@ -284,10 +290,19 @@ public:
 		    BitpackingPrimitives::GetRequiredSize(current_string_count + 1, required_minimum_width);
 
 		// TODO switch to a symbol table per RowGroup, saves a bit of space
-		idx_t required_space = sizeof(fsst_compression_header_t) + current_dict_size + dict_offsets_size + string_len +
-		                       fsst_serialized_symbol_table_size;
+		return sizeof(fsst_compression_header_t) + current_dict_size + dict_offsets_size + string_len +
+		       fsst_serialized_symbol_table_size;
+	}
 
-		return required_space <= Storage::BLOCK_SIZE;
+	// Checks if there is enough space, if there is, sets last_fitting_size
+	bool HasEnoughSpace(size_t string_len) {
+		auto required_size = GetRequiredSize(string_len);
+
+		if (required_size <= Storage::BLOCK_SIZE) {
+			last_fitting_size = required_size;
+			return true;
+		}
+		return false;
 	}
 
 	void Flush(bool final = false) {
@@ -295,7 +310,7 @@ public:
 
 		auto segment_size = Finalize();
 		auto &state = checkpointer.GetCheckpointState();
-		state.FlushSegment(move(current_segment), segment_size);
+		state.FlushSegment(std::move(current_segment), segment_size);
 
 		if (!final) {
 			CreateEmptySegment(next_start);
@@ -313,6 +328,10 @@ public:
 		auto total_size = sizeof(fsst_compression_header_t) + compressed_index_buffer_size + current_dictionary.size +
 		                  fsst_serialized_symbol_table_size;
 
+		if (total_size != last_fitting_size) {
+			throw InternalException("FSST string compression failed due to incorrect size calculation");
+		}
+
 		// calculate ptr and offsets
 		auto base_ptr = handle.Ptr();
 		auto header_ptr = (fsst_compression_header_t *)base_ptr;
@@ -334,11 +353,6 @@ public:
 		Store<uint32_t>(symbol_table_offset, (data_ptr_t)&header_ptr->fsst_symbol_table_offset);
 		Store<uint32_t>((uint32_t)current_width, (data_ptr_t)&header_ptr->bitpacking_width);
 
-		if (symbol_table_offset + fsst_serialized_symbol_table_size >
-		    current_dictionary.end - current_dictionary.size) {
-			throw InternalException("FSST string compression failed due to incorrect size calculation");
-		}
-
 		if (total_size >= FSSTStorage::COMPACTION_FLUSH_LIMIT) {
 			// the block is full enough, don't bother moving around the dictionary
 			return Storage::BLOCK_SIZE;
@@ -369,8 +383,9 @@ public:
 	// Buffers and map for current segment
 	std::vector<uint32_t> index_buffer;
 
-	size_t max_compressed_string_length = 0;
-	bitpacking_width_t current_width = 0;
+	size_t max_compressed_string_length;
+	bitpacking_width_t current_width;
+	idx_t last_fitting_size;
 
 	duckdb_fsst_encoder_t *fsst_encoder = nullptr;
 	unsigned char fsst_serialized_symbol_table[sizeof(duckdb_fsst_decoder_t)];
@@ -515,7 +530,7 @@ unique_ptr<SegmentScanState> FSSTStorage::StringInitScan(ColumnSegment &segment)
 		state->duckdb_fsst_decoder = nullptr;
 	}
 
-	return move(state);
+	return std::move(state);
 }
 
 void DeltaDecodeIndices(uint32_t *buffer_in, uint32_t *buffer_out, idx_t decode_count, uint32_t last_known_value) {

package/src/duckdb/src/storage/compression/rle.cpp

@@ -151,7 +151,7 @@ struct RLECompressState : public CompressionState {
 		auto &type = checkpointer.GetType();
 		auto column_segment = ColumnSegment::CreateTransientSegment(db, type, row_start);
 		column_segment->function = function;
-		current_segment = move(column_segment);
+		current_segment = std::move(column_segment);
 		auto &buffer_manager = BufferManager::GetBufferManager(db);
 		handle = buffer_manager.Pin(current_segment->block);
 	}
@@ -202,7 +202,7 @@ struct RLECompressState : public CompressionState {
 		handle.Destroy();
 
 		auto &state = checkpointer.GetCheckpointState();
-		state.FlushSegment(move(current_segment), total_segment_size);
+		state.FlushSegment(std::move(current_segment), total_segment_size);
 	}
 
 	void Finalize() {
@@ -282,7 +282,7 @@ struct RLEScanState : public SegmentScanState {
 template <class T>
 unique_ptr<SegmentScanState> RLEInitScan(ColumnSegment &segment) {
 	auto result = make_unique<RLEScanState<T>>(segment);
-	return move(result);
+	return std::move(result);
 }
 
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/storage/compression/string_uncompressed.cpp

@@ -12,7 +12,7 @@ namespace duckdb {
 UncompressedStringSegmentState::~UncompressedStringSegmentState() {
 	while (head) {
 		// prevent deep recursion here
-		head = move(head->next);
+		head = std::move(head->next);
 	}
 }
 
@@ -64,7 +64,7 @@ unique_ptr<SegmentScanState> UncompressedStringStorage::StringInitScan(ColumnSeg
 	auto result = make_unique<StringScanState>();
 	auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
 	result->handle = buffer_manager.Pin(segment.block);
-	return move(result);
+	return std::move(result);
 }
 
 //===--------------------------------------------------------------------===//
@@ -108,7 +108,7 @@ BufferHandle &ColumnFetchState::GetOrInsertHandle(ColumnSegment &segment) {
 		// not pinned yet: pin it
 		auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
 		auto handle = buffer_manager.Pin(segment.block);
-		auto entry = handles.insert(make_pair(primary_id, move(handle)));
+		auto entry = handles.insert(make_pair(primary_id, std::move(handle)));
 		return entry.first->second;
 	} else {
 		// already pinned: use the pinned handle
@@ -252,9 +252,9 @@ void UncompressedStringStorage::WriteStringMemory(ColumnSegment &segment, string
 		// allocate an in-memory buffer for it
 		handle = buffer_manager.Allocate(alloc_size, false, &block);
 		state.overflow_blocks[block->BlockId()] = new_block.get();
-		new_block->block = move(block);
-		new_block->next = move(state.head);
-		state.head = move(new_block);
+		new_block->block = std::move(block);
+		new_block->next = std::move(state.head);
+		state.head = std::move(new_block);
 	} else {
 		// string fits, copy it into the current block
 		handle = buffer_manager.Pin(state.head->block);
@@ -329,7 +329,7 @@ string_t UncompressedStringStorage::ReadOverflowString(ColumnSegment &segment, V
 		             uncompressed_size);
 
 		auto final_buffer = decompressed_target_handle.Ptr();
-		StringVector::AddHandle(result, move(decompressed_target_handle));
+		StringVector::AddHandle(result, std::move(decompressed_target_handle));
 		return ReadString(final_buffer, 0, uncompressed_size);
 	} else {
 		// read the overflow string from memory
@@ -338,7 +338,7 @@ string_t UncompressedStringStorage::ReadOverflowString(ColumnSegment &segment, V
 		D_ASSERT(entry != state.overflow_blocks.end());
 		auto handle = buffer_manager.Pin(entry->second->block);
 		auto final_buffer = handle.Ptr();
-		StringVector::AddHandle(result, move(handle));
+		StringVector::AddHandle(result, std::move(handle));
 		return ReadStringWithLength(final_buffer, offset);
 	}
 }

package/src/duckdb/src/storage/compression/validity_uncompressed.cpp

@@ -211,7 +211,7 @@ unique_ptr<SegmentScanState> ValidityInitScan(ColumnSegment &segment) {
 	auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
 	result->handle = buffer_manager.Pin(segment.block);
 	result->block_id = segment.block->BlockId();
-	return move(result);
+	return std::move(result);
 }
 
 //===--------------------------------------------------------------------===//
@@ -395,7 +395,7 @@ void ValidityFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row
 static unique_ptr<CompressionAppendState> ValidityInitAppend(ColumnSegment &segment) {
 	auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
 	auto handle = buffer_manager.Pin(segment.block);
-	return make_unique<CompressionAppendState>(move(handle));
+	return make_unique<CompressionAppendState>(std::move(handle));
 }
 
 unique_ptr<CompressedSegmentState> ValidityInitSegment(ColumnSegment &segment, block_id_t block_id) {