duckdb 0.7.2-dev2320.0 → 0.7.2-dev2410.0

package/src/duckdb/src/storage/compression/dictionary_compression.cpp

@@ -208,7 +208,7 @@ public:
 		// Copy string to dict
 		current_dictionary.size += str.GetSize();
 		auto dict_pos = current_end_ptr - current_dictionary.size;
-		memcpy(dict_pos, str.GetDataUnsafe(), str.GetSize());
+		memcpy(dict_pos, str.GetData(), str.GetSize());
 		current_dictionary.Verify();
 		D_ASSERT(current_dictionary.end == Storage::BLOCK_SIZE);
 

package/src/duckdb/src/storage/compression/fsst.cpp

@@ -155,7 +155,7 @@ idx_t FSSTStorage::StringFinalAnalyze(AnalyzeState &state_p) {
 	vector<unsigned char *> fsst_string_ptrs;
 	for (auto &str : state.fsst_strings) {
 		fsst_string_sizes.push_back(str.GetSize());
-		fsst_string_ptrs.push_back((unsigned char *)str.GetDataUnsafe());
+		fsst_string_ptrs.push_back((unsigned char *)str.GetData());
 	}
 
 	state.fsst_encoder = duckdb_fsst_create(string_count, &fsst_string_sizes[0], &fsst_string_ptrs[0], 0);
@@ -431,7 +431,7 @@ void FSSTStorage::Compress(CompressionState &state_p, Vector &scan_vector, idx_t
 		total_count++;
 		total_size += data[idx].GetSize();
 		sizes_in.push_back(data[idx].GetSize());
-		strings_in.push_back((unsigned char *)data[idx].GetDataUnsafe());
+		strings_in.push_back((unsigned char *)data[idx].GetData());
 	}
 
 	// Only Nulls or empty strings in this vector, nothing to compress
@@ -669,7 +669,7 @@ void FSSTStorage::StringFetchRow(ColumnSegment &segment, ColumnFetchState &state
 		    segment, dict, result, base_ptr, delta_decode_buffer[offsets.unused_delta_decoded_values], string_length);
 
 		result_data[result_idx] = FSSTPrimitives::DecompressValue(
-		    (void *)&decoder, result, (unsigned char *)compressed_string.GetDataUnsafe(), compressed_string.GetSize());
+		    (void *)&decoder, result, (unsigned char *)compressed_string.GetData(), compressed_string.GetSize());
 	} else {
 		// There's no fsst symtable, this only happens for empty strings or nulls, we can just emit an empty string
 		result_data[result_idx] = string_t(nullptr, 0);

package/src/duckdb/src/storage/compression/string_uncompressed.cpp

@@ -267,7 +267,7 @@ void UncompressedStringStorage::WriteStringMemory(ColumnSegment &segment, string
 	auto ptr = handle.Ptr() + state.head->offset;
 	Store<uint32_t>(string.GetSize(), ptr);
 	ptr += sizeof(uint32_t);
-	memcpy(ptr, string.GetDataUnsafe(), string.GetSize());
+	memcpy(ptr, string.GetData(), string.GetSize());
 	state.head->offset += total_length;
 }
 

package/src/duckdb/src/storage/meta_block_writer.cpp

@@ -35,6 +35,10 @@ BlockPointer MetaBlockWriter::GetBlockPointer() {
 }
 
 void MetaBlockWriter::Flush() {
+	if (offset < block->size) {
+		// clear remaining bytes of block (if any)
+		memset(block->buffer + offset, 0, block->size - offset);
+	}
 	AdvanceBlock();
 	block = nullptr;
 }

package/src/duckdb/src/storage/partial_block_manager.cpp

@@ -54,9 +54,14 @@ bool PartialBlockManager::GetPartialBlock(idx_t segment_size, unique_ptr<Partial
 }
 
 void PartialBlockManager::RegisterPartialBlock(PartialBlockAllocation &&allocation) {
-	auto &state(allocation.partial_block->state);
+	auto &state = allocation.partial_block->state;
 	if (state.block_use_count < max_use_count) {
-		auto new_size = AlignValue(allocation.allocation_size + state.offset_in_block);
+		auto unaligned_size = allocation.allocation_size + state.offset_in_block;
+		auto new_size = AlignValue(unaligned_size);
+		if (new_size != unaligned_size) {
+			// register the uninitialized region so we can correctly initialize it before writing to disk
+			allocation.partial_block->AddUninitializedRegion(unaligned_size, new_size);
+		}
 		state.offset_in_block = new_size;
 		auto new_space_left = state.block_size - new_size;
 		// check if the block is STILL partially filled after adding the segment_size
@@ -65,22 +70,24 @@ void PartialBlockManager::RegisterPartialBlock(PartialBlockAllocation &&allocati
 			partially_filled_blocks.insert(make_pair(new_space_left, std::move(allocation.partial_block)));
 		}
 	}
+	idx_t free_space = state.block_size - state.offset_in_block;
 	auto block_to_free = std::move(allocation.partial_block);
 	if (!block_to_free && partially_filled_blocks.size() > MAX_BLOCK_MAP_SIZE) {
 		// Free the page with the least space free.
 		auto itr = partially_filled_blocks.begin();
 		block_to_free = std::move(itr->second);
+		free_space = state.block_size - itr->first;
 		partially_filled_blocks.erase(itr);
 	}
 	// Flush any block that we're not going to reuse.
 	if (block_to_free) {
-		block_to_free->Flush();
+		block_to_free->Flush(free_space);
 	}
 }
 
 void PartialBlockManager::FlushPartialBlocks() {
 	for (auto &e : partially_filled_blocks) {
-		e.second->Flush();
+		e.second->Flush(e.first);
 	}
 	partially_filled_blocks.clear();
 }

package/src/duckdb/src/storage/single_file_block_manager.cpp

@@ -107,15 +107,15 @@ T DeserializeHeaderStructure(data_ptr_t ptr) {
 	return T::Deserialize(source);
 }
 
-SingleFileBlockManager::SingleFileBlockManager(AttachedDatabase &db, string path_p, bool read_only, bool use_direct_io)
+SingleFileBlockManager::SingleFileBlockManager(AttachedDatabase &db, string path_p, StorageManagerOptions options)
     : BlockManager(BufferManager::GetBufferManager(db)), db(db), path(std::move(path_p)),
       header_buffer(Allocator::Get(db), FileBufferType::MANAGED_BUFFER,
                     Storage::FILE_HEADER_SIZE - Storage::BLOCK_HEADER_SIZE),
-      iteration_count(0), read_only(read_only), use_direct_io(use_direct_io) {
+      iteration_count(0), options(options) {
 }
 
 void SingleFileBlockManager::GetFileFlags(uint8_t &flags, FileLockType &lock, bool create_new) {
-	if (read_only) {
+	if (options.read_only) {
 		D_ASSERT(!create_new);
 		flags = FileFlags::FILE_FLAGS_READ;
 		lock = FileLockType::READ_LOCK;
@@ -126,7 +126,7 @@ void SingleFileBlockManager::GetFileFlags(uint8_t &flags, FileLockType &lock, bo
 			flags |= FileFlags::FILE_FLAGS_FILE_CREATE;
 		}
 	}
-	if (use_direct_io) {
+	if (options.use_direct_io) {
 		flags |= FileFlags::FILE_FLAGS_DIRECT_IO;
 	}
 }
@@ -241,7 +241,7 @@ void SingleFileBlockManager::Initialize(DatabaseHeader &header) {
 }
 
 void SingleFileBlockManager::LoadFreeList() {
-	if (read_only) {
+	if (options.read_only) {
 		// no need to load free list for read only db
 		return;
 	}
@@ -343,13 +343,20 @@ idx_t SingleFileBlockManager::FreeBlocks() {
 	return free_list.size();
 }
 
+unique_ptr<Block> SingleFileBlockManager::ConvertBlock(block_id_t block_id, FileBuffer &source_buffer) {
+	D_ASSERT(source_buffer.AllocSize() == Storage::BLOCK_ALLOC_SIZE);
+	return make_uniq<Block>(source_buffer, block_id);
+}
+
 unique_ptr<Block> SingleFileBlockManager::CreateBlock(block_id_t block_id, FileBuffer *source_buffer) {
+	unique_ptr<Block> result;
 	if (source_buffer) {
-		D_ASSERT(source_buffer->AllocSize() == Storage::BLOCK_ALLOC_SIZE);
-		return make_uniq<Block>(*source_buffer, block_id);
+		result = ConvertBlock(block_id, *source_buffer);
 	} else {
-		return make_uniq<Block>(Allocator::Get(db), block_id);
+		result = make_uniq<Block>(Allocator::Get(db), block_id);
 	}
+	result->Initialize(options.debug_initialize);
+	return result;
 }
 
 void SingleFileBlockManager::Read(Block &block) {
@@ -459,7 +466,7 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
 		throw FatalException("Checkpoint aborted after free list write because of PRAGMA checkpoint_abort flag");
 	}
 
-	if (!use_direct_io) {
+	if (!options.use_direct_io) {
 		// if we are not using Direct IO we need to fsync BEFORE we write the header to ensure that all the previous
 		// blocks are written as well
 		handle->Sync();

package/src/duckdb/src/storage/standard_buffer_manager.cpp

@@ -20,14 +20,17 @@ struct BufferAllocatorData : PrivateAllocatorData {
 
 unique_ptr<FileBuffer> StandardBufferManager::ConstructManagedBuffer(idx_t size, unique_ptr<FileBuffer> &&source,
                                                                      FileBufferType type) {
+	unique_ptr<FileBuffer> result;
 	if (source) {
 		auto tmp = std::move(source);
 		D_ASSERT(tmp->AllocSize() == BufferManager::GetAllocSize(size));
-		return make_uniq<FileBuffer>(*tmp, type);
+		result = make_uniq<FileBuffer>(*tmp, type);
 	} else {
 		// no re-usable buffer: allocate a new buffer
-		return make_uniq<FileBuffer>(Allocator::Get(db), type, size);
+		result = make_uniq<FileBuffer>(Allocator::Get(db), type, size);
 	}
+	result->Initialize(DBConfig::GetConfig(db).options.debug_initialize);
+	return result;
 }
 
 class TemporaryFileManager;

package/src/duckdb/src/storage/statistics/string_stats.cpp

@@ -116,7 +116,7 @@ static void ConstructValue(const_data_ptr_t data, idx_t size, data_t target[]) {
 }
 
 void StringStats::Update(BaseStatistics &stats, const string_t &value) {
-	auto data = (const_data_ptr_t)value.GetDataUnsafe();
+	auto data = (const_data_ptr_t)value.GetData();
 	auto size = value.GetSize();
 
 	//! we can only fit 8 bytes, so we might need to trim our string
@@ -238,7 +238,7 @@ void StringStats::Verify(const BaseStatistics &stats, Vector &vector, const Sele
 			continue;
 		}
 		auto value = data[index];
-		auto data = value.GetDataUnsafe();
+		auto data = value.GetData();
 		auto len = value.GetSize();
 		// LCOV_EXCL_START
 		if (string_data.has_max_string_length && len > string_data.max_string_length) {

package/src/duckdb/src/storage/storage_manager.cpp

@@ -95,6 +95,11 @@ void SingleFileStorageManager::LoadDatabase() {
 	auto &fs = FileSystem::Get(db);
 	auto &config = DBConfig::Get(db);
 	bool truncate_wal = false;
+
+	StorageManagerOptions options;
+	options.read_only = read_only;
+	options.use_direct_io = config.options.use_direct_io;
+	options.debug_initialize = config.options.debug_initialize;
 	// first check if the database exists
 	if (!fs.FileExists(path)) {
 		if (read_only) {
@@ -107,13 +112,13 @@ void SingleFileStorageManager::LoadDatabase() {
 			fs.RemoveFile(wal_path);
 		}
 		// initialize the block manager while creating a new db file
-		auto sf_block_manager = make_uniq<SingleFileBlockManager>(db, path, read_only, config.options.use_direct_io);
+		auto sf_block_manager = make_uniq<SingleFileBlockManager>(db, path, options);
 		sf_block_manager->CreateNewDatabase();
 		block_manager = std::move(sf_block_manager);
 		table_io_manager = make_uniq<SingleFileTableIOManager>(*block_manager);
 	} else {
 		// initialize the block manager while loading the current db file
-		auto sf_block_manager = make_uniq<SingleFileBlockManager>(db, path, read_only, config.options.use_direct_io);
+		auto sf_block_manager = make_uniq<SingleFileBlockManager>(db, path, options);
 		sf_block_manager->LoadExistingDatabase();
 		block_manager = std::move(sf_block_manager);
 		table_io_manager = make_uniq<SingleFileTableIOManager>(*block_manager);

package/src/duckdb/src/storage/table/column_checkpoint_state.cpp

@@ -50,16 +50,36 @@ public:
 	BlockManager &block_manager;
 	vector<PartialColumnSegment> tail_segments;
 
+private:
+	struct UninitializedRegion {
+		idx_t start;
+		idx_t end;
+	};
+	vector<UninitializedRegion> uninitialized_regions;
+
 public:
 	bool IsFlushed() {
 		// first_segment is zeroed on Flush
 		return !first_segment;
 	}
 
-	void Flush() override {
+	void AddUninitializedRegion(idx_t start, idx_t end) override {
+		uninitialized_regions.push_back({start, end});
+	}
+
+	void Flush(idx_t free_space_left) override {
 		// At this point, we've already copied all data from tail_segments
 		// into the page owned by first_segment. We flush all segment data to
 		// disk with the following call.
+		if (free_space_left > 0 || !uninitialized_regions.empty()) {
+			auto handle = block_manager.buffer_manager.Pin(first_segment->block);
+			// memset any uninitialized regions
+			for (auto &uninitialized : uninitialized_regions) {
+				memset(handle.Ptr() + uninitialized.start, 0, uninitialized.end - uninitialized.start);
+			}
+			// memset any free space at the end of the block to 0 prior to writing to disk
+			memset(handle.Ptr() + Storage::BLOCK_SIZE - free_space_left, 0, free_space_left);
+		}
 		first_data->IncrementVersion();
 		first_segment->ConvertToPersistent(&block_manager, state.block_id);
 		// Now that the page is persistent, update tail_segments to point to the

package/src/duckdb/ub_src_execution_operator_join.cpp

@@ -1,5 +1,7 @@
 #include "src/execution/operator/join/outer_join_marker.cpp"
 
+#include "src/execution/operator/join/physical_asof_join.cpp"
+
 #include "src/execution/operator/join/physical_blockwise_nl_join.cpp"
 
 #include "src/execution/operator/join/physical_comparison_join.cpp"

package/src/statement.cpp

@@ -377,8 +377,8 @@ struct RunPreparedTask : public Task {
 					// query results, the string data is owned by the QueryResult
 					auto result_ref_ptr = new std::shared_ptr<duckdb::QueryResult>(result_ptr);
 
-					auto array_buffer = Napi::ArrayBuffer::New(env, (void *)blob.GetDataUnsafe(), blob.GetSize(),
-					                                           deleter, result_ref_ptr);
+					auto array_buffer =
+					    Napi::ArrayBuffer::New(env, (void *)blob.GetData(), blob.GetSize(), deleter, result_ref_ptr);
 
 					auto typed_array = Napi::Uint8Array::New(env, blob.GetSize(), array_buffer, 0);
 
@@ -615,7 +615,7 @@ struct GetNextArrowIpcTask : public Task {
 			delete static_cast<unique_ptr<duckdb::DataChunk> *>(hint);
 		};
 		auto array_buffer =
-		    Napi::ArrayBuffer::New(env, (void *)blob.GetDataUnsafe(), blob.GetSize(), deleter, data_chunk_ptr);
+		    Napi::ArrayBuffer::New(env, (void *)blob.GetData(), blob.GetSize(), deleter, data_chunk_ptr);
 
 		deferred.Resolve(array_buffer);
 	}