duckdb 0.5.2-dev1144.0 → 0.5.2-dev1149.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.5.2-dev1144.0",
+  "version": "0.5.2-dev1149.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -15914,11 +15914,21 @@ void FieldReader::Finalize() {
 
 namespace duckdb {
 
-FileBuffer::FileBuffer(Allocator &allocator, FileBufferType type, uint64_t bufsiz)
-    : allocator(allocator), type(type), malloced_buffer(nullptr) {
-	SetMallocedSize(bufsiz);
-	malloced_buffer = allocator.AllocateData(malloced_size);
-	Construct(bufsiz);
+FileBuffer::FileBuffer(Allocator &allocator, FileBufferType type, uint64_t user_size)
+    : allocator(allocator), type(type) {
+	Init();
+	if (user_size) {
+		Resize(user_size);
+	}
+}
+
+void FileBuffer::Init() {
+	buffer = nullptr;
+	size = 0;
+	internal_buffer = nullptr;
+	internal_size = 0;
+	malloced_buffer = nullptr;
+	malloced_size = 0;
 }
 
 FileBuffer::FileBuffer(FileBuffer &source, FileBufferType type_p) : allocator(source.allocator), type(type_p) {
@@ -15930,12 +15940,7 @@ FileBuffer::FileBuffer(FileBuffer &source, FileBufferType type_p) : allocator(so
 	malloced_buffer = source.malloced_buffer;
 	malloced_size = source.malloced_size;
 
-	source.buffer = nullptr;
-	source.size = 0;
-	source.internal_buffer = nullptr;
-	source.internal_size = 0;
-	source.malloced_buffer = nullptr;
-	source.malloced_size = 0;
+	source.Init();
 }
 
 FileBuffer::~FileBuffer() {
@@ -15945,30 +15950,42 @@ FileBuffer::~FileBuffer() {
 	allocator.FreeData(malloced_buffer, malloced_size);
 }
 
-void FileBuffer::SetMallocedSize(uint64_t &bufsiz) {
-	// make room for the block header (if this is not the db file header)
-	if (type == FileBufferType::MANAGED_BUFFER && bufsiz != Storage::FILE_HEADER_SIZE) {
-		bufsiz += Storage::BLOCK_HEADER_SIZE;
+void FileBuffer::ReallocBuffer(size_t new_size) {
+	if (malloced_buffer) {
+		malloced_buffer = allocator.ReallocateData(malloced_buffer, malloced_size, new_size);
+	} else {
+		malloced_buffer = allocator.AllocateData(new_size);
 	}
-	malloced_size = bufsiz;
-}
-
-void FileBuffer::Construct(uint64_t bufsiz) {
 	if (!malloced_buffer) {
 		throw std::bad_alloc();
 	}
+	malloced_size = new_size;
 	internal_buffer = malloced_buffer;
 	internal_size = malloced_size;
-	buffer = internal_buffer + Storage::BLOCK_HEADER_SIZE;
-	size = internal_size - Storage::BLOCK_HEADER_SIZE;
+	// Caller must update these.
+	buffer = nullptr;
+	size = 0;
 }
 
-void FileBuffer::Resize(uint64_t bufsiz) {
-	D_ASSERT(type == FileBufferType::MANAGED_BUFFER);
-	auto old_size = malloced_size;
-	SetMallocedSize(bufsiz);
-	malloced_buffer = allocator.ReallocateData(malloced_buffer, old_size, malloced_size);
-	Construct(bufsiz);
+void FileBuffer::Resize(uint64_t new_size) {
+	{
+		// TODO: All the logic here is specific to SingleFileBlockManager.
+		// and should be moved there, via a specific implementation of FileBuffer.
+		//
+		// make room for the block header (if this is not the db file header)
+		if (type == FileBufferType::MANAGED_BUFFER && new_size != Storage::FILE_HEADER_SIZE) {
+			new_size += Storage::BLOCK_HEADER_SIZE;
+			// If we don't write/read an entire block, our checksum won't match.
+			new_size = AlignValue<uint32_t, Storage::BLOCK_ALLOC_SIZE>(new_size);
+		}
+		new_size = AlignValue<uint32_t, Storage::SECTOR_SIZE>(new_size);
+		ReallocBuffer(new_size);
+	}
+
+	if (new_size > 0) {
+		buffer = internal_buffer + Storage::BLOCK_HEADER_SIZE;
+		size = internal_size - Storage::BLOCK_HEADER_SIZE;
+	}
 }
 
 void FileBuffer::Read(FileHandle &handle, uint64_t location) {
@@ -41725,8 +41742,8 @@ void ColumnDataAllocator::AllocateBlock() {
 	D_ASSERT(type == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR);
 	BlockMetaData data;
 	data.size = 0;
-	data.capacity = Storage::BLOCK_ALLOC_SIZE;
-	data.handle = alloc.buffer_manager->RegisterMemory(Storage::BLOCK_ALLOC_SIZE, false);
+	data.capacity = Storage::BLOCK_SIZE;
+	data.handle = alloc.buffer_manager->RegisterMemory(Storage::BLOCK_SIZE, false);
 	blocks.push_back(move(data));
 }
 
@@ -61613,6 +61630,7 @@ void JoinHashTable::InitializePointerTable() {
 		// allocate the HT if not yet done
 		hash_map = buffer_manager.Allocate(capacity * sizeof(data_ptr_t));
 	}
+	D_ASSERT(hash_map.GetFileBuffer().size >= capacity * sizeof(data_ptr_t));
 
 	// initialize HT with all-zero entries
 	memset(hash_map.Ptr(), 0, capacity * sizeof(data_ptr_t));
@@ -194234,12 +194252,16 @@ bool ArenaAllocator::IsEmpty() {
 namespace duckdb {
 
 Block::Block(Allocator &allocator, block_id_t id)
-    : FileBuffer(allocator, FileBufferType::BLOCK, Storage::BLOCK_ALLOC_SIZE), id(id) {
+    : FileBuffer(allocator, FileBufferType::BLOCK, Storage::BLOCK_SIZE), id(id) {
+}
+
+Block::Block(Allocator &allocator, block_id_t id, uint32_t internal_size)
+    : FileBuffer(allocator, FileBufferType::BLOCK, internal_size), id(id) {
+	D_ASSERT((GetMallocedSize() & (Storage::SECTOR_SIZE - 1)) == 0);
 }
 
 Block::Block(FileBuffer &source, block_id_t id) : FileBuffer(source, FileBufferType::BLOCK), id(id) {
-	D_ASSERT(GetMallocedSize() == Storage::BLOCK_ALLOC_SIZE);
-	D_ASSERT(size == Storage::BLOCK_SIZE);
+	D_ASSERT((GetMallocedSize() & (Storage::SECTOR_SIZE - 1)) == 0);
 }
 
 } // namespace duckdb
@@ -194478,7 +194500,8 @@ BlockHandle::~BlockHandle() {
 	block_manager.UnregisterBlock(block_id, can_destroy);
 }
 
-unique_ptr<Block> AllocateBlock(Allocator &allocator, unique_ptr<FileBuffer> reusable_buffer, block_id_t block_id) {
+unique_ptr<Block> AllocateBlock(BlockManager &block_manager, unique_ptr<FileBuffer> reusable_buffer,
+                                block_id_t block_id) {
 	if (reusable_buffer) {
 		// re-usable buffer: re-use it
 		if (reusable_buffer->type == FileBufferType::BLOCK) {
@@ -194487,18 +194510,19 @@ unique_ptr<Block> AllocateBlock(Allocator &allocator, unique_ptr<FileBuffer> reu
 			block.id = block_id;
 			return unique_ptr_cast<FileBuffer, Block>(move(reusable_buffer));
 		}
-		auto block = make_unique<Block>(*reusable_buffer, block_id);
+		auto block = block_manager.CreateBlock(block_id, reusable_buffer.get());
 		reusable_buffer.reset();
 		return block;
 	} else {
 		// no re-usable buffer: allocate a new block
-		return make_unique<Block>(allocator, block_id);
+		return block_manager.CreateBlock(block_id, nullptr);
 	}
 }
 
-unique_ptr<FileBuffer> AllocateManagedBuffer(DatabaseInstance &db, unique_ptr<FileBuffer> reusable_buffer, idx_t size) {
-	if (reusable_buffer) {
-		auto tmp = move(reusable_buffer);
+unique_ptr<FileBuffer> BufferManager::ConstructManagedBuffer(idx_t size, unique_ptr<FileBuffer> &&source) {
+	if (source) {
+		auto tmp = move(source);
+		D_ASSERT(tmp->size == size);
 		return make_unique<FileBuffer>(*tmp, FileBufferType::MANAGED_BUFFER);
 	} else {
 		// no re-usable buffer: allocate a new buffer
@@ -194515,8 +194539,7 @@ BufferHandle BlockHandle::Load(shared_ptr<BlockHandle> &handle, unique_ptr<FileB
 
 	auto &block_manager = handle->block_manager;
 	if (handle->block_id < MAXIMUM_BLOCK) {
-		auto block = AllocateBlock(Allocator::Get(block_manager.buffer_manager.GetDatabase()), move(reusable_buffer),
-		                           handle->block_id);
+		auto block = AllocateBlock(block_manager, move(reusable_buffer), handle->block_id);
 		block_manager.Read(*block);
 		handle->buffer = move(block);
 	} else {
@@ -194670,6 +194693,10 @@ shared_ptr<BlockHandle> BlockManager::ConvertToPersistent(block_id_t block_id, s
 	D_ASSERT(old_block->state == BlockState::BLOCK_LOADED);
 	D_ASSERT(old_block->buffer);
 
+	// Temp buffers can be larger than the storage block size. But persistent buffers
+	// cannot.
+	D_ASSERT(old_block->buffer->AllocSize() <= Storage::BLOCK_ALLOC_SIZE);
+
 	// register a block with the new block id
 	auto new_block = RegisterBlock(block_id);
 	D_ASSERT(new_block->state == BlockState::BLOCK_UNLOADED);
@@ -194677,7 +194704,7 @@ shared_ptr<BlockHandle> BlockManager::ConvertToPersistent(block_id_t block_id, s
 
 	// move the data from the old block into data for the new block
 	new_block->state = BlockState::BLOCK_LOADED;
-	new_block->buffer = make_unique<Block>(*old_block->buffer, block_id);
+	new_block->buffer = CreateBlock(block_id, old_block->buffer.get());
 
 	// clear the old buffer and unload it
 	old_block->buffer.reset();
@@ -194695,7 +194722,7 @@ shared_ptr<BlockHandle> BlockManager::ConvertToPersistent(block_id_t block_id, s
 }
 
 shared_ptr<BlockHandle> BufferManager::RegisterMemory(idx_t block_size, bool can_destroy) {
-	auto alloc_size = block_size + Storage::BLOCK_HEADER_SIZE;
+	auto alloc_size = AlignValue<idx_t, 1 << 12>(block_size + Storage::BLOCK_HEADER_SIZE);
 	// first evict blocks until we have enough memory to store this buffer
 	unique_ptr<FileBuffer> reusable_buffer;
 	if (!EvictBlocks(alloc_size, maximum_memory, &reusable_buffer)) {
@@ -194703,7 +194730,7 @@ shared_ptr<BlockHandle> BufferManager::RegisterMemory(idx_t block_size, bool can
 		                           GetUsedMemory(), GetMaxMemory(), InMemoryWarning());
 	}
 
-	auto buffer = AllocateManagedBuffer(db, move(reusable_buffer), block_size);
+	auto buffer = ConstructManagedBuffer(block_size, move(reusable_buffer));
 
 	// create a new block pointer for this block
 	return make_shared<BlockHandle>(*temp_block_manager, ++temporary_id, move(buffer), can_destroy, block_size);
@@ -194876,9 +194903,9 @@ void BufferManager::SetLimit(idx_t limit) {
 //===--------------------------------------------------------------------===//
 // Temporary File Management
 //===--------------------------------------------------------------------===//
-unique_ptr<FileBuffer> ReadTemporaryBufferInternal(DatabaseInstance &db, FileHandle &handle, idx_t position, idx_t size,
-                                                   unique_ptr<FileBuffer> reusable_buffer) {
-	auto buffer = AllocateManagedBuffer(db, move(reusable_buffer), size);
+unique_ptr<FileBuffer> ReadTemporaryBufferInternal(BufferManager &buffer_manager, FileHandle &handle, idx_t position,
+                                                   idx_t size, block_id_t id, unique_ptr<FileBuffer> reusable_buffer) {
+	auto buffer = buffer_manager.ConstructManagedBuffer(size, move(reusable_buffer));
 	buffer->Read(handle, position);
 	return buffer;
 }
@@ -194999,8 +195026,9 @@ public:
 
 	unique_ptr<FileBuffer> ReadTemporaryBuffer(block_id_t id, idx_t block_index,
 	                                           unique_ptr<FileBuffer> reusable_buffer) {
-		auto buffer = ReadTemporaryBufferInternal(db, *handle, GetPositionInFile(block_index), Storage::BLOCK_SIZE,
-		                                          move(reusable_buffer));
+		auto buffer =
+		    ReadTemporaryBufferInternal(BufferManager::GetBufferManager(db), *handle, GetPositionInFile(block_index),
+		                                Storage::BLOCK_SIZE, id, move(reusable_buffer));
 		{
 			// remove the block (and potentially truncate the temp file)
 			TemporaryFileLock lock(file_lock);
@@ -195238,7 +195266,7 @@ unique_ptr<FileBuffer> BufferManager::ReadTemporaryBuffer(block_id_t id, unique_
 	handle->Read(&block_size, sizeof(idx_t), 0);
 
 	// now allocate a buffer of this size and read the data into that buffer
-	auto buffer = ReadTemporaryBufferInternal(db, *handle, sizeof(idx_t), block_size, move(reusable_buffer));
+	auto buffer = ReadTemporaryBufferInternal(*this, *handle, sizeof(idx_t), block_size, id, move(reusable_buffer));
 
 	handle.reset();
 	DeleteTemporaryFile(id);
@@ -195950,6 +195978,7 @@ WriteOverflowStringsToDisk::WriteOverflowStringsToDisk(BlockManager &block_manag
 
 WriteOverflowStringsToDisk::~WriteOverflowStringsToDisk() {
 	if (offset > 0) {
+		handle.GetFileBuffer().Resize(offset);
 		block_manager.Write(handle.GetFileBuffer(), block_id);
 	}
 }

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "0e999f15d"
-#define DUCKDB_VERSION "v0.5.2-dev1144"
+#define DUCKDB_SOURCE_ID "596c5fe1e"
+#define DUCKDB_VERSION "v0.5.2-dev1149"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -1935,11 +1935,11 @@ enum class FileBufferType : uint8_t { BLOCK = 1, MANAGED_BUFFER = 2 };
 //! The FileBuffer represents a buffer that can be read or written to a Direct IO FileHandle.
 class FileBuffer {
 public:
-	//! Allocates a buffer of the specified size that is sector-aligned. bufsiz must be a multiple of
-	//! FileSystemConstants::FILE_BUFFER_BLOCK_SIZE. The content in this buffer can be written to FileHandles that have
-	//! been opened with DIRECT_IO on all operating systems, however, the entire buffer must be written to the file.
-	//! Note that the returned size is 8 bytes less than the allocation size to account for the checksum.
-	FileBuffer(Allocator &allocator, FileBufferType type, uint64_t bufsiz);
+	//! Allocates a buffer of the specified size, with room for additional header bytes
+	//! (typically 8 bytes). On return, this->AllocSize() >= this->size >= user_size.
+	//! Our allocation size will always be page-aligned, which is necessary to support
+	//! DIRECT_IO
+	FileBuffer(Allocator &allocator, FileBufferType type, uint64_t user_size);
 	FileBuffer(FileBuffer &source, FileBufferType type);
 
 	virtual ~FileBuffer();
@@ -1957,18 +1957,20 @@ public:
 	void Read(FileHandle &handle, uint64_t location);
 	//! Read into the FileBuffer from the specified location. Automatically verifies the checksum, and throws an
 	//! exception if the checksum does not match correctly.
-	void ReadAndChecksum(FileHandle &handle, uint64_t location);
+	virtual void ReadAndChecksum(FileHandle &handle, uint64_t location);
 	//! Write the contents of the FileBuffer to the specified location.
 	void Write(FileHandle &handle, uint64_t location);
 	//! Write the contents of the FileBuffer to the specified location. Automatically adds a checksum of the contents of
 	//! the filebuffer in front of the written data.
-	void ChecksumAndWrite(FileHandle &handle, uint64_t location);
+	virtual void ChecksumAndWrite(FileHandle &handle, uint64_t location);
 
 	void Clear();
 
-	void Resize(uint64_t bufsiz);
+	// Same rules as the constructor. We will add room for a header, in additio to
+	// the requested user bytes. We will then sector-align the result.
+	virtual void Resize(uint64_t user_size);
 
-	uint64_t AllocSize() {
+	uint64_t AllocSize() const {
 		return internal_size;
 	}
 
@@ -1978,6 +1980,8 @@ protected:
 	//! The aligned size as passed to the constructor. This is the size that is read or written to disk.
 	uint64_t internal_size;
 
+	void ReallocBuffer(size_t malloc_size);
+
 private:
 	//! The buffer that was actually malloc'd, i.e. the pointer that must be freed when the FileBuffer is destroyed
 	data_ptr_t malloced_buffer;
@@ -1987,10 +1991,7 @@ protected:
 	uint64_t GetMallocedSize() {
 		return malloced_size;
 	}
-	//! Sets malloced_size given the requested buffer size
-	void SetMallocedSize(uint64_t &bufsiz);
-	//! Constructs the Filebuffer object
-	void Construct(uint64_t bufsiz);
+	void Init();
 };
 
 } // namespace duckdb
@@ -3849,7 +3850,7 @@ using block_id_t = int64_t;
 //! file.
 struct MainHeader {
 	static constexpr idx_t MAGIC_BYTE_SIZE = 4;
-	static constexpr idx_t MAGIC_BYTE_OFFSET = sizeof(uint64_t);
+	static constexpr idx_t MAGIC_BYTE_OFFSET = Storage::BLOCK_HEADER_SIZE;
 	static constexpr idx_t FLAG_COUNT = 4;
 	// the magic bytes in front of the file
 	// should be "DUCK"
@@ -20585,6 +20586,7 @@ namespace duckdb {
 class Block : public FileBuffer {
 public:
 	Block(Allocator &allocator, block_id_t id);
+	Block(Allocator &allocator, block_id_t id, uint32_t internal_size);
 	Block(FileBuffer &source, block_id_t id);
 
 	block_id_t id;
@@ -20593,8 +20595,8 @@ public:
 struct BlockPointer {
 	BlockPointer(block_id_t block_id_p, uint32_t offset_p) : block_id(block_id_p), offset(offset_p) {};
 	BlockPointer() {};
-	block_id_t block_id;
-	uint32_t offset;
+	block_id_t block_id {0};
+	uint32_t offset {0};
 };
 
 } // namespace duckdb
@@ -25219,6 +25221,11 @@ public:
 		return db;
 	}
 
+	//! Construct a managed buffer.
+	//! The block_id is just used for internal tracking. It doesn't map to any actual
+	//! BlockManager.
+	virtual unique_ptr<FileBuffer> ConstructManagedBuffer(idx_t size, unique_ptr<FileBuffer> &&source);
+
 private:
 	//! Evict blocks until the currently used memory + extra_memory fit, returns false if this was not possible
 	//! (i.e. not enough blocks could be evicted)