duckdb 0.6.2-dev1770.0 → 0.6.2-dev1772.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.6.2-dev1770.0",
+  "version": "0.6.2-dev1772.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/common/file_buffer.cpp

@@ -23,8 +23,6 @@ void FileBuffer::Init() {
 	size = 0;
 	internal_buffer = nullptr;
 	internal_size = 0;
-	malloced_buffer = nullptr;
-	malloced_size = 0;
 }
 
 FileBuffer::FileBuffer(FileBuffer &source, FileBufferType type_p) : allocator(source.allocator), type(type_p) {
@@ -33,31 +31,29 @@ FileBuffer::FileBuffer(FileBuffer &source, FileBufferType type_p) : allocator(so
 	size = source.size;
 	internal_buffer = source.internal_buffer;
 	internal_size = source.internal_size;
-	malloced_buffer = source.malloced_buffer;
-	malloced_size = source.malloced_size;
 
 	source.Init();
 }
 
 FileBuffer::~FileBuffer() {
-	if (!malloced_buffer) {
+	if (!internal_buffer) {
 		return;
 	}
-	allocator.FreeData(malloced_buffer, malloced_size);
+	allocator.FreeData(internal_buffer, internal_size);
 }
 
 void FileBuffer::ReallocBuffer(size_t new_size) {
-	if (malloced_buffer) {
-		malloced_buffer = allocator.ReallocateData(malloced_buffer, malloced_size, new_size);
+	data_ptr_t new_buffer;
+	if (internal_buffer) {
+		new_buffer = allocator.ReallocateData(internal_buffer, internal_size, new_size);
 	} else {
-		malloced_buffer = allocator.AllocateData(new_size);
+		new_buffer = allocator.AllocateData(new_size);
 	}
-	if (!malloced_buffer) {
+	if (!new_buffer) {
 		throw std::bad_alloc();
 	}
-	malloced_size = new_size;
-	internal_buffer = malloced_buffer;
-	internal_size = malloced_size;
+	internal_buffer = new_buffer;
+	internal_size = new_size;
 	// Caller must update these.
 	buffer = nullptr;
 	size = 0;
@@ -92,32 +88,11 @@ void FileBuffer::Read(FileHandle &handle, uint64_t location) {
 	handle.Read(internal_buffer, internal_size, location);
 }
 
-void FileBuffer::ReadAndChecksum(FileHandle &handle, uint64_t location) {
-	// read the buffer from disk
-	Read(handle, location);
-	// compute the checksum
-	auto stored_checksum = Load<uint64_t>(internal_buffer);
-	uint64_t computed_checksum = Checksum(buffer, size);
-	// verify the checksum
-	if (stored_checksum != computed_checksum) {
-		throw IOException("Corrupt database file: computed checksum %llu does not match stored checksum %llu in block",
-		                  computed_checksum, stored_checksum);
-	}
-}
-
 void FileBuffer::Write(FileHandle &handle, uint64_t location) {
 	D_ASSERT(type != FileBufferType::TINY_BUFFER);
 	handle.Write(internal_buffer, internal_size, location);
 }
 
-void FileBuffer::ChecksumAndWrite(FileHandle &handle, uint64_t location) {
-	// compute the checksum and write it to the start of the buffer (if not temp buffer)
-	uint64_t checksum = Checksum(buffer, size);
-	Store<uint64_t>(checksum, internal_buffer);
-	// now write the buffer
-	Write(handle, location);
-}
-
 void FileBuffer::Clear() {
 	memset(internal_buffer, 0, internal_size);
 }

package/src/duckdb/src/execution/index/art/swizzleable_pointer.cpp

@@ -11,8 +11,8 @@ SwizzleablePointer::~SwizzleablePointer() {
 
 SwizzleablePointer::SwizzleablePointer(duckdb::MetaBlockReader &reader) {
 	idx_t block_id = reader.Read<block_id_t>();
-	idx_t offset = reader.Read<uint32_t>();
-	if (block_id == DConstants::INVALID_INDEX || offset == DConstants::INVALID_INDEX) {
+	uint32_t offset = reader.Read<uint32_t>();
+	if (block_id == DConstants::INVALID_INDEX || offset == (uint32_t)DConstants::INVALID_INDEX) {
 		pointer = 0;
 		return;
 	}

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.6.2-dev1770"
+#define DUCKDB_VERSION "0.6.2-dev1772"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "864ff1c719"
+#define DUCKDB_SOURCE_ID "d25a77fff4"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/common/file_buffer.hpp

@@ -39,14 +39,8 @@ public:
 public:
 	//! Read into the FileBuffer from the specified location.
 	void Read(FileHandle &handle, uint64_t location);
-	//! Read into the FileBuffer from the specified location. Automatically verifies the checksum, and throws an
-	//! exception if the checksum does not match correctly.
-	virtual void ReadAndChecksum(FileHandle &handle, uint64_t location);
 	//! Write the contents of the FileBuffer to the specified location.
 	void Write(FileHandle &handle, uint64_t location);
-	//! Write the contents of the FileBuffer to the specified location. Automatically adds a checksum of the contents of
-	//! the filebuffer in front of the written data.
-	virtual void ChecksumAndWrite(FileHandle &handle, uint64_t location);
 
 	void Clear();
 
@@ -57,6 +51,9 @@ public:
 	uint64_t AllocSize() const {
 		return internal_size;
 	}
+	data_ptr_t InternalBuffer() {
+		return internal_buffer;
+	}
 
 	struct MemoryRequirement {
 		idx_t alloc_size;
@@ -72,16 +69,6 @@ protected:
 	uint64_t internal_size;
 
 	void ReallocBuffer(size_t malloc_size);
-
-private:
-	//! The buffer that was actually malloc'd, i.e. the pointer that must be freed when the FileBuffer is destroyed
-	data_ptr_t malloced_buffer;
-	uint64_t malloced_size;
-
-protected:
-	uint64_t GetMallocedSize() {
-		return malloced_size;
-	}
 	void Init();
 };
 

package/src/duckdb/src/include/duckdb/storage/buffer_manager.hpp

@@ -85,10 +85,8 @@ public:
 	}
 
 	//! Construct a managed buffer.
-	//! The block_id is just used for internal tracking. It doesn't map to any actual
-	//! BlockManager.
-	virtual unique_ptr<FileBuffer> ConstructManagedBuffer(idx_t size, unique_ptr<FileBuffer> &&source,
-	                                                      FileBufferType type = FileBufferType::MANAGED_BUFFER);
+	unique_ptr<FileBuffer> ConstructManagedBuffer(idx_t size, unique_ptr<FileBuffer> &&source,
+	                                              FileBufferType type = FileBufferType::MANAGED_BUFFER);
 
 	DUCKDB_API void ReserveMemory(idx_t size);
 	DUCKDB_API void FreeReservedMemory(idx_t size);

package/src/duckdb/src/include/duckdb/storage/index.hpp

@@ -118,6 +118,7 @@ public:
 	}
 	//! Serializes the index and returns the pair of block_id offset positions
 	virtual BlockPointer Serialize(duckdb::MetaBlockWriter &writer);
+	BlockPointer GetBlockPointer();
 
 	//! Returns block/offset of where index was most recently serialized.
 	BlockPointer GetSerializedDataPointer() const {

package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp

@@ -59,6 +59,9 @@ private:
 
 	void Initialize(DatabaseHeader &header);
 
+	void ReadAndChecksum(FileBuffer &handle, uint64_t location) const;
+	void ChecksumAndWrite(FileBuffer &handle, uint64_t location) const;
+
 	//! Return the blocks to which we will write the free list and modified blocks
 	vector<block_id_t> GetFreeListBlocks();
 

package/src/duckdb/src/include/duckdb/storage/write_ahead_log.hpp

@@ -12,11 +12,11 @@
 #include "duckdb/common/types/data_chunk.hpp"
 #include "duckdb/common/enums/wal_type.hpp"
 #include "duckdb/common/serializer/buffered_file_writer.hpp"
-#include "duckdb/catalog/catalog_entry/sequence_catalog_entry.hpp"
-#include "duckdb/storage/storage_info.hpp"
-
 #include "duckdb/catalog/catalog_entry/scalar_macro_catalog_entry.hpp"
+#include "duckdb/catalog/catalog_entry/sequence_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/table_macro_catalog_entry.hpp"
+#include "duckdb/main/attached_database.hpp"
+#include "duckdb/storage/storage_info.hpp"
 
 namespace duckdb {
 
@@ -38,8 +38,8 @@ class TransactionManager;
 class ReplayState {
 public:
 	ReplayState(AttachedDatabase &db, ClientContext &context, Deserializer &source)
-	    : db(db), context(context), catalog(Catalog::GetCatalog(context, INVALID_CATALOG)), source(source),
-	      current_table(nullptr), deserialize_only(false), checkpoint_id(INVALID_BLOCK) {
+	    : db(db), context(context), catalog(db.GetCatalog()), source(source), current_table(nullptr),
+	      deserialize_only(false), checkpoint_id(INVALID_BLOCK) {
 	}
 
 	AttachedDatabase &db;

package/src/duckdb/src/storage/block.cpp

@@ -9,11 +9,11 @@ Block::Block(Allocator &allocator, block_id_t id)
 
 Block::Block(Allocator &allocator, block_id_t id, uint32_t internal_size)
     : FileBuffer(allocator, FileBufferType::BLOCK, internal_size), id(id) {
-	D_ASSERT((GetMallocedSize() & (Storage::SECTOR_SIZE - 1)) == 0);
+	D_ASSERT((AllocSize() & (Storage::SECTOR_SIZE - 1)) == 0);
 }
 
 Block::Block(FileBuffer &source, block_id_t id) : FileBuffer(source, FileBufferType::BLOCK), id(id) {
-	D_ASSERT((GetMallocedSize() & (Storage::SECTOR_SIZE - 1)) == 0);
+	D_ASSERT((AllocSize() & (Storage::SECTOR_SIZE - 1)) == 0);
 }
 
 } // namespace duckdb

package/src/duckdb/src/storage/single_file_block_manager.cpp

@@ -1,6 +1,7 @@
 #include "duckdb/storage/single_file_block_manager.hpp"
 
 #include "duckdb/common/allocator.hpp"
+#include "duckdb/common/checksum.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/serializer/buffered_deserializer.hpp"
 #include "duckdb/common/serializer/buffered_serializer.hpp"
@@ -142,7 +143,7 @@ SingleFileBlockManager::SingleFileBlockManager(AttachedDatabase &db, string path
 
 		SerializeHeaderStructure<MainHeader>(main_header, header_buffer.buffer);
 		// now write the header to the file
-		header_buffer.ChecksumAndWrite(*handle, 0);
+		ChecksumAndWrite(header_buffer, 0);
 		header_buffer.Clear();
 
 		// write the database headers
@@ -155,14 +156,14 @@ SingleFileBlockManager::SingleFileBlockManager(AttachedDatabase &db, string path
 		h1.free_list = INVALID_BLOCK;
 		h1.block_count = 0;
 		SerializeHeaderStructure<DatabaseHeader>(h1, header_buffer.buffer);
-		header_buffer.ChecksumAndWrite(*handle, Storage::FILE_HEADER_SIZE);
+		ChecksumAndWrite(header_buffer, Storage::FILE_HEADER_SIZE);
 		// header 2
 		h2.iteration = 0;
 		h2.meta_block = INVALID_BLOCK;
 		h2.free_list = INVALID_BLOCK;
 		h2.block_count = 0;
 		SerializeHeaderStructure<DatabaseHeader>(h2, header_buffer.buffer);
-		header_buffer.ChecksumAndWrite(*handle, Storage::FILE_HEADER_SIZE * 2);
+		ChecksumAndWrite(header_buffer, Storage::FILE_HEADER_SIZE * 2);
 		// ensure that writing to disk is completed before returning
 		handle->Sync();
 		// we start with h2 as active_header, this way our initial write will be in h1
@@ -172,14 +173,14 @@ SingleFileBlockManager::SingleFileBlockManager(AttachedDatabase &db, string path
 	} else {
 		MainHeader::CheckMagicBytes(*handle);
 		// otherwise, we check the metadata of the file
-		header_buffer.ReadAndChecksum(*handle, 0);
+		ReadAndChecksum(header_buffer, 0);
 		DeserializeHeaderStructure<MainHeader>(header_buffer.buffer);
 
 		// read the database headers from disk
 		DatabaseHeader h1, h2;
-		header_buffer.ReadAndChecksum(*handle, Storage::FILE_HEADER_SIZE);
+		ReadAndChecksum(header_buffer, Storage::FILE_HEADER_SIZE);
 		h1 = DeserializeHeaderStructure<DatabaseHeader>(header_buffer.buffer);
-		header_buffer.ReadAndChecksum(*handle, Storage::FILE_HEADER_SIZE * 2);
+		ReadAndChecksum(header_buffer, Storage::FILE_HEADER_SIZE * 2);
 		h2 = DeserializeHeaderStructure<DatabaseHeader>(header_buffer.buffer);
 		// check the header with the highest iteration count
 		if (h1.iteration > h2.iteration) {
@@ -195,6 +196,27 @@ SingleFileBlockManager::SingleFileBlockManager(AttachedDatabase &db, string path
 	}
 }
 
+void SingleFileBlockManager::ReadAndChecksum(FileBuffer &block, uint64_t location) const {
+	// read the buffer from disk
+	block.Read(*handle, location);
+	// compute the checksum
+	auto stored_checksum = Load<uint64_t>(block.InternalBuffer());
+	uint64_t computed_checksum = Checksum(block.buffer, block.size);
+	// verify the checksum
+	if (stored_checksum != computed_checksum) {
+		throw IOException("Corrupt database file: computed checksum %llu does not match stored checksum %llu in block",
+		                  computed_checksum, stored_checksum);
+	}
+}
+
+void SingleFileBlockManager::ChecksumAndWrite(FileBuffer &block, uint64_t location) const {
+	// compute the checksum and write it to the start of the buffer (if not temp buffer)
+	uint64_t checksum = Checksum(block.buffer, block.size);
+	Store<uint64_t>(checksum, block.InternalBuffer());
+	// now write the buffer
+	block.Write(*handle, location);
+}
+
 void SingleFileBlockManager::Initialize(DatabaseHeader &header) {
 	free_list_id = header.free_list;
 	meta_block = header.meta_block;
@@ -317,12 +339,12 @@ unique_ptr<Block> SingleFileBlockManager::CreateBlock(block_id_t block_id, FileB
 void SingleFileBlockManager::Read(Block &block) {
 	D_ASSERT(block.id >= 0);
 	D_ASSERT(std::find(free_list.begin(), free_list.end(), block.id) == free_list.end());
-	block.ReadAndChecksum(*handle, BLOCK_START + block.id * Storage::BLOCK_ALLOC_SIZE);
+	ReadAndChecksum(block, BLOCK_START + block.id * Storage::BLOCK_ALLOC_SIZE);
 }
 
 void SingleFileBlockManager::Write(FileBuffer &buffer, block_id_t block_id) {
 	D_ASSERT(block_id >= 0);
-	buffer.ChecksumAndWrite(*handle, BLOCK_START + block_id * Storage::BLOCK_ALLOC_SIZE);
+	ChecksumAndWrite(buffer, BLOCK_START + block_id * Storage::BLOCK_ALLOC_SIZE);
 }
 
 vector<block_id_t> SingleFileBlockManager::GetFreeListBlocks() {
@@ -431,8 +453,7 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
 	Store<DatabaseHeader>(header, header_buffer.buffer);
 	// now write the header to the file, active_header determines whether we write to h1 or h2
 	// note that if active_header is h1 we write to h2, and vice versa
-	header_buffer.ChecksumAndWrite(*handle,
-	                               active_header == 1 ? Storage::FILE_HEADER_SIZE : Storage::FILE_HEADER_SIZE * 2);
+	ChecksumAndWrite(header_buffer, active_header == 1 ? Storage::FILE_HEADER_SIZE : Storage::FILE_HEADER_SIZE * 2);
 	// switch active header to the other header
 	active_header = 1 - active_header;
 	//! Ensure the header write ends up on disk