duckdb 0.5.2-dev1229.0 → 0.5.2-dev1241.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.5.2-dev1229.0",
+  "version": "0.5.2-dev1241.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -6538,6 +6538,7 @@ void AllocatedData::Reset() {
 	}
 	D_ASSERT(allocator);
 	allocator->FreeData(pointer, allocated_size);
+	allocated_size = 0;
 	pointer = nullptr;
 }
 
@@ -15982,23 +15983,27 @@ void FileBuffer::ReallocBuffer(size_t new_size) {
 }
 
 void FileBuffer::Resize(uint64_t new_size) {
+	idx_t header_size = Storage::BLOCK_HEADER_SIZE;
 	{
 		// TODO: All the logic here is specific to SingleFileBlockManager.
 		// and should be moved there, via a specific implementation of FileBuffer.
 		//
 		// make room for the block header (if this is not the db file header)
-		if (type == FileBufferType::MANAGED_BUFFER && new_size != Storage::FILE_HEADER_SIZE) {
+		if (type == FileBufferType::TINY_BUFFER) {
+			header_size = 0;
+		}
+		if (type == FileBufferType::MANAGED_BUFFER) {
 			new_size += Storage::BLOCK_HEADER_SIZE;
-			// If we don't write/read an entire block, our checksum won't match.
-			new_size = AlignValue<uint32_t, Storage::BLOCK_ALLOC_SIZE>(new_size);
 		}
-		new_size = AlignValue<uint32_t, Storage::SECTOR_SIZE>(new_size);
+		if (type != FileBufferType::TINY_BUFFER) {
+			new_size = AlignValue<uint32_t, Storage::SECTOR_SIZE>(new_size);
+		}
 		ReallocBuffer(new_size);
 	}
 
 	if (new_size > 0) {
-		buffer = internal_buffer + Storage::BLOCK_HEADER_SIZE;
-		size = internal_size - Storage::BLOCK_HEADER_SIZE;
+		buffer = internal_buffer + header_size;
+		size = internal_size - header_size;
 	}
 }
 
@@ -78786,7 +78791,7 @@ public:
 	void CreateNewCollection(TableCatalogEntry *table, const vector<LogicalType> &insert_types) {
 		auto &table_info = table->storage->info;
 		auto &block_manager = TableIOManager::Get(*table->storage).GetBlockManagerForRowData();
-		current_collection = make_unique<RowGroupCollection>(table_info, block_manager, insert_types, 0);
+		current_collection = make_unique<RowGroupCollection>(table_info, block_manager, insert_types, MAX_ROW_ID);
 		current_collection->InitializeEmpty();
 		current_collection->InitializeAppend(current_append_state);
 		written_to_disk = false;
@@ -78888,7 +78893,7 @@ struct CollectionMerger {
 			auto &table_info = storage.info;
 			auto &block_manager = TableIOManager::Get(storage).GetBlockManagerForRowData();
 			auto types = storage.GetTypes();
-			new_collection = make_unique<RowGroupCollection>(table_info, block_manager, types, 0);
+			new_collection = make_unique<RowGroupCollection>(table_info, block_manager, types, MAX_ROW_ID);
 			TableAppendState append_state;
 			new_collection->InitializeEmpty();
 			new_collection->InitializeAppend(append_state);
@@ -79709,7 +79714,8 @@ SinkResultType PhysicalInsert::Sink(ExecutionContext &context, GlobalSinkState &
 		if (!lstate.local_collection) {
 			auto &table_info = table->storage->info;
 			auto &block_manager = TableIOManager::Get(*table->storage).GetBlockManagerForRowData();
-			lstate.local_collection = make_unique<RowGroupCollection>(table_info, block_manager, insert_types, 0);
+			lstate.local_collection =
+			    make_unique<RowGroupCollection>(table_info, block_manager, insert_types, MAX_ROW_ID);
 			lstate.local_collection->InitializeEmpty();
 			lstate.local_collection->InitializeAppend(lstate.local_append_state);
 			lstate.writer = make_unique<OptimisticDataWriter>(gstate.table->storage.get());
@@ -196037,10 +196043,9 @@ BlockHandle::BlockHandle(BlockManager &block_manager, block_id_t block_id_p, uni
                          bool can_destroy_p, idx_t block_size)
     : block_manager(block_manager), readers(0), block_id(block_id_p), eviction_timestamp(0), can_destroy(can_destroy_p),
       unswizzled(nullptr) {
-	D_ASSERT(block_size >= Storage::BLOCK_SIZE);
 	buffer = move(buffer_p);
 	state = BlockState::BLOCK_LOADED;
-	memory_usage = block_size + Storage::BLOCK_HEADER_SIZE;
+	memory_usage = buffer->type == FileBufferType::TINY_BUFFER ? block_size : block_size + Storage::BLOCK_HEADER_SIZE;
 }
 
 BlockHandle::~BlockHandle() {
@@ -196048,9 +196053,10 @@ BlockHandle::~BlockHandle() {
 	unswizzled = nullptr;
 	auto &buffer_manager = block_manager.buffer_manager;
 	// no references remain to this block: erase
-	if (state == BlockState::BLOCK_LOADED) {
+	if (buffer && state == BlockState::BLOCK_LOADED) {
 		// the block is still loaded in memory: erase it
 		buffer.reset();
+		D_ASSERT(buffer_manager.current_memory >= memory_usage);
 		buffer_manager.current_memory -= memory_usage;
 	}
 	block_manager.UnregisterBlock(block_id, can_destroy);
@@ -196075,14 +196081,15 @@ unique_ptr<Block> AllocateBlock(BlockManager &block_manager, unique_ptr<FileBuff
 	}
 }
 
-unique_ptr<FileBuffer> BufferManager::ConstructManagedBuffer(idx_t size, unique_ptr<FileBuffer> &&source) {
+unique_ptr<FileBuffer> BufferManager::ConstructManagedBuffer(idx_t size, unique_ptr<FileBuffer> &&source,
+                                                             FileBufferType type) {
 	if (source) {
 		auto tmp = move(source);
 		D_ASSERT(tmp->size == size);
-		return make_unique<FileBuffer>(*tmp, FileBufferType::MANAGED_BUFFER);
+		return make_unique<FileBuffer>(*tmp, type);
 	} else {
 		// no re-usable buffer: allocate a new buffer
-		return make_unique<FileBuffer>(Allocator::Get(db), FileBufferType::MANAGED_BUFFER, size);
+		return make_unique<FileBuffer>(Allocator::Get(db), type, size);
 	}
 }
 
@@ -196116,10 +196123,10 @@ unique_ptr<FileBuffer> BlockHandle::UnloadAndTakeBlock() {
 	}
 	D_ASSERT(!unswizzled);
 	D_ASSERT(CanUnload());
-	D_ASSERT(memory_usage >= Storage::BLOCK_ALLOC_SIZE);
 
 	if (block_id >= MAXIMUM_BLOCK && !can_destroy) {
 		// temporary block that cannot be destroyed: write to temporary file
+		D_ASSERT(memory_usage >= Storage::BLOCK_ALLOC_SIZE);
 		block_manager.buffer_manager.WriteTemporaryBuffer(block_id, *buffer);
 	}
 	block_manager.buffer_manager.current_memory -= memory_usage;
@@ -196277,8 +196284,20 @@ shared_ptr<BlockHandle> BlockManager::ConvertToPersistent(block_id_t block_id, s
 	return new_block;
 }
 
+shared_ptr<BlockHandle> BufferManager::RegisterSmallMemory(idx_t block_size) {
+	if (!EvictBlocks(block_size, maximum_memory, nullptr)) {
+		throw OutOfMemoryException("could not allocate block of %lld bytes (%lld/%lld used) %s", block_size,
+		                           GetUsedMemory(), GetMaxMemory(), InMemoryWarning());
+	}
+	auto buffer = ConstructManagedBuffer(block_size, nullptr, FileBufferType::TINY_BUFFER);
+
+	// create a new block pointer for this block
+	return make_shared<BlockHandle>(*temp_block_manager, ++temporary_id, move(buffer), false, block_size);
+}
+
 shared_ptr<BlockHandle> BufferManager::RegisterMemory(idx_t block_size, bool can_destroy) {
-	auto alloc_size = AlignValue<idx_t, 1 << 12>(block_size + Storage::BLOCK_HEADER_SIZE);
+	D_ASSERT(block_size >= Storage::BLOCK_SIZE);
+	auto alloc_size = AlignValue<idx_t, Storage::SECTOR_SIZE>(block_size + Storage::BLOCK_HEADER_SIZE);
 	// first evict blocks until we have enough memory to store this buffer
 	unique_ptr<FileBuffer> reusable_buffer;
 	if (!EvictBlocks(alloc_size, maximum_memory, &reusable_buffer)) {
@@ -196313,6 +196332,7 @@ void BufferManager::ReAllocate(shared_ptr<BlockHandle> &handle, idx_t block_size
 		}
 	} else {
 		// no need to evict blocks
+		D_ASSERT(current_memory >= idx_t(-required_memory));
 		current_memory -= idx_t(-required_memory);
 	}
 
@@ -196334,6 +196354,7 @@ BufferHandle BufferManager::Pin(shared_ptr<BlockHandle> &handle) {
 		}
 		required_memory = handle->memory_usage;
 	}
+	D_ASSERT(required_memory >= Storage::BLOCK_SIZE);
 	// evict blocks until we have space for the current block
 	unique_ptr<FileBuffer> reusable_buffer;
 	if (!EvictBlocks(required_memory, maximum_memory, &reusable_buffer)) {
@@ -196345,6 +196366,7 @@ BufferHandle BufferManager::Pin(shared_ptr<BlockHandle> &handle) {
 	if (handle->state == BlockState::BLOCK_LOADED) {
 		// the block is loaded, increment the reader count and return a pointer to the handle
 		handle->readers++;
+		D_ASSERT(current_memory >= required_memory);
 		current_memory -= required_memory;
 		return handle->Load(handle);
 	}
@@ -196363,6 +196385,9 @@ void BufferManager::AddToEvictionQueue(shared_ptr<BlockHandle> &handle) {
 
 void BufferManager::Unpin(shared_ptr<BlockHandle> &handle) {
 	lock_guard<mutex> lock(handle->lock);
+	if (!handle->buffer || handle->buffer->type == FileBufferType::TINY_BUFFER) {
+		return;
+	}
 	D_ASSERT(handle->readers > 0);
 	handle->readers--;
 	if (handle->readers == 0) {
@@ -196378,6 +196403,7 @@ bool BufferManager::EvictBlocks(idx_t extra_memory, idx_t memory_limit, unique_p
 	while (current_memory > memory_limit) {
 		// get a block to unpin from the queue
 		if (!queue->q.try_dequeue(node)) {
+			D_ASSERT(current_memory >= extra_memory);
 			current_memory -= extra_memory;
 			return false;
 		}
@@ -196797,10 +196823,9 @@ void BufferManager::WriteTemporaryBuffer(block_id_t block_id, FileBuffer &buffer
 		temp_directory_handle->GetTempFile().WriteTemporaryBuffer(block_id, buffer);
 		return;
 	}
-
-	D_ASSERT(buffer.size > Storage::BLOCK_SIZE);
 	// get the path to write to
 	auto path = GetTemporaryPath(block_id);
+	D_ASSERT(buffer.size > Storage::BLOCK_SIZE);
 	// create the file and write the size followed by the buffer contents
 	auto &fs = FileSystem::GetFileSystem(db);
 	auto handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE);
@@ -196876,6 +196901,7 @@ data_ptr_t BufferManager::BufferAllocatorAllocate(PrivateAllocatorData *private_
 
 void BufferManager::BufferAllocatorFree(PrivateAllocatorData *private_data, data_ptr_t pointer, idx_t size) {
 	auto &data = (BufferAllocatorData &)*private_data;
+	D_ASSERT(data.manager.current_memory >= size);
 	data.manager.current_memory -= size;
 	return Allocator::Get(data.manager.db).FreeData(pointer, size);
 }
@@ -197534,7 +197560,6 @@ WriteOverflowStringsToDisk::WriteOverflowStringsToDisk(BlockManager &block_manag
 
 WriteOverflowStringsToDisk::~WriteOverflowStringsToDisk() {
 	if (offset > 0) {
-		handle.GetFileBuffer().Resize(offset);
 		block_manager.Write(handle.GetFileBuffer(), block_id);
 	}
 }
@@ -203289,7 +203314,7 @@ idx_t FixedSizeAppend(CompressionAppendState &append_state, ColumnSegment &segme
 	D_ASSERT(segment.GetBlockOffset() == 0);
 
 	auto target_ptr = append_state.handle.Ptr();
-	idx_t max_tuple_count = Storage::BLOCK_SIZE / sizeof(T);
+	idx_t max_tuple_count = segment.SegmentSize() / sizeof(T);
 	idx_t copy_count = MinValue<idx_t>(count, max_tuple_count - segment.count);
 
 	AppendLoop<T>(stats, target_ptr, segment.count, data, offset, copy_count);
@@ -204802,7 +204827,7 @@ unique_ptr<CompressedSegmentState> UncompressedStringStorage::StringInitSegment(
 		auto handle = buffer_manager.Pin(segment.block);
 		StringDictionaryContainer dictionary;
 		dictionary.size = 0;
-		dictionary.end = Storage::BLOCK_SIZE;
+		dictionary.end = segment.SegmentSize();
 		SetDictionary(segment, handle, dictionary);
 	}
 	return make_unique<UncompressedStringSegmentState>();
@@ -204812,16 +204837,16 @@ idx_t UncompressedStringStorage::FinalizeAppend(ColumnSegment &segment, SegmentS
 	auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
 	auto handle = buffer_manager.Pin(segment.block);
 	auto dict = GetDictionary(segment, handle);
-	D_ASSERT(dict.end == Storage::BLOCK_SIZE);
+	D_ASSERT(dict.end == segment.SegmentSize());
 	// compute the total size required to store this segment
 	auto offset_size = DICTIONARY_HEADER_SIZE + segment.count * sizeof(int32_t);
 	auto total_size = offset_size + dict.size;
 	if (total_size >= COMPACTION_FLUSH_LIMIT) {
 		// the block is full enough, don't bother moving around the dictionary
-		return Storage::BLOCK_SIZE;
+		return segment.SegmentSize();
 	}
 	// the block has space left: figure out how much space we can save
-	auto move_amount = Storage::BLOCK_SIZE - total_size;
+	auto move_amount = segment.SegmentSize() - total_size;
 	// move the dictionary so it lines up exactly with the offsets
 	auto dataptr = handle.Ptr();
 	memmove(dataptr + offset_size, dataptr + dict.end - dict.size, dict.size);
@@ -204868,10 +204893,10 @@ StringDictionaryContainer UncompressedStringStorage::GetDictionary(ColumnSegment
 
 idx_t UncompressedStringStorage::RemainingSpace(ColumnSegment &segment, BufferHandle &handle) {
 	auto dictionary = GetDictionary(segment, handle);
-	D_ASSERT(dictionary.end == Storage::BLOCK_SIZE);
+	D_ASSERT(dictionary.end == segment.SegmentSize());
 	idx_t used_space = dictionary.size + segment.count * sizeof(int32_t) + DICTIONARY_HEADER_SIZE;
-	D_ASSERT(Storage::BLOCK_SIZE >= used_space);
-	return Storage::BLOCK_SIZE - used_space;
+	D_ASSERT(segment.SegmentSize() >= used_space);
+	return segment.SegmentSize() - used_space;
 }
 
 void UncompressedStringStorage::WriteString(ColumnSegment &segment, string_t string, block_id_t &result_block,
@@ -205503,7 +205528,7 @@ unique_ptr<CompressedSegmentState> ValidityInitSegment(ColumnSegment &segment, b
 	auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
 	if (block_id == INVALID_BLOCK) {
 		auto handle = buffer_manager.Pin(segment.block);
-		memset(handle.Ptr(), 0xFF, Storage::BLOCK_SIZE);
+		memset(handle.Ptr(), 0xFF, segment.SegmentSize());
 	}
 	return nullptr;
 }
@@ -205513,7 +205538,7 @@ idx_t ValidityAppend(CompressionAppendState &append_state, ColumnSegment &segmen
 	D_ASSERT(segment.GetBlockOffset() == 0);
 	auto &validity_stats = (ValidityStatistics &)*stats.statistics;
 
-	auto max_tuples = Storage::BLOCK_SIZE / ValidityMask::STANDARD_MASK_SIZE * STANDARD_VECTOR_SIZE;
+	auto max_tuples = segment.SegmentSize() / ValidityMask::STANDARD_MASK_SIZE * STANDARD_VECTOR_SIZE;
 	idx_t append_count = MinValue<idx_t>(vcount, max_tuples - segment.count);
 	if (data.validity.AllValid()) {
 		// no null values: skip append
@@ -205560,7 +205585,7 @@ void ValidityRevertAppend(ColumnSegment &segment, idx_t start_row) {
 		revert_start = start_bit / 8;
 	}
 	// for the rest, we just memset
-	memset(handle.Ptr() + revert_start, 0xFF, Storage::BLOCK_SIZE - revert_start);
+	memset(handle.Ptr() + revert_start, 0xFF, segment.SegmentSize() - revert_start);
 }
 
 //===--------------------------------------------------------------------===//
@@ -207713,8 +207738,9 @@ T DeserializeHeaderStructure(data_ptr_t ptr) {
 SingleFileBlockManager::SingleFileBlockManager(DatabaseInstance &db, string path_p, bool read_only, bool create_new,
                                                bool use_direct_io)
     : BlockManager(BufferManager::GetBufferManager(db)), db(db), path(move(path_p)),
-      header_buffer(Allocator::Get(db), FileBufferType::MANAGED_BUFFER, Storage::FILE_HEADER_SIZE), iteration_count(0),
-      read_only(read_only), use_direct_io(use_direct_io) {
+      header_buffer(Allocator::Get(db), FileBufferType::MANAGED_BUFFER,
+                    Storage::FILE_HEADER_SIZE - Storage::BLOCK_HEADER_SIZE),
+      iteration_count(0), read_only(read_only), use_direct_io(use_direct_io) {
 	uint8_t flags;
 	FileLockType lock;
 	if (read_only) {
@@ -209893,6 +209919,12 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 			pstate->AddSegmentToTail(&column_data, segment.get(), offset_in_block);
 		} else {
 			// Create a new block for future reuse.
+			if (segment->SegmentSize() != Storage::BLOCK_SIZE) {
+				// the segment is smaller than the block size
+				// allocate a new block and copy the data over
+				D_ASSERT(segment->SegmentSize() < Storage::BLOCK_SIZE);
+				segment->Resize(Storage::BLOCK_SIZE);
+			}
 			D_ASSERT(offset_in_block == 0);
 			allocation.partial_block = make_unique<PartialBlockForCheckpoint>(
 			    &column_data, segment.get(), *allocation.block_manager, allocation.state);
@@ -210454,7 +210486,15 @@ unique_ptr<BaseStatistics> ColumnData::GetUpdateStatistics() {
 }
 
 void ColumnData::AppendTransientSegment(SegmentLock &l, idx_t start_row) {
-	auto new_segment = ColumnSegment::CreateTransientSegment(GetDatabase(), type, start_row);
+	idx_t segment_size = Storage::BLOCK_SIZE;
+	if (start_row == idx_t(MAX_ROW_ID)) {
+#if STANDARD_VECTOR_SIZE < 1024
+		segment_size = 1024 * GetTypeIdSize(type.InternalType());
+#else
+		segment_size = STANDARD_VECTOR_SIZE * GetTypeIdSize(type.InternalType());
+#endif
+	}
+	auto new_segment = ColumnSegment::CreateTransientSegment(GetDatabase(), type, start_row, segment_size);
 	data.AppendSegment(l, move(new_segment));
 }
 
@@ -210952,18 +210992,25 @@ unique_ptr<ColumnSegment> ColumnSegment::CreatePersistentSegment(DatabaseInstanc
 		function = config.GetCompressionFunction(compression_type, type.InternalType());
 		block = block_manager.RegisterBlock(block_id);
 	}
+	auto segment_size = Storage::BLOCK_SIZE;
 	return make_unique<ColumnSegment>(db, block, type, ColumnSegmentType::PERSISTENT, start, count, function,
-	                                  move(statistics), block_id, offset);
+	                                  move(statistics), block_id, offset, segment_size);
 }
 
 unique_ptr<ColumnSegment> ColumnSegment::CreateTransientSegment(DatabaseInstance &db, const LogicalType &type,
-                                                                idx_t start) {
+                                                                idx_t start, idx_t segment_size) {
 	auto &config = DBConfig::GetConfig(db);
 	auto function = config.GetCompressionFunction(CompressionType::COMPRESSION_UNCOMPRESSED, type.InternalType());
+	auto &buffer_manager = BufferManager::GetBufferManager(db);
+	shared_ptr<BlockHandle> block;
 	// transient: allocate a buffer for the uncompressed segment
-	auto block = BufferManager::GetBufferManager(db).RegisterMemory(Storage::BLOCK_SIZE, false);
+	if (segment_size < Storage::BLOCK_SIZE) {
+		block = buffer_manager.RegisterSmallMemory(segment_size);
+	} else {
+		block = buffer_manager.RegisterMemory(segment_size, false);
+	}
 	return make_unique<ColumnSegment>(db, block, type, ColumnSegmentType::TRANSIENT, start, 0, function, nullptr,
-	                                  INVALID_BLOCK, 0);
+	                                  INVALID_BLOCK, 0, segment_size);
 }
 
 unique_ptr<ColumnSegment> ColumnSegment::CreateSegment(ColumnSegment &other, idx_t start) {
@@ -210972,10 +211019,11 @@ unique_ptr<ColumnSegment> ColumnSegment::CreateSegment(ColumnSegment &other, idx
 
 ColumnSegment::ColumnSegment(DatabaseInstance &db, shared_ptr<BlockHandle> block, LogicalType type_p,
                              ColumnSegmentType segment_type, idx_t start, idx_t count, CompressionFunction *function_p,
-                             unique_ptr<BaseStatistics> statistics, block_id_t block_id_p, idx_t offset_p)
+                             unique_ptr<BaseStatistics> statistics, block_id_t block_id_p, idx_t offset_p,
+                             idx_t segment_size_p)
     : SegmentBase(start, count), db(db), type(move(type_p)), type_size(GetTypeIdSize(type.InternalType())),
       segment_type(segment_type), function(function_p), stats(type, move(statistics)), block(move(block)),
-      block_id(block_id_p), offset(offset_p) {
+      block_id(block_id_p), offset(offset_p), segment_size(segment_size_p) {
 	D_ASSERT(function);
 	if (function->init_segment) {
 		segment_state = function->init_segment(*this, block_id);
@@ -210985,7 +211033,8 @@ ColumnSegment::ColumnSegment(DatabaseInstance &db, shared_ptr<BlockHandle> block
 ColumnSegment::ColumnSegment(ColumnSegment &other, idx_t start)
     : SegmentBase(start, other.count), db(other.db), type(move(other.type)), type_size(other.type_size),
       segment_type(other.segment_type), function(other.function), stats(move(other.stats)), block(move(other.block)),
-      block_id(other.block_id), offset(other.offset), segment_state(move(other.segment_state)) {
+      block_id(other.block_id), offset(other.offset), segment_size(other.segment_size),
+      segment_state(move(other.segment_state)) {
 }
 
 ColumnSegment::~ColumnSegment() {
@@ -211033,6 +211082,23 @@ void ColumnSegment::FetchRow(ColumnFetchState &state, row_t row_id, Vector &resu
 //===--------------------------------------------------------------------===//
 // Append
 //===--------------------------------------------------------------------===//
+idx_t ColumnSegment::SegmentSize() const {
+	return segment_size;
+}
+
+void ColumnSegment::Resize(idx_t new_size) {
+	D_ASSERT(new_size > this->segment_size);
+	D_ASSERT(offset == 0);
+	auto &buffer_manager = BufferManager::GetBufferManager(db);
+	auto new_block = buffer_manager.RegisterMemory(Storage::BLOCK_SIZE, false);
+	auto old_handle = buffer_manager.Pin(block);
+	auto new_handle = buffer_manager.Pin(new_block);
+	memcpy(new_handle.Ptr(), old_handle.Ptr(), segment_size);
+	this->block_id = new_block->BlockId();
+	this->block = move(new_block);
+	this->segment_size = new_size;
+}
+
 void ColumnSegment::InitializeAppend(ColumnAppendState &state) {
 	D_ASSERT(segment_type == ColumnSegmentType::TRANSIENT);
 	if (!function->init_append) {

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "e4ee601c8"
-#define DUCKDB_VERSION "v0.5.2-dev1229"
+#define DUCKDB_SOURCE_ID "ed8eafda9"
+#define DUCKDB_VERSION "v0.5.2-dev1241"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -1942,7 +1942,7 @@ namespace duckdb {
 class Allocator;
 struct FileHandle;
 
-enum class FileBufferType : uint8_t { BLOCK = 1, MANAGED_BUFFER = 2 };
+enum class FileBufferType : uint8_t { BLOCK = 1, MANAGED_BUFFER = 2, TINY_BUFFER = 3 };
 
 //! The FileBuffer represents a buffer that can be read or written to a Direct IO FileHandle.
 class FileBuffer {
@@ -25307,12 +25307,16 @@ class BufferManager {
 
 public:
 	BufferManager(DatabaseInstance &db, string temp_directory, idx_t maximum_memory);
-	~BufferManager();
+	virtual ~BufferManager();
 
 	//! Register an in-memory buffer of arbitrary size, as long as it is >= BLOCK_SIZE. can_destroy signifies whether or
 	//! not the buffer can be destroyed when unpinned, or whether or not it needs to be written to a temporary file so
 	//! it can be reloaded. The resulting buffer will already be allocated, but needs to be pinned in order to be used.
 	shared_ptr<BlockHandle> RegisterMemory(idx_t block_size, bool can_destroy);
+	//! Registers an in-memory buffer that cannot be unloaded until it is destroyed
+	//! This buffer can be small (smaller than BLOCK_SIZE)
+	//! Unpin and pin are nops on this block of memory
+	shared_ptr<BlockHandle> RegisterSmallMemory(idx_t block_size);
 
 	//! Allocate an in-memory buffer with a single pin.
 	//! The allocated memory is released when the buffer handle is destroyed.
@@ -25353,7 +25357,8 @@ public:
 	//! Construct a managed buffer.
 	//! The block_id is just used for internal tracking. It doesn't map to any actual
 	//! BlockManager.
-	virtual unique_ptr<FileBuffer> ConstructManagedBuffer(idx_t size, unique_ptr<FileBuffer> &&source);
+	virtual unique_ptr<FileBuffer> ConstructManagedBuffer(idx_t size, unique_ptr<FileBuffer> &&source,
+	                                                      FileBufferType type = FileBufferType::MANAGED_BUFFER);
 
 private:
 	//! Evict blocks until the currently used memory + extra_memory fit, returns false if this was not possible
@@ -26166,7 +26171,8 @@ public:
 	                                                         block_id_t id, idx_t offset, const LogicalType &type_p,
 	                                                         idx_t start, idx_t count, CompressionType compression_type,
 	                                                         unique_ptr<BaseStatistics> statistics);
-	static unique_ptr<ColumnSegment> CreateTransientSegment(DatabaseInstance &db, const LogicalType &type, idx_t start);
+	static unique_ptr<ColumnSegment> CreateTransientSegment(DatabaseInstance &db, const LogicalType &type, idx_t start,
+	                                                        idx_t segment_size = Storage::BLOCK_SIZE);
 	static unique_ptr<ColumnSegment> CreateSegment(ColumnSegment &other, idx_t start);
 
 public:
@@ -26182,6 +26188,11 @@ public:
 	//! Skip a scan forward to the row_index specified in the scan state
 	void Skip(ColumnScanState &state);
 
+	// The maximum size of the buffer (in bytes)
+	idx_t SegmentSize() const;
+	//! Resize the block
+	void Resize(idx_t segment_size);
+
 	//! Initialize an append of this segment. Appends are only supported on transient segments.
 	void InitializeAppend(ColumnAppendState &state);
 	//! Appends a (part of) vector to the segment, returns the amount of entries successfully appended
@@ -26227,7 +26238,7 @@ public:
 public:
 	ColumnSegment(DatabaseInstance &db, shared_ptr<BlockHandle> block, LogicalType type, ColumnSegmentType segment_type,
 	              idx_t start, idx_t count, CompressionFunction *function, unique_ptr<BaseStatistics> statistics,
-	              block_id_t block_id, idx_t offset);
+	              block_id_t block_id, idx_t offset, idx_t segment_size);
 	ColumnSegment(ColumnSegment &other, idx_t start);
 
 private:
@@ -26239,6 +26250,8 @@ private:
 	block_id_t block_id;
 	//! The offset into the block (persistent segment only)
 	idx_t offset;
+	//! The allocated segment size
+	idx_t segment_size;
 	//! Storage associated with the compressed segment
 	unique_ptr<CompressedSegmentState> segment_state;
 };