duckdb 0.8.2-dev4514.0 → 0.8.2-dev4572.0

package/src/duckdb/src/storage/data_pointer.cpp

@@ -0,0 +1,20 @@
+#include "duckdb/storage/data_pointer.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
+#include "duckdb/main/config.hpp"
+#include "duckdb/function/compression_function.hpp"
+
+namespace duckdb {
+
+unique_ptr<ColumnSegmentState> ColumnSegmentState::Deserialize(Deserializer &deserializer) {
+	auto compression_type = deserializer.Get<CompressionType>();
+	auto &db = deserializer.Get<DatabaseInstance &>();
+	auto &type = deserializer.Get<LogicalType &>();
+	auto compression_function = DBConfig::GetConfig(db).GetCompressionFunction(compression_type, type.InternalType());
+	if (!compression_function || !compression_function->deserialize_state) {
+		throw SerializationException("Deserializing a ColumnSegmentState but could not find deserialize method");
+	}
+	return compression_function->deserialize_state(deserializer);
+}
+
+} // namespace duckdb

package/src/duckdb/src/storage/local_storage.cpp

@@ -74,17 +74,13 @@ LocalTableStorage::~LocalTableStorage() {
 
 void LocalTableStorage::InitializeScan(CollectionScanState &state, optional_ptr<TableFilterSet> table_filters) {
 	if (row_groups->GetTotalRows() == 0) {
-		// nothing to scan
-		return;
+		throw InternalException("No rows in LocalTableStorage row group for scan");
 	}
 	row_groups->InitializeScan(state, state.GetColumnIds(), table_filters.get());
 }
 
 idx_t LocalTableStorage::EstimatedSize() {
 	idx_t appended_rows = row_groups->GetTotalRows() - deleted_rows;
-	if (appended_rows == 0) {
-		return 0;
-	}
 	idx_t row_size = 0;
 	auto &types = row_groups->GetTypes();
 	for (auto &type : types) {
@@ -169,10 +165,10 @@ void LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, TableAppen
 			} catch (Exception &ex) {
 				error = PreservedError(ex);
 				return false;
-			} catch (std::exception &ex) {
+			} catch (std::exception &ex) { // LCOV_EXCL_START
 				error = PreservedError(ex);
 				return false;
-			}
+			} // LCOV_EXCL_STOP
 
 			current_row += chunk.size();
 			if (current_row >= append_state.current_row) {

package/src/duckdb/src/storage/metadata/metadata_manager.cpp

@@ -3,6 +3,7 @@
 #include "duckdb/storage/buffer/block_handle.hpp"
 #include "duckdb/common/serializer/write_stream.hpp"
 #include "duckdb/common/serializer/read_stream.hpp"
+#include "duckdb/storage/database_size.hpp"
 
 namespace duckdb {
 
@@ -81,11 +82,13 @@ block_id_t MetadataManager::AllocateNewBlock() {
 	auto new_block_id = GetNextBlockId();
 
 	MetadataBlock new_block;
-	buffer_manager.Allocate(Storage::BLOCK_SIZE, false, &new_block.block);
+	auto handle = buffer_manager.Allocate(Storage::BLOCK_SIZE, false, &new_block.block);
 	new_block.block_id = new_block_id;
 	for (idx_t i = 0; i < METADATA_BLOCK_COUNT; i++) {
 		new_block.free_blocks.push_back(METADATA_BLOCK_COUNT - i - 1);
 	}
+	// zero-initialize the handle
+	memset(handle.Ptr(), 0, Storage::BLOCK_SIZE);
 	AddBlock(std::move(new_block));
 	return new_block_id;
 }
@@ -176,11 +179,6 @@ void MetadataManager::Flush() {
 	for (auto &kv : blocks) {
 		auto &block = kv.second;
 		auto handle = buffer_manager.Pin(block.block);
-		// zero-initialize any free blocks
-		for (auto free_block : block.free_blocks) {
-			memset(handle.Ptr() + free_block * MetadataManager::METADATA_BLOCK_SIZE, 0,
-			       MetadataManager::METADATA_BLOCK_SIZE);
-		}
 		// there are a few bytes left-over at the end of the block, zero-initialize them
 		memset(handle.Ptr() + total_metadata_size, 0, Storage::BLOCK_SIZE - total_metadata_size);
 		D_ASSERT(kv.first == block.block_id);
@@ -255,7 +253,6 @@ void MetadataBlock::FreeBlocksFromInteger(idx_t free_list) {
 }
 
 void MetadataManager::MarkBlocksAsModified() {
-
 	// for any blocks that were modified in the last checkpoint - set them to free blocks currently
 	for (auto &kv : modified_blocks) {
 		auto block_id = kv.first;
@@ -266,14 +263,14 @@ void MetadataManager::MarkBlocksAsModified() {
 		idx_t current_free_blocks = block.FreeBlocksToInteger();
 		// merge the current set of free blocks with the modified blocks
 		idx_t new_free_blocks = current_free_blocks | modified_list;
-		//			if (new_free_blocks == NumericLimits<idx_t>::Maximum()) {
-		//				// if new free_blocks is all blocks - mark entire block as modified
-		//				blocks.erase(entry);
-		//				block_manager.MarkBlockAsModified(block_id);
-		//			} else {
-		// set the new set of free blocks
-		block.FreeBlocksFromInteger(new_free_blocks);
-		//			}
+		if (new_free_blocks == NumericLimits<idx_t>::Maximum()) {
+			// if new free_blocks is all blocks - mark entire block as modified
+			blocks.erase(entry);
+			block_manager.MarkBlockAsModified(block_id);
+		} else {
+			// set the new set of free blocks
+			block.FreeBlocksFromInteger(new_free_blocks);
+		}
 	}
 
 	modified_blocks.clear();
@@ -301,6 +298,23 @@ void MetadataManager::ClearModifiedBlocks(const vector<MetaBlockPointer> &pointe
 	}
 }
 
+vector<MetadataBlockInfo> MetadataManager::GetMetadataInfo() const {
+	vector<MetadataBlockInfo> result;
+	for (auto &block : blocks) {
+		MetadataBlockInfo block_info;
+		block_info.block_id = block.second.block_id;
+		block_info.total_blocks = MetadataManager::METADATA_BLOCK_COUNT;
+		for (auto free_block : block.second.free_blocks) {
+			block_info.free_list.push_back(free_block);
+		}
+		std::sort(block_info.free_list.begin(), block_info.free_list.end());
+		result.push_back(std::move(block_info));
+	}
+	std::sort(result.begin(), result.end(),
+	          [](const MetadataBlockInfo &a, const MetadataBlockInfo &b) { return a.block_id < b.block_id; });
+	return result;
+}
+
 block_id_t MetadataManager::GetNextBlockId() {
 	return block_manager.GetFreeBlockId();
 }

package/src/duckdb/src/storage/serialization/serialize_storage.cpp

@@ -29,6 +29,7 @@ void DataPointer::Serialize(Serializer &serializer) const {
 	serializer.WriteProperty(102, "block_pointer", block_pointer);
 	serializer.WriteProperty(103, "compression_type", compression_type);
 	serializer.WriteProperty(104, "statistics", statistics);
+	serializer.WriteProperty(105, "segment_state", segment_state);
 }
 
 DataPointer DataPointer::Deserialize(Deserializer &deserializer) {
@@ -42,6 +43,9 @@ DataPointer DataPointer::Deserialize(Deserializer &deserializer) {
 	result.tuple_count = tuple_count;
 	result.block_pointer = block_pointer;
 	result.compression_type = compression_type;
+	deserializer.Set<CompressionType>(compression_type);
+	deserializer.ReadProperty(105, "segment_state", result.segment_state);
+	deserializer.Unset<CompressionType>();
 	return result;
 }
 

package/src/duckdb/src/storage/single_file_block_manager.cpp

@@ -391,18 +391,23 @@ void SingleFileBlockManager::Truncate() {
 vector<MetadataHandle> SingleFileBlockManager::GetFreeListBlocks() {
 	vector<MetadataHandle> free_list_blocks;
 
-	auto free_list_size = sizeof(uint64_t) + sizeof(block_id_t) * (free_list.size() + modified_blocks.size());
-	auto multi_use_blocks_size = sizeof(uint64_t) + (sizeof(block_id_t) + sizeof(uint32_t)) * multi_use_blocks.size();
-	auto metadata_blocks = sizeof(uint64_t) + (sizeof(idx_t) * 2) * GetMetadataManager().BlockCount();
-	auto total_size = free_list_size + multi_use_blocks_size + metadata_blocks;
-
-	// reserve the blocks that we are going to write
+	// reserve all blocks that we are going to write the free list to
 	// since these blocks are no longer free we cannot just include them in the free list!
 	auto block_size = MetadataManager::METADATA_BLOCK_SIZE - sizeof(idx_t);
-	while (total_size > 0) {
+	idx_t allocated_size = 0;
+	while (true) {
+		auto free_list_size = sizeof(uint64_t) + sizeof(block_id_t) * (free_list.size() + modified_blocks.size());
+		auto multi_use_blocks_size =
+		    sizeof(uint64_t) + (sizeof(block_id_t) + sizeof(uint32_t)) * multi_use_blocks.size();
+		auto metadata_blocks =
+		    sizeof(uint64_t) + (sizeof(block_id_t) + sizeof(idx_t)) * GetMetadataManager().BlockCount();
+		auto total_size = free_list_size + multi_use_blocks_size + metadata_blocks;
+		if (total_size < allocated_size) {
+			break;
+		}
 		auto free_list_handle = GetMetadataManager().AllocateHandle();
 		free_list_blocks.push_back(std::move(free_list_handle));
-		total_size -= MinValue<idx_t>(total_size, block_size);
+		allocated_size += block_size;
 	}
 
 	return free_list_blocks;
@@ -434,13 +439,14 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
 	auto free_list_blocks = GetFreeListBlocks();
 
 	// now handle the free list
+	auto &metadata_manager = GetMetadataManager();
 	// add all modified blocks to the free list: they can now be written to again
+	metadata_manager.MarkBlocksAsModified();
 	for (auto &block : modified_blocks) {
 		free_list.insert(block);
 	}
 	modified_blocks.clear();
 
-	auto &metadata_manager = GetMetadataManager();
 	if (!free_list_blocks.empty()) {
 		// there are blocks to write, either in the free_list or in the modified_blocks
 		// we write these blocks specifically to the free_list_blocks

package/src/duckdb/src/storage/storage_info.cpp

@@ -2,7 +2,7 @@
 
 namespace duckdb {
 
-const uint64_t VERSION_NUMBER = 60;
+const uint64_t VERSION_NUMBER = 61;
 
 struct StorageVersionInfo {
 	const char *version_name;

package/src/duckdb/src/storage/storage_manager.cpp

@@ -260,6 +260,11 @@ DatabaseSize SingleFileStorageManager::GetDatabaseSize() {
 	return ds;
 }
 
+vector<MetadataBlockInfo> SingleFileStorageManager::GetMetadataInfo() {
+	auto &metadata_manager = block_manager->GetMetadataManager();
+	return metadata_manager.GetMetadataInfo();
+}
+
 bool SingleFileStorageManager::AutomaticCheckpoint(idx_t estimated_wal_bytes) {
 	auto log = GetWriteAheadLog();
 	if (!log) {

package/src/duckdb/src/storage/table/column_checkpoint_state.cpp

@@ -181,6 +181,9 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 	}
 	data_pointer.tuple_count = tuple_count;
 	data_pointer.compression_type = segment->function.get().type;
+	if (segment->function.get().serialize_state) {
+		data_pointer.segment_state = segment->function.get().serialize_state(*segment);
+	}
 
 	// append the segment to the new segment tree
 	new_tree.AppendSegment(std::move(segment));

package/src/duckdb/src/storage/table/column_data.cpp

@@ -402,12 +402,7 @@ void ColumnData::AppendTransientSegment(SegmentLock &l, idx_t start_row) {
 void ColumnData::CommitDropColumn() {
 	for (auto &segment_p : data.Segments()) {
 		auto &segment = segment_p;
-		if (segment.segment_type == ColumnSegmentType::PERSISTENT) {
-			auto block_id = segment.GetBlockId();
-			if (block_id != INVALID_BLOCK) {
-				block_manager.MarkBlockAsModified(block_id);
-			}
-		}
+		segment.CommitDropSegment();
 	}
 }
 
@@ -453,12 +448,18 @@ unique_ptr<ColumnCheckpointState> ColumnData::Checkpoint(RowGroup &row_group,
 
 void ColumnData::DeserializeColumn(Deserializer &deserializer) {
 	// load the data pointers for the column
-	this->count = 0;
+	deserializer.Set<DatabaseInstance &>(info.db.GetDatabase());
 	deserializer.Set<LogicalType &>(type);
 
-	deserializer.ReadList(100, "data_pointers", [&](Deserializer::List &list, idx_t i) {
-		auto data_pointer = list.ReadElement<DataPointer>();
+	vector<DataPointer> data_pointers;
+	deserializer.ReadProperty(100, "data_pointers", data_pointers);
+
+	deserializer.Unset<DatabaseInstance>();
+	deserializer.Unset<LogicalType>();
 
+	// construct the segments based on the data pointers
+	this->count = 0;
+	for (auto &data_pointer : data_pointers) {
 		// Update the count and statistics
 		this->count += data_pointer.tuple_count;
 		if (stats) {
@@ -469,12 +470,10 @@ void ColumnData::DeserializeColumn(Deserializer &deserializer) {
 		auto segment = ColumnSegment::CreatePersistentSegment(
 		    GetDatabase(), block_manager, data_pointer.block_pointer.block_id, data_pointer.block_pointer.offset, type,
 		    data_pointer.row_start, data_pointer.tuple_count, data_pointer.compression_type,
-		    std::move(data_pointer.statistics));
+		    std::move(data_pointer.statistics), std::move(data_pointer.segment_state));
 
 		data.AppendSegment(std::move(segment));
-	});
-
-	deserializer.Unset<LogicalType>();
+	}
 }
 
 shared_ptr<ColumnData> ColumnData::Deserialize(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
@@ -530,10 +529,14 @@ void ColumnData::GetColumnSegmentInfo(idx_t row_group_index, vector<idx_t> col_p
 		} else {
 			column_info.persistent = false;
 		}
+		auto segment_state = segment->GetSegmentState();
+		if (segment_state) {
+			column_info.segment_info = segment_state->GetSegmentInfo();
+		}
 		result.emplace_back(column_info);
 
 		segment_idx++;
-		segment = (ColumnSegment *)data.GetNextSegment(segment);
+		segment = data.GetNextSegment(segment);
 	}
 }
 

package/src/duckdb/src/storage/table/column_data_checkpointer.cpp

@@ -168,16 +168,9 @@ void ColumnDataCheckpointer::WriteToDisk() {
 	// first we check the current segments
 	// if there are any persistent segments, we will mark their old block ids as modified
 	// since the segments will be rewritten their old on disk data is no longer required
-	auto &block_manager = col_data.GetBlockManager();
 	for (idx_t segment_idx = 0; segment_idx < nodes.size(); segment_idx++) {
 		auto segment = nodes[segment_idx].node.get();
-		if (segment->segment_type == ColumnSegmentType::PERSISTENT) {
-			// persistent segment has updates: mark it as modified and rewrite the block with the merged updates
-			auto block_id = segment->GetBlockId();
-			if (block_id != INVALID_BLOCK) {
-				block_manager.MarkBlockAsModified(block_id);
-			}
-		}
+		segment->CommitDropSegment();
 	}
 
 	// now we need to write our segment
@@ -231,6 +224,9 @@ void ColumnDataCheckpointer::WritePersistentSegments() {
 		pointer.row_start = segment->start;
 		pointer.tuple_count = segment->count;
 		pointer.compression_type = segment->function.get().type;
+		if (segment->function.get().serialize_state) {
+			pointer.segment_state = segment->function.get().serialize_state(*segment);
+		}
 
 		// merge the persistent stats into the global column stats
 		state.global_stats->Merge(segment->stats.statistics);

package/src/duckdb/src/storage/table/column_segment.cpp

@@ -9,6 +9,7 @@
 #include "duckdb/planner/filter/constant_filter.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
+#include "duckdb/storage/data_pointer.hpp"
 
 #include <cstring>
 
@@ -18,7 +19,8 @@ unique_ptr<ColumnSegment> ColumnSegment::CreatePersistentSegment(DatabaseInstanc
                                                                  block_id_t block_id, idx_t offset,
                                                                  const LogicalType &type, idx_t start, idx_t count,
                                                                  CompressionType compression_type,
-                                                                 BaseStatistics statistics) {
+                                                                 BaseStatistics statistics,
+                                                                 unique_ptr<ColumnSegmentState> segment_state) {
 	auto &config = DBConfig::GetConfig(db);
 	optional_ptr<CompressionFunction> function;
 	shared_ptr<BlockHandle> block;
@@ -31,7 +33,7 @@ unique_ptr<ColumnSegment> ColumnSegment::CreatePersistentSegment(DatabaseInstanc
 	}
 	auto segment_size = Storage::BLOCK_SIZE;
 	return make_uniq<ColumnSegment>(db, std::move(block), type, ColumnSegmentType::PERSISTENT, start, count, *function,
-	                                std::move(statistics), block_id, offset, segment_size);
+	                                std::move(statistics), block_id, offset, segment_size, std::move(segment_state));
 }
 
 unique_ptr<ColumnSegment> ColumnSegment::CreateTransientSegment(DatabaseInstance &db, const LogicalType &type,
@@ -56,13 +58,14 @@ unique_ptr<ColumnSegment> ColumnSegment::CreateSegment(ColumnSegment &other, idx
 
 ColumnSegment::ColumnSegment(DatabaseInstance &db, shared_ptr<BlockHandle> block, LogicalType type_p,
                              ColumnSegmentType segment_type, idx_t start, idx_t count, CompressionFunction &function_p,
-                             BaseStatistics statistics, block_id_t block_id_p, idx_t offset_p, idx_t segment_size_p)
+                             BaseStatistics statistics, block_id_t block_id_p, idx_t offset_p, idx_t segment_size_p,
+                             unique_ptr<ColumnSegmentState> segment_state)
     : SegmentBase<ColumnSegment>(start, count), db(db), type(std::move(type_p)),
       type_size(GetTypeIdSize(type.InternalType())), segment_type(segment_type), function(function_p),
       stats(std::move(statistics)), block(std::move(block)), block_id(block_id_p), offset(offset_p),
       segment_size(segment_size_p) {
 	if (function.get().init_segment) {
-		segment_state = function.get().init_segment(*this, block_id);
+		this->segment_state = function.get().init_segment(*this, block_id, segment_state.get());
 	}
 }
 
@@ -190,11 +193,6 @@ void ColumnSegment::ConvertToPersistent(optional_ptr<BlockManager> block_manager
 		// instead of copying the data we alter some metadata so the buffer points to an on-disk block
 		block = block_manager->ConvertToPersistent(block_id, std::move(block));
 	}
-
-	segment_state.reset();
-	if (function.get().init_segment) {
-		segment_state = function.get().init_segment(*this, block_id);
-	}
 }
 
 void ColumnSegment::MarkAsPersistent(shared_ptr<BlockHandle> block_p, uint32_t offset_p) {
@@ -204,10 +202,21 @@ void ColumnSegment::MarkAsPersistent(shared_ptr<BlockHandle> block_p, uint32_t o
 	block_id = block_p->BlockId();
 	offset = offset_p;
 	block = std::move(block_p);
+}
 
-	segment_state.reset();
-	if (function.get().init_segment) {
-		segment_state = function.get().init_segment(*this, block_id);
+//===--------------------------------------------------------------------===//
+// Drop Segment
+//===--------------------------------------------------------------------===//
+void ColumnSegment::CommitDropSegment() {
+	if (segment_type != ColumnSegmentType::PERSISTENT) {
+		// not persistent
+		return;
+	}
+	if (block_id != INVALID_BLOCK) {
+		GetBlockManager().MarkBlockAsModified(block_id);
+	}
+	if (function.get().cleanup_state) {
+		function.get().cleanup_state(*this);
 	}
 }
 

package/src/duckdb/ub_src_function_table_system.cpp

@@ -32,6 +32,8 @@
 
 #include "src/function/table/system/pragma_database_size.cpp"
 
+#include "src/function/table/system/pragma_metadata_info.cpp"
+
 #include "src/function/table/system/pragma_storage_info.cpp"
 
 #include "src/function/table/system/pragma_table_info.cpp"

package/src/duckdb/ub_src_storage.cpp

@@ -6,6 +6,8 @@
 
 #include "src/storage/block.cpp"
 
+#include "src/storage/data_pointer.cpp"
+
 #include "src/storage/data_table.cpp"
 
 #include "src/storage/index.cpp"

package/src/duckdb_node.hpp

@@ -138,6 +138,7 @@ public:
 	static Napi::Object NewInstance(const Napi::Value &db);
 
 public:
+	Napi::Value Close(const Napi::CallbackInfo &info);
 	Napi::Value Prepare(const Napi::CallbackInfo &info);
 	Napi::Value Exec(const Napi::CallbackInfo &info);
 	Napi::Value RegisterUdf(const Napi::CallbackInfo &info);

package/test/close_hang.test.ts

@@ -0,0 +1,39 @@
+import 'chai-as-promised';
+import {exec as _exec} from "child_process";
+import fs from "fs-extra";
+import path from "path";
+import {Database, OPEN_READWRITE} from "..";
+import {promisify} from "util";
+import {expect} from "chai";
+
+const exec = promisify(_exec);
+
+it("close hang", async function main() {
+  if (process.platform == 'win32') this.skip();
+
+  const databasePath = path.join(__dirname, "tmp", "close_hang.db");
+  const pathExists = await fs.pathExists(databasePath);
+  if (pathExists) {
+    await fs.remove(databasePath);
+  }
+  await fs.mkdirp(path.dirname(databasePath));
+
+  const db = await new Promise<Database>((resolve, reject) => {
+    let db: Database = new Database(
+        databasePath,
+        OPEN_READWRITE,
+        (err: unknown) => (err ? reject(err) : resolve(db)),
+    );
+  });
+  try {
+    await new Promise((resolve) =>
+        db.exec("create table foo(bar int)", resolve),
+    );
+  } finally {
+    await new Promise((resolve) => db.close(resolve));
+  }
+
+  // exit code 1 and stdout blank means no open handles
+  await expect(exec(
+      `lsof ${databasePath}`)).to.eventually.be.rejected.and.to.include({ 'code': 1 ,'stdout': ''});
+});