duckdb 1.1.0 → 1.1.1-dev3.0

package/src/duckdb/src/storage/standard_buffer_manager.cpp

@@ -126,9 +126,8 @@ shared_ptr<BlockHandle> StandardBufferManager::RegisterTransientMemory(const idx
 		return RegisterSmallMemory(size);
 	}
 
-	shared_ptr<BlockHandle> block;
-	Allocate(MemoryTag::IN_MEMORY_TABLE, size, false, &block);
-	return block;
+	auto buffer_handle = Allocate(MemoryTag::IN_MEMORY_TABLE, size, false);
+	return buffer_handle.GetBlockHandle();
 }
 
 shared_ptr<BlockHandle> StandardBufferManager::RegisterSmallMemory(const idx_t size) {
@@ -164,17 +163,14 @@ shared_ptr<BlockHandle> StandardBufferManager::RegisterMemory(MemoryTag tag, idx
 	                                    destroy_buffer_upon, alloc_size, std::move(res));
 }
 
-BufferHandle StandardBufferManager::Allocate(MemoryTag tag, idx_t block_size, bool can_destroy,
-                                             shared_ptr<BlockHandle> *block) {
-	shared_ptr<BlockHandle> local_block;
-	auto block_ptr = block ? block : &local_block;
-	*block_ptr = RegisterMemory(tag, block_size, can_destroy);
+BufferHandle StandardBufferManager::Allocate(MemoryTag tag, idx_t block_size, bool can_destroy) {
+	auto block = RegisterMemory(tag, block_size, can_destroy);
 
 #ifdef DUCKDB_DEBUG_DESTROY_BLOCKS
 	// Initialize the memory with garbage data
-	WriteGarbageIntoBuffer(*(*block_ptr)->buffer);
+	WriteGarbageIntoBuffer(*block->buffer);
 #endif
-	return Pin(*block_ptr);
+	return Pin(block);
 }
 
 void StandardBufferManager::ReAllocate(shared_ptr<BlockHandle> &handle, idx_t block_size) {
@@ -253,7 +249,7 @@ void StandardBufferManager::BatchRead(vector<shared_ptr<BlockHandle>> &handles,
 			}
 			auto block_ptr =
 			    intermediate_buffer.GetFileBuffer().InternalBuffer() + block_idx * block_manager.GetBlockAllocSize();
-			buf = BlockHandle::LoadFromBuffer(handle, block_ptr, std::move(reusable_buffer));
+			buf = handle->LoadFromBuffer(block_ptr, std::move(reusable_buffer));
 			handle->readers = 1;
 			handle->memory_charge = std::move(reservation);
 		}
@@ -314,7 +310,7 @@ BufferHandle StandardBufferManager::Pin(shared_ptr<BlockHandle> &handle) {
 		if (handle->state == BlockState::BLOCK_LOADED) {
 			// the block is loaded, increment the reader count and set the BufferHandle
 			handle->readers++;
-			buf = handle->Load(handle);
+			buf = handle->Load();
 		}
 		required_memory = handle->memory_usage;
 	}
@@ -335,11 +331,11 @@ BufferHandle StandardBufferManager::Pin(shared_ptr<BlockHandle> &handle) {
 			// the block is loaded, increment the reader count and return a pointer to the handle
 			handle->readers++;
 			reservation.Resize(0);
-			buf = handle->Load(handle);
+			buf = handle->Load();
 		} else {
 			// now we can actually load the current block
 			D_ASSERT(handle->readers == 0);
-			buf = handle->Load(handle, std::move(reusable_buffer));
+			buf = handle->Load(std::move(reusable_buffer));
 			handle->readers = 1;
 			handle->memory_charge = std::move(reservation);
 			// in the case of a variable sized block, the buffer may be smaller than a full block.
@@ -480,33 +476,34 @@ void StandardBufferManager::WriteTemporaryBuffer(MemoryTag tag, block_id_t block
 	buffer.Write(*handle, sizeof(idx_t));
 }
 
-unique_ptr<FileBuffer> StandardBufferManager::ReadTemporaryBuffer(MemoryTag tag, block_id_t id,
+unique_ptr<FileBuffer> StandardBufferManager::ReadTemporaryBuffer(MemoryTag tag, BlockHandle &block,
                                                                   unique_ptr<FileBuffer> reusable_buffer) {
 	D_ASSERT(!temporary_directory.path.empty());
 	D_ASSERT(temporary_directory.handle.get());
+	auto id = block.BlockId();
 	if (temporary_directory.handle->GetTempFile().HasTemporaryBuffer(id)) {
-		evicted_data_per_tag[uint8_t(tag)] -= GetBlockSize();
+		// This is a block that was offloaded to a regular .tmp file, the file contains blocks of a fixed size
 		return temporary_directory.handle->GetTempFile().ReadTemporaryBuffer(id, std::move(reusable_buffer));
 	}
 
-	// Open the temporary file and read its size.
+	// This block contains data of variable size so we need to open it and read it to get its size.
 	idx_t block_size;
 	auto path = GetTemporaryPath(id);
 	auto &fs = FileSystem::GetFileSystem(db);
 	auto handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ);
 	handle->Read(&block_size, sizeof(idx_t), 0);
-	evicted_data_per_tag[uint8_t(tag)] -= block_size;
 
 	// Allocate a buffer of the file's size and read the data into that buffer.
 	auto buffer = ReadTemporaryBufferInternal(*this, *handle, sizeof(idx_t), block_size, std::move(reusable_buffer));
 	handle.reset();
 
 	// Delete the file and return the buffer.
-	DeleteTemporaryFile(id);
+	DeleteTemporaryFile(block);
 	return buffer;
 }
 
-void StandardBufferManager::DeleteTemporaryFile(block_id_t id) {
+void StandardBufferManager::DeleteTemporaryFile(BlockHandle &block) {
+	auto id = block.BlockId();
 	if (temporary_directory.path.empty()) {
 		// no temporary directory specified: nothing to delete
 		return;
@@ -520,6 +517,7 @@ void StandardBufferManager::DeleteTemporaryFile(block_id_t id) {
 	}
 	// check if we should delete the file from the shared pool of files, or from the general file system
 	if (temporary_directory.handle->GetTempFile().HasTemporaryBuffer(id)) {
+		evicted_data_per_tag[uint8_t(block.GetMemoryTag())] -= GetBlockSize();
 		temporary_directory.handle->GetTempFile().DeleteTemporaryBuffer(id);
 		return;
 	}
@@ -528,6 +526,7 @@ void StandardBufferManager::DeleteTemporaryFile(block_id_t id) {
 	auto &fs = FileSystem::GetFileSystem(db);
 	auto path = GetTemporaryPath(id);
 	if (fs.FileExists(path)) {
+		evicted_data_per_tag[uint8_t(block.GetMemoryTag())] -= block.GetMemoryUsage();
 		auto handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ);
 		auto content_size = handle->GetFileSize();
 		handle.reset();

package/src/duckdb/src/storage/statistics/column_statistics.cpp

@@ -20,8 +20,7 @@ shared_ptr<ColumnStatistics> ColumnStatistics::CreateEmptyStats(const LogicalTyp
 
 void ColumnStatistics::Merge(ColumnStatistics &other) {
 	stats.Merge(other.stats);
-	if (distinct_stats) {
-		D_ASSERT(other.distinct_stats);
+	if (distinct_stats && other.distinct_stats) {
 		distinct_stats->Merge(*other.distinct_stats);
 	}
 }

package/src/duckdb/src/storage/table/column_data.cpp

@@ -92,7 +92,10 @@ void ColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx)
 	state.last_offset = 0;
 }
 
-ScanVectorType ColumnData::GetVectorScanType(ColumnScanState &state, idx_t scan_count) {
+ScanVectorType ColumnData::GetVectorScanType(ColumnScanState &state, idx_t scan_count, Vector &result) {
+	if (result.GetVectorType() != VectorType::FLAT_VECTOR) {
+		return ScanVectorType::SCAN_ENTIRE_VECTOR;
+	}
 	if (HasUpdates()) {
 		// if we have updates we need to merge in the updates
 		// always need to scan flat vectors
@@ -230,7 +233,7 @@ void ColumnData::UpdateInternal(TransactionData transaction, idx_t column_index,
 template <bool SCAN_COMMITTED, bool ALLOW_UPDATES>
 idx_t ColumnData::ScanVector(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
                              idx_t target_scan) {
-	auto scan_type = GetVectorScanType(state, target_scan);
+	auto scan_type = GetVectorScanType(state, target_scan, result);
 	auto scan_count = ScanVector(state, result, target_scan, scan_type);
 	if (scan_type != ScanVectorType::SCAN_ENTIRE_VECTOR) {
 		// if we are scanning an entire vector we cannot have updates

package/src/duckdb/src/storage/table/column_segment.cpp

@@ -161,8 +161,8 @@ void ColumnSegment::Resize(idx_t new_size) {
 
 	auto &buffer_manager = BufferManager::GetBufferManager(db);
 	auto old_handle = buffer_manager.Pin(block);
-	shared_ptr<BlockHandle> new_block;
-	auto new_handle = buffer_manager.Allocate(MemoryTag::IN_MEMORY_TABLE, new_size, false, &new_block);
+	auto new_handle = buffer_manager.Allocate(MemoryTag::IN_MEMORY_TABLE, new_size);
+	auto new_block = new_handle.GetBlockHandle();
 	memcpy(new_handle.Ptr(), old_handle.Ptr(), segment_size);
 
 	this->block_id = new_block->BlockId();

package/src/duckdb/src/storage/table/row_group_collection.cpp

@@ -848,9 +848,10 @@ public:
 		if (total_append_count != merge_rows) {
 			throw InternalException("Mismatch in row group count vs verify count in RowGroupCollection::Checkpoint");
 		}
-		// merging is complete - schedule checkpoint tasks of the target row groups
+		// merging is complete - execute checkpoint tasks of the target row groups
 		for (idx_t i = 0; i < target_count; i++) {
-			collection.ScheduleCheckpointTask(checkpoint_state, segment_idx + i);
+			auto checkpoint_task = collection.GetCheckpointTask(checkpoint_state, segment_idx + i);
+			checkpoint_task->ExecuteTask();
 		}
 	}
 
@@ -886,7 +887,7 @@ void RowGroupCollection::InitializeVacuumState(CollectionCheckpointState &checkp
 }
 
 bool RowGroupCollection::ScheduleVacuumTasks(CollectionCheckpointState &checkpoint_state, VacuumState &state,
-                                             idx_t segment_idx) {
+                                             idx_t segment_idx, bool schedule_vacuum) {
 	static constexpr const idx_t MAX_MERGE_COUNT = 3;
 
 	if (!state.can_vacuum_deletes) {
@@ -902,6 +903,9 @@ bool RowGroupCollection::ScheduleVacuumTasks(CollectionCheckpointState &checkpoi
 		D_ASSERT(!checkpoint_state.segments[segment_idx].node);
 		return false;
 	}
+	if (!schedule_vacuum) {
+		return false;
+	}
 	idx_t merge_rows;
 	idx_t next_idx = 0;
 	idx_t merge_count;
@@ -952,9 +956,9 @@ bool RowGroupCollection::ScheduleVacuumTasks(CollectionCheckpointState &checkpoi
 //===--------------------------------------------------------------------===//
 // Checkpoint
 //===--------------------------------------------------------------------===//
-void RowGroupCollection::ScheduleCheckpointTask(CollectionCheckpointState &checkpoint_state, idx_t segment_idx) {
-	auto checkpoint_task = make_uniq<CheckpointTask>(checkpoint_state, segment_idx);
-	checkpoint_state.executor.ScheduleTask(std::move(checkpoint_task));
+unique_ptr<CheckpointTask> RowGroupCollection::GetCheckpointTask(CollectionCheckpointState &checkpoint_state,
+                                                                 idx_t segment_idx) {
+	return make_uniq<CheckpointTask>(checkpoint_state, segment_idx);
 }
 
 void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &global_stats) {
@@ -970,13 +974,12 @@ void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &gl
 	auto &config = DBConfig::GetConfig(writer.GetDatabase());
 	for (idx_t segment_idx = 0; segment_idx < segments.size(); segment_idx++) {
 		auto &entry = segments[segment_idx];
-		if (total_vacuum_tasks < config.options.max_vacuum_tasks) {
-			auto vacuum_tasks = ScheduleVacuumTasks(checkpoint_state, vacuum_state, segment_idx);
-			if (vacuum_tasks) {
-				// vacuum tasks were scheduled - don't schedule a checkpoint task yet
-				total_vacuum_tasks++;
-				continue;
-			}
+		auto vacuum_tasks = ScheduleVacuumTasks(checkpoint_state, vacuum_state, segment_idx,
+		                                        total_vacuum_tasks < config.options.max_vacuum_tasks);
+		if (vacuum_tasks) {
+			// vacuum tasks were scheduled - don't schedule a checkpoint task yet
+			total_vacuum_tasks++;
+			continue;
 		}
 		if (!entry.node) {
 			// row group was vacuumed/dropped - skip
@@ -984,7 +987,8 @@ void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &gl
 		}
 		// schedule a checkpoint task for this row group
 		entry.node->MoveToCollection(*this, vacuum_state.row_start);
-		ScheduleCheckpointTask(checkpoint_state, segment_idx);
+		auto checkpoint_task = GetCheckpointTask(checkpoint_state, segment_idx);
+		checkpoint_state.executor.ScheduleTask(std::move(checkpoint_task));
 		vacuum_state.row_start += entry.node->count;
 	}
 	// all tasks have been scheduled - execute tasks until we are done

package/src/duckdb/src/storage/table/standard_column_data.cpp

@@ -21,16 +21,16 @@ void StandardColumnData::SetStart(idx_t new_start) {
 	validity.SetStart(new_start);
 }
 
-ScanVectorType StandardColumnData::GetVectorScanType(ColumnScanState &state, idx_t scan_count) {
+ScanVectorType StandardColumnData::GetVectorScanType(ColumnScanState &state, idx_t scan_count, Vector &result) {
 	// if either the current column data, or the validity column data requires flat vectors, we scan flat vectors
-	auto scan_type = ColumnData::GetVectorScanType(state, scan_count);
+	auto scan_type = ColumnData::GetVectorScanType(state, scan_count, result);
 	if (scan_type == ScanVectorType::SCAN_FLAT_VECTOR) {
 		return ScanVectorType::SCAN_FLAT_VECTOR;
 	}
 	if (state.child_states.empty()) {
 		return scan_type;
 	}
-	return validity.GetVectorScanType(state.child_states[0], scan_count);
+	return validity.GetVectorScanType(state.child_states[0], scan_count, result);
 }
 
 void StandardColumnData::InitializePrefetch(PrefetchState &prefetch_state, ColumnScanState &scan_state, idx_t rows) {

package/src/duckdb/src/storage/wal_replay.cpp

@@ -554,9 +554,8 @@ void WriteAheadLogDeserializer::ReplayCreateIndex() {
 		for (idx_t j = 0; j < data_info.allocation_sizes.size(); j++) {
 
 			// read the data into a buffer handle
-			shared_ptr<BlockHandle> block_handle;
-			buffer_manager.Allocate(MemoryTag::ART_INDEX, block_manager->GetBlockSize(), false, &block_handle);
-			auto buffer_handle = buffer_manager.Pin(block_handle);
+			auto buffer_handle = buffer_manager.Allocate(MemoryTag::ART_INDEX, block_manager->GetBlockSize(), false);
+			auto block_handle = buffer_handle.GetBlockHandle();
 			auto data_ptr = buffer_handle.Ptr();
 
 			list.ReadElement<bool>(data_ptr, data_info.allocation_sizes[j]);

package/src/duckdb/third_party/libpg_query/include/common/keywords.hpp

@@ -14,6 +14,7 @@
 #pragma once
 
 #include <cstdint>
+#include "pg_simplified_token.hpp"
 
 /* Keyword categories --- should match lists in gram.y */
 #define UNRESERVED_KEYWORD		0

package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp

@@ -1287,6 +1287,7 @@ typedef struct PGSelectStmt {
 	 */
 	PGSetOperation op;         /* type of set op */
 	bool all;                  /* ALL specified? */
+	bool from_first;           /* FROM first or SELECT first */
 	struct PGNode *larg; /* left child */
 	struct PGNode *rarg; /* right child */
 	                           /* Eventually add fields for CORRESPONDING spec here */

package/src/duckdb/third_party/libpg_query/include/parser/parser.hpp

@@ -17,7 +17,6 @@
 #include "nodes/parsenodes.hpp"
 #include "pg_simplified_token.hpp"
 #include <vector>
-
 namespace duckdb_libpgquery {
 
 typedef enum PGBackslashQuoteType {
@@ -29,7 +28,7 @@ typedef enum PGBackslashQuoteType {
 /* Primary entry point for the raw parsing functions */
 PGList *raw_parser(const char *str);
 
-bool is_keyword(const char *str);
+PGKeywordCategory is_keyword(const char *str);
 std::vector<PGKeyword> keyword_list();
 
 std::vector<PGSimplifiedToken> tokenize(const char *str);

package/src/duckdb/third_party/libpg_query/include/pg_simplified_token.hpp

@@ -20,12 +20,14 @@ struct PGSimplifiedToken {
 };
 
 enum class PGKeywordCategory : uint8_t {
-	PG_KEYWORD_RESERVED,
-	PG_KEYWORD_UNRESERVED,
-	PG_KEYWORD_TYPE_FUNC,
-	PG_KEYWORD_COL_NAME
+	PG_KEYWORD_UNRESERVED = 0,
+	PG_KEYWORD_COL_NAME = 1,
+	PG_KEYWORD_TYPE_FUNC= 2,
+	PG_KEYWORD_RESERVED = 3,
+	PG_KEYWORD_NONE = 4
 };
 
+
 struct PGKeyword {
 	std::string text;
 	PGKeywordCategory category;

package/src/duckdb/third_party/libpg_query/include/postgres_parser.hpp

@@ -28,7 +28,7 @@ public:
 	void Parse(const std::string &query);
 	static duckdb::vector<duckdb_libpgquery::PGSimplifiedToken> Tokenize(const std::string &query);
 
-	static bool IsKeyword(const std::string &text);
+	static duckdb_libpgquery::PGKeywordCategory IsKeyword(const std::string &text);
 	static duckdb::vector<duckdb_libpgquery::PGKeyword> KeywordList();
 
 	static void SetPreserveIdentifierCase(bool downcase);

package/src/duckdb/third_party/libpg_query/postgres_parser.cpp

@@ -34,7 +34,7 @@ PostgresParser::~PostgresParser()  {
     duckdb_libpgquery::pg_parser_cleanup();
 }
 
-bool PostgresParser::IsKeyword(const std::string &text) {
+duckdb_libpgquery::PGKeywordCategory PostgresParser::IsKeyword(const std::string &text) {
 	return duckdb_libpgquery::is_keyword(text.c_str());
 }