duckdb 0.7.2-dev2366.0 → 0.7.2-dev2410.0

package/src/duckdb/src/main/settings/settings.cpp

@@ -142,7 +142,7 @@ Value DebugForceNoCrossProduct::GetSetting(ClientContext &context) {
 }
 
 //===--------------------------------------------------------------------===//
-// Debug Ordered Aggregate Threshold
+// Ordered Aggregate Threshold
 //===--------------------------------------------------------------------===//
 
 void OrderedAggregateThreshold::ResetLocal(ClientContext &context) {
@@ -185,6 +185,21 @@ Value DebugWindowMode::GetSetting(ClientContext &context) {
 	return Value();
 }
 
+//===--------------------------------------------------------------------===//
+// Debug AsOf Join
+//===--------------------------------------------------------------------===//
+void DebugAsOfIEJoin::ResetLocal(ClientContext &context) {
+	ClientConfig::GetConfig(context).force_no_cross_product = ClientConfig().force_asof_iejoin;
+}
+
+void DebugAsOfIEJoin::SetLocal(ClientContext &context, const Value &input) {
+	ClientConfig::GetConfig(context).force_asof_iejoin = input.GetValue<bool>();
+}
+
+Value DebugAsOfIEJoin::GetSetting(ClientContext &context) {
+	return Value::BOOLEAN(ClientConfig::GetConfig(context).force_asof_iejoin);
+}
+
 //===--------------------------------------------------------------------===//
 // Default Collation
 //===--------------------------------------------------------------------===//
@@ -253,14 +268,18 @@ void DefaultNullOrderSetting::SetGlobal(DatabaseInstance *db, DBConfig &config,
 	auto parameter = StringUtil::Lower(input.ToString());
 
 	if (parameter == "nulls_first" || parameter == "nulls first" || parameter == "null first" || parameter == "first") {
-		config.options.default_null_order = OrderByNullType::NULLS_FIRST;
+		config.options.default_null_order = DefaultOrderByNullType::NULLS_FIRST;
 	} else if (parameter == "nulls_last" || parameter == "nulls last" || parameter == "null last" ||
 	           parameter == "last") {
-		config.options.default_null_order = OrderByNullType::NULLS_LAST;
+		config.options.default_null_order = DefaultOrderByNullType::NULLS_LAST;
+	} else if (parameter == "nulls_first_on_asc_last_on_desc" || parameter == "sqlite" || parameter == "mysql") {
+		config.options.default_null_order = DefaultOrderByNullType::NULLS_FIRST_ON_ASC_LAST_ON_DESC;
+	} else if (parameter == "nulls_last_on_asc_first_on_desc" || parameter == "postgres") {
+		config.options.default_null_order = DefaultOrderByNullType::NULLS_LAST_ON_ASC_FIRST_ON_DESC;
 	} else {
-		throw ParserException(
-		    "Unrecognized parameter for option NULL_ORDER \"%s\", expected either NULLS FIRST or NULLS LAST",
-		    parameter);
+		throw ParserException("Unrecognized parameter for option NULL_ORDER \"%s\", expected either NULLS FIRST, NULLS "
+		                      "LAST, SQLite, MySQL or Postgres",
+		                      parameter);
 	}
 }
 
@@ -271,10 +290,14 @@ void DefaultNullOrderSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config
 Value DefaultNullOrderSetting::GetSetting(ClientContext &context) {
 	auto &config = DBConfig::GetConfig(context);
 	switch (config.options.default_null_order) {
-	case OrderByNullType::NULLS_FIRST:
+	case DefaultOrderByNullType::NULLS_FIRST:
 		return "nulls_first";
-	case OrderByNullType::NULLS_LAST:
+	case DefaultOrderByNullType::NULLS_LAST:
 		return "nulls_last";
+	case DefaultOrderByNullType::NULLS_FIRST_ON_ASC_LAST_ON_DESC:
+		return "nulls_first_on_asc_last_on_desc";
+	case DefaultOrderByNullType::NULLS_LAST_ON_ASC_FIRST_ON_DESC:
+		return "nulls_last_on_asc_first_on_desc";
 	default:
 		throw InternalException("Unknown null order setting");
 	}

package/src/duckdb/src/planner/binder/expression/bind_aggregate_expression.cpp

@@ -215,11 +215,8 @@ BindResult BaseSelectBinder::BindAggregate(FunctionExpression &aggr, optional_pt
 		auto &config = DBConfig::GetConfig(context);
 		for (auto &order : aggr.order_bys->orders) {
 			auto &order_expr = (BoundExpression &)*order.expression;
-			const auto sense =
-			    (order.type == OrderType::ORDER_DEFAULT) ? config.options.default_order_type : order.type;
-			const auto null_order = (order.null_order == OrderByNullType::ORDER_DEFAULT)
-			                            ? config.options.default_null_order
-			                            : order.null_order;
+			const auto sense = config.ResolveOrder(order.type);
+			const auto null_order = config.ResolveNullOrder(sense, order.null_order);
 			order_bys->orders.emplace_back(sense, null_order, std::move(order_expr.expr));
 		}
 	}

package/src/duckdb/src/planner/binder/expression/bind_window_expression.cpp

@@ -64,14 +64,6 @@ static LogicalType ResolveWindowExpressionType(ExpressionType window_type, const
 	}
 }
 
-static inline OrderType ResolveOrderType(const DBConfig &config, OrderType type) {
-	return (type == OrderType::ORDER_DEFAULT) ? config.options.default_order_type : type;
-}
-
-static inline OrderByNullType ResolveNullOrder(const DBConfig &config, OrderByNullType null_order) {
-	return (null_order == OrderByNullType::ORDER_DEFAULT) ? config.options.default_null_order : null_order;
-}
-
 static unique_ptr<Expression> GetExpression(unique_ptr<ParsedExpression> &expr) {
 	if (!expr) {
 		return nullptr;
@@ -231,12 +223,12 @@ BindResult BaseSelectBinder::BindWindow(WindowExpression &window, idx_t depth) {
 	LogicalType start_type = LogicalType::BIGINT;
 	if (window.start == WindowBoundary::EXPR_PRECEDING_RANGE) {
 		D_ASSERT(window.orders.size() == 1);
-		range_sense = ResolveOrderType(config, window.orders[0].type);
+		range_sense = config.ResolveOrder(window.orders[0].type);
 		const auto name = (range_sense == OrderType::ASCENDING) ? "-" : "+";
 		start_type = BindRangeExpression(context, name, window.start_expr, window.orders[0].expression);
 	} else if (window.start == WindowBoundary::EXPR_FOLLOWING_RANGE) {
 		D_ASSERT(window.orders.size() == 1);
-		range_sense = ResolveOrderType(config, window.orders[0].type);
+		range_sense = config.ResolveOrder(window.orders[0].type);
 		const auto name = (range_sense == OrderType::ASCENDING) ? "+" : "-";
 		start_type = BindRangeExpression(context, name, window.start_expr, window.orders[0].expression);
 	}
@@ -244,12 +236,12 @@ BindResult BaseSelectBinder::BindWindow(WindowExpression &window, idx_t depth) {
 	LogicalType end_type = LogicalType::BIGINT;
 	if (window.end == WindowBoundary::EXPR_PRECEDING_RANGE) {
 		D_ASSERT(window.orders.size() == 1);
-		range_sense = ResolveOrderType(config, window.orders[0].type);
+		range_sense = config.ResolveOrder(window.orders[0].type);
 		const auto name = (range_sense == OrderType::ASCENDING) ? "-" : "+";
 		end_type = BindRangeExpression(context, name, window.end_expr, window.orders[0].expression);
 	} else if (window.end == WindowBoundary::EXPR_FOLLOWING_RANGE) {
 		D_ASSERT(window.orders.size() == 1);
-		range_sense = ResolveOrderType(config, window.orders[0].type);
+		range_sense = config.ResolveOrder(window.orders[0].type);
 		const auto name = (range_sense == OrderType::ASCENDING) ? "+" : "-";
 		end_type = BindRangeExpression(context, name, window.end_expr, window.orders[0].expression);
 	}
@@ -276,8 +268,8 @@ BindResult BaseSelectBinder::BindWindow(WindowExpression &window, idx_t depth) {
 	}
 
 	for (auto &order : window.orders) {
-		auto type = ResolveOrderType(config, order.type);
-		auto null_order = ResolveNullOrder(config, order.null_order);
+		auto type = config.ResolveOrder(order.type);
+		auto null_order = config.ResolveNullOrder(type, order.null_order);
 		auto expression = GetExpression(order.expression);
 		result->orders.emplace_back(type, null_order, std::move(expression));
 	}

package/src/duckdb/src/planner/binder/query_node/bind_select_node.cpp

@@ -158,11 +158,8 @@ void Binder::BindModifiers(OrderBinder &order_binder, QueryNode &statement, Boun
 				vector<unique_ptr<ParsedExpression>> order_list;
 				order_binders[0]->ExpandStarExpression(std::move(order_node.expression), order_list);
 
-				auto type =
-				    order_node.type == OrderType::ORDER_DEFAULT ? config.options.default_order_type : order_node.type;
-				auto null_order = order_node.null_order == OrderByNullType::ORDER_DEFAULT
-				                      ? config.options.default_null_order
-				                      : order_node.null_order;
+				auto type = config.ResolveOrder(order_node.type);
+				auto null_order = config.ResolveNullOrder(type, order_node.null_order);
 				for (auto &order_expr : order_list) {
 					auto bound_expr = BindOrderExpression(order_binder, std::move(order_expr));
 					if (!bound_expr) {

package/src/duckdb/src/storage/buffer/block_manager.cpp

@@ -33,7 +33,6 @@ void BlockManager::ClearMetaBlockHandles() {
 }
 
 shared_ptr<BlockHandle> BlockManager::ConvertToPersistent(block_id_t block_id, shared_ptr<BlockHandle> old_block) {
-
 	// pin the old block to ensure we have it loaded in memory
 	auto old_handle = buffer_manager.Pin(old_block);
 	D_ASSERT(old_block->state == BlockState::BLOCK_LOADED);
@@ -50,7 +49,7 @@ shared_ptr<BlockHandle> BlockManager::ConvertToPersistent(block_id_t block_id, s
 
 	// move the data from the old block into data for the new block
 	new_block->state = BlockState::BLOCK_LOADED;
-	new_block->buffer = CreateBlock(block_id, old_block->buffer.get());
+	new_block->buffer = ConvertBlock(block_id, *old_block->buffer);
 	new_block->memory_usage = old_block->memory_usage;
 	new_block->memory_charge = std::move(old_block->memory_charge);
 

package/src/duckdb/src/storage/meta_block_writer.cpp

@@ -35,6 +35,10 @@ BlockPointer MetaBlockWriter::GetBlockPointer() {
 }
 
 void MetaBlockWriter::Flush() {
+	if (offset < block->size) {
+		// clear remaining bytes of block (if any)
+		memset(block->buffer + offset, 0, block->size - offset);
+	}
 	AdvanceBlock();
 	block = nullptr;
 }

package/src/duckdb/src/storage/partial_block_manager.cpp

@@ -54,9 +54,14 @@ bool PartialBlockManager::GetPartialBlock(idx_t segment_size, unique_ptr<Partial
 }
 
 void PartialBlockManager::RegisterPartialBlock(PartialBlockAllocation &&allocation) {
-	auto &state(allocation.partial_block->state);
+	auto &state = allocation.partial_block->state;
 	if (state.block_use_count < max_use_count) {
-		auto new_size = AlignValue(allocation.allocation_size + state.offset_in_block);
+		auto unaligned_size = allocation.allocation_size + state.offset_in_block;
+		auto new_size = AlignValue(unaligned_size);
+		if (new_size != unaligned_size) {
+			// register the uninitialized region so we can correctly initialize it before writing to disk
+			allocation.partial_block->AddUninitializedRegion(unaligned_size, new_size);
+		}
 		state.offset_in_block = new_size;
 		auto new_space_left = state.block_size - new_size;
 		// check if the block is STILL partially filled after adding the segment_size
@@ -65,22 +70,24 @@ void PartialBlockManager::RegisterPartialBlock(PartialBlockAllocation &&allocati
 			partially_filled_blocks.insert(make_pair(new_space_left, std::move(allocation.partial_block)));
 		}
 	}
+	idx_t free_space = state.block_size - state.offset_in_block;
 	auto block_to_free = std::move(allocation.partial_block);
 	if (!block_to_free && partially_filled_blocks.size() > MAX_BLOCK_MAP_SIZE) {
 		// Free the page with the least space free.
 		auto itr = partially_filled_blocks.begin();
 		block_to_free = std::move(itr->second);
+		free_space = state.block_size - itr->first;
 		partially_filled_blocks.erase(itr);
 	}
 	// Flush any block that we're not going to reuse.
 	if (block_to_free) {
-		block_to_free->Flush();
+		block_to_free->Flush(free_space);
 	}
 }
 
 void PartialBlockManager::FlushPartialBlocks() {
 	for (auto &e : partially_filled_blocks) {
-		e.second->Flush();
+		e.second->Flush(e.first);
 	}
 	partially_filled_blocks.clear();
 }

package/src/duckdb/src/storage/single_file_block_manager.cpp

@@ -107,15 +107,15 @@ T DeserializeHeaderStructure(data_ptr_t ptr) {
 	return T::Deserialize(source);
 }
 
-SingleFileBlockManager::SingleFileBlockManager(AttachedDatabase &db, string path_p, bool read_only, bool use_direct_io)
+SingleFileBlockManager::SingleFileBlockManager(AttachedDatabase &db, string path_p, StorageManagerOptions options)
     : BlockManager(BufferManager::GetBufferManager(db)), db(db), path(std::move(path_p)),
       header_buffer(Allocator::Get(db), FileBufferType::MANAGED_BUFFER,
                     Storage::FILE_HEADER_SIZE - Storage::BLOCK_HEADER_SIZE),
-      iteration_count(0), read_only(read_only), use_direct_io(use_direct_io) {
+      iteration_count(0), options(options) {
 }
 
 void SingleFileBlockManager::GetFileFlags(uint8_t &flags, FileLockType &lock, bool create_new) {
-	if (read_only) {
+	if (options.read_only) {
 		D_ASSERT(!create_new);
 		flags = FileFlags::FILE_FLAGS_READ;
 		lock = FileLockType::READ_LOCK;
@@ -126,7 +126,7 @@ void SingleFileBlockManager::GetFileFlags(uint8_t &flags, FileLockType &lock, bo
 			flags |= FileFlags::FILE_FLAGS_FILE_CREATE;
 		}
 	}
-	if (use_direct_io) {
+	if (options.use_direct_io) {
 		flags |= FileFlags::FILE_FLAGS_DIRECT_IO;
 	}
 }
@@ -241,7 +241,7 @@ void SingleFileBlockManager::Initialize(DatabaseHeader &header) {
 }
 
 void SingleFileBlockManager::LoadFreeList() {
-	if (read_only) {
+	if (options.read_only) {
 		// no need to load free list for read only db
 		return;
 	}
@@ -343,13 +343,20 @@ idx_t SingleFileBlockManager::FreeBlocks() {
 	return free_list.size();
 }
 
+unique_ptr<Block> SingleFileBlockManager::ConvertBlock(block_id_t block_id, FileBuffer &source_buffer) {
+	D_ASSERT(source_buffer.AllocSize() == Storage::BLOCK_ALLOC_SIZE);
+	return make_uniq<Block>(source_buffer, block_id);
+}
+
 unique_ptr<Block> SingleFileBlockManager::CreateBlock(block_id_t block_id, FileBuffer *source_buffer) {
+	unique_ptr<Block> result;
 	if (source_buffer) {
-		D_ASSERT(source_buffer->AllocSize() == Storage::BLOCK_ALLOC_SIZE);
-		return make_uniq<Block>(*source_buffer, block_id);
+		result = ConvertBlock(block_id, *source_buffer);
 	} else {
-		return make_uniq<Block>(Allocator::Get(db), block_id);
+		result = make_uniq<Block>(Allocator::Get(db), block_id);
 	}
+	result->Initialize(options.debug_initialize);
+	return result;
 }
 
 void SingleFileBlockManager::Read(Block &block) {
@@ -459,7 +466,7 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
 		throw FatalException("Checkpoint aborted after free list write because of PRAGMA checkpoint_abort flag");
 	}
 
-	if (!use_direct_io) {
+	if (!options.use_direct_io) {
 		// if we are not using Direct IO we need to fsync BEFORE we write the header to ensure that all the previous
 		// blocks are written as well
 		handle->Sync();

package/src/duckdb/src/storage/standard_buffer_manager.cpp

@@ -20,14 +20,17 @@ struct BufferAllocatorData : PrivateAllocatorData {
 
 unique_ptr<FileBuffer> StandardBufferManager::ConstructManagedBuffer(idx_t size, unique_ptr<FileBuffer> &&source,
                                                                      FileBufferType type) {
+	unique_ptr<FileBuffer> result;
 	if (source) {
 		auto tmp = std::move(source);
 		D_ASSERT(tmp->AllocSize() == BufferManager::GetAllocSize(size));
-		return make_uniq<FileBuffer>(*tmp, type);
+		result = make_uniq<FileBuffer>(*tmp, type);
 	} else {
 		// no re-usable buffer: allocate a new buffer
-		return make_uniq<FileBuffer>(Allocator::Get(db), type, size);
+		result = make_uniq<FileBuffer>(Allocator::Get(db), type, size);
 	}
+	result->Initialize(DBConfig::GetConfig(db).options.debug_initialize);
+	return result;
 }
 
 class TemporaryFileManager;

package/src/duckdb/src/storage/storage_manager.cpp

@@ -95,6 +95,11 @@ void SingleFileStorageManager::LoadDatabase() {
 	auto &fs = FileSystem::Get(db);
 	auto &config = DBConfig::Get(db);
 	bool truncate_wal = false;
+
+	StorageManagerOptions options;
+	options.read_only = read_only;
+	options.use_direct_io = config.options.use_direct_io;
+	options.debug_initialize = config.options.debug_initialize;
 	// first check if the database exists
 	if (!fs.FileExists(path)) {
 		if (read_only) {
@@ -107,13 +112,13 @@ void SingleFileStorageManager::LoadDatabase() {
 			fs.RemoveFile(wal_path);
 		}
 		// initialize the block manager while creating a new db file
-		auto sf_block_manager = make_uniq<SingleFileBlockManager>(db, path, read_only, config.options.use_direct_io);
+		auto sf_block_manager = make_uniq<SingleFileBlockManager>(db, path, options);
 		sf_block_manager->CreateNewDatabase();
 		block_manager = std::move(sf_block_manager);
 		table_io_manager = make_uniq<SingleFileTableIOManager>(*block_manager);
 	} else {
 		// initialize the block manager while loading the current db file
-		auto sf_block_manager = make_uniq<SingleFileBlockManager>(db, path, read_only, config.options.use_direct_io);
+		auto sf_block_manager = make_uniq<SingleFileBlockManager>(db, path, options);
 		sf_block_manager->LoadExistingDatabase();
 		block_manager = std::move(sf_block_manager);
 		table_io_manager = make_uniq<SingleFileTableIOManager>(*block_manager);

package/src/duckdb/src/storage/table/column_checkpoint_state.cpp

@@ -50,16 +50,36 @@ public:
 	BlockManager &block_manager;
 	vector<PartialColumnSegment> tail_segments;
 
+private:
+	struct UninitializedRegion {
+		idx_t start;
+		idx_t end;
+	};
+	vector<UninitializedRegion> uninitialized_regions;
+
 public:
 	bool IsFlushed() {
 		// first_segment is zeroed on Flush
 		return !first_segment;
 	}
 
-	void Flush() override {
+	void AddUninitializedRegion(idx_t start, idx_t end) override {
+		uninitialized_regions.push_back({start, end});
+	}
+
+	void Flush(idx_t free_space_left) override {
 		// At this point, we've already copied all data from tail_segments
 		// into the page owned by first_segment. We flush all segment data to
 		// disk with the following call.
+		if (free_space_left > 0 || !uninitialized_regions.empty()) {
+			auto handle = block_manager.buffer_manager.Pin(first_segment->block);
+			// memset any uninitialized regions
+			for (auto &uninitialized : uninitialized_regions) {
+				memset(handle.Ptr() + uninitialized.start, 0, uninitialized.end - uninitialized.start);
+			}
+			// memset any free space at the end of the block to 0 prior to writing to disk
+			memset(handle.Ptr() + Storage::BLOCK_SIZE - free_space_left, 0, free_space_left);
+		}
 		first_data->IncrementVersion();
 		first_segment->ConvertToPersistent(&block_manager, state.block_id);
 		// Now that the page is persistent, update tail_segments to point to the

package/src/duckdb/ub_src_execution_operator_join.cpp

@@ -1,5 +1,7 @@
 #include "src/execution/operator/join/outer_join_marker.cpp"
 
+#include "src/execution/operator/join/physical_asof_join.cpp"
+
 #include "src/execution/operator/join/physical_blockwise_nl_join.cpp"
 
 #include "src/execution/operator/join/physical_comparison_join.cpp"