duckdb 0.7.2-dev2366.0 → 0.7.2-dev2430.0

package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp

@@ -1,4 +1,5 @@
 #include "duckdb/execution/operator/aggregate/physical_window.hpp"
+#include "duckdb/execution/operator/join/physical_asof_join.hpp"
 #include "duckdb/execution/operator/join/physical_iejoin.hpp"
 #include "duckdb/execution/operator/projection/physical_projection.hpp"
 #include "duckdb/execution/physical_plan_generator.hpp"
@@ -39,7 +40,11 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalAsOfJoin &
 	}
 	D_ASSERT(asof_idx < op.conditions.size());
 
-	//	Temporary implementation: IEJoin of Window
+	if (!ClientConfig::GetConfig(context).force_asof_iejoin) {
+		return make_uniq<PhysicalAsOfJoin>(op, std::move(left), std::move(right));
+	}
+
+	//	Debug implementation: IEJoin of Window
 	//	LEAD(asof_column, 1, infinity) OVER (PARTITION BY equi_column... ORDER BY asof_column) AS asof_temp
 	auto &asof_comp = op.conditions[asof_idx];
 	auto &asof_column = asof_comp.right;
@@ -47,6 +52,7 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalAsOfJoin &
 	auto asof_temp = make_uniq<BoundWindowExpression>(ExpressionType::WINDOW_LEAD, asof_type, nullptr, nullptr);
 	asof_temp->children.emplace_back(asof_column->Copy());
 	asof_temp->offset_expr = make_uniq<BoundConstantExpression>(Value::BIGINT(1));
+	// TODO: If infinities are not supported for a type, fake them by looking at LHS statistics?
 	asof_temp->default_expr = make_uniq<BoundConstantExpression>(Value::Infinity(asof_type));
 	for (auto equi_idx : equi_indexes) {
 		asof_temp->partitions.emplace_back(op.conditions[equi_idx].right->Copy());

package/src/duckdb/src/function/scalar/list/list_sort.cpp

@@ -244,83 +244,57 @@ static unique_ptr<FunctionData> ListSortBind(ClientContext &context, ScalarFunct
 	return make_uniq<ListSortBindData>(order, null_order, bound_function.return_type, child_type, context);
 }
 
-static OrderByNullType GetNullOrder(ClientContext &context, vector<unique_ptr<Expression>> &arguments, idx_t idx) {
-
-	if (!arguments[idx]->IsFoldable()) {
-		throw InvalidInputException("Null sorting order must be a constant");
-	}
-	Value null_order_value = ExpressionExecutor::EvaluateScalar(context, *arguments[idx]);
-	auto null_order_name = StringUtil::Upper(null_order_value.ToString());
-	const auto null_order_arg = EnumSerializer::StringToEnum<OrderByNullType>(null_order_name.c_str());
-	switch (null_order_arg) {
-	case OrderByNullType::NULLS_FIRST:
-	case OrderByNullType::NULLS_LAST:
-		return null_order_arg;
-	case OrderByNullType::ORDER_DEFAULT:
-		return DBConfig::GetConfig(context).options.default_null_order;
-	default:
-		throw InvalidInputException("Null sorting order must be either NULLS FIRST, NULLS LAST or DEFAULT");
+template <class T>
+static T GetOrder(ClientContext &context, Expression &expr) {
+	if (!expr.IsFoldable()) {
+		throw InvalidInputException("Sorting order must be a constant");
 	}
+	Value order_value = ExpressionExecutor::EvaluateScalar(context, expr);
+	auto order_name = StringUtil::Upper(order_value.ToString());
+	return EnumSerializer::StringToEnum<T>(order_name.c_str());
 }
 
 static unique_ptr<FunctionData> ListNormalSortBind(ClientContext &context, ScalarFunction &bound_function,
                                                    vector<unique_ptr<Expression>> &arguments) {
-
-	D_ASSERT(!bound_function.arguments.empty() && bound_function.arguments.size() <= 3);
 	D_ASSERT(!arguments.empty() && arguments.size() <= 3);
-
-	// set default values
-	auto &config = DBConfig::GetConfig(context);
-	auto order = config.options.default_order_type;
-	auto null_order = config.options.default_null_order;
+	auto order = OrderType::ORDER_DEFAULT;
+	auto null_order = OrderByNullType::ORDER_DEFAULT;
 
 	// get the sorting order
 	if (arguments.size() >= 2) {
-
-		if (!arguments[1]->IsFoldable()) {
-			throw InvalidInputException("Sorting order must be a constant");
-		}
-		Value order_value = ExpressionExecutor::EvaluateScalar(context, *arguments[1]);
-
-		const auto order_name = StringUtil::Upper(order_value.ToString());
-		const auto order_arg = EnumSerializer::StringToEnum<OrderType>(order_name.c_str());
-		switch (order_arg) {
-		case OrderType::ASCENDING:
-		case OrderType::DESCENDING:
-			order = order_arg;
-			break;
-		case OrderType::ORDER_DEFAULT:
-			break;
-		default:
-			throw InvalidInputException("Sorting order must be either ASC, DESC or DEFAULT");
-		}
+		order = GetOrder<OrderType>(context, *arguments[1]);
 	}
-
 	// get the null sorting order
 	if (arguments.size() == 3) {
-		null_order = GetNullOrder(context, arguments, 2);
+		null_order = GetOrder<OrderByNullType>(context, *arguments[2]);
 	}
-
+	auto &config = DBConfig::GetConfig(context);
+	order = config.ResolveOrder(order);
+	null_order = config.ResolveNullOrder(order, null_order);
 	return ListSortBind(context, bound_function, arguments, order, null_order);
 }
 
 static unique_ptr<FunctionData> ListReverseSortBind(ClientContext &context, ScalarFunction &bound_function,
                                                     vector<unique_ptr<Expression>> &arguments) {
+	auto order = OrderType::ORDER_DEFAULT;
+	auto null_order = OrderByNullType::ORDER_DEFAULT;
 
-	D_ASSERT(bound_function.arguments.size() == 1 || bound_function.arguments.size() == 2);
-	D_ASSERT(arguments.size() == 1 || arguments.size() == 2);
-
-	// set (reverse) default values
-	auto &config = DBConfig::GetConfig(context);
-	auto order =
-	    (config.options.default_order_type == OrderType::ASCENDING) ? OrderType::DESCENDING : OrderType::ASCENDING;
-	auto null_order = config.options.default_null_order;
-
-	// get the null sorting order
 	if (arguments.size() == 2) {
-		null_order = GetNullOrder(context, arguments, 1);
+		null_order = GetOrder<OrderByNullType>(context, *arguments[1]);
 	}
-
+	auto &config = DBConfig::GetConfig(context);
+	order = config.ResolveOrder(order);
+	switch (order) {
+	case OrderType::ASCENDING:
+		order = OrderType::DESCENDING;
+		break;
+	case OrderType::DESCENDING:
+		order = OrderType::ASCENDING;
+		break;
+	default:
+		throw InternalException("Unexpected order type in list reverse sort");
+	}
+	null_order = config.ResolveNullOrder(order, null_order);
 	return ListSortBind(context, bound_function, arguments, order, null_order);
 }
 

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.7.2-dev2366"
+#define DUCKDB_VERSION "0.7.2-dev2430"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "1d7b9c315e"
+#define DUCKDB_SOURCE_ID "f429595834"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/common/enums/debug_initialize.hpp

@@ -0,0 +1,17 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/enums/debug_initialize.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/constants.hpp"
+
+namespace duckdb {
+
+enum class DebugInitialize : uint8_t { NO_INITIALIZE = 0, DEBUG_ZERO_INITIALIZE = 1, DEBUG_ONE_INITIALIZE = 2 };
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/enums/order_type.hpp

@@ -17,4 +17,12 @@ enum class OrderType : uint8_t { INVALID = 0, ORDER_DEFAULT = 1, ASCENDING = 2,
 
 enum class OrderByNullType : uint8_t { INVALID = 0, ORDER_DEFAULT = 1, NULLS_FIRST = 2, NULLS_LAST = 3 };
 
+enum class DefaultOrderByNullType : uint8_t {
+	INVALID = 0,
+	NULLS_FIRST = 2,
+	NULLS_LAST = 3,
+	NULLS_FIRST_ON_ASC_LAST_ON_DESC = 4,
+	NULLS_LAST_ON_ASC_FIRST_ON_DESC = 5
+};
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp

@@ -58,6 +58,7 @@ enum class PhysicalOperatorType : uint8_t {
 	DELIM_JOIN,
 	INDEX_JOIN,
 	POSITIONAL_JOIN,
+	ASOF_JOIN,
 	// -----------------------------
 	// SetOps
 	// -----------------------------

package/src/duckdb/src/include/duckdb/common/file_buffer.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/common/constants.hpp"
+#include "duckdb/common/enums/debug_initialize.hpp"
 
 namespace duckdb {
 class Allocator;
@@ -62,6 +63,8 @@ public:
 
 	MemoryRequirement CalculateMemory(uint64_t user_size);
 
+	void Initialize(DebugInitialize info);
+
 protected:
 	//! The pointer to the internal buffer that will be read or written, including the buffer header
 	data_ptr_t internal_buffer;

package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp

@@ -49,6 +49,9 @@ public:
 	//! Select using a cutoff on the radix bits of the hash
 	static idx_t Select(Vector &hashes, const SelectionVector *sel, idx_t count, idx_t radix_bits, idx_t cutoff,
 	                    SelectionVector *true_sel, SelectionVector *false_sel);
+
+	//! Convert hashes to bins
+	static void HashesToBins(Vector &hashes, idx_t radix_bits, Vector &bins, idx_t count);
 };
 
 //! Templated radix partitioning constants, can be templated to the number of radix bits

package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp

@@ -25,6 +25,8 @@ public:
 	PartitionGlobalHashGroup(BufferManager &buffer_manager, const Orders &partitions, const Orders &orders,
 	                         const Types &payload_types, bool external);
 
+	int ComparePartitions(const SBIterator &left, const SBIterator &right) const;
+
 	void ComputeMasks(ValidityMask &partition_mask, ValidityMask &order_mask);
 
 	GlobalSortStatePtr global_sort;
@@ -43,8 +45,12 @@ public:
 	using GroupingPartition = unique_ptr<PartitionedColumnData>;
 	using GroupingAppend = unique_ptr<PartitionedColumnDataAppendState>;
 
-	PartitionGlobalSinkState(ClientContext &context, const vector<unique_ptr<Expression>> &partitions_p,
-	                         const vector<BoundOrderByNode> &orders_p, const Types &payload_types,
+	static void GenerateOrderings(Orders &partitions, Orders &orders,
+	                              const vector<unique_ptr<Expression>> &partition_bys, const Orders &order_bys,
+	                              const vector<unique_ptr<BaseStatistics>> &partitions_stats);
+
+	PartitionGlobalSinkState(ClientContext &context, const vector<unique_ptr<Expression>> &partition_bys,
+	                         const vector<BoundOrderByNode> &order_bys, const Types &payload_types,
 	                         const vector<unique_ptr<BaseStatistics>> &partitions_stats, idx_t estimated_cardinality);
 
 	void UpdateLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append);
@@ -68,6 +74,8 @@ public:
 	const Types payload_types;
 	vector<HashGroupPtr> hash_groups;
 	bool external;
+	//	Reverse lookup from hash bins to non-empty hash groups
+	vector<size_t> bin_groups;
 
 	// OVER() (no sorting)
 	unique_ptr<RowDataCollection> rows;
@@ -121,7 +129,7 @@ class PartitionGlobalMergeState {
 public:
 	using GroupDataPtr = unique_ptr<ColumnDataCollection>;
 
-	explicit PartitionGlobalMergeState(PartitionGlobalSinkState &sink, GroupDataPtr group_data);
+	PartitionGlobalMergeState(PartitionGlobalSinkState &sink, GroupDataPtr group_data, hash_t hash_bin);
 
 	bool IsSorted() const {
 		lock_guard<mutex> guard(lock);
@@ -187,61 +195,4 @@ public:
 	void Schedule() override;
 };
 
-class PartitionGlobalSourceState {
-public:
-	explicit PartitionGlobalSourceState(PartitionGlobalSinkState &gsink_p) : gsink(gsink_p), next_bin(0) {
-	}
-
-	PartitionGlobalSinkState &gsink;
-	//! The output read position.
-	atomic<idx_t> next_bin;
-
-public:
-	idx_t MaxThreads() {
-		// If there is only one partition, we have to process it on one thread.
-		if (!gsink.grouping_data) {
-			return 1;
-		}
-
-		// If there is not a lot of data, process serially.
-		if (gsink.count < STANDARD_ROW_GROUPS_SIZE) {
-			return 1;
-		}
-
-		return gsink.hash_groups.size();
-	}
-};
-
-// Per-thread read state
-class PartitionLocalSourceState {
-public:
-	using HashGroupPtr = unique_ptr<PartitionGlobalHashGroup>;
-
-	explicit PartitionLocalSourceState(PartitionGlobalSinkState &gstate_p);
-
-	void MaterializeSortedData();
-	idx_t GeneratePartition(const idx_t hash_bin);
-
-	PartitionGlobalSinkState &gstate;
-
-	//! The read partition
-	idx_t hash_bin;
-	HashGroupPtr hash_group;
-
-	//! The generated input chunks
-	unique_ptr<RowDataCollection> rows;
-	unique_ptr<RowDataCollection> heap;
-	RowLayout layout;
-	//! The partition boundary mask
-	vector<validity_t> partition_bits;
-	ValidityMask partition_mask;
-	//! The order boundary mask
-	vector<validity_t> order_bits;
-	ValidityMask order_mask;
-	//! The read cursor
-	unique_ptr<RowDataCollectionScanner> scanner;
-	//! Buffer for the inputs
-	DataChunk input_chunk;
-};
-
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/join/outer_join_marker.hpp

@@ -29,7 +29,7 @@ struct OuterJoinLocalScanState {
 
 class OuterJoinMarker {
 public:
-	OuterJoinMarker(bool enabled);
+	explicit OuterJoinMarker(bool enabled);
 
 	bool Enabled() {
 		return enabled;
@@ -60,6 +60,11 @@ public:
 	//! Perform the scan
 	void Scan(OuterJoinGlobalScanState &gstate, OuterJoinLocalScanState &lstate, DataChunk &result);
 
+	//! Read-only matches vector
+	const bool *GetMatches() const {
+		return found_match.get();
+	}
+
 private:
 	bool enabled;
 	unique_ptr<bool[]> found_match;

package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp

@@ -0,0 +1,93 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/join/physical_asof_join.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/execution/operator/join/physical_comparison_join.hpp"
+#include "duckdb/planner/bound_result_modifier.hpp"
+
+namespace duckdb {
+
+//! PhysicalAsOfJoin represents a piecewise merge loop join between
+//! two tables
+class PhysicalAsOfJoin : public PhysicalComparisonJoin {
+public:
+	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::ASOF_JOIN;
+
+public:
+	PhysicalAsOfJoin(LogicalComparisonJoin &op, unique_ptr<PhysicalOperator> left, unique_ptr<PhysicalOperator> right);
+
+	vector<LogicalType> join_key_types;
+	vector<column_t> null_sensitive;
+
+	// Equalities
+	vector<unique_ptr<Expression>> lhs_partitions;
+	vector<unique_ptr<Expression>> rhs_partitions;
+
+	// Inequality Only
+	vector<BoundOrderByNode> lhs_orders;
+	vector<BoundOrderByNode> rhs_orders;
+
+	// Projection mappings
+	vector<column_t> right_projection_map;
+
+public:
+	// Operator Interface
+	unique_ptr<GlobalOperatorState> GetGlobalOperatorState(ClientContext &context) const override;
+	unique_ptr<OperatorState> GetOperatorState(ExecutionContext &context) const override;
+
+	bool ParallelOperator() const override {
+		return true;
+	}
+
+protected:
+	// CachingOperator Interface
+	OperatorResultType ExecuteInternal(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
+	                                   GlobalOperatorState &gstate, OperatorState &state) const override;
+
+public:
+	// Source interface
+	unique_ptr<LocalSourceState> GetLocalSourceState(ExecutionContext &context,
+	                                                 GlobalSourceState &gstate) const override;
+	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
+	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
+	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return IsRightOuterJoin(join_type);
+	}
+	bool ParallelSource() const override {
+		return true;
+	}
+
+public:
+	// Sink Interface
+	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
+	unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
+	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,
+	                    DataChunk &input) const override;
+	void Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const override;
+	SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
+	                          GlobalSinkState &gstate) const override;
+
+	bool IsSink() const override {
+		return true;
+	}
+	bool ParallelSink() const override {
+		return true;
+	}
+
+private:
+	// resolve joins that output max N elements (SEMI, ANTI, MARK)
+	void ResolveSimpleJoin(ExecutionContext &context, DataChunk &input, DataChunk &chunk, OperatorState &state) const;
+	// resolve joins that can potentially output N*M elements (INNER, LEFT, FULL)
+	OperatorResultType ResolveComplexJoin(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
+	                                      OperatorState &state) const;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/physical_operator.hpp

@@ -199,7 +199,7 @@ public:
 	~CachingOperatorState() override {
 	}
 
-	virtual void Finalize(const PhysicalOperator &op, ExecutionContext &context) override {
+	void Finalize(const PhysicalOperator &op, ExecutionContext &context) override {
 	}
 
 	unique_ptr<DataChunk> cached_chunk;

package/src/duckdb/src/include/duckdb/main/client_config.hpp

@@ -71,6 +71,8 @@ struct ClientConfig {
 	bool force_external = false;
 	//! Force disable cross product generation when hyper graph isn't connected, used for testing
 	bool force_no_cross_product = false;
+	//! Force use of IEJoin to implement AsOfJoin, used for testing
+	bool force_asof_iejoin = false;
 	//! If this context should also try to use the available replacement scans
 	//! True by default
 	bool use_replacement_scans = true;

package/src/duckdb/src/include/duckdb/main/config.hpp

@@ -106,8 +106,8 @@ struct DBConfigOptions {
 	string collation = string();
 	//! The order type used when none is specified (default: ASC)
 	OrderType default_order_type = OrderType::ASCENDING;
-	//! Null ordering used when none is specified (default: NULLS FIRST)
-	OrderByNullType default_null_order = OrderByNullType::NULLS_FIRST;
+	//! Null ordering used when none is specified (default: NULLS LAST)
+	DefaultOrderByNullType default_null_order = DefaultOrderByNullType::NULLS_LAST;
 	//! enable COPY and related commands
 	bool enable_external_access = true;
 	//! Whether or not object cache is used
@@ -145,6 +145,8 @@ struct DBConfigOptions {
 	bool enable_fsst_vectors = false;
 	//! Start transactions immediately in all attached databases - instead of lazily when a database is referenced
 	bool immediate_transaction_mode = false;
+	//! Debug setting - how to initialize  blocks in the storage layer when allocating
+	DebugInitialize debug_initialize = DebugInitialize::NO_INITIALIZE;
 	//! The set of unrecognized (other) options
 	unordered_map<string, Value> unrecognized_options;
 
@@ -226,6 +228,9 @@ public:
 	void SetDefaultMaxThreads();
 	void SetDefaultMaxMemory();
 
+	OrderType ResolveOrder(OrderType order_type) const;
+	OrderByNullType ResolveNullOrder(OrderType order_type, OrderByNullType null_type) const;
+
 private:
 	unique_ptr<CompressionFunctionSet> compression_functions;
 	unique_ptr<CastFunctionSet> cast_functions;

package/src/duckdb/src/include/duckdb/main/settings.hpp

@@ -66,9 +66,19 @@ struct DebugForceNoCrossProduct {
 };
 
 struct OrderedAggregateThreshold {
-	static constexpr const char *Name = "ordered_aggregate_threshold";
-	static constexpr const char *Description = "the number of rows to accumulate before sorting, used for tuning";
-	static constexpr const LogicalTypeId InputType = LogicalTypeId::UBIGINT;
+	static constexpr const char *Name = "ordered_aggregate_threshold"; // NOLINT
+	static constexpr const char *Description =                         // NOLINT
+	    "the number of rows to accumulate before sorting, used for tuning";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::UBIGINT; // NOLINT
+	static void SetLocal(ClientContext &context, const Value &parameter);
+	static void ResetLocal(ClientContext &context);
+	static Value GetSetting(ClientContext &context);
+};
+
+struct DebugAsOfIEJoin {
+	static constexpr const char *Name = "debug_asof_iejoin";                                                 // NOLINT
+	static constexpr const char *Description = "DEBUG SETTING: force use of IEJoin to implement AsOf joins"; // NOLINT
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN;                                 // NOLINT
 	static void SetLocal(ClientContext &context, const Value &parameter);
 	static void ResetLocal(ClientContext &context);
 	static Value GetSetting(ClientContext &context);

package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp

@@ -87,8 +87,10 @@ public:
 		// Start with function call
 		string result = schema.empty() ? function_name : schema + "." + function_name;
 		result += "(";
-		result += StringUtil::Join(entry.children, entry.children.size(), ", ",
-		                           [](const unique_ptr<BASE> &child) { return child->ToString(); });
+		if (entry.children.size()) {
+			result += StringUtil::Join(entry.children, entry.children.size(), ", ",
+			                           [](const unique_ptr<BASE> &child) { return child->ToString(); });
+		}
 		// Lead/Lag extra arguments
 		if (entry.offset_expr.get()) {
 			result += ", ";

package/src/duckdb/src/include/duckdb/storage/block_manager.hpp

@@ -33,6 +33,7 @@ public:
 
 public:
 	//! Creates a new block inside the block manager
+	virtual unique_ptr<Block> ConvertBlock(block_id_t block_id, FileBuffer &source_buffer) = 0;
 	virtual unique_ptr<Block> CreateBlock(block_id_t block_id, FileBuffer *source_buffer) = 0;
 	//! Return the next free block id
 	virtual block_id_t GetFreeBlockId() = 0;

package/src/duckdb/src/include/duckdb/storage/in_memory_block_manager.hpp

@@ -20,6 +20,9 @@ public:
 	using BlockManager::BlockManager;
 
 	// LCOV_EXCL_START
+	unique_ptr<Block> ConvertBlock(block_id_t block_id, FileBuffer &source_buffer) override {
+		throw InternalException("Cannot perform IO in in-memory database!");
+	}
 	unique_ptr<Block> CreateBlock(block_id_t block_id, FileBuffer *source_buffer) override {
 		throw InternalException("Cannot perform IO in in-memory database!");
 	}

package/src/duckdb/src/include/duckdb/storage/partial_block_manager.hpp

@@ -44,7 +44,8 @@ struct PartialBlock {
 	PartialBlockState state;
 
 public:
-	virtual void Flush() = 0;
+	virtual void AddUninitializedRegion(idx_t start, idx_t end) = 0;
+	virtual void Flush(idx_t free_space_left) = 0;
 	virtual void Clear() {
 	}
 };

package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp

@@ -15,24 +15,32 @@
 #include "duckdb/common/unordered_set.hpp"
 #include "duckdb/common/set.hpp"
 #include "duckdb/common/vector.hpp"
+#include "duckdb/main/config.hpp"
 
 namespace duckdb {
 
 class DatabaseInstance;
 
+struct StorageManagerOptions {
+	bool read_only = false;
+	bool use_direct_io = false;
+	DebugInitialize debug_initialize = DebugInitialize::NO_INITIALIZE;
+};
+
 //! SingleFileBlockManager is an implementation for a BlockManager which manages blocks in a single file
 class SingleFileBlockManager : public BlockManager {
 	//! The location in the file where the block writing starts
 	static constexpr uint64_t BLOCK_START = Storage::FILE_HEADER_SIZE * 3;
 
 public:
-	SingleFileBlockManager(AttachedDatabase &db, string path, bool read_only, bool use_direct_io);
+	SingleFileBlockManager(AttachedDatabase &db, string path, StorageManagerOptions options);
 
 	void GetFileFlags(uint8_t &flags, FileLockType &lock, bool create_new);
 	void CreateNewDatabase();
 	void LoadExistingDatabase();
 
 	//! Creates a new Block using the specified block_id and returns a pointer
+	unique_ptr<Block> ConvertBlock(block_id_t block_id, FileBuffer &source_buffer) override;
 	unique_ptr<Block> CreateBlock(block_id_t block_id, FileBuffer *source_buffer) override;
 	//! Return the next free block id
 	block_id_t GetFreeBlockId() override;
@@ -96,10 +104,8 @@ private:
 	block_id_t free_list_id;
 	//! The current header iteration count
 	uint64_t iteration_count;
-	//! Whether or not the db is opened in read-only mode
-	bool read_only;
-	//! Whether or not to use Direct IO to read the blocks
-	bool use_direct_io;
+	//! The storage manager options
+	StorageManagerOptions options;
 	//! Lock for performing various operations in the single file block manager
 	mutex block_lock;
 };

package/src/duckdb/src/main/config.cpp

@@ -54,6 +54,7 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_GLOBAL(DebugCheckpointAbort),
                                                  DUCKDB_LOCAL(DebugForceExternal),
                                                  DUCKDB_LOCAL(DebugForceNoCrossProduct),
+                                                 DUCKDB_LOCAL(DebugAsOfIEJoin),
                                                  DUCKDB_GLOBAL(DebugWindowMode),
                                                  DUCKDB_GLOBAL_LOCAL(DefaultCollationSetting),
                                                  DUCKDB_GLOBAL(DefaultOrderSetting),
@@ -359,4 +360,29 @@ bool DBConfig::operator!=(const DBConfig &other) {
 	return !(other.options == options);
 }
 
+OrderType DBConfig::ResolveOrder(OrderType order_type) const {
+	if (order_type != OrderType::ORDER_DEFAULT) {
+		return order_type;
+	}
+	return options.default_order_type;
+}
+
+OrderByNullType DBConfig::ResolveNullOrder(OrderType order_type, OrderByNullType null_type) const {
+	if (null_type != OrderByNullType::ORDER_DEFAULT) {
+		return null_type;
+	}
+	switch (options.default_null_order) {
+	case DefaultOrderByNullType::NULLS_FIRST:
+		return OrderByNullType::NULLS_FIRST;
+	case DefaultOrderByNullType::NULLS_LAST:
+		return OrderByNullType::NULLS_LAST;
+	case DefaultOrderByNullType::NULLS_FIRST_ON_ASC_LAST_ON_DESC:
+		return order_type == OrderType::ASCENDING ? OrderByNullType::NULLS_FIRST : OrderByNullType::NULLS_LAST;
+	case DefaultOrderByNullType::NULLS_LAST_ON_ASC_FIRST_ON_DESC:
+		return order_type == OrderType::ASCENDING ? OrderByNullType::NULLS_LAST : OrderByNullType::NULLS_FIRST;
+	default:
+		throw InternalException("Unknown null order setting");
+	}
+}
+
 } // namespace duckdb