duckdb 0.6.2-dev1770.0 → 0.6.2-dev1794.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.6.2-dev1770.0",
+  "version": "0.6.2-dev1794.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/common/enums/optimizer_type.cpp

@@ -18,6 +18,7 @@ static DefaultOptimizerType internal_optimizer_types[] = {
     {"in_clause", OptimizerType::IN_CLAUSE},
     {"join_order", OptimizerType::JOIN_ORDER},
     {"deliminator", OptimizerType::DELIMINATOR},
+    {"unnest_rewriter", OptimizerType::UNNEST_REWRITER},
     {"unused_columns", OptimizerType::UNUSED_COLUMNS},
     {"statistics_propagation", OptimizerType::STATISTICS_PROPAGATION},
     {"common_subexpressions", OptimizerType::COMMON_SUBEXPRESSIONS},

package/src/duckdb/src/common/file_buffer.cpp

@@ -23,8 +23,6 @@ void FileBuffer::Init() {
 	size = 0;
 	internal_buffer = nullptr;
 	internal_size = 0;
-	malloced_buffer = nullptr;
-	malloced_size = 0;
 }
 
 FileBuffer::FileBuffer(FileBuffer &source, FileBufferType type_p) : allocator(source.allocator), type(type_p) {
@@ -33,31 +31,29 @@ FileBuffer::FileBuffer(FileBuffer &source, FileBufferType type_p) : allocator(so
 	size = source.size;
 	internal_buffer = source.internal_buffer;
 	internal_size = source.internal_size;
-	malloced_buffer = source.malloced_buffer;
-	malloced_size = source.malloced_size;
 
 	source.Init();
 }
 
 FileBuffer::~FileBuffer() {
-	if (!malloced_buffer) {
+	if (!internal_buffer) {
 		return;
 	}
-	allocator.FreeData(malloced_buffer, malloced_size);
+	allocator.FreeData(internal_buffer, internal_size);
 }
 
 void FileBuffer::ReallocBuffer(size_t new_size) {
-	if (malloced_buffer) {
-		malloced_buffer = allocator.ReallocateData(malloced_buffer, malloced_size, new_size);
+	data_ptr_t new_buffer;
+	if (internal_buffer) {
+		new_buffer = allocator.ReallocateData(internal_buffer, internal_size, new_size);
 	} else {
-		malloced_buffer = allocator.AllocateData(new_size);
+		new_buffer = allocator.AllocateData(new_size);
 	}
-	if (!malloced_buffer) {
+	if (!new_buffer) {
 		throw std::bad_alloc();
 	}
-	malloced_size = new_size;
-	internal_buffer = malloced_buffer;
-	internal_size = malloced_size;
+	internal_buffer = new_buffer;
+	internal_size = new_size;
 	// Caller must update these.
 	buffer = nullptr;
 	size = 0;
@@ -92,32 +88,11 @@ void FileBuffer::Read(FileHandle &handle, uint64_t location) {
 	handle.Read(internal_buffer, internal_size, location);
 }
 
-void FileBuffer::ReadAndChecksum(FileHandle &handle, uint64_t location) {
-	// read the buffer from disk
-	Read(handle, location);
-	// compute the checksum
-	auto stored_checksum = Load<uint64_t>(internal_buffer);
-	uint64_t computed_checksum = Checksum(buffer, size);
-	// verify the checksum
-	if (stored_checksum != computed_checksum) {
-		throw IOException("Corrupt database file: computed checksum %llu does not match stored checksum %llu in block",
-		                  computed_checksum, stored_checksum);
-	}
-}
-
 void FileBuffer::Write(FileHandle &handle, uint64_t location) {
 	D_ASSERT(type != FileBufferType::TINY_BUFFER);
 	handle.Write(internal_buffer, internal_size, location);
 }
 
-void FileBuffer::ChecksumAndWrite(FileHandle &handle, uint64_t location) {
-	// compute the checksum and write it to the start of the buffer (if not temp buffer)
-	uint64_t checksum = Checksum(buffer, size);
-	Store<uint64_t>(checksum, internal_buffer);
-	// now write the buffer
-	Write(handle, location);
-}
-
 void FileBuffer::Clear() {
 	memset(internal_buffer, 0, internal_size);
 }

package/src/duckdb/src/execution/index/art/swizzleable_pointer.cpp

@@ -11,8 +11,8 @@ SwizzleablePointer::~SwizzleablePointer() {
 
 SwizzleablePointer::SwizzleablePointer(duckdb::MetaBlockReader &reader) {
 	idx_t block_id = reader.Read<block_id_t>();
-	idx_t offset = reader.Read<uint32_t>();
-	if (block_id == DConstants::INVALID_INDEX || offset == DConstants::INVALID_INDEX) {
+	uint32_t offset = reader.Read<uint32_t>();
+	if (block_id == DConstants::INVALID_INDEX || offset == (uint32_t)DConstants::INVALID_INDEX) {
 		pointer = 0;
 		return;
 	}

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.6.2-dev1770"
+#define DUCKDB_VERSION "0.6.2-dev1794"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "864ff1c719"
+#define DUCKDB_SOURCE_ID "d8add1ee3f"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp

@@ -21,6 +21,7 @@ enum class OptimizerType : uint32_t {
 	IN_CLAUSE,
 	JOIN_ORDER,
 	DELIMINATOR,
+	UNNEST_REWRITER,
 	UNUSED_COLUMNS,
 	STATISTICS_PROPAGATION,
 	COMMON_SUBEXPRESSIONS,

package/src/duckdb/src/include/duckdb/common/file_buffer.hpp

@@ -39,14 +39,8 @@ public:
 public:
 	//! Read into the FileBuffer from the specified location.
 	void Read(FileHandle &handle, uint64_t location);
-	//! Read into the FileBuffer from the specified location. Automatically verifies the checksum, and throws an
-	//! exception if the checksum does not match correctly.
-	virtual void ReadAndChecksum(FileHandle &handle, uint64_t location);
 	//! Write the contents of the FileBuffer to the specified location.
 	void Write(FileHandle &handle, uint64_t location);
-	//! Write the contents of the FileBuffer to the specified location. Automatically adds a checksum of the contents of
-	//! the filebuffer in front of the written data.
-	virtual void ChecksumAndWrite(FileHandle &handle, uint64_t location);
 
 	void Clear();
 
@@ -57,6 +51,9 @@ public:
 	uint64_t AllocSize() const {
 		return internal_size;
 	}
+	data_ptr_t InternalBuffer() {
+		return internal_buffer;
+	}
 
 	struct MemoryRequirement {
 		idx_t alloc_size;
@@ -72,16 +69,6 @@ protected:
 	uint64_t internal_size;
 
 	void ReallocBuffer(size_t malloc_size);
-
-private:
-	//! The buffer that was actually malloc'd, i.e. the pointer that must be freed when the FileBuffer is destroyed
-	data_ptr_t malloced_buffer;
-	uint64_t malloced_size;
-
-protected:
-	uint64_t GetMallocedSize() {
-		return malloced_size;
-	}
 	void Init();
 };
 

package/src/duckdb/src/include/duckdb/optimizer/unnest_rewriter.hpp

@@ -0,0 +1,84 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/optimizer/unnest_rewriter.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/planner/logical_operator.hpp"
+#include "duckdb/common/pair.hpp"
+
+namespace duckdb {
+
+class Optimizer;
+
+struct ReplaceBinding {
+	ReplaceBinding() {};
+	ReplaceBinding(ColumnBinding old_binding, ColumnBinding new_binding)
+	    : old_binding(old_binding), new_binding(new_binding) {
+	}
+	ColumnBinding old_binding;
+	ColumnBinding new_binding;
+};
+
+struct LHSBinding {
+	LHSBinding() {};
+	LHSBinding(ColumnBinding binding, LogicalType type) : binding(binding), type(type) {
+	}
+	ColumnBinding binding;
+	LogicalType type;
+	string alias;
+};
+
+//! The UnnestRewriterPlanUpdater updates column bindings after changing the operator plan
+class UnnestRewriterPlanUpdater : LogicalOperatorVisitor {
+public:
+	UnnestRewriterPlanUpdater() {
+	}
+	//! Update each operator of the plan after moving an UNNEST into a projection
+	void VisitOperator(LogicalOperator &op) override;
+	//! Visit an expression and update its column bindings after moving and UNNEST into a projection
+	void VisitExpression(unique_ptr<Expression> *expression) override;
+
+	//! Contains all bindings that need to be updated
+	vector<ReplaceBinding> replace_bindings;
+};
+
+//! The UnnestRewriter optimizer traverses the logical operator tree and rewrites duplicate
+//! eliminated joins that contain UNNESTs by moving the UNNESTs into the projection of
+//! the SELECT
+class UnnestRewriter {
+public:
+	UnnestRewriter() {
+	}
+	//! Rewrite duplicate eliminated joins with UNNESTs
+	unique_ptr<LogicalOperator> Optimize(unique_ptr<LogicalOperator> op);
+
+private:
+	//! Find delim joins that contain an UNNEST
+	void FindCandidates(unique_ptr<LogicalOperator> *op_ptr, vector<unique_ptr<LogicalOperator> *> &candidates);
+	//! Rewrite a delim join that contains an UNNEST
+	bool RewriteCandidate(unique_ptr<LogicalOperator> *candidate);
+	//! Update the bindings of the RHS sequence of LOGICAL_PROJECTION(s)
+	void UpdateRHSBindings(unique_ptr<LogicalOperator> *plan_ptr, unique_ptr<LogicalOperator> *candidate,
+	                       UnnestRewriterPlanUpdater &updater);
+	//! Update the bindings of the BOUND_UNNEST expression of the LOGICAL_UNNEST
+	void UpdateBoundUnnestBindings(UnnestRewriterPlanUpdater &updater, unique_ptr<LogicalOperator> *candidate);
+
+	//! Store all delim columns of the delim join
+	void GetDelimColumns(LogicalOperator &op);
+	//! Store all LHS expressions of the LOGICAL_PROJECTION
+	void GetLHSExpressions(LogicalOperator &op);
+
+	//! Keep track of the delim columns to find the correct UNNEST column
+	vector<ColumnBinding> delim_columns;
+	//! Store the column bindings of the LHS child of the LOGICAL_DELIM_JOIN
+	vector<LHSBinding> lhs_bindings;
+	//! Stores the table index of the former child of the LOGICAL_UNNEST
+	idx_t overwritten_tbl_idx;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/operator/logical_unnest.hpp

@@ -12,8 +12,7 @@
 
 namespace duckdb {
 
-//! LogicalAggregate represents an aggregate operation with (optional) GROUP BY
-//! operator.
+//! LogicalUnnest represents the logical UNNEST operator.
 class LogicalUnnest : public LogicalOperator {
 public:
 	explicit LogicalUnnest(idx_t unnest_index)

package/src/duckdb/src/include/duckdb/storage/buffer_manager.hpp

@@ -85,10 +85,8 @@ public:
 	}
 
 	//! Construct a managed buffer.
-	//! The block_id is just used for internal tracking. It doesn't map to any actual
-	//! BlockManager.
-	virtual unique_ptr<FileBuffer> ConstructManagedBuffer(idx_t size, unique_ptr<FileBuffer> &&source,
-	                                                      FileBufferType type = FileBufferType::MANAGED_BUFFER);
+	unique_ptr<FileBuffer> ConstructManagedBuffer(idx_t size, unique_ptr<FileBuffer> &&source,
+	                                              FileBufferType type = FileBufferType::MANAGED_BUFFER);
 
 	DUCKDB_API void ReserveMemory(idx_t size);
 	DUCKDB_API void FreeReservedMemory(idx_t size);

package/src/duckdb/src/include/duckdb/storage/index.hpp

@@ -118,6 +118,7 @@ public:
 	}
 	//! Serializes the index and returns the pair of block_id offset positions
 	virtual BlockPointer Serialize(duckdb::MetaBlockWriter &writer);
+	BlockPointer GetBlockPointer();
 
 	//! Returns block/offset of where index was most recently serialized.
 	BlockPointer GetSerializedDataPointer() const {

package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp

@@ -59,6 +59,9 @@ private:
 
 	void Initialize(DatabaseHeader &header);
 
+	void ReadAndChecksum(FileBuffer &handle, uint64_t location) const;
+	void ChecksumAndWrite(FileBuffer &handle, uint64_t location) const;
+
 	//! Return the blocks to which we will write the free list and modified blocks
 	vector<block_id_t> GetFreeListBlocks();
 

package/src/duckdb/src/include/duckdb/storage/write_ahead_log.hpp

@@ -12,11 +12,11 @@
 #include "duckdb/common/types/data_chunk.hpp"
 #include "duckdb/common/enums/wal_type.hpp"
 #include "duckdb/common/serializer/buffered_file_writer.hpp"
-#include "duckdb/catalog/catalog_entry/sequence_catalog_entry.hpp"
-#include "duckdb/storage/storage_info.hpp"
-
 #include "duckdb/catalog/catalog_entry/scalar_macro_catalog_entry.hpp"
+#include "duckdb/catalog/catalog_entry/sequence_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/table_macro_catalog_entry.hpp"
+#include "duckdb/main/attached_database.hpp"
+#include "duckdb/storage/storage_info.hpp"
 
 namespace duckdb {
 
@@ -38,8 +38,8 @@ class TransactionManager;
 class ReplayState {
 public:
 	ReplayState(AttachedDatabase &db, ClientContext &context, Deserializer &source)
-	    : db(db), context(context), catalog(Catalog::GetCatalog(context, INVALID_CATALOG)), source(source),
-	      current_table(nullptr), deserialize_only(false), checkpoint_id(INVALID_BLOCK) {
+	    : db(db), context(context), catalog(db.GetCatalog()), source(source), current_table(nullptr),
+	      deserialize_only(false), checkpoint_id(INVALID_BLOCK) {
 	}
 
 	AttachedDatabase &db;

package/src/duckdb/src/optimizer/optimizer.cpp

@@ -9,6 +9,7 @@
 #include "duckdb/optimizer/common_aggregate_optimizer.hpp"
 #include "duckdb/optimizer/cse_optimizer.hpp"
 #include "duckdb/optimizer/deliminator.hpp"
+#include "duckdb/optimizer/unnest_rewriter.hpp"
 #include "duckdb/optimizer/expression_heuristics.hpp"
 #include "duckdb/optimizer/filter_pullup.hpp"
 #include "duckdb/optimizer/filter_pushdown.hpp"
@@ -111,6 +112,13 @@ unique_ptr<LogicalOperator> Optimizer::Optimize(unique_ptr<LogicalOperator> plan
 		plan = deliminator.Optimize(std::move(plan));
 	});
 
+	// rewrites UNNESTs in DelimJoins by moving them to the projection
+	RunOptimizer(OptimizerType::UNNEST_REWRITER, [&]() {
+		UnnestRewriter unnest_rewriter;
+		plan = unnest_rewriter.Optimize(std::move(plan));
+	});
+
+	// removes unused columns
 	RunOptimizer(OptimizerType::UNUSED_COLUMNS, [&]() {
 		RemoveUnusedColumns unused(binder, context, true);
 		unused.VisitOperator(*plan);

package/src/duckdb/src/optimizer/unnest_rewriter.cpp

@@ -0,0 +1,312 @@
+#include "duckdb/optimizer/unnest_rewriter.hpp"
+
+#include "duckdb/common/pair.hpp"
+#include "duckdb/planner/operator/logical_delim_get.hpp"
+#include "duckdb/planner/operator/logical_delim_join.hpp"
+#include "duckdb/planner/operator/logical_unnest.hpp"
+#include "duckdb/planner/operator/logical_projection.hpp"
+#include "duckdb/planner/operator/logical_window.hpp"
+#include "duckdb/planner/expression/bound_unnest_expression.hpp"
+#include "duckdb/planner/expression/bound_columnref_expression.hpp"
+
+namespace duckdb {
+
+void UnnestRewriterPlanUpdater::VisitOperator(LogicalOperator &op) {
+	VisitOperatorChildren(op);
+	VisitOperatorExpressions(op);
+}
+
+void UnnestRewriterPlanUpdater::VisitExpression(unique_ptr<Expression> *expression) {
+
+	auto &expr = *expression;
+
+	if (expr->expression_class == ExpressionClass::BOUND_COLUMN_REF) {
+
+		auto &bound_column_ref = (BoundColumnRefExpression &)*expr;
+		for (idx_t i = 0; i < replace_bindings.size(); i++) {
+			if (bound_column_ref.binding == replace_bindings[i].old_binding) {
+				bound_column_ref.binding = replace_bindings[i].new_binding;
+			}
+			// previously pointing to the LOGICAL_DELIM_GET
+			if (bound_column_ref.binding.table_index == replace_bindings[i].old_binding.table_index &&
+			    replace_bindings[i].old_binding.column_index == DConstants::INVALID_INDEX) {
+				bound_column_ref.binding = replace_bindings[i].new_binding;
+			}
+		}
+	}
+
+	VisitExpressionChildren(**expression);
+}
+
+unique_ptr<LogicalOperator> UnnestRewriter::Optimize(unique_ptr<LogicalOperator> op) {
+
+	UnnestRewriterPlanUpdater updater;
+	vector<unique_ptr<LogicalOperator> *> candidates;
+	FindCandidates(&op, candidates);
+
+	// rewrite the plan and update the bindings
+	for (auto &candidate : candidates) {
+
+		// rearrange the logical operators
+		if (RewriteCandidate(candidate)) {
+			// update the bindings of the BOUND_UNNEST expression
+			UpdateBoundUnnestBindings(updater, candidate);
+			// update the sequence of LOGICAL_PROJECTION(s)
+			UpdateRHSBindings(&op, candidate, updater);
+			// reset
+			delim_columns.clear();
+			lhs_bindings.clear();
+		}
+	}
+
+	return op;
+}
+
+void UnnestRewriter::FindCandidates(unique_ptr<LogicalOperator> *op_ptr,
+                                    vector<unique_ptr<LogicalOperator> *> &candidates) {
+	auto op = op_ptr->get();
+	// search children before adding, so that we add candidates bottom-up
+	for (auto &child : op->children) {
+		FindCandidates(&child, candidates);
+	}
+
+	// search for operator that has a LOGICAL_DELIM_JOIN as its child
+	if (op->children.size() != 1) {
+		return;
+	}
+	if (op->children[0]->type != LogicalOperatorType::LOGICAL_DELIM_JOIN) {
+		return;
+	}
+
+	// found a delim join
+	auto &delim_join = (LogicalDelimJoin &)*op->children[0];
+	// only support INNER delim joins
+	if (delim_join.join_type != JoinType::INNER) {
+		return;
+	}
+	// INNER delim join must have exactly one condition
+	if (delim_join.conditions.size() != 1) {
+		return;
+	}
+
+	// LHS child is a window
+	if (delim_join.children[0]->type != LogicalOperatorType::LOGICAL_WINDOW) {
+		return;
+	}
+
+	// RHS child must be projection(s) followed by an UNNEST
+	auto curr_op = &delim_join.children[1];
+	while (curr_op->get()->type == LogicalOperatorType::LOGICAL_PROJECTION) {
+		if (curr_op->get()->children.size() != 1) {
+			break;
+		}
+		curr_op = &curr_op->get()->children[0];
+	}
+
+	if (curr_op->get()->type == LogicalOperatorType::LOGICAL_UNNEST) {
+		candidates.push_back(op_ptr);
+	}
+	return;
+}
+
+bool UnnestRewriter::RewriteCandidate(unique_ptr<LogicalOperator> *candidate) {
+
+	auto &topmost_op = (LogicalOperator &)**candidate;
+	if (topmost_op.type != LogicalOperatorType::LOGICAL_PROJECTION &&
+	    topmost_op.type != LogicalOperatorType::LOGICAL_WINDOW &&
+	    topmost_op.type != LogicalOperatorType::LOGICAL_FILTER &&
+	    topmost_op.type != LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY &&
+	    topmost_op.type != LogicalOperatorType::LOGICAL_UNNEST) {
+		return false;
+	}
+
+	// get the LOGICAL_DELIM_JOIN, which is a child of the candidate
+	D_ASSERT(topmost_op.children.size() == 1);
+	auto &delim_join = *(topmost_op.children[0]);
+	D_ASSERT(delim_join.type == LogicalOperatorType::LOGICAL_DELIM_JOIN);
+	GetDelimColumns(delim_join);
+
+	// LHS of the LOGICAL_DELIM_JOIN is a LOGICAL_WINDOW that contains a LOGICAL_PROJECTION
+	// this lhs_proj later becomes the child of the UNNEST
+	auto &window = *delim_join.children[0];
+	auto &lhs_op = window.children[0];
+	GetLHSExpressions(*lhs_op);
+
+	// find the LOGICAL_UNNEST
+	// and get the path down to the LOGICAL_UNNEST
+	vector<unique_ptr<LogicalOperator> *> path_to_unnest;
+	auto curr_op = &(delim_join.children[1]);
+	while (curr_op->get()->type == LogicalOperatorType::LOGICAL_PROJECTION) {
+		path_to_unnest.push_back(curr_op);
+		curr_op = &curr_op->get()->children[0];
+	}
+
+	// store the table index of the child of the LOGICAL_UNNEST
+	// then update the plan by making the lhs_proj the child of the LOGICAL_UNNEST
+	D_ASSERT(curr_op->get()->type == LogicalOperatorType::LOGICAL_UNNEST);
+	auto &unnest = (LogicalUnnest &)*curr_op->get();
+	D_ASSERT(unnest.children[0]->type == LogicalOperatorType::LOGICAL_DELIM_GET);
+	overwritten_tbl_idx = ((LogicalDelimGet &)*unnest.children[0]).table_index;
+	unnest.children[0] = std::move(lhs_op);
+
+	// replace the LOGICAL_DELIM_JOIN with its RHS child operator
+	topmost_op.children[0] = std::move(*path_to_unnest.front());
+	return true;
+}
+
+void UnnestRewriter::UpdateRHSBindings(unique_ptr<LogicalOperator> *plan_ptr, unique_ptr<LogicalOperator> *candidate,
+                                       UnnestRewriterPlanUpdater &updater) {
+
+	auto &topmost_op = (LogicalOperator &)**candidate;
+	idx_t shift = lhs_bindings.size();
+
+	vector<unique_ptr<LogicalOperator> *> path_to_unnest;
+	auto curr_op = &(topmost_op.children[0]);
+	while (curr_op->get()->type == LogicalOperatorType::LOGICAL_PROJECTION) {
+
+		path_to_unnest.push_back(curr_op);
+		D_ASSERT(curr_op->get()->type == LogicalOperatorType::LOGICAL_PROJECTION);
+		auto &proj = (LogicalProjection &)*curr_op->get();
+
+		// pop the two last expressions from all projections (delim_idx and UNNEST column)
+		D_ASSERT(proj.expressions.size() > 2);
+		proj.expressions.pop_back();
+		proj.expressions.pop_back();
+
+		// store all shifted current bindings
+		idx_t tbl_idx = proj.table_index;
+		for (idx_t i = 0; i < proj.expressions.size(); i++) {
+			ReplaceBinding replace_binding(ColumnBinding(tbl_idx, i), ColumnBinding(tbl_idx, i + shift));
+			updater.replace_bindings.push_back(replace_binding);
+		}
+
+		curr_op = &curr_op->get()->children[0];
+	}
+
+	// update all bindings by shifting them
+	updater.VisitOperator(*plan_ptr->get());
+	updater.replace_bindings.clear();
+
+	// update all bindings coming from the LHS to RHS bindings
+	D_ASSERT(topmost_op.children[0]->type == LogicalOperatorType::LOGICAL_PROJECTION);
+	auto &top_proj = (LogicalProjection &)*topmost_op.children[0];
+	for (idx_t i = 0; i < lhs_bindings.size(); i++) {
+		ReplaceBinding replace_binding(lhs_bindings[i].binding, ColumnBinding(top_proj.table_index, i));
+		updater.replace_bindings.push_back(replace_binding);
+	}
+
+	// temporarily remove the BOUND_UNNEST and the child of the LOGICAL_UNNEST from the plan
+	D_ASSERT(curr_op->get()->type == LogicalOperatorType::LOGICAL_UNNEST);
+	auto &unnest = (LogicalUnnest &)*curr_op->get();
+	auto temp_bound_unnest = std::move(unnest.expressions[0]);
+	auto temp_unnest_child = std::move(unnest.children[0]);
+	unnest.expressions.clear();
+	unnest.children.clear();
+	// update the bindings of the plan
+	updater.VisitOperator(*plan_ptr->get());
+	updater.replace_bindings.clear();
+	// add the child again
+	unnest.expressions.push_back(std::move(temp_bound_unnest));
+	unnest.children.push_back(std::move(temp_unnest_child));
+
+	// add the LHS expressions to each LOGICAL_PROJECTION
+	for (idx_t i = path_to_unnest.size(); i > 0; i--) {
+
+		D_ASSERT(path_to_unnest[i - 1]->get()->type == LogicalOperatorType::LOGICAL_PROJECTION);
+		auto &proj = (LogicalProjection &)*path_to_unnest[i - 1]->get();
+
+		// temporarily store the existing expressions
+		vector<unique_ptr<Expression>> existing_expressions;
+		for (idx_t expr_idx = 0; expr_idx < proj.expressions.size(); expr_idx++) {
+			existing_expressions.push_back(std::move(proj.expressions[expr_idx]));
+		}
+
+		proj.expressions.clear();
+
+		// add the new expressions
+		for (idx_t expr_idx = 0; expr_idx < lhs_bindings.size(); expr_idx++) {
+			auto new_expr = make_unique<BoundColumnRefExpression>(
+			    lhs_bindings[expr_idx].alias, lhs_bindings[expr_idx].type, lhs_bindings[expr_idx].binding);
+			proj.expressions.push_back(std::move(new_expr));
+
+			// update the table index
+			lhs_bindings[expr_idx].binding.table_index = proj.table_index;
+			lhs_bindings[expr_idx].binding.column_index = expr_idx;
+		}
+
+		// add the existing expressions again
+		for (idx_t expr_idx = 0; expr_idx < existing_expressions.size(); expr_idx++) {
+			proj.expressions.push_back(std::move(existing_expressions[expr_idx]));
+		}
+	}
+}
+
+void UnnestRewriter::UpdateBoundUnnestBindings(UnnestRewriterPlanUpdater &updater,
+                                               unique_ptr<LogicalOperator> *candidate) {
+
+	auto &topmost_op = (LogicalOperator &)**candidate;
+
+	// traverse LOGICAL_PROJECTION(s)
+	auto curr_op = &(topmost_op.children[0]);
+	while (curr_op->get()->type == LogicalOperatorType::LOGICAL_PROJECTION) {
+		curr_op = &curr_op->get()->children[0];
+	}
+
+	// found the LOGICAL_UNNEST
+	D_ASSERT(curr_op->get()->type == LogicalOperatorType::LOGICAL_UNNEST);
+	auto &unnest = (LogicalUnnest &)*curr_op->get();
+
+	auto unnest_child_cols = unnest.children[0]->GetColumnBindings();
+	for (idx_t delim_col_idx = 0; delim_col_idx < delim_columns.size(); delim_col_idx++) {
+		for (idx_t child_col_idx = 0; child_col_idx < unnest_child_cols.size(); child_col_idx++) {
+			if (delim_columns[delim_col_idx].table_index == unnest_child_cols[child_col_idx].table_index) {
+				ColumnBinding old_binding(overwritten_tbl_idx, DConstants::INVALID_INDEX);
+				updater.replace_bindings.emplace_back(ReplaceBinding(old_binding, delim_columns[delim_col_idx]));
+				break;
+			}
+		}
+	}
+
+	// update bindings
+	D_ASSERT(unnest.expressions.size() == 1);
+	updater.VisitExpression(&unnest.expressions[0]);
+	updater.replace_bindings.clear();
+}
+
+void UnnestRewriter::GetDelimColumns(LogicalOperator &op) {
+
+	D_ASSERT(op.type == LogicalOperatorType::LOGICAL_DELIM_JOIN);
+	auto &delim_join = (LogicalDelimJoin &)op;
+	for (idx_t i = 0; i < delim_join.duplicate_eliminated_columns.size(); i++) {
+		auto &expr = *delim_join.duplicate_eliminated_columns[i];
+		D_ASSERT(expr.type == ExpressionType::BOUND_COLUMN_REF);
+		auto &bound_colref_expr = (BoundColumnRefExpression &)expr;
+		delim_columns.push_back(bound_colref_expr.binding);
+	}
+}
+
+void UnnestRewriter::GetLHSExpressions(LogicalOperator &op) {
+
+	op.ResolveOperatorTypes();
+	auto col_bindings = op.GetColumnBindings();
+	D_ASSERT(op.types.size() == col_bindings.size());
+
+	bool set_alias = false;
+	// we can easily extract the alias for LOGICAL_PROJECTION(s)
+	if (op.type == LogicalOperatorType::LOGICAL_PROJECTION) {
+		auto &proj = (LogicalProjection &)op;
+		if (proj.expressions.size() == op.types.size()) {
+			set_alias = true;
+		}
+	}
+
+	for (idx_t i = 0; i < op.types.size(); i++) {
+		lhs_bindings.emplace_back(LHSBinding(col_bindings[i], op.types[i]));
+		if (set_alias) {
+			auto &proj = (LogicalProjection &)op;
+			lhs_bindings.back().alias = proj.expressions[i]->alias;
+		}
+	}
+}
+
+} // namespace duckdb

package/src/duckdb/src/storage/block.cpp

@@ -9,11 +9,11 @@ Block::Block(Allocator &allocator, block_id_t id)
 
 Block::Block(Allocator &allocator, block_id_t id, uint32_t internal_size)
     : FileBuffer(allocator, FileBufferType::BLOCK, internal_size), id(id) {
-	D_ASSERT((GetMallocedSize() & (Storage::SECTOR_SIZE - 1)) == 0);
+	D_ASSERT((AllocSize() & (Storage::SECTOR_SIZE - 1)) == 0);
 }
 
 Block::Block(FileBuffer &source, block_id_t id) : FileBuffer(source, FileBufferType::BLOCK), id(id) {
-	D_ASSERT((GetMallocedSize() & (Storage::SECTOR_SIZE - 1)) == 0);
+	D_ASSERT((AllocSize() & (Storage::SECTOR_SIZE - 1)) == 0);
 }
 
 } // namespace duckdb

package/src/duckdb/src/storage/single_file_block_manager.cpp

@@ -1,6 +1,7 @@
 #include "duckdb/storage/single_file_block_manager.hpp"
 
 #include "duckdb/common/allocator.hpp"
+#include "duckdb/common/checksum.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/serializer/buffered_deserializer.hpp"
 #include "duckdb/common/serializer/buffered_serializer.hpp"
@@ -142,7 +143,7 @@ SingleFileBlockManager::SingleFileBlockManager(AttachedDatabase &db, string path
 
 		SerializeHeaderStructure<MainHeader>(main_header, header_buffer.buffer);
 		// now write the header to the file
-		header_buffer.ChecksumAndWrite(*handle, 0);
+		ChecksumAndWrite(header_buffer, 0);
 		header_buffer.Clear();
 
 		// write the database headers
@@ -155,14 +156,14 @@ SingleFileBlockManager::SingleFileBlockManager(AttachedDatabase &db, string path
 		h1.free_list = INVALID_BLOCK;
 		h1.block_count = 0;
 		SerializeHeaderStructure<DatabaseHeader>(h1, header_buffer.buffer);
-		header_buffer.ChecksumAndWrite(*handle, Storage::FILE_HEADER_SIZE);
+		ChecksumAndWrite(header_buffer, Storage::FILE_HEADER_SIZE);
 		// header 2
 		h2.iteration = 0;
 		h2.meta_block = INVALID_BLOCK;
 		h2.free_list = INVALID_BLOCK;
 		h2.block_count = 0;
 		SerializeHeaderStructure<DatabaseHeader>(h2, header_buffer.buffer);
-		header_buffer.ChecksumAndWrite(*handle, Storage::FILE_HEADER_SIZE * 2);
+		ChecksumAndWrite(header_buffer, Storage::FILE_HEADER_SIZE * 2);
 		// ensure that writing to disk is completed before returning
 		handle->Sync();
 		// we start with h2 as active_header, this way our initial write will be in h1
@@ -172,14 +173,14 @@ SingleFileBlockManager::SingleFileBlockManager(AttachedDatabase &db, string path
 	} else {
 		MainHeader::CheckMagicBytes(*handle);
 		// otherwise, we check the metadata of the file
-		header_buffer.ReadAndChecksum(*handle, 0);
+		ReadAndChecksum(header_buffer, 0);
 		DeserializeHeaderStructure<MainHeader>(header_buffer.buffer);
 
 		// read the database headers from disk
 		DatabaseHeader h1, h2;
-		header_buffer.ReadAndChecksum(*handle, Storage::FILE_HEADER_SIZE);
+		ReadAndChecksum(header_buffer, Storage::FILE_HEADER_SIZE);
 		h1 = DeserializeHeaderStructure<DatabaseHeader>(header_buffer.buffer);
-		header_buffer.ReadAndChecksum(*handle, Storage::FILE_HEADER_SIZE * 2);
+		ReadAndChecksum(header_buffer, Storage::FILE_HEADER_SIZE * 2);
 		h2 = DeserializeHeaderStructure<DatabaseHeader>(header_buffer.buffer);
 		// check the header with the highest iteration count
 		if (h1.iteration > h2.iteration) {
@@ -195,6 +196,27 @@ SingleFileBlockManager::SingleFileBlockManager(AttachedDatabase &db, string path
 	}
 }
 
+void SingleFileBlockManager::ReadAndChecksum(FileBuffer &block, uint64_t location) const {
+	// read the buffer from disk
+	block.Read(*handle, location);
+	// compute the checksum
+	auto stored_checksum = Load<uint64_t>(block.InternalBuffer());
+	uint64_t computed_checksum = Checksum(block.buffer, block.size);
+	// verify the checksum
+	if (stored_checksum != computed_checksum) {
+		throw IOException("Corrupt database file: computed checksum %llu does not match stored checksum %llu in block",
+		                  computed_checksum, stored_checksum);
+	}
+}
+
+void SingleFileBlockManager::ChecksumAndWrite(FileBuffer &block, uint64_t location) const {
+	// compute the checksum and write it to the start of the buffer (if not temp buffer)
+	uint64_t checksum = Checksum(block.buffer, block.size);
+	Store<uint64_t>(checksum, block.InternalBuffer());
+	// now write the buffer
+	block.Write(*handle, location);
+}
+
 void SingleFileBlockManager::Initialize(DatabaseHeader &header) {
 	free_list_id = header.free_list;
 	meta_block = header.meta_block;
@@ -317,12 +339,12 @@ unique_ptr<Block> SingleFileBlockManager::CreateBlock(block_id_t block_id, FileB
 void SingleFileBlockManager::Read(Block &block) {
 	D_ASSERT(block.id >= 0);
 	D_ASSERT(std::find(free_list.begin(), free_list.end(), block.id) == free_list.end());
-	block.ReadAndChecksum(*handle, BLOCK_START + block.id * Storage::BLOCK_ALLOC_SIZE);
+	ReadAndChecksum(block, BLOCK_START + block.id * Storage::BLOCK_ALLOC_SIZE);
 }
 
 void SingleFileBlockManager::Write(FileBuffer &buffer, block_id_t block_id) {
 	D_ASSERT(block_id >= 0);
-	buffer.ChecksumAndWrite(*handle, BLOCK_START + block_id * Storage::BLOCK_ALLOC_SIZE);
+	ChecksumAndWrite(buffer, BLOCK_START + block_id * Storage::BLOCK_ALLOC_SIZE);
 }
 
 vector<block_id_t> SingleFileBlockManager::GetFreeListBlocks() {
@@ -431,8 +453,7 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
 	Store<DatabaseHeader>(header, header_buffer.buffer);
 	// now write the header to the file, active_header determines whether we write to h1 or h2
 	// note that if active_header is h1 we write to h2, and vice versa
-	header_buffer.ChecksumAndWrite(*handle,
-	                               active_header == 1 ? Storage::FILE_HEADER_SIZE : Storage::FILE_HEADER_SIZE * 2);
+	ChecksumAndWrite(header_buffer, active_header == 1 ? Storage::FILE_HEADER_SIZE : Storage::FILE_HEADER_SIZE * 2);
 	// switch active header to the other header
 	active_header = 1 - active_header;
 	//! Ensure the header write ends up on disk

package/src/duckdb/ub_src_optimizer.cpp

@@ -4,6 +4,8 @@
 
 #include "src/optimizer/deliminator.cpp"
 
+#include "src/optimizer/unnest_rewriter.cpp"
+
 #include "src/optimizer/column_lifetime_analyzer.cpp"
 
 #include "src/optimizer/expression_heuristics.cpp"