duckdb 0.8.2-dev11.0 → 0.8.2-dev1212.0

package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp

@@ -5,34 +5,67 @@
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
 #include "duckdb/planner/expression/bound_operator_expression.hpp"
 #include "duckdb/planner/expression/bound_subquery_expression.hpp"
+#include "duckdb/planner/query_node/bound_select_node.hpp"
 #include "duckdb/planner/expression_iterator.hpp"
+#include "duckdb/planner/tableref/bound_joinref.hpp"
+#include "duckdb/planner/operator/logical_dependent_join.hpp"
 
 namespace duckdb {
 
 RewriteCorrelatedExpressions::RewriteCorrelatedExpressions(ColumnBinding base_binding,
-                                                           column_binding_map_t<idx_t> &correlated_map)
-    : base_binding(base_binding), correlated_map(correlated_map) {
+                                                           column_binding_map_t<idx_t> &correlated_map,
+                                                           idx_t lateral_depth, bool recursive_rewrite)
+    : base_binding(base_binding), correlated_map(correlated_map), lateral_depth(lateral_depth),
+      recursive_rewrite(recursive_rewrite) {
 }
 
 void RewriteCorrelatedExpressions::VisitOperator(LogicalOperator &op) {
+	if (recursive_rewrite) {
+		// Update column bindings from left child of lateral to right child
+		if (op.type == LogicalOperatorType::LOGICAL_DEPENDENT_JOIN) {
+			D_ASSERT(op.children.size() == 2);
+			VisitOperator(*op.children[0]);
+			lateral_depth++;
+			VisitOperator(*op.children[1]);
+			lateral_depth--;
+		} else {
+			VisitOperatorChildren(op);
+		}
+	}
+	// update the bindings in the correlated columns of the dependendent join
+	if (op.type == LogicalOperatorType::LOGICAL_DEPENDENT_JOIN) {
+		auto &plan = op.Cast<LogicalDependentJoin>();
+		for (auto &corr : plan.correlated_columns) {
+			auto entry = correlated_map.find(corr.binding);
+			if (entry != correlated_map.end()) {
+				corr.binding = ColumnBinding(base_binding.table_index, base_binding.column_index + entry->second);
+			}
+		}
+	}
 	VisitOperatorExpressions(op);
 }
 
 unique_ptr<Expression> RewriteCorrelatedExpressions::VisitReplace(BoundColumnRefExpression &expr,
                                                                   unique_ptr<Expression> *expr_ptr) {
-	if (expr.depth == 0) {
+	if (expr.depth <= lateral_depth) {
+		// Indicates local correlations not relevant for the current the rewrite
 		return nullptr;
 	}
 	// correlated column reference
 	// replace with the entry referring to the duplicate eliminated scan
-	// if this assertion occurs it generally means the correlated expressions were not propagated correctly
-	// through different binders
-	D_ASSERT(expr.depth == 1);
+	// if this assertion occurs it generally means the bindings are inappropriate set in the binder or
+	// we either missed to account for lateral binder or over-counted for the lateral binder
+	D_ASSERT(expr.depth == 1 + lateral_depth);
 	auto entry = correlated_map.find(expr.binding);
 	D_ASSERT(entry != correlated_map.end());
 
 	expr.binding = ColumnBinding(base_binding.table_index, base_binding.column_index + entry->second);
-	expr.depth = 0;
+	if (recursive_rewrite) {
+		D_ASSERT(expr.depth > 1);
+		expr.depth--;
+	} else {
+		expr.depth = 0;
+	}
 	return nullptr;
 }
 
@@ -53,6 +86,21 @@ RewriteCorrelatedExpressions::RewriteCorrelatedRecursive::RewriteCorrelatedRecur
     : parent(parent), base_binding(base_binding), correlated_map(correlated_map) {
 }
 
+void RewriteCorrelatedExpressions::RewriteCorrelatedRecursive::RewriteJoinRefRecursive(BoundTableRef &ref) {
+	// recursively rewrite bindings in the correlated columns for the table ref and all the children
+	if (ref.type == TableReferenceType::JOIN) {
+		auto &bound_join = ref.Cast<BoundJoinRef>();
+		for (auto &corr : bound_join.correlated_columns) {
+			auto entry = correlated_map.find(corr.binding);
+			if (entry != correlated_map.end()) {
+				corr.binding = ColumnBinding(base_binding.table_index, base_binding.column_index + entry->second);
+			}
+		}
+		RewriteJoinRefRecursive(*bound_join.left);
+		RewriteJoinRefRecursive(*bound_join.right);
+	}
+}
+
 void RewriteCorrelatedExpressions::RewriteCorrelatedRecursive::RewriteCorrelatedSubquery(
     BoundSubqueryExpression &expr) {
 	// rewrite the binding in the correlated list of the subquery)
@@ -62,6 +110,16 @@ void RewriteCorrelatedExpressions::RewriteCorrelatedRecursive::RewriteCorrelated
 			corr.binding = ColumnBinding(base_binding.table_index, base_binding.column_index + entry->second);
 		}
 	}
+	// TODO: Cleanup and find a better way to do this
+	auto &node = *expr.subquery;
+	if (node.type == QueryNodeType::SELECT_NODE) {
+		// Found an unplanned select node, need to update column bindings correlated columns in the from tables
+		auto &bound_select = node.Cast<BoundSelectNode>();
+		if (bound_select.from_table) {
+			BoundTableRef &table_ref = *bound_select.from_table;
+			RewriteJoinRefRecursive(table_ref);
+		}
+	}
 	// now rewrite any correlated BoundColumnRef expressions inside the subquery
 	ExpressionIterator::EnumerateQueryNodeChildren(*expr.subquery,
 	                                               [&](Expression &child) { RewriteCorrelatedExpressions(child); });

package/src/duckdb/src/storage/arena_allocator.cpp

@@ -109,7 +109,6 @@ data_ptr_t ArenaAllocator::ReallocateAligned(data_ptr_t pointer, idx_t old_size,
 }
 
 void ArenaAllocator::Reset() {
-
 	if (head) {
 		// destroy all chunks except the current one
 		if (head->next) {
@@ -148,7 +147,7 @@ ArenaChunk *ArenaAllocator::GetTail() {
 	return tail;
 }
 
-bool ArenaAllocator::IsEmpty() {
+bool ArenaAllocator::IsEmpty() const {
 	return head == nullptr;
 }
 

package/src/duckdb/src/storage/buffer/block_manager.cpp

@@ -79,4 +79,7 @@ void BlockManager::UnregisterBlock(block_id_t block_id, bool can_destroy) {
 	}
 }
 
+void BlockManager::Truncate() {
+}
+
 } // namespace duckdb

package/src/duckdb/src/storage/checkpoint_manager.cpp

@@ -111,6 +111,9 @@ void SingleFileCheckpointWriter::CreateCheckpoint() {
 	// truncate the WAL
 	wal->Truncate(0);
 
+	// truncate the file
+	block_manager.Truncate();
+
 	// mark all blocks written as part of the metadata as modified
 	metadata_writer->MarkWrittenBlocks();
 	table_metadata_writer->MarkWrittenBlocks();

package/src/duckdb/src/storage/compression/rle.cpp

@@ -272,7 +272,6 @@ struct RLEScanState : public SegmentScanState {
 	}
 
 	BufferHandle handle;
-	uint32_t rle_offset;
 	idx_t entry_pos;
 	idx_t position_in_entry;
 	uint32_t rle_count_offset;

package/src/duckdb/src/storage/data_table.cpp

@@ -56,7 +56,7 @@ DataTable::DataTable(AttachedDatabase &db, shared_ptr<TableIOManager> table_io_m
 	row_groups->Verify();
 }
 
-DataTable::DataTable(ClientContext &context, DataTable &parent, ColumnDefinition &new_column, Expression *default_value)
+DataTable::DataTable(ClientContext &context, DataTable &parent, ColumnDefinition &new_column, Expression &default_value)
     : info(parent.info), db(parent.db), is_root(true) {
 	// add the column definitions from this DataTable
 	for (auto &column_def : parent.column_definitions) {

package/src/duckdb/src/storage/local_storage.cpp

@@ -61,11 +61,11 @@ LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &paren
 }
 
 LocalTableStorage::LocalTableStorage(ClientContext &context, DataTable &new_dt, LocalTableStorage &parent,
-                                     ColumnDefinition &new_column, optional_ptr<Expression> default_value)
+                                     ColumnDefinition &new_column, Expression &default_value)
     : table_ref(new_dt), allocator(Allocator::Get(new_dt.db)), deleted_rows(parent.deleted_rows),
       optimistic_writer(new_dt, parent.optimistic_writer), optimistic_writers(std::move(parent.optimistic_writers)),
       merged_storage(parent.merged_storage) {
-	row_groups = parent.row_groups->AddColumn(context, new_column, default_value.get());
+	row_groups = parent.row_groups->AddColumn(context, new_column, default_value);
 	parent.row_groups.reset();
 	indexes.Move(parent.indexes);
 }
@@ -508,7 +508,7 @@ void LocalStorage::MoveStorage(DataTable &old_dt, DataTable &new_dt) {
 }
 
 void LocalStorage::AddColumn(DataTable &old_dt, DataTable &new_dt, ColumnDefinition &new_column,
-                             optional_ptr<Expression> default_value) {
+                             Expression &default_value) {
 	// check if there are any pending appends for the old version of the table
 	auto storage = table_manager.MoveEntry(old_dt);
 	if (!storage) {

package/src/duckdb/src/storage/single_file_block_manager.cpp

@@ -368,6 +368,29 @@ void SingleFileBlockManager::Write(FileBuffer &buffer, block_id_t block_id) {
 	ChecksumAndWrite(buffer, BLOCK_START + block_id * Storage::BLOCK_ALLOC_SIZE);
 }
 
+void SingleFileBlockManager::Truncate() {
+	BlockManager::Truncate();
+	idx_t blocks_to_truncate = 0;
+	// reverse iterate over the free-list
+	for (auto entry = free_list.rbegin(); entry != free_list.rend(); entry++) {
+		auto block_id = *entry;
+		if (block_id + 1 != max_block) {
+			break;
+		}
+		blocks_to_truncate++;
+		max_block--;
+	}
+	if (blocks_to_truncate == 0) {
+		// nothing to truncate
+		return;
+	}
+	// truncate the file
+	for (idx_t i = 0; i < blocks_to_truncate; i++) {
+		free_list.erase(max_block + i);
+	}
+	handle->Truncate(BLOCK_START + max_block * Storage::BLOCK_ALLOC_SIZE);
+}
+
 vector<block_id_t> SingleFileBlockManager::GetFreeListBlocks() {
 	vector<block_id_t> free_list_blocks;
 

package/src/duckdb/src/storage/statistics/string_stats.cpp

@@ -1,10 +1,11 @@
 #include "duckdb/storage/statistics/string_stats.hpp"
-#include "duckdb/storage/statistics/base_statistics.hpp"
+
 #include "duckdb/common/field_writer.hpp"
-#include "utf8proc_wrapper.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/main/error_manager.hpp"
+#include "duckdb/storage/statistics/base_statistics.hpp"
+#include "utf8proc_wrapper.hpp"
 
 namespace duckdb {
 
@@ -67,6 +68,24 @@ bool StringStats::CanContainUnicode(const BaseStatistics &stats) {
 	return StringStats::GetDataUnsafe(stats).has_unicode;
 }
 
+string GetStringMinMaxValue(const data_t data[]) {
+	idx_t len;
+	for (len = 0; len < StringStatsData::MAX_STRING_MINMAX_SIZE; len++) {
+		if (!data[len]) {
+			break;
+		}
+	}
+	return string(const_char_ptr_cast(data), len);
+}
+
+string StringStats::Min(const BaseStatistics &stats) {
+	return GetStringMinMaxValue(StringStats::GetDataUnsafe(stats).min);
+}
+
+string StringStats::Max(const BaseStatistics &stats) {
+	return GetStringMinMaxValue(StringStats::GetDataUnsafe(stats).max);
+}
+
 void StringStats::ResetMaxStringLength(BaseStatistics &stats) {
 	StringStats::GetDataUnsafe(stats).has_max_string_length = false;
 }

package/src/duckdb/src/storage/storage_info.cpp

@@ -2,7 +2,7 @@
 
 namespace duckdb {
 
-const uint64_t VERSION_NUMBER = 51;
+const uint64_t VERSION_NUMBER = 52;
 
 struct StorageVersionInfo {
 	const char *version_name;

package/src/duckdb/src/storage/storage_manager.cpp

@@ -94,8 +94,13 @@ void SingleFileStorageManager::LoadDatabase() {
 		table_io_manager = make_uniq<SingleFileTableIOManager>(*block_manager);
 		return;
 	}
-
-	string wal_path = path + ".wal";
+	std::size_t question_mark_pos = path.find('?');
+	auto wal_path = path;
+	if (question_mark_pos != std::string::npos) {
+		wal_path.insert(question_mark_pos, ".wal");
+	} else {
+		wal_path += ".wal";
+	}
 	auto &fs = FileSystem::Get(db);
 	auto &config = DBConfig::Get(db);
 	bool truncate_wal = false;

package/src/duckdb/src/storage/table/chunk_info.cpp

@@ -89,6 +89,10 @@ void ChunkConstantInfo::Serialize(Serializer &serializer) {
 	serializer.Write<idx_t>(start);
 }
 
+idx_t ChunkConstantInfo::GetCommittedDeletedCount(idx_t max_count) {
+	return delete_id < TRANSACTION_ID_START ? max_count : 0;
+}
+
 unique_ptr<ChunkInfo> ChunkConstantInfo::Deserialize(Deserializer &source) {
 	auto start = source.Read<idx_t>();
 
@@ -244,6 +248,19 @@ void ChunkVectorInfo::Serialize(Serializer &serializer) {
 	serializer.WriteData(data_ptr_cast(deleted_tuples), sizeof(bool) * STANDARD_VECTOR_SIZE);
 }
 
+idx_t ChunkVectorInfo::GetCommittedDeletedCount(idx_t max_count) {
+	if (!any_deleted) {
+		return 0;
+	}
+	idx_t delete_count = 0;
+	for (idx_t i = 0; i < max_count; i++) {
+		if (deleted[i] < TRANSACTION_ID_START) {
+			delete_count++;
+		}
+	}
+	return delete_count;
+}
+
 unique_ptr<ChunkInfo> ChunkVectorInfo::Deserialize(Deserializer &source) {
 	auto start = source.Read<idx_t>();
 

package/src/duckdb/src/storage/table/row_group.cpp

@@ -67,6 +67,21 @@ void VersionNode::SetStart(idx_t start) {
 	}
 }
 
+idx_t VersionNode::GetCommittedDeletedCount(idx_t count) {
+	idx_t deleted_count = 0;
+	for (idx_t r = 0, i = 0; r < count; r += STANDARD_VECTOR_SIZE, i++) {
+		if (!info[i]) {
+			continue;
+		}
+		idx_t max_count = MinValue<idx_t>(STANDARD_VECTOR_SIZE, count - r);
+		if (max_count == 0) {
+			break;
+		}
+		deleted_count += info[i]->GetCommittedDeletedCount(max_count);
+	}
+	return deleted_count;
+}
+
 RowGroup::~RowGroup() {
 }
 
@@ -112,10 +127,6 @@ ColumnData &RowGroup::GetColumn(storage_t c) {
 	return *columns[c];
 }
 
-DatabaseInstance &RowGroup::GetDatabase() {
-	return GetCollection().GetDatabase();
-}
-
 BlockManager &RowGroup::GetBlockManager() {
 	return GetCollection().GetBlockManager();
 }
@@ -270,7 +281,7 @@ unique_ptr<RowGroup> RowGroup::AlterType(RowGroupCollection &new_collection, con
 }
 
 unique_ptr<RowGroup> RowGroup::AddColumn(RowGroupCollection &new_collection, ColumnDefinition &new_column,
-                                         ExpressionExecutor &executor, Expression *default_value, Vector &result) {
+                                         ExpressionExecutor &executor, Expression &default_value, Vector &result) {
 	Verify();
 
 	// construct a new column data for the new column
@@ -285,10 +296,8 @@ unique_ptr<RowGroup> RowGroup::AddColumn(RowGroupCollection &new_collection, Col
 		added_column->InitializeAppend(state);
 		for (idx_t i = 0; i < rows_to_write; i += STANDARD_VECTOR_SIZE) {
 			idx_t rows_in_this_vector = MinValue<idx_t>(rows_to_write - i, STANDARD_VECTOR_SIZE);
-			if (default_value) {
-				dummy_chunk.SetCardinality(rows_in_this_vector);
-				executor.ExecuteExpression(dummy_chunk, result);
-			}
+			dummy_chunk.SetCardinality(rows_in_this_vector);
+			executor.ExecuteExpression(dummy_chunk, result);
 			added_column->Append(state, result, rows_in_this_vector);
 		}
 	}
@@ -796,6 +805,13 @@ RowGroupWriteData RowGroup::WriteToDisk(PartialBlockManager &manager,
 	return result;
 }
 
+bool RowGroup::AllDeleted() {
+	if (!version_info) {
+		return false;
+	}
+	return version_info->GetCommittedDeletedCount(count) == count;
+}
+
 RowGroupPointer RowGroup::Checkpoint(RowGroupWriter &writer, TableStatistics &global_stats) {
 	RowGroupPointer row_group_pointer;
 

package/src/duckdb/src/storage/table/row_group_collection.cpp

@@ -67,10 +67,6 @@ AttachedDatabase &RowGroupCollection::GetAttached() {
 	return GetTableInfo().db;
 }
 
-DatabaseInstance &RowGroupCollection::GetDatabase() {
-	return GetAttached().GetDatabase();
-}
-
 //===--------------------------------------------------------------------===//
 // Initialize
 //===--------------------------------------------------------------------===//
@@ -97,10 +93,6 @@ RowGroup *RowGroupCollection::GetRowGroup(int64_t index) {
 	return (RowGroup *)row_groups->GetSegmentByIndex(index);
 }
 
-idx_t RowGroupCollection::RowGroupCount() {
-	return row_groups->GetSegmentCount();
-}
-
 void RowGroupCollection::Verify() {
 #ifdef DEBUG
 	idx_t current_total_rows = 0;
@@ -600,11 +592,24 @@ void RowGroupCollection::UpdateColumn(TransactionData transaction, Vector &row_i
 // Checkpoint
 //===--------------------------------------------------------------------===//
 void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &global_stats) {
-	for (auto &row_group : row_groups->Segments()) {
-		auto rowg_writer = writer.GetRowGroupWriter(row_group);
-		auto pointer = row_group.Checkpoint(*rowg_writer, global_stats);
-		writer.AddRowGroup(std::move(pointer), std::move(rowg_writer));
+	bool can_vacuum_deletes = info->indexes.Empty();
+	idx_t start = this->row_start;
+	auto segments = row_groups->MoveSegments();
+	auto l = row_groups->Lock();
+	for (auto &entry : segments) {
+		auto &row_group = *entry.node;
+		if (can_vacuum_deletes && row_group.AllDeleted()) {
+			row_group.CommitDrop();
+			continue;
+		}
+		row_group.MoveToCollection(*this, start);
+		auto row_group_writer = writer.GetRowGroupWriter(row_group);
+		auto pointer = row_group.Checkpoint(*row_group_writer, global_stats);
+		writer.AddRowGroup(std::move(pointer), std::move(row_group_writer));
+		row_groups->AppendSegment(l, std::move(entry.node));
+		start += row_group.count;
 	}
+	total_rows = start;
 }
 
 //===--------------------------------------------------------------------===//
@@ -637,7 +642,7 @@ vector<ColumnSegmentInfo> RowGroupCollection::GetColumnSegmentInfo() {
 // Alter
 //===--------------------------------------------------------------------===//
 shared_ptr<RowGroupCollection> RowGroupCollection::AddColumn(ClientContext &context, ColumnDefinition &new_column,
-                                                             Expression *default_value) {
+                                                             Expression &default_value) {
 	idx_t new_column_idx = types.size();
 	auto new_types = types;
 	new_types.push_back(new_column.GetType());
@@ -647,11 +652,7 @@ shared_ptr<RowGroupCollection> RowGroupCollection::AddColumn(ClientContext &cont
 	ExpressionExecutor executor(context);
 	DataChunk dummy_chunk;
 	Vector default_vector(new_column.GetType());
-	if (!default_value) {
-		FlatVector::Validity(default_vector).SetAllInvalid(STANDARD_VECTOR_SIZE);
-	} else {
-		executor.AddExpression(*default_value);
-	}
+	executor.AddExpression(default_value);
 
 	result->stats.InitializeAddColumn(stats, new_column.GetType());
 	auto &new_column_stats = result->stats.GetStats(new_column_idx);

package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h

@@ -80,7 +80,7 @@ namespace duckdb_moodycamel { namespace details {
 	static const thread_id_t invalid_thread_id2 = 0xFFFFFFFFU;	// Not technically guaranteed to be invalid, but is never used in practice. Note that all Win32 thread IDs are presently multiples of 4.
 	static inline thread_id_t thread_id() { return static_cast<thread_id_t>(::GetCurrentThreadId()); }
 } }
-#elif defined(__arm__) || defined(_M_ARM) || defined(__aarch64__) || (defined(__APPLE__) && TARGET_OS_IPHONE)
+#elif defined(__arm__) || defined(_M_ARM) || defined(__aarch64__) || (defined(__APPLE__) && TARGET_OS_IPHONE) || defined(__MVS__)
 namespace duckdb_moodycamel { namespace details {
 	static_assert(sizeof(std::thread::id) == 4 || sizeof(std::thread::id) == 8, "std::thread::id is expected to be either 4 or 8 bytes");
 	
@@ -194,7 +194,7 @@ namespace duckdb_moodycamel { namespace details {
 // VS2013 doesn't support `thread_local`, and MinGW-w64 w/ POSIX threading has a crippling bug: http://sourceforge.net/p/mingw-w64/bugs/445
 // g++ <=4.7 doesn't support thread_local either.
 // Finally, iOS/ARM doesn't have support for it either, and g++/ARM allows it to compile but it's unconfirmed to actually work
-#if (!defined(_MSC_VER) || _MSC_VER >= 1900) && (!defined(__MINGW32__) && !defined(__MINGW64__) || !defined(__WINPTHREADS_VERSION)) && (!defined(__GNUC__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) && (!defined(__APPLE__) || !TARGET_OS_IPHONE) && !defined(__arm__) && !defined(_M_ARM) && !defined(__aarch64__)
+#if (!defined(_MSC_VER) || _MSC_VER >= 1900) && (!defined(__MINGW32__) && !defined(__MINGW64__) || !defined(__WINPTHREADS_VERSION)) && (!defined(__GNUC__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) && (!defined(__APPLE__) || !TARGET_OS_IPHONE) && !defined(__arm__) && !defined(_M_ARM) && !defined(__aarch64__) && !defined(__MVS__)
 // Assume `thread_local` is fully supported in all other C++11 compilers/platforms
 //#define MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED    // always disabled for now since several users report having problems with it on
 #endif

package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h

@@ -27,6 +27,8 @@ extern "C" {
 #elif defined(__unix__)
 #include <semaphore.h>
 #include <chrono>
+#elif defined(__MVS__)
+#include <zos-semaphore.h>
 #endif
 
 namespace duckdb_moodycamel
@@ -254,6 +256,80 @@ public:
 		}
 	}
 };
+#elif defined(__MVS__)
+//---------------------------------------------------------
+// Semaphore (MVS aka z/OS)
+//---------------------------------------------------------
+class Semaphore
+{
+private:
+	sem_t m_sema;
+
+	Semaphore(const Semaphore& other) MOODYCAMEL_DELETE_FUNCTION;
+	Semaphore& operator=(const Semaphore& other) MOODYCAMEL_DELETE_FUNCTION;
+
+public:
+	Semaphore(int initialCount = 0)
+	{
+		assert(initialCount >= 0);
+		int rc = sem_init(&m_sema, 0, initialCount);
+		assert(rc == 0);
+		(void)rc;
+	}
+
+	~Semaphore()
+	{
+		sem_destroy(&m_sema);
+	}
+
+	bool wait()
+	{
+		// http://stackoverflow.com/questions/2013181/gdb-causes-sem-wait-to-fail-with-eintr-error
+		int rc;
+		do {
+			rc = sem_wait(&m_sema);
+		} while (rc == -1 && errno == EINTR);
+		return rc == 0;
+	}
+
+	bool try_wait()
+	{
+		int rc;
+		do {
+			rc = sem_trywait(&m_sema);
+		} while (rc == -1 && errno == EINTR);
+		return rc == 0;
+	}
+
+	bool timed_wait(std::uint64_t usecs)
+	{
+		struct timespec ts;
+		const int usecs_in_1_sec = 1000000;
+		const int nsecs_in_1_sec = 1000000000;
+
+		ts.tv_sec = usecs / usecs_in_1_sec;
+		ts.tv_nsec = (usecs % usecs_in_1_sec) * 1000;
+
+		int rc;
+		do {
+			rc = sem_timedwait(&m_sema, &ts);
+		} while (rc == -1 && errno == EINTR);
+		return rc == 0;
+	}
+
+	void signal()
+	{
+		while (sem_post(&m_sema) == -1);
+	}
+
+	void signal(int count)
+	{
+		while (count-- > 0)
+		{
+			while (sem_post(&m_sema) == -1);
+		}
+	}
+};
 #else
 #error Unsupported platform! (No semaphore wrapper available)
 #endif

package/src/duckdb/third_party/fast_float/fast_float/fast_float.h

@@ -130,6 +130,8 @@ from_chars_result from_chars(const char *first, const char *last,
 #include <machine/endian.h>
 #elif defined(sun) || defined(__sun)
 #include <sys/byteorder.h>
+#elif defined(__MVS__)
+#include <sys/endian.h>
 #else
 #include <endian.h>
 #endif

package/src/duckdb/third_party/httplib/httplib.hpp

@@ -183,7 +183,16 @@ using socket_t = SOCKET;
 
 #include <arpa/inet.h>
 #include <cstring>
+#ifndef __MVS__
 #include <ifaddrs.h>
+#endif
+#ifdef __MVS__
+#include <net/if.h>
+#include <strings.h>
+#ifndef NI_MAXHOST
+#define NI_MAXHOST 1025
+#endif
+#endif
 #include <netdb.h>
 #include <netinet/in.h>
 #ifdef __linux__
@@ -2668,7 +2677,7 @@ inline bool bind_ip_address(socket_t sock, const char *host) {
 	return ret;
 }
 
-#if !defined _WIN32 && !defined ANDROID
+#if !defined _WIN32 && !defined ANDROID && !defined __MVS__
 #define USE_IF2IP
 #endif
 

package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp

@@ -1058,10 +1058,19 @@ typedef struct PGOnConflictClause {
  *
  * We don't currently support the SEARCH or CYCLE clause.
  */
+
+typedef enum PGCTEMaterialize
+{
+	PGCTEMaterializeDefault,		/* no option specified */
+	PGCTEMaterializeAlways,		/* MATERIALIZED */
+	PGCTEMaterializeNever			/* NOT MATERIALIZED */
+} PGCTEMaterialize;
+
 typedef struct PGCommonTableExpr {
 	PGNodeTag type;
 	char *ctename;         /* query name (never qualified) */
 	PGList *aliascolnames; /* optional list of column names */
+	PGCTEMaterialize ctematerialized; /* is this an optimization fence? */
 	/* SelectStmt/InsertStmt/etc before parse analysis, PGQuery afterwards: */
 	PGNode *ctequery; /* the CTE's subquery */
 	int location;     /* token location, or -1 if unknown */

package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp

@@ -1049,6 +1049,7 @@ typedef union YYSTYPE
 	PGAlias				*alias;
 	PGRangeVar			*range;
 	PGIntoClause			*into;
+	PGCTEMaterialize			ctematerialize;
 	PGWithClause			*with;
 	PGInferClause			*infer;
 	PGOnConflictClause	*onconflict;
@@ -1068,7 +1069,7 @@ typedef union YYSTYPE
 	PGInsertColumnOrder bynameorposition;
 }
 /* Line 1529 of yacc.c.  */
-#line 1072 "third_party/libpg_query/grammar/grammar_out.hpp"
+#line 1073 "third_party/libpg_query/grammar/grammar_out.hpp"
 	YYSTYPE;
 # define yystype YYSTYPE /* obsolescent; will be withdrawn */
 # define YYSTYPE_IS_DECLARED 1