duckdb 0.8.2-dev33.0 → 0.8.2-dev3334.0

package/src/duckdb/src/storage/table/list_column_data.cpp

@@ -89,16 +89,19 @@ idx_t ListColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t co
 	D_ASSERT(scan_count > 0);
 	validity.ScanCount(state.child_states[0], result, count);
 
-	auto data = FlatVector::GetData<uint64_t>(offset_vector);
-	auto last_entry = data[scan_count - 1];
+	UnifiedVectorFormat offsets;
+	offset_vector.ToUnifiedFormat(scan_count, offsets);
+	auto data = UnifiedVectorFormat::GetData<uint64_t>(offsets);
+	auto last_entry = data[offsets.sel->get_index(scan_count - 1)];
 
 	// shift all offsets so they are 0 at the first entry
 	auto result_data = FlatVector::GetData<list_entry_t>(result);
 	auto base_offset = state.last_offset;
 	idx_t current_offset = 0;
 	for (idx_t i = 0; i < scan_count; i++) {
+		auto offset_index = offsets.sel->get_index(i);
 		result_data[i].offset = current_offset;
-		result_data[i].length = data[i] - current_offset - base_offset;
+		result_data[i].length = data[offset_index] - current_offset - base_offset;
 		current_offset += result_data[i].length;
 	}
 

package/src/duckdb/src/storage/table/persistent_table_data.cpp

@@ -3,8 +3,7 @@
 
 namespace duckdb {
 
-PersistentTableData::PersistentTableData(idx_t column_count)
-    : total_rows(0), row_group_count(0), block_id(INVALID_BLOCK), offset(0) {
+PersistentTableData::PersistentTableData(idx_t column_count) : total_rows(0), row_group_count(0) {
 }
 
 PersistentTableData::~PersistentTableData() {

package/src/duckdb/src/storage/table/row_group.cpp

@@ -11,7 +11,7 @@
 #include "duckdb/planner/table_filter.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/storage/checkpoint/table_data_writer.hpp"
-#include "duckdb/storage/meta_block_reader.hpp"
+#include "duckdb/storage/metadata/metadata_reader.hpp"
 #include "duckdb/transaction/duck_transaction_manager.hpp"
 #include "duckdb/main/database.hpp"
 #include "duckdb/main/attached_database.hpp"
@@ -67,6 +67,21 @@ void VersionNode::SetStart(idx_t start) {
 	}
 }
 
+idx_t VersionNode::GetCommittedDeletedCount(idx_t count) {
+	idx_t deleted_count = 0;
+	for (idx_t r = 0, i = 0; r < count; r += STANDARD_VECTOR_SIZE, i++) {
+		if (!info[i]) {
+			continue;
+		}
+		idx_t max_count = MinValue<idx_t>(STANDARD_VECTOR_SIZE, count - r);
+		if (max_count == 0) {
+			break;
+		}
+		deleted_count += info[i]->GetCommittedDeletedCount(max_count);
+	}
+	return deleted_count;
+}
+
 RowGroup::~RowGroup() {
 }
 
@@ -101,21 +116,16 @@ ColumnData &RowGroup::GetColumn(storage_t c) {
 	if (column_pointers.size() != columns.size()) {
 		throw InternalException("Lazy loading a column but the pointer was not set");
 	}
-	auto &block_manager = GetCollection().GetBlockManager();
+	auto &metadata_manager = GetCollection().GetMetadataManager();
 	auto &types = GetCollection().GetTypes();
 	auto &block_pointer = column_pointers[c];
-	MetaBlockReader column_data_reader(block_manager, block_pointer.block_id);
-	column_data_reader.offset = block_pointer.offset;
+	MetadataReader column_data_reader(metadata_manager, block_pointer);
 	this->columns[c] =
 	    ColumnData::Deserialize(GetBlockManager(), GetTableInfo(), c, start, column_data_reader, types[c], nullptr);
 	is_loaded[c] = true;
 	return *columns[c];
 }
 
-DatabaseInstance &RowGroup::GetDatabase() {
-	return GetCollection().GetDatabase();
-}
-
 BlockManager &RowGroup::GetBlockManager() {
 	return GetCollection().GetBlockManager();
 }
@@ -270,7 +280,7 @@ unique_ptr<RowGroup> RowGroup::AlterType(RowGroupCollection &new_collection, con
 }
 
 unique_ptr<RowGroup> RowGroup::AddColumn(RowGroupCollection &new_collection, ColumnDefinition &new_column,
-                                         ExpressionExecutor &executor, Expression *default_value, Vector &result) {
+                                         ExpressionExecutor &executor, Expression &default_value, Vector &result) {
 	Verify();
 
 	// construct a new column data for the new column
@@ -285,10 +295,8 @@ unique_ptr<RowGroup> RowGroup::AddColumn(RowGroupCollection &new_collection, Col
 		added_column->InitializeAppend(state);
 		for (idx_t i = 0; i < rows_to_write; i += STANDARD_VECTOR_SIZE) {
 			idx_t rows_in_this_vector = MinValue<idx_t>(rows_to_write - i, STANDARD_VECTOR_SIZE);
-			if (default_value) {
-				dummy_chunk.SetCardinality(rows_in_this_vector);
-				executor.ExecuteExpression(dummy_chunk, result);
-			}
+			dummy_chunk.SetCardinality(rows_in_this_vector);
+			executor.ExecuteExpression(dummy_chunk, result);
 			added_column->Append(state, result, rows_in_this_vector);
 		}
 	}
@@ -796,6 +804,13 @@ RowGroupWriteData RowGroup::WriteToDisk(PartialBlockManager &manager,
 	return result;
 }
 
+bool RowGroup::AllDeleted() {
+	if (!version_info) {
+		return false;
+	}
+	return version_info->GetCommittedDeletedCount(count) == count;
+}
+
 RowGroupPointer RowGroup::Checkpoint(RowGroupWriter &writer, TableStatistics &global_stats) {
 	RowGroupPointer row_group_pointer;
 
@@ -816,7 +831,7 @@ RowGroupPointer RowGroup::Checkpoint(RowGroupWriter &writer, TableStatistics &gl
 	for (auto &state : result.states) {
 		// get the current position of the table data writer
 		auto &data_writer = writer.GetPayloadWriter();
-		auto pointer = data_writer.GetBlockPointer();
+		auto pointer = data_writer.GetMetaBlockPointer();
 
 		// store the stats and the data pointers in the row group pointers
 		row_group_pointer.data_pointers.push_back(pointer);
@@ -882,8 +897,8 @@ void RowGroup::Serialize(RowGroupPointer &pointer, Serializer &main_serializer)
 	writer.WriteField<uint64_t>(pointer.tuple_count);
 	auto &serializer = writer.GetSerializer();
 	for (auto &data_pointer : pointer.data_pointers) {
-		serializer.Write<block_id_t>(data_pointer.block_id);
-		serializer.Write<uint64_t>(data_pointer.offset);
+		serializer.Write<idx_t>(data_pointer.block_pointer);
+		serializer.Write<uint32_t>(data_pointer.offset);
 	}
 	CheckpointDeletes(pointer.versions.get(), serializer);
 	writer.Finalize();
@@ -901,9 +916,9 @@ RowGroupPointer RowGroup::Deserialize(Deserializer &main_source, const vector<Lo
 
 	auto &source = reader.GetSource();
 	for (idx_t i = 0; i < physical_columns; i++) {
-		BlockPointer pointer;
-		pointer.block_id = source.Read<block_id_t>();
-		pointer.offset = source.Read<uint64_t>();
+		MetaBlockPointer pointer;
+		pointer.block_pointer = source.Read<idx_t>();
+		pointer.offset = source.Read<uint32_t>();
 		result.data_pointers.push_back(pointer);
 	}
 	result.versions = DeserializeDeletes(source);

package/src/duckdb/src/storage/table/row_group_collection.cpp

@@ -6,7 +6,7 @@
 #include "duckdb/planner/constraints/bound_not_null_constraint.hpp"
 #include "duckdb/storage/checkpoint/table_data_writer.hpp"
 #include "duckdb/storage/table/row_group_segment_tree.hpp"
-#include "duckdb/storage/meta_block_reader.hpp"
+#include "duckdb/storage/metadata/metadata_reader.hpp"
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/table_storage_info.hpp"
@@ -27,8 +27,7 @@ void RowGroupSegmentTree::Initialize(PersistentTableData &data) {
 	current_row_group = 0;
 	max_row_group = data.row_group_count;
 	finished_loading = false;
-	reader = make_uniq<MetaBlockReader>(collection.GetBlockManager(), data.block_id);
-	reader->offset = data.offset;
+	reader = make_uniq<MetadataReader>(collection.GetMetadataManager(), data.block_pointer);
 }
 
 unique_ptr<RowGroup> RowGroupSegmentTree::LoadSegment() {
@@ -67,8 +66,8 @@ AttachedDatabase &RowGroupCollection::GetAttached() {
 	return GetTableInfo().db;
 }
 
-DatabaseInstance &RowGroupCollection::GetDatabase() {
-	return GetAttached().GetDatabase();
+MetadataManager &RowGroupCollection::GetMetadataManager() {
+	return GetBlockManager().GetMetadataManager();
 }
 
 //===--------------------------------------------------------------------===//
@@ -97,10 +96,6 @@ RowGroup *RowGroupCollection::GetRowGroup(int64_t index) {
 	return (RowGroup *)row_groups->GetSegmentByIndex(index);
 }
 
-idx_t RowGroupCollection::RowGroupCount() {
-	return row_groups->GetSegmentCount();
-}
-
 void RowGroupCollection::Verify() {
 #ifdef DEBUG
 	idx_t current_total_rows = 0;
@@ -600,11 +595,24 @@ void RowGroupCollection::UpdateColumn(TransactionData transaction, Vector &row_i
 // Checkpoint
 //===--------------------------------------------------------------------===//
 void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &global_stats) {
-	for (auto &row_group : row_groups->Segments()) {
-		auto rowg_writer = writer.GetRowGroupWriter(row_group);
-		auto pointer = row_group.Checkpoint(*rowg_writer, global_stats);
-		writer.AddRowGroup(std::move(pointer), std::move(rowg_writer));
+	bool can_vacuum_deletes = info->indexes.Empty();
+	idx_t start = this->row_start;
+	auto segments = row_groups->MoveSegments();
+	auto l = row_groups->Lock();
+	for (auto &entry : segments) {
+		auto &row_group = *entry.node;
+		if (can_vacuum_deletes && row_group.AllDeleted()) {
+			row_group.CommitDrop();
+			continue;
+		}
+		row_group.MoveToCollection(*this, start);
+		auto row_group_writer = writer.GetRowGroupWriter(row_group);
+		auto pointer = row_group.Checkpoint(*row_group_writer, global_stats);
+		writer.AddRowGroup(std::move(pointer), std::move(row_group_writer));
+		row_groups->AppendSegment(l, std::move(entry.node));
+		start += row_group.count;
 	}
+	total_rows = start;
 }
 
 //===--------------------------------------------------------------------===//
@@ -637,7 +645,7 @@ vector<ColumnSegmentInfo> RowGroupCollection::GetColumnSegmentInfo() {
 // Alter
 //===--------------------------------------------------------------------===//
 shared_ptr<RowGroupCollection> RowGroupCollection::AddColumn(ClientContext &context, ColumnDefinition &new_column,
-                                                             Expression *default_value) {
+                                                             Expression &default_value) {
 	idx_t new_column_idx = types.size();
 	auto new_types = types;
 	new_types.push_back(new_column.GetType());
@@ -647,11 +655,7 @@ shared_ptr<RowGroupCollection> RowGroupCollection::AddColumn(ClientContext &cont
 	ExpressionExecutor executor(context);
 	DataChunk dummy_chunk;
 	Vector default_vector(new_column.GetType());
-	if (!default_value) {
-		FlatVector::Validity(default_vector).SetAllInvalid(STANDARD_VECTOR_SIZE);
-	} else {
-		executor.AddExpression(*default_value);
-	}
+	executor.AddExpression(default_value);
 
 	result->stats.InitializeAddColumn(stats, new_column.GetType());
 	auto &new_column_stats = result->stats.GetStats(new_column_idx);

package/src/duckdb/src/storage/table/update_segment.cpp

@@ -427,7 +427,7 @@ void UpdateSegment::FetchRow(TransactionData transaction, idx_t row_id, Vector &
 	if (!root->info[vector_index]) {
 		return;
 	}
-	idx_t row_in_vector = row_id - vector_index * STANDARD_VECTOR_SIZE;
+	idx_t row_in_vector = (row_id - column_data.start) - vector_index * STANDARD_VECTOR_SIZE;
 	fetch_row_function(transaction.start_time, transaction.transaction_id, root->info[vector_index]->info.get(),
 	                   row_in_vector, result, result_idx);
 }

package/src/duckdb/src/storage/table_index_list.cpp

@@ -79,7 +79,7 @@ vector<column_t> TableIndexList::GetRequiredColumns() {
 	return result;
 }
 
-vector<BlockPointer> TableIndexList::SerializeIndexes(duckdb::MetaBlockWriter &writer) {
+vector<BlockPointer> TableIndexList::SerializeIndexes(duckdb::MetadataWriter &writer) {
 	vector<BlockPointer> blocks_info;
 	for (auto &index : indexes) {
 		blocks_info.emplace_back(index->Serialize(writer));

package/src/duckdb/src/storage/wal_replay.cpp

@@ -36,7 +36,6 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
 	// first deserialize the WAL to look for a checkpoint flag
 	// if there is a checkpoint flag, we might have already flushed the contents of the WAL to disk
 	ReplayState checkpoint_state(database, *con.context, *initial_reader);
-	initial_reader->SetCatalog(checkpoint_state.catalog);
 	checkpoint_state.deserialize_only = true;
 	try {
 		while (true) {
@@ -61,7 +60,7 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
 		return false;
 	} // LCOV_EXCL_STOP
 	initial_reader.reset();
-	if (checkpoint_state.checkpoint_id != INVALID_BLOCK) {
+	if (checkpoint_state.checkpoint_id.IsValid()) {
 		// there is a checkpoint flag: check if we need to deserialize the WAL
 		auto &manager = database.GetStorageManager();
 		if (manager.IsCheckpointClean(checkpoint_state.checkpoint_id)) {
@@ -73,7 +72,6 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
 
 	// we need to recover from the WAL: actually set up the replay state
 	BufferedFileReader reader(FileSystem::Get(database), path.c_str(), con.context.get());
-	reader.SetCatalog(checkpoint_state.catalog);
 	ReplayState state(database, *con.context, reader);
 
 	// replay the WAL
@@ -195,7 +193,7 @@ void ReplayState::ReplayEntry(WALType entry_type) {
 // Replay Table
 //===--------------------------------------------------------------------===//
 void ReplayState::ReplayCreateTable() {
-	auto info = TableCatalogEntry::Deserialize(source, context);
+	auto info = TableCatalogEntry::Deserialize(source);
 	if (deserialize_only) {
 		return;
 	}
@@ -234,12 +232,12 @@ void ReplayState::ReplayAlter() {
 // Replay View
 //===--------------------------------------------------------------------===//
 void ReplayState::ReplayCreateView() {
-	auto entry = ViewCatalogEntry::Deserialize(source, context);
+	auto entry = CatalogEntry::Deserialize(source);
 	if (deserialize_only) {
 		return;
 	}
 
-	catalog.CreateView(context, *entry);
+	catalog.CreateView(context, entry->Cast<CreateViewInfo>());
 }
 
 void ReplayState::ReplayDropView() {
@@ -284,7 +282,7 @@ void ReplayState::ReplayDropSchema() {
 void ReplayState::ReplayCreateType() {
 	auto info = TypeCatalogEntry::Deserialize(source);
 	info->on_conflict = OnCreateConflict::IGNORE_ON_CONFLICT;
-	catalog.CreateType(context, *info);
+	catalog.CreateType(context, info->Cast<CreateTypeInfo>());
 }
 
 void ReplayState::ReplayDropType() {
@@ -309,7 +307,7 @@ void ReplayState::ReplayCreateSequence() {
 		return;
 	}
 
-	catalog.CreateSequence(context, *entry);
+	catalog.CreateSequence(context, entry->Cast<CreateSequenceInfo>());
 }
 
 void ReplayState::ReplayDropSequence() {
@@ -345,12 +343,12 @@ void ReplayState::ReplaySequenceValue() {
 // Replay Macro
 //===--------------------------------------------------------------------===//
 void ReplayState::ReplayCreateMacro() {
-	auto entry = ScalarMacroCatalogEntry::Deserialize(source, context);
+	auto entry = ScalarMacroCatalogEntry::Deserialize(source);
 	if (deserialize_only) {
 		return;
 	}
 
-	catalog.CreateFunction(context, *entry);
+	catalog.CreateFunction(context, entry->Cast<CreateMacroInfo>());
 }
 
 void ReplayState::ReplayDropMacro() {
@@ -369,12 +367,12 @@ void ReplayState::ReplayDropMacro() {
 // Replay Table Macro
 //===--------------------------------------------------------------------===//
 void ReplayState::ReplayCreateTableMacro() {
-	auto entry = TableMacroCatalogEntry::Deserialize(source, context);
+	auto entry = TableMacroCatalogEntry::Deserialize(source);
 	if (deserialize_only) {
 		return;
 	}
 
-	catalog.CreateFunction(context, *entry);
+	catalog.CreateFunction(context, entry->Cast<CreateMacroInfo>());
 }
 
 void ReplayState::ReplayDropTableMacro() {
@@ -393,30 +391,31 @@ void ReplayState::ReplayDropTableMacro() {
 // Replay Index
 //===--------------------------------------------------------------------===//
 void ReplayState::ReplayCreateIndex() {
-	auto info = IndexCatalogEntry::Deserialize(source, context);
+	auto info = IndexCatalogEntry::Deserialize(source);
 	if (deserialize_only) {
 		return;
 	}
+	auto &index_info = info->Cast<CreateIndexInfo>();
 
 	// get the physical table to which we'll add the index
-	auto &table = catalog.GetEntry<TableCatalogEntry>(context, info->schema, info->table->table_name);
+	auto &table = catalog.GetEntry<TableCatalogEntry>(context, info->schema, index_info.table);
 	auto &data_table = table.GetStorage();
 
 	// bind the parsed expressions
-	if (info->expressions.empty()) {
-		for (auto &parsed_expr : info->parsed_expressions) {
-			info->expressions.push_back(parsed_expr->Copy());
+	if (index_info.expressions.empty()) {
+		for (auto &parsed_expr : index_info.parsed_expressions) {
+			index_info.expressions.push_back(parsed_expr->Copy());
 		}
 	}
 	auto binder = Binder::CreateBinder(context);
-	auto expressions = binder->BindCreateIndexExpressions(table, *info);
+	auto expressions = binder->BindCreateIndexExpressions(table, index_info);
 
 	// create the empty index
 	unique_ptr<Index> index;
-	switch (info->index_type) {
+	switch (index_info.index_type) {
 	case IndexType::ART: {
-		index = make_uniq<ART>(info->column_ids, TableIOManager::Get(data_table), expressions, info->constraint_type,
-		                       data_table.db);
+		index = make_uniq<ART>(index_info.column_ids, TableIOManager::Get(data_table), expressions,
+		                       index_info.constraint_type, data_table.db);
 		break;
 	}
 	default:
@@ -424,10 +423,10 @@ void ReplayState::ReplayCreateIndex() {
 	}
 
 	// add the index to the catalog
-	auto &index_entry = catalog.CreateIndex(context, *info)->Cast<DuckIndexEntry>();
+	auto &index_entry = catalog.CreateIndex(context, index_info)->Cast<DuckIndexEntry>();
 	index_entry.index = index.get();
 	index_entry.info = data_table.info;
-	for (auto &parsed_expr : info->parsed_expressions) {
+	for (auto &parsed_expr : index_info.parsed_expressions) {
 		index_entry.parsed_expressions.push_back(parsed_expr->Copy());
 	}
 
@@ -524,7 +523,8 @@ void ReplayState::ReplayUpdate() {
 }
 
 void ReplayState::ReplayCheckpoint() {
-	checkpoint_id = source.Read<block_id_t>();
+	checkpoint_id.block_pointer = source.Read<idx_t>();
+	checkpoint_id.offset = source.Read<uint32_t>();
 }
 
 } // namespace duckdb

package/src/duckdb/src/storage/write_ahead_log.cpp

@@ -48,9 +48,10 @@ void WriteAheadLog::Delete() {
 //===--------------------------------------------------------------------===//
 // Write Entries
 //===--------------------------------------------------------------------===//
-void WriteAheadLog::WriteCheckpoint(block_id_t meta_block) {
+void WriteAheadLog::WriteCheckpoint(MetaBlockPointer meta_block) {
 	writer->Write<WALType>(WALType::CHECKPOINT);
-	writer->Write<block_id_t>(meta_block);
+	writer->Write<idx_t>(meta_block.block_pointer);
+	writer->Write<uint32_t>(meta_block.offset);
 }
 
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/verification/prepared_statement_verifier.cpp

@@ -23,6 +23,9 @@ void PreparedStatementVerifier::Extract() {
 	ParsedExpressionIterator::EnumerateQueryNodeChildren(
 	    *select.node, [&](unique_ptr<ParsedExpression> &child) { ConvertConstants(child); });
 	statement->n_param = values.size();
+	for (auto &kv : values) {
+		statement->named_param_map[kv.first] = 0;
+	}
 	// create the PREPARE and EXECUTE statements
 	string name = "__duckdb_verification_prepared_statement";
 	auto prepare = make_uniq<PrepareStatement>();
@@ -31,7 +34,7 @@ void PreparedStatementVerifier::Extract() {
 
 	auto execute = make_uniq<ExecuteStatement>();
 	execute->name = name;
-	execute->values = std::move(values);
+	execute->named_values = std::move(values);
 
 	auto dealloc = make_uniq<DropStatement>();
 	dealloc->info->type = CatalogType::PREPARED_STATEMENT;
@@ -49,19 +52,21 @@ void PreparedStatementVerifier::ConvertConstants(unique_ptr<ParsedExpression> &c
 		child->alias = string();
 		// check if the value already exists
 		idx_t index = values.size();
-		for (idx_t v_idx = 0; v_idx < values.size(); v_idx++) {
-			if (values[v_idx]->Equals(*child)) {
-				// duplicate value! refer to the original value
-				index = v_idx;
-				break;
-			}
-		}
-		if (index == values.size()) {
-			values.push_back(std::move(child));
+		auto identifier = std::to_string(index + 1);
+		const auto predicate = [&](const std::pair<const string, unique_ptr<ParsedExpression>> &pair) {
+			return pair.second->Equals(*child.get());
+		};
+		auto result = std::find_if(values.begin(), values.end(), predicate);
+		if (result == values.end()) {
+			// If it doesn't exist yet, add it
+			values[identifier] = std::move(child);
+		} else {
+			identifier = result->first;
 		}
+
 		// replace it with an expression
 		auto parameter = make_uniq<ParameterExpression>();
-		parameter->parameter_nr = index + 1;
+		parameter->identifier = identifier;
 		parameter->alias = alias;
 		child = std::move(parameter);
 		return;

package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h

@@ -80,7 +80,7 @@ namespace duckdb_moodycamel { namespace details {
 	static const thread_id_t invalid_thread_id2 = 0xFFFFFFFFU;	// Not technically guaranteed to be invalid, but is never used in practice. Note that all Win32 thread IDs are presently multiples of 4.
 	static inline thread_id_t thread_id() { return static_cast<thread_id_t>(::GetCurrentThreadId()); }
 } }
-#elif defined(__arm__) || defined(_M_ARM) || defined(__aarch64__) || (defined(__APPLE__) && TARGET_OS_IPHONE)
+#elif defined(__arm__) || defined(_M_ARM) || defined(__aarch64__) || (defined(__APPLE__) && TARGET_OS_IPHONE) || defined(__MVS__)
 namespace duckdb_moodycamel { namespace details {
 	static_assert(sizeof(std::thread::id) == 4 || sizeof(std::thread::id) == 8, "std::thread::id is expected to be either 4 or 8 bytes");
 	
@@ -194,7 +194,7 @@ namespace duckdb_moodycamel { namespace details {
 // VS2013 doesn't support `thread_local`, and MinGW-w64 w/ POSIX threading has a crippling bug: http://sourceforge.net/p/mingw-w64/bugs/445
 // g++ <=4.7 doesn't support thread_local either.
 // Finally, iOS/ARM doesn't have support for it either, and g++/ARM allows it to compile but it's unconfirmed to actually work
-#if (!defined(_MSC_VER) || _MSC_VER >= 1900) && (!defined(__MINGW32__) && !defined(__MINGW64__) || !defined(__WINPTHREADS_VERSION)) && (!defined(__GNUC__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) && (!defined(__APPLE__) || !TARGET_OS_IPHONE) && !defined(__arm__) && !defined(_M_ARM) && !defined(__aarch64__)
+#if (!defined(_MSC_VER) || _MSC_VER >= 1900) && (!defined(__MINGW32__) && !defined(__MINGW64__) || !defined(__WINPTHREADS_VERSION)) && (!defined(__GNUC__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) && (!defined(__APPLE__) || !TARGET_OS_IPHONE) && !defined(__arm__) && !defined(_M_ARM) && !defined(__aarch64__) && !defined(__MVS__)
 // Assume `thread_local` is fully supported in all other C++11 compilers/platforms
 //#define MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED    // always disabled for now since several users report having problems with it on
 #endif

package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h

@@ -27,6 +27,9 @@ extern "C" {
 #elif defined(__unix__)
 #include <semaphore.h>
 #include <chrono>
+#elif defined(__MVS__)
+#include <zos-semaphore.h>
+#include <chrono>
 #endif
 
 namespace duckdb_moodycamel
@@ -160,9 +163,9 @@ public:
 		}
 	}
 };
-#elif defined(__unix__)
+#elif defined(__unix__) || defined(__MVS__)
 //---------------------------------------------------------
-// Semaphore (POSIX, Linux)
+// Semaphore (POSIX, Linux, zOS aka MVS)
 //---------------------------------------------------------
 class Semaphore
 {

package/src/duckdb/third_party/fast_float/fast_float/fast_float.h

@@ -130,6 +130,8 @@ from_chars_result from_chars(const char *first, const char *last,
 #include <machine/endian.h>
 #elif defined(sun) || defined(__sun)
 #include <sys/byteorder.h>
+#elif defined(__MVS__)
+#include <sys/endian.h>
 #else
 #include <endian.h>
 #endif

package/src/duckdb/third_party/httplib/httplib.hpp

@@ -183,7 +183,16 @@ using socket_t = SOCKET;
 
 #include <arpa/inet.h>
 #include <cstring>
+#ifndef __MVS__
 #include <ifaddrs.h>
+#endif
+#ifdef __MVS__
+#include <net/if.h>
+#include <strings.h>
+#ifndef NI_MAXHOST
+#define NI_MAXHOST 1025
+#endif
+#endif
 #include <netdb.h>
 #include <netinet/in.h>
 #ifdef __linux__
@@ -2668,7 +2677,7 @@ inline bool bind_ip_address(socket_t sock, const char *host) {
 	return ret;
 }
 
-#if !defined _WIN32 && !defined ANDROID
+#if !defined _WIN32 && !defined ANDROID && !defined __MVS__
 #define USE_IF2IP
 #endif
 

package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp

@@ -325,6 +325,7 @@ typedef struct PGAIndices {
 	bool is_slice; /* true if slice (i.e., colon present) */
 	PGNode *lidx;  /* slice lower bound, if any */
 	PGNode *uidx;  /* subscript, or slice upper bound if any */
+	PGNode *step;  /* slice step, if any */
 } PGAIndices;
 
 /*
@@ -1058,10 +1059,19 @@ typedef struct PGOnConflictClause {
  *
  * We don't currently support the SEARCH or CYCLE clause.
  */
+
+typedef enum PGCTEMaterialize
+{
+	PGCTEMaterializeDefault,		/* no option specified */
+	PGCTEMaterializeAlways,		/* MATERIALIZED */
+	PGCTEMaterializeNever			/* NOT MATERIALIZED */
+} PGCTEMaterialize;
+
 typedef struct PGCommonTableExpr {
 	PGNodeTag type;
 	char *ctename;         /* query name (never qualified) */
 	PGList *aliascolnames; /* optional list of column names */
+	PGCTEMaterialize ctematerialized; /* is this an optimization fence? */
 	/* SelectStmt/InsertStmt/etc before parse analysis, PGQuery afterwards: */
 	PGNode *ctequery; /* the CTE's subquery */
 	int location;     /* token location, or -1 if unknown */

package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp

@@ -1049,6 +1049,7 @@ typedef union YYSTYPE
 	PGAlias				*alias;
 	PGRangeVar			*range;
 	PGIntoClause			*into;
+	PGCTEMaterialize			ctematerialize;
 	PGWithClause			*with;
 	PGInferClause			*infer;
 	PGOnConflictClause	*onconflict;
@@ -1068,7 +1069,7 @@ typedef union YYSTYPE
 	PGInsertColumnOrder bynameorposition;
 }
 /* Line 1529 of yacc.c.  */
-#line 1072 "third_party/libpg_query/grammar/grammar_out.hpp"
+#line 1073 "third_party/libpg_query/grammar/grammar_out.hpp"
 	YYSTYPE;
 # define yystype YYSTYPE /* obsolescent; will be withdrawn */
 # define YYSTYPE_IS_DECLARED 1

package/src/duckdb/third_party/libpg_query/pg_functions.cpp

@@ -8,6 +8,9 @@
 #include <mutex>
 #include <cstring>
 
+#ifdef __MVS__
+#include <zos-tls.h>
+#endif
 
 // max parse tree size approx 100 MB, should be enough
 #define PG_MALLOC_SIZE 10240
@@ -26,7 +29,17 @@ struct pg_parser_state_str {
 	size_t malloc_ptr_size;
 };
 
+#ifdef __MVS__
+// --------------------------------------------------------
+// Permanent - WIP
+// static __tlssim<parser_state> pg_parser_state_impl();
+// #define pg_parser_state (*pg_parser_state_impl.access())
+// --------------------------------------------------------
+// Temporary
+static parser_state pg_parser_state;
+#else
 static __thread parser_state pg_parser_state;
+#endif
 
 #ifndef __GNUC__
 __thread PGNode *duckdb_newNodeMacroHolder;