duckdb 0.8.2-dev145.0 → 0.8.2-dev1493.0

package/src/duckdb/extension/parquet/parquet_writer.cpp

@@ -29,6 +29,30 @@ using duckdb_parquet::format::PageType;
 using ParquetRowGroup = duckdb_parquet::format::RowGroup;
 using duckdb_parquet::format::Type;
 
+ChildFieldIDs::ChildFieldIDs() {
+	ids = make_uniq<case_insensitive_map_t<FieldID>>();
+}
+
+ChildFieldIDs ChildFieldIDs::Copy() const {
+	ChildFieldIDs result;
+	for (const auto &id : *ids) {
+		result.ids->emplace(id.first, id.second.Copy());
+	}
+	return result;
+}
+
+FieldID::FieldID() : set(false) {
+}
+
+FieldID::FieldID(int32_t field_id_p) : set(true), field_id(field_id_p) {
+}
+
+FieldID FieldID::Copy() const {
+	auto result = set ? FieldID(field_id) : FieldID();
+	result.child_field_ids = child_field_ids.Copy();
+	return result;
+}
+
 class MyTransport : public TTransport {
 public:
 	explicit MyTransport(Serializer &serializer) : serializer(serializer) {
@@ -226,8 +250,9 @@ void VerifyUniqueNames(const vector<string> &names) {
 }
 
 ParquetWriter::ParquetWriter(FileSystem &fs, string file_name_p, vector<LogicalType> types_p, vector<string> names_p,
-                             CompressionCodec::type codec)
-    : file_name(std::move(file_name_p)), sql_types(std::move(types_p)), column_names(std::move(names_p)), codec(codec) {
+                             CompressionCodec::type codec, ChildFieldIDs field_ids_p)
+    : file_name(std::move(file_name_p)), sql_types(std::move(types_p)), column_names(std::move(names_p)), codec(codec),
+      field_ids(std::move(field_ids_p)) {
 	// initialize the file writer
 	writer = make_uniq<BufferedFileWriter>(fs, file_name.c_str(),
 	                                       FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW);
@@ -257,11 +282,18 @@ ParquetWriter::ParquetWriter(FileSystem &fs, string file_name_p, vector<LogicalT
 	vector<string> schema_path;
 	for (idx_t i = 0; i < sql_types.size(); i++) {
 		column_writers.push_back(ColumnWriter::CreateWriterRecursive(file_meta_data.schema, *this, sql_types[i],
-		                                                             unique_names[i], schema_path));
+		                                                             unique_names[i], schema_path, &field_ids));
 	}
 }
 
 void ParquetWriter::PrepareRowGroup(ColumnDataCollection &buffer, PreparedRowGroup &result) {
+	// We write 8 columns at a time so that iterating over ColumnDataCollection is more efficient
+	static constexpr idx_t COLUMNS_PER_PASS = 8;
+
+	// We want these to be in-memory/hybrid so we don't have to copy over strings to the dictionary
+	D_ASSERT(buffer.GetAllocatorType() == ColumnDataAllocatorType::IN_MEMORY_ALLOCATOR ||
+	         buffer.GetAllocatorType() == ColumnDataAllocatorType::HYBRID);
+
 	// set up a new row group for this chunk collection
 	auto &row_group = result.row_group;
 	row_group.num_rows = buffer.Count();
@@ -270,24 +302,52 @@ void ParquetWriter::PrepareRowGroup(ColumnDataCollection &buffer, PreparedRowGro
 	auto &states = result.states;
 	// iterate over each of the columns of the chunk collection and write them
 	D_ASSERT(buffer.ColumnCount() == column_writers.size());
-	for (idx_t col_idx = 0; col_idx < buffer.ColumnCount(); col_idx++) {
-		const auto &col_writer = column_writers[col_idx];
-		auto write_state = col_writer->InitializeWriteState(row_group, buffer.GetAllocator());
-		if (col_writer->HasAnalyze()) {
-			for (auto &chunk : buffer.Chunks()) {
-				col_writer->Analyze(*write_state, nullptr, chunk.data[col_idx], chunk.size());
+	for (idx_t col_idx = 0; col_idx < buffer.ColumnCount(); col_idx += COLUMNS_PER_PASS) {
+		const auto next = MinValue<idx_t>(buffer.ColumnCount() - col_idx, COLUMNS_PER_PASS);
+		vector<column_t> column_ids;
+		vector<reference<ColumnWriter>> col_writers;
+		vector<unique_ptr<ColumnWriterState>> write_states;
+		for (idx_t i = 0; i < next; i++) {
+			column_ids.emplace_back(col_idx + i);
+			col_writers.emplace_back(*column_writers[column_ids.back()]);
+			write_states.emplace_back(col_writers.back().get().InitializeWriteState(row_group));
+		}
+
+		for (auto &chunk : buffer.Chunks({column_ids})) {
+			for (idx_t i = 0; i < next; i++) {
+				if (col_writers[i].get().HasAnalyze()) {
+					col_writers[i].get().Analyze(*write_states[i], nullptr, chunk.data[i], chunk.size());
+				}
+			}
+		}
+
+		for (idx_t i = 0; i < next; i++) {
+			if (col_writers[i].get().HasAnalyze()) {
+				col_writers[i].get().FinalizeAnalyze(*write_states[i]);
+			}
+		}
+
+		for (auto &chunk : buffer.Chunks({column_ids})) {
+			for (idx_t i = 0; i < next; i++) {
+				col_writers[i].get().Prepare(*write_states[i], nullptr, chunk.data[i], chunk.size());
 			}
-			col_writer->FinalizeAnalyze(*write_state);
 		}
-		for (auto &chunk : buffer.Chunks()) {
-			col_writer->Prepare(*write_state, nullptr, chunk.data[col_idx], chunk.size());
+
+		for (idx_t i = 0; i < next; i++) {
+			col_writers[i].get().BeginWrite(*write_states[i]);
 		}
-		col_writer->BeginWrite(*write_state);
-		for (auto &chunk : buffer.Chunks()) {
-			col_writer->Write(*write_state, chunk.data[col_idx], chunk.size());
+
+		for (auto &chunk : buffer.Chunks({column_ids})) {
+			for (idx_t i = 0; i < next; i++) {
+				col_writers[i].get().Write(*write_states[i], chunk.data[i], chunk.size());
+			}
+		}
+
+		for (auto &write_state : write_states) {
+			states.push_back(std::move(write_state));
 		}
-		states.push_back(std::move(write_state));
 	}
+	result.heaps = buffer.GetHeapReferences();
 }
 
 void ParquetWriter::FlushRowGroup(PreparedRowGroup &prepared) {
@@ -307,6 +367,8 @@ void ParquetWriter::FlushRowGroup(PreparedRowGroup &prepared) {
 	// append the row group to the file meta data
 	file_meta_data.row_groups.push_back(row_group);
 	file_meta_data.num_rows += row_group.num_rows;
+
+	prepared.heaps.clear();
 }
 
 void ParquetWriter::Flush(ColumnDataCollection &buffer) {
@@ -316,6 +378,7 @@ void ParquetWriter::Flush(ColumnDataCollection &buffer) {
 
 	PreparedRowGroup prepared_row_group;
 	PrepareRowGroup(buffer, prepared_row_group);
+	buffer.Reset();
 
 	FlushRowGroup(prepared_row_group);
 }

package/src/duckdb/extension/parquet/zstd_file_system.cpp

@@ -155,7 +155,7 @@ void ZstdStreamWrapper::Close() {
 
 class ZStdFile : public CompressedFile {
 public:
-	ZStdFile(duckdb::unique_ptr<FileHandle> child_handle_p, const string &path, bool write)
+	ZStdFile(unique_ptr<FileHandle> child_handle_p, const string &path, bool write)
 	    : CompressedFile(zstd_fs, std::move(child_handle_p), path) {
 		Initialize(write);
 	}
@@ -163,7 +163,7 @@ public:
 	ZStdFileSystem zstd_fs;
 };
 
-unique_ptr<FileHandle> ZStdFileSystem::OpenCompressedFile(duckdb::unique_ptr<FileHandle> handle, bool write) {
+unique_ptr<FileHandle> ZStdFileSystem::OpenCompressedFile(unique_ptr<FileHandle> handle, bool write) {
 	auto path = handle->path;
 	return make_uniq<ZStdFile>(std::move(handle), path, write);
 }

package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp

@@ -298,7 +298,7 @@ unique_ptr<CatalogEntry> DuckTableEntry::AddColumn(ClientContext &context, AddCo
 	auto binder = Binder::CreateBinder(context);
 	auto bound_create_info = binder->BindCreateTableInfo(std::move(create_info));
 	auto new_storage =
-	    make_shared<DataTable>(context, *storage, info.new_column, bound_create_info->bound_defaults.back().get());
+	    make_shared<DataTable>(context, *storage, info.new_column, *bound_create_info->bound_defaults.back());
 	return make_uniq<DuckTableEntry>(catalog, schema, *bound_create_info, new_storage);
 }
 

package/src/duckdb/src/catalog/catalog_search_path.cpp

@@ -1,10 +1,10 @@
 #include "duckdb/catalog/catalog_search_path.hpp"
 
+#include "duckdb/catalog/catalog.hpp"
 #include "duckdb/common/constants.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/main/client_context.hpp"
-#include "duckdb/catalog/catalog.hpp"
 #include "duckdb/main/database_manager.hpp"
 
 namespace duckdb {
@@ -249,13 +249,14 @@ void CatalogSearchPath::SetPaths(vector<CatalogSearchEntry> new_paths) {
 bool CatalogSearchPath::SchemaInSearchPath(ClientContext &context, const string &catalog_name,
                                            const string &schema_name) {
 	for (auto &path : paths) {
-		if (path.schema != schema_name) {
+		if (!StringUtil::CIEquals(path.schema, schema_name)) {
 			continue;
 		}
-		if (path.catalog == catalog_name) {
+		if (StringUtil::CIEquals(path.catalog, catalog_name)) {
 			return true;
 		}
-		if (IsInvalidCatalog(path.catalog) && catalog_name == DatabaseManager::GetDefaultDatabase(context)) {
+		if (IsInvalidCatalog(path.catalog) &&
+		    StringUtil::CIEquals(catalog_name, DatabaseManager::GetDefaultDatabase(context))) {
 			return true;
 		}
 	}

package/src/duckdb/src/catalog/default/default_functions.cpp

@@ -99,6 +99,19 @@ static DefaultMacro internal_macros[] = {
 	{DEFAULT_SCHEMA, "count_if", {"l", nullptr}, "sum(if(l, 1, 0))"},
 	{DEFAULT_SCHEMA, "split_part", {"string", "delimiter", "position", nullptr}, "coalesce(string_split(string, delimiter)[position],'')"},
 
+    // FIXME implement as actual function if we encounter a lot of performance issues. Complexity now: n * m, with hashing possibly n + m
+    {DEFAULT_SCHEMA, "list_intersect", {"l1", "l2", nullptr}, "list_filter(l1, (x) -> list_contains(l2, x))"},
+    {DEFAULT_SCHEMA, "array_intersect", {"l1", "l2", nullptr}, "list_intersect(l1, l2)"},
+
+    {DEFAULT_SCHEMA, "list_has_any", {"l1", "l2", nullptr}, "CASE WHEN l1 IS NULL THEN NULL WHEN l2 IS NULL THEN NULL WHEN len(list_intersect(l1, l2)) > 0 THEN true ELSE false END"},
+    {DEFAULT_SCHEMA, "array_has_any", {"l1", "l2", nullptr}, "list_has_any(l1, l2)" },
+    {DEFAULT_SCHEMA, "&&", {"l1", "l2", nullptr}, "list_has_any(l1, l2)" }, // "&&" is the operator for "list_has_any
+
+    {DEFAULT_SCHEMA, "list_has_all", {"l1", "l2", nullptr}, "CASE WHEN l1 IS NULL THEN NULL WHEN l2 IS NULL THEN NULL WHEN len(list_intersect(l2, l1)) = len(list_filter(l2, x -> x IS NOT NULL)) THEN true ELSE false END"},
+    {DEFAULT_SCHEMA, "array_has_all", {"l1", "l2", nullptr}, "list_has_all(l1, l2)" },
+    {DEFAULT_SCHEMA, "@>", {"l1", "l2", nullptr}, "list_has_all(l1, l2)" }, // "@>" is the operator for "list_has_all
+    {DEFAULT_SCHEMA, "<@", {"l1", "l2", nullptr}, "list_has_all(l2, l1)" }, // "<@" is the operator for "list_has_all
+
 	// algebraic list aggregates
 	{DEFAULT_SCHEMA, "list_avg", {"l", nullptr}, "list_aggr(l, 'avg')"},
 	{DEFAULT_SCHEMA, "list_var_samp", {"l", nullptr}, "list_aggr(l, 'var_samp')"},
@@ -205,6 +218,9 @@ unique_ptr<CatalogEntry> DefaultFunctionGenerator::CreateDefaultEntry(ClientCont
 vector<string> DefaultFunctionGenerator::GetDefaultEntries() {
 	vector<string> result;
 	for (idx_t index = 0; internal_macros[index].name != nullptr; index++) {
+		if (StringUtil::Lower(internal_macros[index].name) != internal_macros[index].name) {
+			throw InternalException("Default macro name %s should be lowercase", internal_macros[index].name);
+		}
 		if (internal_macros[index].schema == schema.name) {
 			result.emplace_back(internal_macros[index].name);
 		}

package/src/duckdb/src/common/adbc/adbc.cpp

@@ -44,10 +44,23 @@ duckdb_adbc::AdbcStatusCode duckdb_adbc_init(size_t count, struct duckdb_adbc::A
 	driver->ConnectionRollback = duckdb_adbc::ConnectionRollback;
 	driver->ConnectionReadPartition = duckdb_adbc::ConnectionReadPartition;
 	driver->StatementExecutePartitions = duckdb_adbc::StatementExecutePartitions;
+	driver->ConnectionGetTableSchema = duckdb_adbc::ConnectionGetTableSchema;
+	driver->StatementSetSubstraitPlan = duckdb_adbc::StatementSetSubstraitPlan;
+
 	return ADBC_STATUS_OK;
 }
 
 namespace duckdb_adbc {
+
+struct DuckDBAdbcStatementWrapper {
+	::duckdb_connection connection;
+	::duckdb_arrow result;
+	::duckdb_prepared_statement statement;
+	char *ingestion_table_name;
+	ArrowArrayStream *ingestion_stream;
+};
+static AdbcStatusCode QueryInternal(struct AdbcConnection *connection, struct ArrowArrayStream *out, const char *query,
+                                    struct AdbcError *error);
 AdbcStatusCode SetErrorMaybe(const void *result, AdbcError *error, const std::string &error_message) {
 	if (!error) {
 		return ADBC_STATUS_INVALID_ARGUMENT;
@@ -109,6 +122,28 @@ AdbcStatusCode DatabaseNew(struct AdbcDatabase *database, struct AdbcError *erro
 	return CheckResult(res, error, "Failed to allocate");
 }
 
+AdbcStatusCode StatementSetSubstraitPlan(struct AdbcStatement *statement, const uint8_t *plan, size_t length,
+                                         struct AdbcError *error) {
+	if (!statement) {
+		SetError(error, "Statement is not set");
+		return ADBC_STATUS_INVALID_ARGUMENT;
+	}
+	if (!plan) {
+		SetError(error, "Substrait Plan is not set");
+		return ADBC_STATUS_INVALID_ARGUMENT;
+	}
+	if (length == 0) {
+		SetError(error, "Can't execute plan with size = 0");
+		return ADBC_STATUS_INVALID_ARGUMENT;
+	}
+	auto wrapper = reinterpret_cast<DuckDBAdbcStatementWrapper *>(statement->private_data);
+	auto plan_str = std::string(reinterpret_cast<const char *>(plan), length);
+	auto query = "CALL from_substrait('" + plan_str + "'::BLOB)";
+	auto res = duckdb_prepare(wrapper->connection, query.c_str(), &wrapper->statement);
+	auto error_msg = duckdb_prepare_error(wrapper->statement);
+	return CheckResult(res, error, error_msg);
+}
+
 AdbcStatusCode DatabaseSetOption(struct AdbcDatabase *database, const char *key, const char *value,
                                  struct AdbcError *error) {
 	auto status = SetErrorMaybe(database, error, "Missing database object");
@@ -159,6 +194,44 @@ AdbcStatusCode DatabaseRelease(struct AdbcDatabase *database, struct AdbcError *
 	return ADBC_STATUS_OK;
 }
 
+AdbcStatusCode ConnectionGetTableSchema(struct AdbcConnection *connection, const char *catalog, const char *db_schema,
+                                        const char *table_name, struct ArrowSchema *schema, struct AdbcError *error) {
+	if (!connection) {
+		SetError(error, "Connection is not set");
+		return ADBC_STATUS_INVALID_ARGUMENT;
+	}
+	if (catalog != nullptr && strlen(catalog) > 0) {
+		// In DuckDB this is the name of the database, not sure what's the expected functionality here, so for now,
+		// scream.
+		SetError(error, "Catalog Name is not used in DuckDB. It must be set to nullptr or an empty string");
+		return ADBC_STATUS_NOT_IMPLEMENTED;
+	} else if (db_schema == nullptr) {
+		SetError(error, "AdbcConnectionGetTableSchema: must provide db_schema");
+		return ADBC_STATUS_INVALID_ARGUMENT;
+	} else if (table_name == nullptr) {
+		SetError(error, "AdbcConnectionGetTableSchema: must provide table_name");
+		return ADBC_STATUS_INVALID_ARGUMENT;
+	} else if (strlen(table_name) == 0) {
+		SetError(error, "AdbcConnectionGetTableSchema: must provide table_name");
+		return ADBC_STATUS_INVALID_ARGUMENT;
+	}
+	ArrowArrayStream arrow_stream;
+
+	std::string query = "SELECT * FROM ";
+	if (strlen(db_schema) > 0) {
+		query += std::string(db_schema) + ".";
+	}
+	query += std::string(table_name) + " LIMIT 0;";
+
+	auto success = QueryInternal(connection, &arrow_stream, query.c_str(), error);
+	if (success != ADBC_STATUS_OK) {
+		return success;
+	}
+	arrow_stream.get_schema(&arrow_stream, schema);
+	arrow_stream.release(&arrow_stream);
+	return ADBC_STATUS_OK;
+}
+
 AdbcStatusCode ConnectionNew(struct AdbcConnection *connection, struct AdbcError *error) {
 	auto status = SetErrorMaybe(connection, error, "Missing connection object");
 	if (status != ADBC_STATUS_OK) {
@@ -400,14 +473,6 @@ AdbcStatusCode Ingest(duckdb_connection connection, const char *table_name, stru
 	return ADBC_STATUS_OK;
 }
 
-struct DuckDBAdbcStatementWrapper {
-	::duckdb_connection connection;
-	::duckdb_arrow result;
-	::duckdb_prepared_statement statement;
-	char *ingestion_table_name;
-	ArrowArrayStream *ingestion_stream;
-};
-
 AdbcStatusCode StatementNew(struct AdbcConnection *connection, struct AdbcStatement *statement,
                             struct AdbcError *error) {
 
@@ -581,8 +646,8 @@ AdbcStatusCode StatementSetOption(struct AdbcStatement *statement, const char *k
 	return ADBC_STATUS_INVALID_ARGUMENT;
 }
 
-static AdbcStatusCode QueryInternal(struct AdbcConnection *connection, struct ArrowArrayStream *out, const char *query,
-                                    struct AdbcError *error) {
+AdbcStatusCode QueryInternal(struct AdbcConnection *connection, struct ArrowArrayStream *out, const char *query,
+                             struct AdbcError *error) {
 	AdbcStatement statement;
 
 	auto status = StatementNew(connection, &statement, error);

package/src/duckdb/src/common/adbc/driver_manager.cpp

@@ -109,8 +109,9 @@ struct ManagerDriverState {
 static AdbcStatusCode ReleaseDriver(struct AdbcDriver *driver, struct AdbcError *error) {
 	AdbcStatusCode status = ADBC_STATUS_OK;
 
-	if (!driver->private_manager)
+	if (!driver->private_manager) {
 		return status;
+	}
 	ManagerDriverState *state = reinterpret_cast<ManagerDriverState *>(driver->private_manager);
 
 	if (state->driver_release) {
@@ -138,11 +139,6 @@ AdbcStatusCode ConnectionGetInfo(struct AdbcConnection *connection, uint32_t *in
 	return ADBC_STATUS_NOT_IMPLEMENTED;
 }
 
-AdbcStatusCode ConnectionGetTableSchema(struct AdbcConnection *, const char *, const char *, const char *,
-                                        struct ArrowSchema *, struct AdbcError *error) {
-	return ADBC_STATUS_NOT_IMPLEMENTED;
-}
-
 AdbcStatusCode StatementBind(struct AdbcStatement *, struct ArrowArray *, struct ArrowSchema *,
                              struct AdbcError *error) {
 	return ADBC_STATUS_NOT_IMPLEMENTED;
@@ -152,9 +148,6 @@ AdbcStatusCode StatementGetParameterSchema(struct AdbcStatement *statement, stru
                                            struct AdbcError *error) {
 	return ADBC_STATUS_NOT_IMPLEMENTED;
 }
-AdbcStatusCode StatementSetSubstraitPlan(struct AdbcStatement *, const uint8_t *, size_t, struct AdbcError *error) {
-	return ADBC_STATUS_NOT_IMPLEMENTED;
-}
 
 /// Temporary state while the database is being configured.
 struct TempDatabase {
@@ -351,15 +344,17 @@ AdbcStatusCode AdbcConnectionInit(struct AdbcConnection *connection, struct Adbc
 	delete args;
 
 	auto status = database->private_driver->ConnectionNew(connection, error);
-	if (status != ADBC_STATUS_OK)
+	if (status != ADBC_STATUS_OK) {
 		return status;
+	}
 	connection->private_driver = database->private_driver;
 
 	for (const auto &option : options) {
 		status = database->private_driver->ConnectionSetOption(connection, option.first.c_str(), option.second.c_str(),
 		                                                       error);
-		if (status != ADBC_STATUS_OK)
+		if (status != ADBC_STATUS_OK) {
 			return status;
+		}
 	}
 	return connection->private_driver->ConnectionInit(connection, database, error);
 }

package/src/duckdb/src/common/allocator.cpp

@@ -14,7 +14,13 @@
 #include <execinfo.h>
 #endif
 
-#if defined(BUILD_JEMALLOC_EXTENSION) && !defined(WIN32)
+#ifndef USE_JEMALLOC
+#if defined(DUCKDB_EXTENSION_JEMALLOC_LINKED) && DUCKDB_EXTENSION_JEMALLOC_LINKED && !defined(WIN32)
+#define USE_JEMALLOC
+#endif
+#endif
+
+#ifdef USE_JEMALLOC
 #include "jemalloc_extension.hpp"
 #endif
 
@@ -89,7 +95,7 @@ PrivateAllocatorData::~PrivateAllocatorData() {
 //===--------------------------------------------------------------------===//
 // Allocator
 //===--------------------------------------------------------------------===//
-#if defined(BUILD_JEMALLOC_EXTENSION) && !defined(WIN32)
+#ifdef USE_JEMALLOC
 Allocator::Allocator()
     : Allocator(JemallocExtension::Allocate, JemallocExtension::Free, JemallocExtension::Reallocate, nullptr) {
 }
@@ -177,6 +183,12 @@ Allocator &Allocator::DefaultAllocator() {
 	return *DefaultAllocatorReference();
 }
 
+void Allocator::ThreadFlush(idx_t threshold) {
+#ifdef USE_JEMALLOC
+	JemallocExtension::ThreadFlush(threshold);
+#endif
+}
+
 //===--------------------------------------------------------------------===//
 // Debug Info (extended)
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/common/arrow/arrow_appender.cpp

@@ -153,7 +153,9 @@ struct ArrowIntervalConverter {
 template <class TGT, class SRC = TGT, class OP = ArrowScalarConverter>
 struct ArrowScalarBaseData {
 	static void Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
+		D_ASSERT(to >= from);
 		idx_t size = to - from;
+		D_ASSERT(size <= input_size);
 		UnifiedVectorFormat format;
 		input.ToUnifiedFormat(input_size, format);
 
@@ -489,16 +491,9 @@ struct ArrowListData {
 		SelectionVector child_sel(child_indices.data());
 		auto &child = ListVector::GetEntry(input);
 		auto child_size = child_indices.size();
-		if (size != input_size) {
-			// Let's avoid doing this
-			Vector child_copy(child.GetType());
-			child_copy.Slice(child, child_sel, child_size);
-			append_data.child_data[0]->append_vector(*append_data.child_data[0], child_copy, 0, child_size, child_size);
-		} else {
-			// We don't care about the vector, slice it
-			child.Slice(child_sel, child_size);
-			append_data.child_data[0]->append_vector(*append_data.child_data[0], child, 0, child_size, child_size);
-		}
+		Vector child_copy(child.GetType());
+		child_copy.Slice(child, child_sel, child_size);
+		append_data.child_data[0]->append_vector(*append_data.child_data[0], child_copy, 0, child_size, child_size);
 		append_data.row_count += size;
 	}
 
@@ -791,7 +786,6 @@ ArrowArray ArrowAppender::Finalize() {
 
 	// Configure root array
 	result.length = row_count;
-	result.n_children = types.size();
 	result.n_buffers = 1;
 	result.buffers = root_holder->buffers.data(); // there is no actual buffer there since we don't have NULLs
 	result.offset = 0;

package/src/duckdb/src/common/assert.cpp

@@ -4,6 +4,9 @@
 namespace duckdb {
 
 void DuckDBAssertInternal(bool condition, const char *condition_name, const char *file, int linenr) {
+#ifdef DISABLE_ASSERTIONS
+	return;
+#endif
 	if (condition) {
 		return;
 	}

package/src/duckdb/src/common/enum_util.cpp

@@ -879,6 +879,8 @@ const char *EnumUtil::ToChars<QueryNodeType>(QueryNodeType value) {
 		return "BOUND_SUBQUERY_NODE";
 	case QueryNodeType::RECURSIVE_CTE_NODE:
 		return "RECURSIVE_CTE_NODE";
+	case QueryNodeType::CTE_NODE:
+		return "CTE_NODE";
 	default:
 		throw NotImplementedException(StringUtil::Format("Enum value: '%d' not implemented", value));
 	}
@@ -898,6 +900,9 @@ QueryNodeType EnumUtil::FromString<QueryNodeType>(const char *value) {
 	if (StringUtil::Equals(value, "RECURSIVE_CTE_NODE")) {
 		return QueryNodeType::RECURSIVE_CTE_NODE;
 	}
+	if (StringUtil::Equals(value, "CTE_NODE")) {
+		return QueryNodeType::RECURSIVE_CTE_NODE;
+	}
 	throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value));
 }
 
@@ -2112,6 +2117,33 @@ LogicalTypeId EnumUtil::FromString<LogicalTypeId>(const char *value) {
 	throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value));
 }
 
+template <>
+CTEMaterialize EnumUtil::FromString<CTEMaterialize>(const char *value) {
+	if (StringUtil::Equals(value, "CTE_MATERIALIZE_DEFAULT")) {
+		return CTEMaterialize::CTE_MATERIALIZE_DEFAULT;
+	} else if (StringUtil::Equals(value, "CTE_MATERIALIZE_ALWAYS")) {
+		return CTEMaterialize::CTE_MATERIALIZE_ALWAYS;
+	} else if (StringUtil::Equals(value, "CTE_MATERIALIZE_NEVER")) {
+		return CTEMaterialize::CTE_MATERIALIZE_NEVER;
+	} else {
+		throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value));
+	}
+}
+
+template <>
+const char *EnumUtil::ToChars<CTEMaterialize>(CTEMaterialize value) {
+	switch (value) {
+	case CTEMaterialize::CTE_MATERIALIZE_DEFAULT:
+		return "CTE_MATERIALIZE_DEFAULT";
+	case CTEMaterialize::CTE_MATERIALIZE_ALWAYS:
+		return "CTE_MATERIALIZE_ALWAYS";
+	case CTEMaterialize::CTE_MATERIALIZE_NEVER:
+		return "CTE_MATERIALIZE_NEVER";
+	default:
+		throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value));
+	}
+}
+
 template <>
 const char *EnumUtil::ToChars<OutputStream>(OutputStream value) {
 	switch (value) {
@@ -2578,6 +2610,8 @@ const char *EnumUtil::ToChars<PhysicalOperatorType>(PhysicalOperatorType value)
 		return "CHUNK_SCAN";
 	case PhysicalOperatorType::RECURSIVE_CTE_SCAN:
 		return "RECURSIVE_CTE_SCAN";
+	case PhysicalOperatorType::CTE_SCAN:
+		return "CTE_SCAN";
 	case PhysicalOperatorType::DELIM_SCAN:
 		return "DELIM_SCAN";
 	case PhysicalOperatorType::EXPRESSION_SCAN:
@@ -2748,6 +2782,9 @@ PhysicalOperatorType EnumUtil::FromString<PhysicalOperatorType>(const char *valu
 	if (StringUtil::Equals(value, "RECURSIVE_CTE_SCAN")) {
 		return PhysicalOperatorType::RECURSIVE_CTE_SCAN;
 	}
+	if (StringUtil::Equals(value, "CTE_SCAN")) {
+		return PhysicalOperatorType::CTE_SCAN;
+	}
 	if (StringUtil::Equals(value, "DELIM_SCAN")) {
 		return PhysicalOperatorType::DELIM_SCAN;
 	}
@@ -4649,6 +4686,8 @@ const char *EnumUtil::ToChars<LogicalOperatorType>(LogicalOperatorType value) {
 		return "LOGICAL_INTERSECT";
 	case LogicalOperatorType::LOGICAL_RECURSIVE_CTE:
 		return "LOGICAL_RECURSIVE_CTE";
+	case LogicalOperatorType::LOGICAL_MATERIALIZED_CTE:
+		return "LOGICAL_MATERIALIZED_CTE";
 	case LogicalOperatorType::LOGICAL_INSERT:
 		return "LOGICAL_INSERT";
 	case LogicalOperatorType::LOGICAL_DELETE:
@@ -4804,6 +4843,9 @@ LogicalOperatorType EnumUtil::FromString<LogicalOperatorType>(const char *value)
 	if (StringUtil::Equals(value, "LOGICAL_RECURSIVE_CTE")) {
 		return LogicalOperatorType::LOGICAL_RECURSIVE_CTE;
 	}
+	if (StringUtil::Equals(value, "LOGICAL_MATERIALIZED_CTE")) {
+		return LogicalOperatorType::LOGICAL_MATERIALIZED_CTE;
+	}
 	if (StringUtil::Equals(value, "LOGICAL_INSERT")) {
 		return LogicalOperatorType::LOGICAL_INSERT;
 	}
@@ -5616,8 +5658,6 @@ ExplainOutputType EnumUtil::FromString<ExplainOutputType>(const char *value) {
 template <>
 const char *EnumUtil::ToChars<NType>(NType value) {
 	switch (value) {
-	case NType::PREFIX_SEGMENT:
-		return "PREFIX_SEGMENT";
 	case NType::LEAF_SEGMENT:
 		return "LEAF_SEGMENT";
 	case NType::LEAF:
@@ -5637,9 +5677,6 @@ const char *EnumUtil::ToChars<NType>(NType value) {
 
 template <>
 NType EnumUtil::FromString<NType>(const char *value) {
-	if (StringUtil::Equals(value, "PREFIX_SEGMENT")) {
-		return NType::PREFIX_SEGMENT;
-	}
 	if (StringUtil::Equals(value, "LEAF_SEGMENT")) {
 		return NType::LEAF_SEGMENT;
 	}

package/src/duckdb/src/common/enums/logical_operator_type.cpp

@@ -22,6 +22,8 @@ string LogicalOperatorToString(LogicalOperatorType type) {
 		return "ANY_JOIN";
 	case LogicalOperatorType::LOGICAL_ASOF_JOIN:
 		return "ASOF_JOIN";
+	case LogicalOperatorType::LOGICAL_DEPENDENT_JOIN:
+		return "DEPENDENT_JOIN";
 	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
 		return "COMPARISON_JOIN";
 	case LogicalOperatorType::LOGICAL_DELIM_JOIN:
@@ -86,6 +88,8 @@ string LogicalOperatorToString(LogicalOperatorType type) {
 		return "VACUUM";
 	case LogicalOperatorType::LOGICAL_RECURSIVE_CTE:
 		return "REC_CTE";
+	case LogicalOperatorType::LOGICAL_MATERIALIZED_CTE:
+		return "CTE";
 	case LogicalOperatorType::LOGICAL_CTE_REF:
 		return "CTE_SCAN";
 	case LogicalOperatorType::LOGICAL_SHOW:

package/src/duckdb/src/common/enums/optimizer_type.cpp

@@ -25,6 +25,8 @@ static DefaultOptimizerType internal_optimizer_types[] = {
     {"common_aggregate", OptimizerType::COMMON_AGGREGATE},
     {"column_lifetime", OptimizerType::COLUMN_LIFETIME},
     {"top_n", OptimizerType::TOP_N},
+    {"compressed_materialization", OptimizerType::COMPRESSED_MATERIALIZATION},
+    {"duplicate_groups", OptimizerType::DUPLICATE_GROUPS},
     {"reorder_filter", OptimizerType::REORDER_FILTER},
     {"extension", OptimizerType::EXTENSION},
     {nullptr, OptimizerType::INVALID}};

package/src/duckdb/src/common/enums/physical_operator_type.cpp

@@ -103,8 +103,12 @@ string PhysicalOperatorToString(PhysicalOperatorType type) {
 		return "VACUUM";
 	case PhysicalOperatorType::RECURSIVE_CTE:
 		return "REC_CTE";
+	case PhysicalOperatorType::CTE:
+		return "CTE";
 	case PhysicalOperatorType::RECURSIVE_CTE_SCAN:
 		return "REC_CTE_SCAN";
+	case PhysicalOperatorType::CTE_SCAN:
+		return "CTE_SCAN";
 	case PhysicalOperatorType::EXPRESSION_SCAN:
 		return "EXPRESSION_SCAN";
 	case PhysicalOperatorType::ALTER:

package/src/duckdb/src/common/exception.cpp

@@ -62,11 +62,11 @@ string Exception::ConstructMessageRecursive(const string &msg, std::vector<Excep
 #ifdef DEBUG
 	// Verify that we have the required amount of values for the message
 	idx_t parameter_count = 0;
-	for (idx_t i = 0; i < msg.size(); i++) {
+	for (idx_t i = 0; i + 1 < msg.size(); i++) {
 		if (msg[i] != '%') {
 			continue;
 		}
-		if (i < msg.size() && msg[i + 1] == '%') {
+		if (msg[i + 1] == '%') {
 			i++;
 			continue;
 		}