duckdb 0.10.3-dev0.0 → 0.10.3-dev13.0

package/src/duckdb/src/execution/operator/persistent/physical_export.cpp

@@ -17,12 +17,23 @@ namespace duckdb {
 
 using std::stringstream;
 
-static void WriteCatalogEntries(stringstream &ss, vector<reference<CatalogEntry>> &entries) {
+void ReorderTableEntries(catalog_entry_vector_t &tables);
+
+static void WriteCatalogEntries(stringstream &ss, catalog_entry_vector_t &entries) {
 	for (auto &entry : entries) {
 		if (entry.get().internal) {
 			continue;
 		}
-		ss << entry.get().ToSQL() << '\n';
+		auto create_info = entry.get().GetInfo();
+		try {
+			// Strip the catalog from the info
+			create_info->catalog.clear();
+			auto to_string = create_info->ToString();
+			ss << to_string;
+		} catch (const NotImplementedException &) {
+			ss << entry.get().ToSQL();
+		}
+		ss << '\n';
 	}
 	ss << '\n';
 }
@@ -31,7 +42,7 @@ static void WriteStringStreamToFile(FileSystem &fs, stringstream &ss, const stri
 	auto ss_string = ss.str();
 	auto handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW |
 	                                    FileLockType::WRITE_LOCK);
-	fs.Write(*handle, (void *)ss_string.c_str(), ss_string.size());
+	fs.Write(*handle, (void *)ss_string.c_str(), NumericCast<int64_t>(ss_string.size()));
 	handle.reset();
 }
 
@@ -39,13 +50,13 @@ static void WriteCopyStatement(FileSystem &fs, stringstream &ss, CopyInfo &info,
                                CopyFunction const &function) {
 	ss << "COPY ";
 
-	if (exported_table.schema_name != DEFAULT_SCHEMA) {
+	//! NOTE: The catalog is explicitly not set here
+	if (exported_table.schema_name != DEFAULT_SCHEMA && !exported_table.schema_name.empty()) {
 		ss << KeywordHelper::WriteOptionallyQuoted(exported_table.schema_name) << ".";
 	}
 
 	auto file_path = StringUtil::Replace(exported_table.file_path, "\\", "/");
 	ss << StringUtil::Format("%s FROM %s (", SQLIdentifier(exported_table.table_name), SQLString(file_path));
-
 	// write the copy options
 	ss << "FORMAT '" << info.format << "'";
 	if (info.format == "csv") {
@@ -60,6 +71,10 @@ static void WriteCopyStatement(FileSystem &fs, stringstream &ss, CopyInfo &info,
 		if (info.options.find("quote") == info.options.end()) {
 			info.options["quote"].push_back(Value("\""));
 		}
+		info.options.erase("force_not_null");
+		for (auto &not_null_column : exported_table.not_null_columns) {
+			info.options["force_not_null"].push_back(not_null_column);
+		}
 	}
 	for (auto &copy_option : info.options) {
 		if (copy_option.first == "force_quote") {
@@ -73,8 +88,15 @@ static void WriteCopyStatement(FileSystem &fs, stringstream &ss, CopyInfo &info,
 		if (copy_option.second.size() == 1) {
 			ss << copy_option.second[0].ToSQLString();
 		} else {
-			// FIXME handle multiple options
-			throw NotImplementedException("FIXME: serialize list of options");
+			// For Lists
+			ss << "(";
+			for (idx_t i = 0; i < copy_option.second.size(); i++) {
+				ss << copy_option.second[i].ToSQLString();
+				if (i != copy_option.second.size() - 1) {
+					ss << ", ";
+				}
+			}
+			ss << ")";
 		}
 	}
 	ss << ");" << '\n';
@@ -113,11 +135,24 @@ void PhysicalExport::ExtractEntries(ClientContext &context, vector<reference<Sch
 				result.tables.push_back(entry);
 			}
 		});
-		schema.Scan(context, CatalogType::SEQUENCE_ENTRY,
-		            [&](CatalogEntry &entry) { result.sequences.push_back(entry); });
-		schema.Scan(context, CatalogType::TYPE_ENTRY,
-		            [&](CatalogEntry &entry) { result.custom_types.push_back(entry); });
-		schema.Scan(context, CatalogType::INDEX_ENTRY, [&](CatalogEntry &entry) { result.indexes.push_back(entry); });
+		schema.Scan(context, CatalogType::SEQUENCE_ENTRY, [&](CatalogEntry &entry) {
+			if (entry.internal) {
+				return;
+			}
+			result.sequences.push_back(entry);
+		});
+		schema.Scan(context, CatalogType::TYPE_ENTRY, [&](CatalogEntry &entry) {
+			if (entry.internal) {
+				return;
+			}
+			result.custom_types.push_back(entry);
+		});
+		schema.Scan(context, CatalogType::INDEX_ENTRY, [&](CatalogEntry &entry) {
+			if (entry.internal) {
+				return;
+			}
+			result.indexes.push_back(entry);
+		});
 		schema.Scan(context, CatalogType::MACRO_ENTRY, [&](CatalogEntry &entry) {
 			if (!entry.internal && entry.type == CatalogType::MACRO_ENTRY) {
 				result.macros.push_back(entry);
@@ -131,6 +166,47 @@ void PhysicalExport::ExtractEntries(ClientContext &context, vector<reference<Sch
 	}
 }
 
+static void AddEntries(catalog_entry_vector_t &all_entries, catalog_entry_vector_t &to_add) {
+	for (auto &entry : to_add) {
+		all_entries.push_back(entry);
+	}
+	to_add.clear();
+}
+
+catalog_entry_vector_t PhysicalExport::GetNaiveExportOrder(ClientContext &context, Catalog &catalog) {
+	// gather all catalog types to export
+	ExportEntries entries;
+	auto schema_list = catalog.GetSchemas(context);
+	PhysicalExport::ExtractEntries(context, schema_list, entries);
+
+	ReorderTableEntries(entries.tables);
+
+	// order macro's by timestamp so nested macro's are imported nicely
+	sort(entries.macros.begin(), entries.macros.end(),
+	     [](const reference<CatalogEntry> &lhs, const reference<CatalogEntry> &rhs) {
+		     return lhs.get().oid < rhs.get().oid;
+	     });
+
+	catalog_entry_vector_t catalog_entries;
+	idx_t size = 0;
+	size += entries.schemas.size();
+	size += entries.custom_types.size();
+	size += entries.sequences.size();
+	size += entries.tables.size();
+	size += entries.views.size();
+	size += entries.indexes.size();
+	size += entries.macros.size();
+	catalog_entries.reserve(size);
+	AddEntries(catalog_entries, entries.schemas);
+	AddEntries(catalog_entries, entries.sequences);
+	AddEntries(catalog_entries, entries.custom_types);
+	AddEntries(catalog_entries, entries.tables);
+	AddEntries(catalog_entries, entries.macros);
+	AddEntries(catalog_entries, entries.views);
+	AddEntries(catalog_entries, entries.indexes);
+	return catalog_entries;
+}
+
 SourceResultType PhysicalExport::GetData(ExecutionContext &context, DataChunk &chunk,
                                          OperatorSourceInput &input) const {
 	auto &state = input.global_state.Cast<ExportSourceState>();

package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp

@@ -17,25 +17,24 @@
 #include "duckdb/execution/index/art/art.hpp"
 #include "duckdb/transaction/duck_transaction.hpp"
 #include "duckdb/storage/table/append_state.hpp"
+#include "duckdb/storage/table/update_state.hpp"
 
 namespace duckdb {
 
-PhysicalInsert::PhysicalInsert(vector<LogicalType> types_p, TableCatalogEntry &table,
-                               physical_index_vector_t<idx_t> column_index_map,
-                               vector<unique_ptr<Expression>> bound_defaults,
-                               vector<unique_ptr<Expression>> set_expressions, vector<PhysicalIndex> set_columns,
-                               vector<LogicalType> set_types, idx_t estimated_cardinality, bool return_chunk,
-                               bool parallel, OnConflictAction action_type,
-                               unique_ptr<Expression> on_conflict_condition_p,
-                               unique_ptr<Expression> do_update_condition_p, unordered_set<column_t> conflict_target_p,
-                               vector<column_t> columns_to_fetch_p)
+PhysicalInsert::PhysicalInsert(
+    vector<LogicalType> types_p, TableCatalogEntry &table, physical_index_vector_t<idx_t> column_index_map,
+    vector<unique_ptr<Expression>> bound_defaults, vector<unique_ptr<BoundConstraint>> bound_constraints_p,
+    vector<unique_ptr<Expression>> set_expressions, vector<PhysicalIndex> set_columns, vector<LogicalType> set_types,
+    idx_t estimated_cardinality, bool return_chunk, bool parallel, OnConflictAction action_type,
+    unique_ptr<Expression> on_conflict_condition_p, unique_ptr<Expression> do_update_condition_p,
+    unordered_set<column_t> conflict_target_p, vector<column_t> columns_to_fetch_p)
     : PhysicalOperator(PhysicalOperatorType::INSERT, std::move(types_p), estimated_cardinality),
       column_index_map(std::move(column_index_map)), insert_table(&table), insert_types(table.GetTypes()),
-      bound_defaults(std::move(bound_defaults)), return_chunk(return_chunk), parallel(parallel),
-      action_type(action_type), set_expressions(std::move(set_expressions)), set_columns(std::move(set_columns)),
-      set_types(std::move(set_types)), on_conflict_condition(std::move(on_conflict_condition_p)),
-      do_update_condition(std::move(do_update_condition_p)), conflict_target(std::move(conflict_target_p)),
-      columns_to_fetch(std::move(columns_to_fetch_p)) {
+      bound_defaults(std::move(bound_defaults)), bound_constraints(std::move(bound_constraints_p)),
+      return_chunk(return_chunk), parallel(parallel), action_type(action_type),
+      set_expressions(std::move(set_expressions)), set_columns(std::move(set_columns)), set_types(std::move(set_types)),
+      on_conflict_condition(std::move(on_conflict_condition_p)), do_update_condition(std::move(do_update_condition_p)),
+      conflict_target(std::move(conflict_target_p)), columns_to_fetch(std::move(columns_to_fetch_p)) {
 
 	if (action_type == OnConflictAction::THROW) {
 		return;
@@ -90,8 +89,9 @@ public:
 class InsertLocalState : public LocalSinkState {
 public:
 	InsertLocalState(ClientContext &context, const vector<LogicalType> &types,
-	                 const vector<unique_ptr<Expression>> &bound_defaults)
-	    : default_executor(context, bound_defaults) {
+	                 const vector<unique_ptr<Expression>> &bound_defaults,
+	                 const vector<unique_ptr<BoundConstraint>> &bound_constraints)
+	    : default_executor(context, bound_defaults), bound_constraints(bound_constraints) {
 		insert_chunk.Initialize(Allocator::Get(context), types);
 	}
 
@@ -105,6 +105,15 @@ public:
 	// Rows in the transaction-local storage that have been updated by a DO UPDATE conflict
 	unordered_set<row_t> updated_local_rows;
 	idx_t update_count = 0;
+	unique_ptr<ConstraintState> constraint_state;
+	const vector<unique_ptr<BoundConstraint>> &bound_constraints;
+
+	ConstraintState &GetConstraintState(DataTable &table, TableCatalogEntry &tableref) {
+		if (!constraint_state) {
+			constraint_state = table.InitializeConstraintState(tableref, bound_constraints);
+		}
+		return *constraint_state;
+	}
 };
 
 unique_ptr<GlobalSinkState> PhysicalInsert::GetGlobalSinkState(ClientContext &context) const {
@@ -125,7 +134,7 @@ unique_ptr<GlobalSinkState> PhysicalInsert::GetGlobalSinkState(ClientContext &co
 }
 
 unique_ptr<LocalSinkState> PhysicalInsert::GetLocalSinkState(ExecutionContext &context) const {
-	return make_uniq<InsertLocalState>(context.client, insert_types, bound_defaults);
+	return make_uniq<InsertLocalState>(context.client, insert_types, bound_defaults, bound_constraints);
 }
 
 void PhysicalInsert::ResolveDefaults(const TableCatalogEntry &table, DataChunk &chunk,
@@ -278,7 +287,8 @@ static idx_t PerformOnConflictAction(ExecutionContext &context, DataChunk &chunk
 	auto &data_table = table.GetStorage();
 	// Perform the update, using the results of the SET expressions
 	if (GLOBAL) {
-		data_table.Update(table, context.client, row_ids, set_columns, update_chunk);
+		auto update_state = data_table.InitializeUpdate(table, context.client, op.bound_constraints);
+		data_table.Update(*update_state, context.client, row_ids, set_columns, update_chunk);
 	} else {
 		auto &local_storage = LocalStorage::Get(context.client, data_table.db);
 		// Perform the update, using the results of the SET expressions
@@ -320,7 +330,8 @@ static idx_t HandleInsertConflicts(TableCatalogEntry &table, ExecutionContext &c
 	ConflictInfo conflict_info(conflict_target);
 	ConflictManager conflict_manager(VerifyExistenceType::APPEND, lstate.insert_chunk.size(), &conflict_info);
 	if (GLOBAL) {
-		data_table.VerifyAppendConstraints(table, context.client, lstate.insert_chunk, &conflict_manager);
+		auto &constraint_state = lstate.GetConstraintState(data_table, table);
+		data_table.VerifyAppendConstraints(constraint_state, context.client, lstate.insert_chunk, &conflict_manager);
 	} else {
 		DataTable::VerifyUniqueIndexes(local_storage.GetIndexes(data_table), context.client, lstate.insert_chunk,
 		                               &conflict_manager);
@@ -380,7 +391,8 @@ static idx_t HandleInsertConflicts(TableCatalogEntry &table, ExecutionContext &c
 			combined_chunk.Slice(sel.Selection(), sel.Count());
 			row_ids.Slice(sel.Selection(), sel.Count());
 			if (GLOBAL) {
-				data_table.VerifyAppendConstraints(table, context.client, combined_chunk, nullptr);
+				auto &constraint_state = lstate.GetConstraintState(data_table, table);
+				data_table.VerifyAppendConstraints(constraint_state, context.client, combined_chunk, nullptr);
 			} else {
 				DataTable::VerifyUniqueIndexes(local_storage.GetIndexes(data_table), context.client,
 				                               lstate.insert_chunk, nullptr);
@@ -406,7 +418,8 @@ idx_t PhysicalInsert::OnConflictHandling(TableCatalogEntry &table, ExecutionCont
                                          InsertLocalState &lstate) const {
 	auto &data_table = table.GetStorage();
 	if (action_type == OnConflictAction::THROW) {
-		data_table.VerifyAppendConstraints(table, context.client, lstate.insert_chunk, nullptr);
+		auto &constraint_state = lstate.GetConstraintState(data_table, table);
+		data_table.VerifyAppendConstraints(constraint_state, context.client, lstate.insert_chunk, nullptr);
 		return 0;
 	}
 	// Check whether any conflicts arise, and if they all meet the conflict_target + condition
@@ -429,7 +442,7 @@ SinkResultType PhysicalInsert::Sink(ExecutionContext &context, DataChunk &chunk,
 
 	if (!parallel) {
 		if (!gstate.initialized) {
-			storage.InitializeLocalAppend(gstate.append_state, context.client);
+			storage.InitializeLocalAppend(gstate.append_state, table, context.client, bound_constraints);
 			gstate.initialized = true;
 		}
 
@@ -440,15 +453,22 @@ SinkResultType PhysicalInsert::Sink(ExecutionContext &context, DataChunk &chunk,
 		gstate.insert_count += lstate.insert_chunk.size();
 		gstate.insert_count += updated_tuples;
 		storage.LocalAppend(gstate.append_state, table, context.client, lstate.insert_chunk, true);
+
+		// We finalize the local append to write the segment node count.
+		if (action_type != OnConflictAction::THROW) {
+			storage.FinalizeLocalAppend(gstate.append_state);
+			gstate.initialized = false;
+		}
+
 	} else {
 		D_ASSERT(!return_chunk);
 		// parallel append
 		if (!lstate.local_collection) {
 			lock_guard<mutex> l(gstate.lock);
-			auto &table_info = storage.info;
+			auto table_info = storage.GetDataTableInfo();
 			auto &block_manager = TableIOManager::Get(storage).GetBlockManagerForRowData();
-			lstate.local_collection =
-			    make_uniq<RowGroupCollection>(table_info, block_manager, insert_types, MAX_ROW_ID);
+			lstate.local_collection = make_uniq<RowGroupCollection>(std::move(table_info), block_manager, insert_types,
+			                                                        NumericCast<idx_t>(MAX_ROW_ID));
 			lstate.local_collection->InitializeEmpty();
 			lstate.local_collection->InitializeAppend(lstate.local_append_state);
 			lstate.writer = &gstate.table.GetStorage().CreateOptimisticWriter(context.client);
@@ -487,7 +507,7 @@ SinkCombineResultType PhysicalInsert::Combine(ExecutionContext &context, Operato
 		// we have few rows - append to the local storage directly
 		auto &table = gstate.table;
 		auto &storage = table.GetStorage();
-		storage.InitializeLocalAppend(gstate.append_state, context.client);
+		storage.InitializeLocalAppend(gstate.append_state, table, context.client, bound_constraints);
 		auto &transaction = DuckTransaction::Get(context.client, table.catalog);
 		lstate.local_collection->Scan(transaction, [&](DataChunk &insert_chunk) {
 			storage.LocalAppend(gstate.append_state, table, context.client, insert_chunk);
@@ -540,7 +560,7 @@ SourceResultType PhysicalInsert::GetData(ExecutionContext &context, DataChunk &c
 	auto &insert_gstate = sink_state->Cast<InsertGlobalState>();
 	if (!return_chunk) {
 		chunk.SetCardinality(1);
-		chunk.SetValue(0, 0, Value::BIGINT(insert_gstate.insert_count));
+		chunk.SetValue(0, 0, Value::BIGINT(NumericCast<int64_t>(insert_gstate.insert_count)));
 		return SourceResultType::FINISHED;
 	}
 

package/src/duckdb/src/execution/operator/persistent/physical_update.cpp

@@ -8,16 +8,20 @@
 #include "duckdb/parallel/thread_context.hpp"
 #include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/storage/data_table.hpp"
+#include "duckdb/storage/table/delete_state.hpp"
+#include "duckdb/storage/table/update_state.hpp"
 
 namespace duckdb {
 
 PhysicalUpdate::PhysicalUpdate(vector<LogicalType> types, TableCatalogEntry &tableref, DataTable &table,
                                vector<PhysicalIndex> columns, vector<unique_ptr<Expression>> expressions,
-                               vector<unique_ptr<Expression>> bound_defaults, idx_t estimated_cardinality,
+                               vector<unique_ptr<Expression>> bound_defaults,
+                               vector<unique_ptr<BoundConstraint>> bound_constraints, idx_t estimated_cardinality,
                                bool return_chunk)
     : PhysicalOperator(PhysicalOperatorType::UPDATE, std::move(types), estimated_cardinality), tableref(tableref),
       table(table), columns(std::move(columns)), expressions(std::move(expressions)),
-      bound_defaults(std::move(bound_defaults)), return_chunk(return_chunk) {
+      bound_defaults(std::move(bound_defaults)), bound_constraints(std::move(bound_constraints)),
+      return_chunk(return_chunk) {
 }
 
 //===--------------------------------------------------------------------===//
@@ -38,8 +42,9 @@ public:
 class UpdateLocalState : public LocalSinkState {
 public:
 	UpdateLocalState(ClientContext &context, const vector<unique_ptr<Expression>> &expressions,
-	                 const vector<LogicalType> &table_types, const vector<unique_ptr<Expression>> &bound_defaults)
-	    : default_executor(context, bound_defaults) {
+	                 const vector<LogicalType> &table_types, const vector<unique_ptr<Expression>> &bound_defaults,
+	                 const vector<unique_ptr<BoundConstraint>> &bound_constraints)
+	    : default_executor(context, bound_defaults), bound_constraints(bound_constraints) {
 		// initialize the update chunk
 		auto &allocator = Allocator::Get(context);
 		vector<LogicalType> update_types;
@@ -55,6 +60,23 @@ public:
 	DataChunk update_chunk;
 	DataChunk mock_chunk;
 	ExpressionExecutor default_executor;
+	unique_ptr<TableDeleteState> delete_state;
+	unique_ptr<TableUpdateState> update_state;
+	const vector<unique_ptr<BoundConstraint>> &bound_constraints;
+
+	TableDeleteState &GetDeleteState(DataTable &table, TableCatalogEntry &tableref, ClientContext &context) {
+		if (!delete_state) {
+			delete_state = table.InitializeDelete(tableref, context, bound_constraints);
+		}
+		return *delete_state;
+	}
+
+	TableUpdateState &GetUpdateState(DataTable &table, TableCatalogEntry &tableref, ClientContext &context) {
+		if (!update_state) {
+			update_state = table.InitializeUpdate(tableref, context, bound_constraints);
+		}
+		return *update_state;
+	}
 };
 
 SinkResultType PhysicalUpdate::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const {
@@ -106,13 +128,14 @@ SinkResultType PhysicalUpdate::Sink(ExecutionContext &context, DataChunk &chunk,
 			// we need to slice here
 			update_chunk.Slice(sel, update_count);
 		}
-		table.Delete(tableref, context.client, row_ids, update_chunk.size());
+		auto &delete_state = lstate.GetDeleteState(table, tableref, context.client);
+		table.Delete(delete_state, context.client, row_ids, update_chunk.size());
 		// for the append we need to arrange the columns in a specific manner (namely the "standard table order")
 		mock_chunk.SetCardinality(update_chunk);
 		for (idx_t i = 0; i < columns.size(); i++) {
 			mock_chunk.data[columns[i].index].Reference(update_chunk.data[i]);
 		}
-		table.LocalAppend(tableref, context.client, mock_chunk);
+		table.LocalAppend(tableref, context.client, mock_chunk, bound_constraints);
 	} else {
 		if (return_chunk) {
 			mock_chunk.SetCardinality(update_chunk);
@@ -120,7 +143,8 @@ SinkResultType PhysicalUpdate::Sink(ExecutionContext &context, DataChunk &chunk,
 				mock_chunk.data[columns[i].index].Reference(update_chunk.data[i]);
 			}
 		}
-		table.Update(tableref, context.client, row_ids, columns, update_chunk);
+		auto &update_state = lstate.GetUpdateState(table, tableref, context.client);
+		table.Update(update_state, context.client, row_ids, columns, update_chunk);
 	}
 
 	if (return_chunk) {
@@ -137,7 +161,8 @@ unique_ptr<GlobalSinkState> PhysicalUpdate::GetGlobalSinkState(ClientContext &co
 }
 
 unique_ptr<LocalSinkState> PhysicalUpdate::GetLocalSinkState(ExecutionContext &context) const {
-	return make_uniq<UpdateLocalState>(context.client, expressions, table.GetTypes(), bound_defaults);
+	return make_uniq<UpdateLocalState>(context.client, expressions, table.GetTypes(), bound_defaults,
+	                                   bound_constraints);
 }
 
 SinkCombineResultType PhysicalUpdate::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
@@ -175,7 +200,7 @@ SourceResultType PhysicalUpdate::GetData(ExecutionContext &context, DataChunk &c
 	auto &g = sink_state->Cast<UpdateGlobalState>();
 	if (!return_chunk) {
 		chunk.SetCardinality(1);
-		chunk.SetValue(0, 0, Value::BIGINT(g.updated_count));
+		chunk.SetValue(0, 0, Value::BIGINT(NumericCast<int64_t>(g.updated_count)));
 		return SourceResultType::FINISHED;
 	}
 

package/src/duckdb/src/execution/operator/projection/physical_unnest.cpp

@@ -204,6 +204,9 @@ static void UnnestVector(UnifiedVectorFormat &child_vector_data, Vector &child_v
 		}
 		break;
 	}
+	case PhysicalType::ARRAY: {
+		throw NotImplementedException("ARRAY type not supported for UNNEST.");
+	}
 	default:
 		throw InternalException("Unimplemented type for UNNEST.");
 	}

package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp

@@ -10,9 +10,8 @@ namespace duckdb {
 
 PhysicalColumnDataScan::PhysicalColumnDataScan(vector<LogicalType> types, PhysicalOperatorType op_type,
                                                idx_t estimated_cardinality,
-                                               unique_ptr<ColumnDataCollection> owned_collection_p)
-    : PhysicalOperator(op_type, std::move(types), estimated_cardinality), collection(owned_collection_p.get()),
-      owned_collection(std::move(owned_collection_p)) {
+                                               optionally_owned_ptr<ColumnDataCollection> collection_p)
+    : PhysicalOperator(op_type, std::move(types), estimated_cardinality), collection(std::move(collection_p)) {
 }
 
 PhysicalColumnDataScan::PhysicalColumnDataScan(vector<LogicalType> types, PhysicalOperatorType op_type,

package/src/duckdb/src/execution/operator/scan/physical_expression_scan.cpp

@@ -1,6 +1,7 @@
 #include "duckdb/execution/operator/scan/physical_expression_scan.hpp"
-#include "duckdb/parallel/thread_context.hpp"
+
 #include "duckdb/execution/expression_executor.hpp"
+#include "duckdb/parallel/thread_context.hpp"
 
 namespace duckdb {
 
@@ -27,8 +28,7 @@ OperatorResultType PhysicalExpressionScan::Execute(ExecutionContext &context, Da
 	for (; chunk.size() + input.size() <= STANDARD_VECTOR_SIZE && state.expression_index < expressions.size();
 	     state.expression_index++) {
 		state.temp_chunk.Reset();
-		EvaluateExpression(context.client, state.expression_index, &input, state.temp_chunk);
-		chunk.Append(state.temp_chunk);
+		EvaluateExpression(context.client, state.expression_index, &input, chunk, &state.temp_chunk);
 	}
 	if (state.expression_index < expressions.size()) {
 		return OperatorResultType::HAVE_MORE_OUTPUT;
@@ -38,15 +38,30 @@ OperatorResultType PhysicalExpressionScan::Execute(ExecutionContext &context, Da
 	}
 }
 
-void PhysicalExpressionScan::EvaluateExpression(ClientContext &context, idx_t expression_idx, DataChunk *child_chunk,
-                                                DataChunk &result) const {
+void PhysicalExpressionScan::EvaluateExpression(ClientContext &context, idx_t expression_idx,
+                                                optional_ptr<DataChunk> child_chunk, DataChunk &result,
+                                                optional_ptr<DataChunk> temp_chunk_ptr) const {
+	if (temp_chunk_ptr) {
+		EvaluateExpressionInternal(context, expression_idx, child_chunk, result, *temp_chunk_ptr);
+	} else {
+		DataChunk temp_chunk;
+		temp_chunk.Initialize(Allocator::Get(context), GetTypes());
+		EvaluateExpressionInternal(context, expression_idx, child_chunk, result, temp_chunk);
+	}
+}
+
+void PhysicalExpressionScan::EvaluateExpressionInternal(ClientContext &context, idx_t expression_idx,
+                                                        optional_ptr<DataChunk> child_chunk, DataChunk &result,
+                                                        DataChunk &temp_chunk) const {
 	ExpressionExecutor executor(context, expressions[expression_idx]);
 	if (child_chunk) {
 		child_chunk->Verify();
-		executor.Execute(*child_chunk, result);
+		executor.Execute(*child_chunk, temp_chunk);
 	} else {
-		executor.Execute(result);
+		executor.Execute(temp_chunk);
 	}
+	// Need to append because "executor" might be holding state (e.g., strings), which go out of scope here
+	result.Append(temp_chunk);
 }
 
 bool PhysicalExpressionScan::IsFoldable() const {

package/src/duckdb/src/execution/operator/schema/physical_attach.cpp

@@ -99,7 +99,7 @@ SourceResultType PhysicalAttach::GetData(ExecutionContext &context, DataChunk &c
 	// get the database type and attach the database
 	db_manager.GetDatabaseType(context.client, db_type, *info, config, unrecognized_option);
 	auto attached_db = db_manager.AttachDatabase(context.client, *info, db_type, access_mode);
-	attached_db->Initialize(&context.client);
+	attached_db->Initialize();
 	return SourceResultType::FINISHED;
 }
 

package/src/duckdb/src/execution/operator/schema/physical_create_art_index.cpp

@@ -4,9 +4,9 @@
 #include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 #include "duckdb/execution/index/art/art_key.hpp"
+#include "duckdb/execution/index/bound_index.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/database_manager.hpp"
-#include "duckdb/storage/index.hpp"
 #include "duckdb/storage/storage_manager.hpp"
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/common/exception/transaction_exception.hpp"
@@ -34,14 +34,14 @@ PhysicalCreateARTIndex::PhysicalCreateARTIndex(LogicalOperator &op, TableCatalog
 class CreateARTIndexGlobalSinkState : public GlobalSinkState {
 public:
 	//! Global index to be added to the table
-	unique_ptr<Index> global_index;
+	unique_ptr<BoundIndex> global_index;
 };
 
 class CreateARTIndexLocalSinkState : public LocalSinkState {
 public:
 	explicit CreateARTIndexLocalSinkState(ClientContext &context) : arena_allocator(Allocator::Get(context)) {};
 
-	unique_ptr<Index> local_index;
+	unique_ptr<BoundIndex> local_index;
 	ArenaAllocator arena_allocator;
 	vector<ARTKey> keys;
 	DataChunk key_chunk;
@@ -104,9 +104,9 @@ SinkResultType PhysicalCreateARTIndex::SinkSorted(Vector &row_identifiers, Opera
 	auto &l_index = l_state.local_index;
 
 	// create an ART from the chunk
-	auto art =
-	    make_uniq<ART>(info->index_name, l_index->index_constraint_type, l_index->column_ids, l_index->table_io_manager,
-	                   l_index->unbound_expressions, storage.db, l_index->Cast<ART>().allocators);
+	auto art = make_uniq<ART>(info->index_name, l_index->GetConstraintType(), l_index->GetColumnIds(),
+	                          l_index->table_io_manager, l_index->unbound_expressions, storage.db,
+	                          l_index->Cast<ART>().allocators);
 	if (!art->ConstructFromSorted(l_state.key_chunk.size(), l_state.keys, row_identifiers)) {
 		throw ConstraintException("Data contains duplicates on indexed column(s)");
 	}
@@ -169,7 +169,7 @@ SinkFinalizeType PhysicalCreateARTIndex::Finalize(Pipeline &pipeline, Event &eve
 
 	auto &schema = table.schema;
 	info->column_ids = storage_ids;
-	auto index_entry = schema.CreateIndex(context, *info, table).get();
+	auto index_entry = schema.CreateIndex(schema.GetCatalogTransaction(context), *info, table).get();
 	if (!index_entry) {
 		D_ASSERT(info->on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT);
 		// index already exists, but error ignored because of IF NOT EXISTS
@@ -178,13 +178,13 @@ SinkFinalizeType PhysicalCreateARTIndex::Finalize(Pipeline &pipeline, Event &eve
 	auto &index = index_entry->Cast<DuckIndexEntry>();
 	index.initial_index_size = state.global_index->GetInMemorySize();
 
-	index.info = make_shared<IndexDataTableInfo>(storage.info, index.name);
+	index.info = make_shared_ptr<IndexDataTableInfo>(storage.GetDataTableInfo(), index.name);
 	for (auto &parsed_expr : info->parsed_expressions) {
 		index.parsed_expressions.push_back(parsed_expr->Copy());
 	}
 
 	// add index to storage
-	storage.info->indexes.AddIndex(std::move(state.global_index));
+	storage.AddIndex(std::move(state.global_index));
 	return SinkFinalizeType::READY;
 }
 

package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp

@@ -200,7 +200,7 @@ void PhysicalRecursiveCTE::BuildPipelines(Pipeline &current, MetaPipeline &meta_
 	initial_state_pipeline.Build(*children[0]);
 
 	// the RHS is the recursive pipeline
-	recursive_meta_pipeline = make_shared<MetaPipeline>(executor, state, this);
+	recursive_meta_pipeline = make_shared_ptr<MetaPipeline>(executor, state, this);
 	recursive_meta_pipeline->SetRecursiveCTE();
 	recursive_meta_pipeline->Build(*children[1]);
 

package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp

@@ -64,7 +64,7 @@ static void ComputeGroupLocationTemplated(UnifiedVectorFormat &group_data, Value
 			// we only need to handle non-null values here
 			if (group_data.validity.RowIsValid(index)) {
 				D_ASSERT(data[index] >= min_val);
-				uintptr_t adjusted_value = (data[index] - min_val) + 1;
+				auto adjusted_value = UnsafeNumericCast<uintptr_t>((data[index] - min_val) + 1);
 				address_data[i] += adjusted_value << current_shift;
 			}
 		}
@@ -72,7 +72,7 @@ static void ComputeGroupLocationTemplated(UnifiedVectorFormat &group_data, Value
 		// no null values: we can directly compute the addresses
 		for (idx_t i = 0; i < count; i++) {
 			auto index = group_data.sel->get_index(i);
-			uintptr_t adjusted_value = (data[index] - min_val) + 1;
+			auto adjusted_value = UnsafeNumericCast<uintptr_t>((data[index] - min_val) + 1);
 			address_data[i] += adjusted_value << current_shift;
 		}
 	}
@@ -149,7 +149,7 @@ void PerfectAggregateHashTable::AddChunk(DataChunk &groups, DataChunk &payload)
 		}
 		// move to the next aggregate
 		payload_idx += input_count;
-		VectorOperations::AddInPlace(addresses, aggregate.payload_size, payload.size());
+		VectorOperations::AddInPlace(addresses, UnsafeNumericCast<int64_t>(aggregate.payload_size), payload.size());
 	}
 }
 
@@ -205,7 +205,8 @@ static void ReconstructGroupVectorTemplated(uint32_t group_values[], Value &min,
 			validity_mask.SetInvalid(i);
 		} else {
 			// otherwise we add the value (minus 1) to the min value
-			data[i] = UnsafeNumericCast<T>(min_data + group_index - 1);
+			data[i] = UnsafeNumericCast<T>(UnsafeNumericCast<int64_t>(min_data) +
+			                               UnsafeNumericCast<int64_t>(group_index) - 1);
 		}
 	}
 }

package/src/duckdb/src/execution/physical_operator.cpp

@@ -122,8 +122,8 @@ unique_ptr<GlobalSinkState> PhysicalOperator::GetGlobalSinkState(ClientContext &
 idx_t PhysicalOperator::GetMaxThreadMemory(ClientContext &context) {
 	// Memory usage per thread should scale with max mem / num threads
 	// We take 1/4th of this, to be conservative
-	idx_t max_memory = BufferManager::GetBufferManager(context).GetQueryMaxMemory();
-	idx_t num_threads = TaskScheduler::GetScheduler(context).NumberOfThreads();
+	auto max_memory = BufferManager::GetBufferManager(context).GetQueryMaxMemory();
+	auto num_threads = NumericCast<idx_t>(TaskScheduler::GetScheduler(context).NumberOfThreads());
 	return (max_memory / num_threads) / 4;
 }
 

package/src/duckdb/src/execution/physical_plan/plan_column_data_get.cpp

@@ -8,10 +8,8 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalColumnData
 	D_ASSERT(op.children.size() == 0);
 	D_ASSERT(op.collection);
 
-	// create a PhysicalChunkScan pointing towards the owned collection
-	auto chunk_scan = make_uniq<PhysicalColumnDataScan>(op.types, PhysicalOperatorType::COLUMN_DATA_SCAN,
-	                                                    op.estimated_cardinality, std::move(op.collection));
-	return std::move(chunk_scan);
+	return make_uniq<PhysicalColumnDataScan>(op.types, PhysicalOperatorType::COLUMN_DATA_SCAN, op.estimated_cardinality,
+	                                         std::move(op.collection));
 }
 
 } // namespace duckdb