duckdb 0.7.2-dev3515.0 → 0.7.2-dev3666.0

package/src/duckdb/src/planner/binder/expression/bind_star_expression.cpp

@@ -8,6 +8,15 @@
 
 namespace duckdb {
 
+string GetColumnsStringValue(ParsedExpression &expr) {
+	if (expr.type == ExpressionType::COLUMN_REF) {
+		auto &colref = expr.Cast<ColumnRefExpression>();
+		return colref.GetColumnName();
+	} else {
+		return expr.ToString();
+	}
+}
+
 bool Binder::FindStarExpression(unique_ptr<ParsedExpression> &expr, StarExpression **star, bool is_root,
                                 bool in_columns) {
 	bool has_star = false;
@@ -33,7 +42,7 @@ bool Binder::FindStarExpression(unique_ptr<ParsedExpression> &expr, StarExpressi
 			vector<Value> values;
 			values.reserve(star_list.size());
 			for (auto &expr : star_list) {
-				values.emplace_back(expr->ToString());
+				values.emplace_back(GetColumnsStringValue(*expr));
 			}
 			D_ASSERT(!values.empty());
 
@@ -139,17 +148,26 @@ void Binder::ExpandStarExpression(unique_ptr<ParsedExpression> expr,
 			}
 			auto &children = ListValue::GetChildren(val);
 			vector<unique_ptr<ParsedExpression>> new_list;
+			// scan the list for all selected columns and construct a lookup table
+			case_insensitive_map_t<bool> selected_set;
 			for (auto &child : children) {
-				auto qname = QualifiedName::Parse(StringValue::Get(child));
-				vector<string> names;
-				if (!qname.catalog.empty()) {
-					names.push_back(qname.catalog);
+				selected_set.insert(make_pair(StringValue::Get(child), false));
+			}
+			// now check the list of all possible expressions and select which ones make it in
+			for (auto &expr : star_list) {
+				auto str = GetColumnsStringValue(*expr);
+				auto entry = selected_set.find(str);
+				if (entry != selected_set.end()) {
+					new_list.push_back(std::move(expr));
+					entry->second = true;
 				}
-				if (!qname.schema.empty()) {
-					names.push_back(qname.schema);
+			}
+			// check if all expressions found a match
+			for (auto &entry : selected_set) {
+				if (!entry.second) {
+					throw BinderException("Column \"%s\" was selected but was not found in the FROM clause",
+					                      entry.first);
 				}
-				names.push_back(qname.name);
-				new_list.push_back(make_uniq<ColumnRefExpression>(std::move(names)));
 			}
 			star_list = std::move(new_list);
 		} else {

package/src/duckdb/src/planner/binder/statement/bind_copy.cpp

@@ -129,10 +129,15 @@ BoundStatement Binder::BindCopyTo(CopyStatement &stmt) {
 	if (per_thread_output && !partition_cols.empty()) {
 		throw NotImplementedException("Can't combine PER_THREAD_OUTPUT and PARTITION_BY for COPY");
 	}
-	bool is_file_and_exists = config.file_system->FileExists(stmt.info->file_path);
-	bool is_stdout = stmt.info->file_path == "/dev/stdout";
-	if (!user_set_use_tmp_file) {
-		use_tmp_file = is_file_and_exists && !per_thread_output && partition_cols.empty() && !is_stdout;
+	bool is_remote_file = config.file_system->IsRemoteFile(stmt.info->file_path);
+	if (is_remote_file) {
+		use_tmp_file = false;
+	} else {
+		bool is_file_and_exists = config.file_system->FileExists(stmt.info->file_path);
+		bool is_stdout = stmt.info->file_path == "/dev/stdout";
+		if (!user_set_use_tmp_file) {
+			use_tmp_file = is_file_and_exists && !per_thread_output && partition_cols.empty() && !is_stdout;
+		}
 	}
 
 	auto unique_column_names = GetUniqueNames(select_node.names);

package/src/duckdb/src/planner/binder/statement/bind_create.cpp

@@ -455,7 +455,7 @@ static bool AnyConstraintReferencesGeneratedColumn(CreateTableInfo &table_info)
 unique_ptr<LogicalOperator> DuckCatalog::BindCreateIndex(Binder &binder, CreateStatement &stmt,
                                                          TableCatalogEntry &table, unique_ptr<LogicalOperator> plan) {
 	D_ASSERT(plan->type == LogicalOperatorType::LOGICAL_GET);
-	auto &base = (CreateIndexInfo &)*stmt.info;
+	auto &base = stmt.info->Cast<CreateIndexInfo>();
 
 	auto &get = plan->Cast<LogicalGet>();
 	// bind the index expressions
@@ -614,6 +614,7 @@ BoundStatement Binder::Bind(CreateStatement &stmt) {
 			// CREATE TYPE mood AS ENUM (SELECT 'happy')
 			auto query_obj = Bind(*create_type_info.query);
 			auto query = std::move(query_obj.plan);
+			create_type_info.query.reset();
 
 			auto &sql_types = query_obj.types;
 			if (sql_types.size() != 1) {

package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp

@@ -256,6 +256,7 @@ unique_ptr<BoundCreateTableInfo> Binder::BindCreateTableInfo(unique_ptr<CreateIn
 	if (base.query) {
 		// construct the result object
 		auto query_obj = Bind(*base.query);
+		base.query.reset();
 		result->query = std::move(query_obj.plan);
 
 		// construct the set of columns based on the names and types of the query

package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp

@@ -94,7 +94,7 @@ unique_ptr<BoundTableRef> Binder::Bind(BaseTableRef &ref) {
 			for (auto &scan : config.replacement_scans) {
 				auto replacement_function = scan.function(context, table_name, scan.data.get());
 				if (replacement_function) {
-					replacement_function->alias = ref.alias.empty() ? ref.table_name : ref.alias;
+					replacement_function->alias = ref.alias.empty() ? replacement_function->alias : ref.alias;
 					if (replacement_function->type == TableReferenceType::TABLE_FUNCTION) {
 						auto &table_function = replacement_function->Cast<TableFunctionRef>();
 						table_function.column_name_alias = ref.column_name_alias;

package/src/duckdb/src/planner/logical_operator.cpp

@@ -344,10 +344,9 @@ unique_ptr<LogicalOperator> LogicalOperator::Deserialize(Deserializer &deseriali
 	case LogicalOperatorType::LOGICAL_ATTACH:
 	case LogicalOperatorType::LOGICAL_TRANSACTION:
 	case LogicalOperatorType::LOGICAL_DROP:
+	case LogicalOperatorType::LOGICAL_DETACH:
 		result = LogicalSimple::Deserialize(state, reader);
 		break;
-	case LogicalOperatorType::LOGICAL_DETACH:
-		throw SerializationException("Logical Detach does not support serialization");
 	case LogicalOperatorType::LOGICAL_EXTENSION_OPERATOR:
 		result = LogicalExtensionOperator::Deserialize(state, reader);
 		break;

package/src/duckdb/src/planner/operator/logical_create_index.cpp

@@ -7,43 +7,34 @@
 namespace duckdb {
 
 void LogicalCreateIndex::Serialize(FieldWriter &writer) const {
-
 	writer.WriteOptional(info);
-	table.Serialize(writer.GetSerializer());
+	writer.WriteString(table.catalog.GetName());
+	writer.WriteString(table.schema.name);
+	writer.WriteString(table.name);
 	FunctionSerializer::SerializeBase<TableFunction>(writer, function, bind_data.get());
 	writer.WriteSerializableList(unbound_expressions);
-
-	writer.Finalize();
 }
 
 unique_ptr<LogicalOperator> LogicalCreateIndex::Deserialize(LogicalDeserializationState &state, FieldReader &reader) {
-
 	auto &context = state.gstate.context;
-	auto catalog_info = TableCatalogEntry::Deserialize(reader.GetSource(), context);
-
-	auto &table =
-	    Catalog::GetEntry<TableCatalogEntry>(context, catalog_info->catalog, catalog_info->schema, catalog_info->table);
-	auto unbound_expressions = reader.ReadRequiredSerializableList<Expression>(state.gstate);
-
-	auto create_info = reader.ReadOptional<CreateInfo>(nullptr);
-	if (create_info->type != CatalogType::INDEX_ENTRY) {
-		throw InternalException("Unexpected type: '%s', expected '%s'", CatalogTypeToString(create_info->type),
-		                        CatalogTypeToString(CatalogType::INDEX_ENTRY));
-	}
-
-	CreateInfo *raw_create_info_ptr = create_info.release();
-	CreateIndexInfo *raw_create_index_info_ptr = static_cast<CreateIndexInfo *>(raw_create_info_ptr);
-	unique_ptr<CreateIndexInfo> uptr_create_index_info = unique_ptr<CreateIndexInfo> {raw_create_index_info_ptr};
-	auto info = unique_ptr<CreateIndexInfo> {static_cast<CreateIndexInfo *>(create_info.release())};
 
+	auto info = reader.ReadOptional<CreateInfo>(nullptr);
+	auto catalog = reader.ReadRequired<string>();
+	auto schema = reader.ReadRequired<string>();
+	auto table_name = reader.ReadRequired<string>();
 	unique_ptr<FunctionData> bind_data;
 	bool has_deserialize;
 	auto function = FunctionSerializer::DeserializeBaseInternal<TableFunction, TableFunctionCatalogEntry>(
 	    reader, state.gstate, CatalogType::TABLE_FUNCTION_ENTRY, bind_data, has_deserialize);
-
-	reader.Finalize();
-	return make_uniq<LogicalCreateIndex>(std::move(bind_data), std::move(info), std::move(unbound_expressions), table,
-	                                     std::move(function));
+	auto unbound_expressions = reader.ReadRequiredSerializableList<Expression>(state.gstate);
+	if (info->type != CatalogType::INDEX_ENTRY) {
+		throw InternalException("Unexpected type: '%s', expected '%s'", CatalogTypeToString(info->type),
+		                        CatalogTypeToString(CatalogType::INDEX_ENTRY));
+	}
+	auto index_info = unique_ptr_cast<CreateInfo, CreateIndexInfo>(std::move(info));
+	auto &table = Catalog::GetEntry<TableCatalogEntry>(context, catalog, schema, table_name);
+	return make_uniq<LogicalCreateIndex>(std::move(bind_data), std::move(index_info), std::move(unbound_expressions),
+	                                     table, std::move(function));
 }
 
 } // namespace duckdb

package/src/duckdb/src/planner/operator/logical_insert.cpp

@@ -25,6 +25,16 @@ void LogicalInsert::Serialize(FieldWriter &writer) const {
 	writer.WriteField(return_chunk);
 	writer.WriteSerializableList(bound_defaults);
 	writer.WriteField(action_type);
+	writer.WriteRegularSerializableList(expected_set_types);
+	writer.WriteList<column_t>(on_conflict_filter);
+	writer.WriteOptional(on_conflict_condition);
+	writer.WriteOptional(do_update_condition);
+	writer.WriteIndexList(set_columns);
+	writer.WriteRegularSerializableList(set_types);
+	writer.WriteField(excluded_table_index);
+	writer.WriteList<column_t>(columns_to_fetch);
+	writer.WriteList<column_t>(source_columns);
+	writer.WriteSerializableList<Expression>(expressions);
 }
 
 unique_ptr<LogicalOperator> LogicalInsert::Deserialize(LogicalDeserializationState &state, FieldReader &reader) {
@@ -42,6 +52,16 @@ unique_ptr<LogicalOperator> LogicalInsert::Deserialize(LogicalDeserializationSta
 	auto return_chunk = reader.ReadRequired<bool>();
 	auto bound_defaults = reader.ReadRequiredSerializableList<Expression>(state.gstate);
 	auto action_type = reader.ReadRequired<OnConflictAction>();
+	auto expected_set_types = reader.ReadRequiredSerializableList<LogicalType, LogicalType>();
+	auto on_conflict_filter = reader.ReadRequiredSet<column_t, unordered_set<column_t>>();
+	auto on_conflict_condition = reader.ReadOptional<Expression>(nullptr, state.gstate);
+	auto do_update_condition = reader.ReadOptional<Expression>(nullptr, state.gstate);
+	auto set_columns = reader.ReadRequiredIndexList<PhysicalIndex>();
+	auto set_types = reader.ReadRequiredSerializableList<LogicalType, LogicalType>();
+	auto excluded_table_index = reader.ReadRequired<idx_t>();
+	auto columns_to_fetch = reader.ReadRequiredList<column_t>();
+	auto source_columns = reader.ReadRequiredList<column_t>();
+	auto expressions = reader.ReadRequiredSerializableList<Expression>(state.gstate);
 
 	auto &catalog = Catalog::GetCatalog(context, info->catalog);
 
@@ -54,6 +74,16 @@ unique_ptr<LogicalOperator> LogicalInsert::Deserialize(LogicalDeserializationSta
 	result->expected_types = expected_types;
 	result->bound_defaults = std::move(bound_defaults);
 	result->action_type = action_type;
+	result->expected_set_types = std::move(expected_set_types);
+	result->on_conflict_filter = std::move(on_conflict_filter);
+	result->on_conflict_condition = std::move(on_conflict_condition);
+	result->do_update_condition = std::move(do_update_condition);
+	result->set_columns = std::move(set_columns);
+	result->set_types = std::move(set_types);
+	result->excluded_table_index = excluded_table_index;
+	result->columns_to_fetch = std::move(columns_to_fetch);
+	result->source_columns = std::move(source_columns);
+	result->expressions = std::move(expressions);
 	return std::move(result);
 }
 

package/src/duckdb/src/planner/operator/logical_simple.cpp

@@ -1,7 +1,11 @@
 #include "duckdb/planner/operator/logical_simple.hpp"
 #include "duckdb/parser/parsed_data/alter_info.hpp"
+#include "duckdb/parser/parsed_data/attach_info.hpp"
 #include "duckdb/parser/parsed_data/drop_info.hpp"
 #include "duckdb/parser/parsed_data/load_info.hpp"
+#include "duckdb/parser/parsed_data/transaction_info.hpp"
+#include "duckdb/parser/parsed_data/vacuum_info.hpp"
+#include "duckdb/parser/parsed_data/detach_info.hpp"
 
 namespace duckdb {
 
@@ -9,16 +13,28 @@ void LogicalSimple::Serialize(FieldWriter &writer) const {
 	writer.WriteField<LogicalOperatorType>(type);
 	switch (type) {
 	case LogicalOperatorType::LOGICAL_ALTER:
-		static_cast<const AlterInfo &>(*info).Serialize(writer.GetSerializer());
+		info->Cast<AlterInfo>().Serialize(writer.GetSerializer());
 		break;
 	case LogicalOperatorType::LOGICAL_DROP:
-		static_cast<const DropInfo &>(*info).Serialize(writer.GetSerializer());
+		info->Cast<DropInfo>().Serialize(writer.GetSerializer());
 		break;
 	case LogicalOperatorType::LOGICAL_LOAD:
-		static_cast<const LoadInfo &>(*info).Serialize(writer.GetSerializer());
+		info->Cast<LoadInfo>().Serialize(writer.GetSerializer());
+		break;
+	case LogicalOperatorType::LOGICAL_VACUUM:
+		info->Cast<VacuumInfo>().Serialize(writer.GetSerializer());
+		break;
+	case LogicalOperatorType::LOGICAL_ATTACH:
+		info->Cast<AttachInfo>().Serialize(writer.GetSerializer());
+		break;
+	case LogicalOperatorType::LOGICAL_DETACH:
+		info->Cast<DetachInfo>().Serialize(writer.GetSerializer());
+		break;
+	case LogicalOperatorType::LOGICAL_TRANSACTION:
+		info->Cast<TransactionInfo>().Serialize(writer.GetSerializer());
 		break;
 	default:
-		throw NotImplementedException(LogicalOperatorToString(type));
+		throw InternalException(LogicalOperatorToString(type));
 	}
 }
 
@@ -35,8 +51,20 @@ unique_ptr<LogicalOperator> LogicalSimple::Deserialize(LogicalDeserializationSta
 	case LogicalOperatorType::LOGICAL_LOAD:
 		parse_info = LoadInfo::Deserialize(reader.GetSource());
 		break;
+	case LogicalOperatorType::LOGICAL_VACUUM:
+		parse_info = VacuumInfo::Deserialize(reader.GetSource());
+		break;
+	case LogicalOperatorType::LOGICAL_ATTACH:
+		parse_info = AttachInfo::Deserialize(reader.GetSource());
+		break;
+	case LogicalOperatorType::LOGICAL_DETACH:
+		parse_info = DetachInfo::Deserialize(reader.GetSource());
+		break;
+	case LogicalOperatorType::LOGICAL_TRANSACTION:
+		parse_info = TransactionInfo::Deserialize(reader.GetSource());
+		break;
 	default:
-		throw NotImplementedException(LogicalOperatorToString(state.type));
+		throw InternalException(LogicalOperatorToString(state.type));
 	}
 	return make_uniq<LogicalSimple>(type, std::move(parse_info));
 }

package/src/duckdb/src/planner/parsed_data/bound_create_table_info.cpp

@@ -6,25 +6,15 @@
 namespace duckdb {
 void BoundCreateTableInfo::Serialize(Serializer &serializer) const {
 	serializer.WriteOptional(base);
-	serializer.WriteList(constraints);
-	serializer.WriteList(bound_constraints);
-	serializer.WriteList(bound_defaults);
-	serializer.WriteOptional(query);
 }
 
 unique_ptr<BoundCreateTableInfo> BoundCreateTableInfo::Deserialize(Deserializer &source,
                                                                    PlanDeserializationState &state) {
-	auto create_info_base = source.ReadOptional<CreateInfo>();
-	// Get schema from the catalog to create BoundCreateTableInfo
-	auto schema_name = create_info_base->schema;
-	auto catalog = create_info_base->catalog;
-	auto &schema_catalog_entry = Catalog::GetSchema(state.context, catalog, schema_name);
-
-	auto result = make_uniq<BoundCreateTableInfo>(schema_catalog_entry, std::move(create_info_base));
-	source.ReadList<Constraint>(result->constraints);
-	source.ReadList<BoundConstraint>(result->bound_constraints);
-	source.ReadList<Expression>(result->bound_defaults, state);
-	result->query = source.ReadOptional<LogicalOperator>(state);
-	return result;
+	auto info = source.ReadOptional<CreateInfo>();
+	auto schema_name = info->schema;
+	auto catalog = info->catalog;
+	auto binder = Binder::CreateBinder(state.context);
+	auto bound_info = binder->BindCreateTableInfo(std::move(info));
+	return bound_info;
 }
 } // namespace duckdb

package/src/duckdb/src/planner/planner.cpp

@@ -143,24 +143,11 @@ void Planner::CreatePlan(unique_ptr<SQLStatement> statement) {
 
 static bool OperatorSupportsSerialization(LogicalOperator &op) {
 	switch (op.type) {
-	case LogicalOperatorType::LOGICAL_INSERT:
-	case LogicalOperatorType::LOGICAL_UPDATE:
-	case LogicalOperatorType::LOGICAL_DELETE:
 	case LogicalOperatorType::LOGICAL_PREPARE:
 	case LogicalOperatorType::LOGICAL_EXECUTE:
-	case LogicalOperatorType::LOGICAL_ALTER:
-	case LogicalOperatorType::LOGICAL_CREATE_TABLE:
-	case LogicalOperatorType::LOGICAL_CREATE_INDEX:
-	case LogicalOperatorType::LOGICAL_CREATE_SEQUENCE:
-	case LogicalOperatorType::LOGICAL_CREATE_VIEW:
-	case LogicalOperatorType::LOGICAL_CREATE_SCHEMA:
-	case LogicalOperatorType::LOGICAL_CREATE_MACRO:
 	case LogicalOperatorType::LOGICAL_PRAGMA:
-	case LogicalOperatorType::LOGICAL_TRANSACTION:
-	case LogicalOperatorType::LOGICAL_CREATE_TYPE:
 	case LogicalOperatorType::LOGICAL_EXPLAIN:
 	case LogicalOperatorType::LOGICAL_COPY_TO_FILE:
-	case LogicalOperatorType::LOGICAL_VACUUM:
 		// unsupported (for now)
 		return false;
 	default:
@@ -175,6 +162,10 @@ static bool OperatorSupportsSerialization(LogicalOperator &op) {
 }
 
 void Planner::VerifyPlan(ClientContext &context, unique_ptr<LogicalOperator> &op, bound_parameter_map_t *map) {
+#ifdef DUCKDB_ALTERNATIVE_VERIFY
+	// if alternate verification is enabled we run the original operator
+	return;
+#endif
 	if (!op || !ClientConfig::GetConfig(context).verify_serializer) {
 		return;
 	}

package/src/duckdb/src/storage/checkpoint_manager.cpp

@@ -337,7 +337,6 @@ void CheckpointWriter::WriteIndex(IndexCatalogEntry &index_catalog) {
 }
 
 void CheckpointReader::ReadIndex(ClientContext &context, MetaBlockReader &reader) {
-
 	// deserialize the index metadata
 	auto info = IndexCatalogEntry::Deserialize(reader, context);
 
@@ -361,10 +360,17 @@ void CheckpointReader::ReadIndex(ClientContext &context, MetaBlockReader &reader
 	}
 
 	// bind the parsed expressions
+	// add the table to the bind context
 	auto binder = Binder::CreateBinder(context);
-	auto &table_ref = info->table->Cast<TableRef>();
-	auto bound_table = binder->Bind(table_ref);
-	D_ASSERT(bound_table->type == TableReferenceType::BASE_TABLE);
+	vector<LogicalType> column_types;
+	vector<string> column_names;
+	for (auto &col : table_catalog.GetColumns().Logical()) {
+		column_types.push_back(col.Type());
+		column_names.push_back(col.Name());
+	}
+	vector<column_t> column_ids;
+	binder->bind_context.AddBaseTable(0, info->table->table_name, column_names, column_types, column_ids,
+	                                  &table_catalog);
 	IndexBinder idx_binder(*binder, context);
 	unbound_expressions.reserve(parsed_expressions.size());
 	for (auto &expr : parsed_expressions) {
@@ -419,7 +425,7 @@ void CheckpointWriter::WriteMacro(ScalarMacroCatalogEntry &macro) {
 }
 
 void CheckpointReader::ReadMacro(ClientContext &context, MetaBlockReader &reader) {
-	auto info = ScalarMacroCatalogEntry::Deserialize(reader, context);
+	auto info = MacroCatalogEntry::Deserialize(reader, context);
 	catalog.CreateFunction(context, *info);
 }
 
@@ -428,7 +434,7 @@ void CheckpointWriter::WriteTableMacro(TableMacroCatalogEntry &macro) {
 }
 
 void CheckpointReader::ReadTableMacro(ClientContext &context, MetaBlockReader &reader) {
-	auto info = TableMacroCatalogEntry::Deserialize(reader, context);
+	auto info = MacroCatalogEntry::Deserialize(reader, context);
 	catalog.CreateFunction(context, *info);
 }
 

package/src/duckdb/src/storage/single_file_block_manager.cpp

@@ -241,10 +241,6 @@ void SingleFileBlockManager::Initialize(DatabaseHeader &header) {
 }
 
 void SingleFileBlockManager::LoadFreeList() {
-	if (options.read_only) {
-		// no need to load free list for read only db
-		return;
-	}
 	if (free_list_id == INVALID_BLOCK) {
 		// no free list
 		return;

package/src/duckdb/src/storage/storage_info.cpp

@@ -2,7 +2,7 @@
 
 namespace duckdb {
 
-const uint64_t VERSION_NUMBER = 50;
+const uint64_t VERSION_NUMBER = 51;
 
 struct StorageVersionInfo {
 	const char *version_name;