duckdb 0.8.2-dev37.0 → 0.8.2-dev3989.0

package/src/duckdb/src/planner/operator/logical_get.cpp

@@ -8,13 +8,19 @@
 #include "duckdb/function/table/table_scan.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/storage/data_table.hpp"
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
 
 namespace duckdb {
 
+LogicalGet::LogicalGet() : LogicalOperator(LogicalOperatorType::LOGICAL_GET) {
+}
+
 LogicalGet::LogicalGet(idx_t table_index, TableFunction function, unique_ptr<FunctionData> bind_data,
                        vector<LogicalType> returned_types, vector<string> returned_names)
     : LogicalOperator(LogicalOperatorType::LOGICAL_GET), table_index(table_index), function(std::move(function)),
-      bind_data(std::move(bind_data)), returned_types(std::move(returned_types)), names(std::move(returned_names)) {
+      bind_data(std::move(bind_data)), returned_types(std::move(returned_types)), names(std::move(returned_names)),
+      extra_info() {
 }
 
 optional_ptr<TableCatalogEntry> LogicalGet::GetTable() const {
@@ -22,7 +28,7 @@ optional_ptr<TableCatalogEntry> LogicalGet::GetTable() const {
 }
 
 string LogicalGet::ParamsToString() const {
-	string result;
+	string result = "";
 	for (auto &kv : table_filters.filters) {
 		auto &column_index = kv.first;
 		auto &filter = kv.second;
@@ -31,10 +37,14 @@ string LogicalGet::ParamsToString() const {
 		}
 		result += "\n";
 	}
+	if (!extra_info.file_filters.empty()) {
+		result += "\n[INFOSEPARATOR]\n";
+		result += "File Filters: " + extra_info.file_filters;
+	}
 	if (!function.to_string) {
-		return string();
+		return result;
 	}
-	return function.to_string(bind_data.get());
+	return result + "\n" + function.to_string(bind_data.get());
 }
 
 vector<ColumnBinding> LogicalGet::GetColumnBindings() {
@@ -99,6 +109,10 @@ void LogicalGet::ResolveTypes() {
 }
 
 idx_t LogicalGet::EstimateCardinality(ClientContext &context) {
+	// join order optimizer does better cardinality estimation.
+	if (has_estimated_cardinality) {
+		return estimated_cardinality;
+	}
 	if (function.cardinality) {
 		auto node_stats = function.cardinality(context, bind_data.get());
 		if (node_stats && node_stats->has_estimated_cardinality) {
@@ -194,6 +208,70 @@ unique_ptr<LogicalOperator> LogicalGet::Deserialize(LogicalDeserializationState
 	return std::move(result);
 }
 
+void LogicalGet::FormatSerialize(FormatSerializer &serializer) const {
+	LogicalOperator::FormatSerialize(serializer);
+	serializer.WriteProperty(200, "table_index", table_index);
+	serializer.WriteProperty(201, "returned_types", returned_types);
+	serializer.WriteProperty(202, "names", names);
+	serializer.WriteProperty(203, "column_ids", column_ids);
+	serializer.WriteProperty(204, "projection_ids", projection_ids);
+	serializer.WriteProperty(205, "table_filters", table_filters);
+	FunctionSerializer::FormatSerialize(serializer, function, bind_data.get());
+	if (!function.format_serialize) {
+		D_ASSERT(!function.format_deserialize);
+		// no serialize method: serialize input values and named_parameters for rebinding purposes
+		serializer.WriteProperty(206, "parameters", parameters);
+		serializer.WriteProperty(207, "named_parameters", named_parameters);
+		serializer.WriteProperty(208, "input_table_types", input_table_types);
+		serializer.WriteProperty(209, "input_table_names", input_table_names);
+	}
+	serializer.WriteProperty(210, "projected_input", projected_input);
+}
+
+unique_ptr<LogicalOperator> LogicalGet::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto result = unique_ptr<LogicalGet>(new LogicalGet());
+	deserializer.ReadProperty(200, "table_index", result->table_index);
+	deserializer.ReadProperty(201, "returned_types", result->returned_types);
+	deserializer.ReadProperty(202, "names", result->names);
+	deserializer.ReadProperty(203, "column_ids", result->column_ids);
+	deserializer.ReadProperty(204, "projection_ids", result->projection_ids);
+	deserializer.ReadProperty(205, "table_filters", result->table_filters);
+	auto entry = FunctionSerializer::FormatDeserializeBase<TableFunction, TableFunctionCatalogEntry>(
+	    deserializer, CatalogType::TABLE_FUNCTION_ENTRY);
+	auto &function = entry.first;
+	auto has_serialize = entry.second;
+
+	unique_ptr<FunctionData> bind_data;
+	if (!has_serialize) {
+		deserializer.ReadProperty(206, "parameters", result->parameters);
+		deserializer.ReadProperty(207, "named_parameters", result->named_parameters);
+		deserializer.ReadProperty(208, "input_table_types", result->input_table_types);
+		deserializer.ReadProperty(209, "input_table_names", result->input_table_names);
+		TableFunctionBindInput input(result->parameters, result->named_parameters, result->input_table_types,
+		                             result->input_table_names, function.function_info.get());
+
+		vector<LogicalType> bind_return_types;
+		vector<string> bind_names;
+		if (!function.bind) {
+			throw InternalException("Table function \"%s\" has neither bind nor (de)serialize", function.name);
+		}
+		bind_data = function.bind(deserializer.Get<ClientContext &>(), input, bind_return_types, bind_names);
+		if (result->returned_types != bind_return_types) {
+			throw SerializationException(
+			    "Table function deserialization failure - bind returned different return types than were serialized");
+		}
+		// names can actually be different because of aliases - only the sizes cannot be different
+		if (result->names.size() != bind_names.size()) {
+			throw SerializationException(
+			    "Table function deserialization failure - bind returned different returned names than were serialized");
+		}
+	} else {
+		bind_data = FunctionSerializer::FunctionDeserialize(deserializer, function);
+	}
+	deserializer.ReadProperty(210, "projected_input", result->projected_input);
+	return std::move(result);
+}
+
 vector<idx_t> LogicalGet::GetTableIndex() const {
 	return vector<idx_t> {table_index};
 }

package/src/duckdb/src/planner/operator/logical_insert.cpp

@@ -12,6 +12,11 @@ LogicalInsert::LogicalInsert(TableCatalogEntry &table, idx_t table_index)
       action_type(OnConflictAction::THROW) {
 }
 
+LogicalInsert::LogicalInsert(ClientContext &context, const string &catalog, const string &schema, const string &table)
+    : LogicalOperator(LogicalOperatorType::LOGICAL_INSERT),
+      table(Catalog::GetEntry<TableCatalogEntry>(context, catalog, schema, table)) {
+}
+
 void LogicalInsert::Serialize(FieldWriter &writer) const {
 	writer.WriteField<idx_t>(insert_values.size());
 	for (auto &entry : insert_values) {
@@ -47,7 +52,7 @@ unique_ptr<LogicalOperator> LogicalInsert::Deserialize(LogicalDeserializationSta
 
 	auto column_index_map = reader.ReadRequiredList<idx_t, physical_index_vector_t<idx_t>>();
 	auto expected_types = reader.ReadRequiredSerializableList<LogicalType, LogicalType>();
-	auto info = TableCatalogEntry::Deserialize(reader.GetSource(), context);
+	auto info = TableCatalogEntry::Deserialize(reader.GetSource());
 	auto table_index = reader.ReadRequired<idx_t>();
 	auto return_chunk = reader.ReadRequired<bool>();
 	auto bound_defaults = reader.ReadRequiredSerializableList<Expression>(state.gstate);
@@ -65,7 +70,8 @@ unique_ptr<LogicalOperator> LogicalInsert::Deserialize(LogicalDeserializationSta
 
 	auto &catalog = Catalog::GetCatalog(context, info->catalog);
 
-	auto &table_catalog_entry = catalog.GetEntry<TableCatalogEntry>(context, info->schema, info->table);
+	auto &table_catalog_entry =
+	    catalog.GetEntry<TableCatalogEntry>(context, info->schema, info->Cast<CreateTableInfo>().table);
 	auto result = make_uniq<LogicalInsert>(table_catalog_entry, table_index);
 	result->type = state.type;
 	result->return_chunk = return_chunk;

package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp

@@ -0,0 +1,22 @@
+#include "duckdb/planner/operator/logical_materialized_cte.hpp"
+
+#include "duckdb/common/field_writer.hpp"
+
+namespace duckdb {
+
+void LogicalMaterializedCTE::Serialize(FieldWriter &writer) const {
+	writer.WriteField(table_index);
+}
+
+unique_ptr<LogicalOperator> LogicalMaterializedCTE::Deserialize(LogicalDeserializationState &state,
+                                                                FieldReader &reader) {
+	auto result = unique_ptr<LogicalMaterializedCTE>(new LogicalMaterializedCTE());
+	result->table_index = reader.ReadRequired<idx_t>();
+	return std::move(result);
+}
+
+vector<idx_t> LogicalMaterializedCTE::GetTableIndex() const {
+	return vector<idx_t> {table_index};
+}
+
+} // namespace duckdb

package/src/duckdb/src/planner/operator/logical_order.cpp

@@ -4,6 +4,45 @@
 
 namespace duckdb {
 
+LogicalOrder::LogicalOrder(vector<BoundOrderByNode> orders)
+    : LogicalOperator(LogicalOperatorType::LOGICAL_ORDER_BY), orders(std::move(orders)) {
+}
+
+vector<ColumnBinding> LogicalOrder::GetColumnBindings() {
+	auto child_bindings = children[0]->GetColumnBindings();
+	if (projections.empty()) {
+		return child_bindings;
+	}
+
+	vector<ColumnBinding> result;
+	for (auto &col_idx : projections) {
+		result.push_back(child_bindings[col_idx]);
+	}
+	return result;
+}
+
+string LogicalOrder::ParamsToString() const {
+	string result = "ORDERS:\n";
+	for (idx_t i = 0; i < orders.size(); i++) {
+		if (i > 0) {
+			result += "\n";
+		}
+		result += orders[i].expression->GetName();
+	}
+	return result;
+}
+
+void LogicalOrder::ResolveTypes() {
+	const auto child_types = children[0]->types;
+	if (projections.empty()) {
+		types = child_types;
+	} else {
+		for (auto &col_idx : projections) {
+			types.push_back(child_types[col_idx]);
+		}
+	}
+}
+
 void LogicalOrder::Serialize(FieldWriter &writer) const {
 	writer.WriteRegularSerializableList(orders);
 	writer.WriteList<idx_t>(projections);

package/src/duckdb/src/planner/operator/logical_pivot.cpp

@@ -4,6 +4,9 @@
 
 namespace duckdb {
 
+LogicalPivot::LogicalPivot() : LogicalOperator(LogicalOperatorType::LOGICAL_PIVOT) {
+}
+
 LogicalPivot::LogicalPivot(idx_t pivot_idx, unique_ptr<LogicalOperator> plan, BoundPivotInfo info_p)
     : LogicalOperator(LogicalOperatorType::LOGICAL_PIVOT), pivot_index(pivot_idx), bound_pivot(std::move(info_p)) {
 	D_ASSERT(plan);

package/src/duckdb/src/planner/operator/logical_recursive_cte.cpp

@@ -12,11 +12,11 @@ void LogicalRecursiveCTE::Serialize(FieldWriter &writer) const {
 }
 
 unique_ptr<LogicalOperator> LogicalRecursiveCTE::Deserialize(LogicalDeserializationState &state, FieldReader &reader) {
-	auto table_index = reader.ReadRequired<idx_t>();
-	auto column_count = reader.ReadRequired<idx_t>();
-	auto union_all = reader.ReadRequired<bool>();
-	// TODO(stephwang): review if unique_ptr<LogicalOperator> plan is needed
-	return unique_ptr<LogicalRecursiveCTE>(new LogicalRecursiveCTE(table_index, column_count, union_all));
+	auto result = unique_ptr<LogicalRecursiveCTE>(new LogicalRecursiveCTE());
+	result->table_index = reader.ReadRequired<idx_t>();
+	result->column_count = reader.ReadRequired<idx_t>();
+	result->union_all = reader.ReadRequired<bool>();
+	return std::move(result);
 }
 
 vector<idx_t> LogicalRecursiveCTE::GetTableIndex() const {

package/src/duckdb/src/planner/operator/logical_sample.cpp

@@ -3,6 +3,9 @@
 
 namespace duckdb {
 
+LogicalSample::LogicalSample() : LogicalOperator(LogicalOperatorType::LOGICAL_SAMPLE) {
+}
+
 LogicalSample::LogicalSample(unique_ptr<SampleOptions> sample_options_p, unique_ptr<LogicalOperator> child)
     : LogicalOperator(LogicalOperatorType::LOGICAL_SAMPLE), sample_options(std::move(sample_options_p)) {
 	children.push_back(std::move(child));

package/src/duckdb/src/planner/operator/logical_update.cpp

@@ -10,6 +10,11 @@ LogicalUpdate::LogicalUpdate(TableCatalogEntry &table)
     : LogicalOperator(LogicalOperatorType::LOGICAL_UPDATE), table(table), table_index(0), return_chunk(false) {
 }
 
+LogicalUpdate::LogicalUpdate(ClientContext &context, const string &catalog, const string &schema, const string &table)
+    : LogicalOperator(LogicalOperatorType::LOGICAL_UPDATE),
+      table(Catalog::GetEntry<TableCatalogEntry>(context, catalog, schema, table)) {
+}
+
 void LogicalUpdate::Serialize(FieldWriter &writer) const {
 	table.Serialize(writer.GetSerializer());
 	writer.WriteField(table_index);
@@ -22,10 +27,11 @@ void LogicalUpdate::Serialize(FieldWriter &writer) const {
 
 unique_ptr<LogicalOperator> LogicalUpdate::Deserialize(LogicalDeserializationState &state, FieldReader &reader) {
 	auto &context = state.gstate.context;
-	auto info = TableCatalogEntry::Deserialize(reader.GetSource(), context);
+	auto info = TableCatalogEntry::Deserialize(reader.GetSource());
 	auto &catalog = Catalog::GetCatalog(context, info->catalog);
 
-	auto &table_catalog_entry = catalog.GetEntry<TableCatalogEntry>(context, info->schema, info->table);
+	auto &table_catalog_entry =
+	    catalog.GetEntry<TableCatalogEntry>(context, info->schema, info->Cast<CreateTableInfo>().table);
 	auto result = make_uniq<LogicalUpdate>(table_catalog_entry);
 	result->table_index = reader.ReadRequired<idx_t>();
 	result->return_chunk = reader.ReadRequired<bool>();

package/src/duckdb/src/planner/parsed_data/bound_create_table_info.cpp

@@ -2,8 +2,11 @@
 #include "duckdb/parser/parsed_data/create_schema_info.hpp"
 #include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp"
 #include "duckdb/common/field_writer.hpp"
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
 
 namespace duckdb {
+
 void BoundCreateTableInfo::Serialize(Serializer &serializer) const {
 	serializer.WriteOptional(base);
 }
@@ -11,10 +14,9 @@ void BoundCreateTableInfo::Serialize(Serializer &serializer) const {
 unique_ptr<BoundCreateTableInfo> BoundCreateTableInfo::Deserialize(Deserializer &source,
                                                                    PlanDeserializationState &state) {
 	auto info = source.ReadOptional<CreateInfo>();
-	auto schema_name = info->schema;
-	auto catalog = info->catalog;
 	auto binder = Binder::CreateBinder(state.context);
 	auto bound_info = binder->BindCreateTableInfo(std::move(info));
 	return bound_info;
 }
+
 } // namespace duckdb

package/src/duckdb/src/planner/planner.cpp

@@ -11,6 +11,8 @@
 #include "duckdb/planner/binder.hpp"
 #include "duckdb/planner/expression/bound_parameter_expression.hpp"
 #include "duckdb/transaction/meta_transaction.hpp"
+#include "duckdb/common/serializer/binary_serializer.hpp"
+#include "duckdb/common/serializer/binary_deserializer.hpp"
 
 namespace duckdb {
 
@@ -81,15 +83,15 @@ void Planner::CreatePlan(SQLStatement &statement) {
 
 	// set up a map of parameter number -> value entries
 	for (auto &kv : bound_parameters.parameters) {
-		auto parameter_index = kv.first;
-		auto &parameter_data = kv.second;
+		auto &identifier = kv.first;
+		auto &param = kv.second;
 		// check if the type of the parameter could be resolved
-		if (!parameter_data->return_type.IsValid()) {
+		if (!param->return_type.IsValid()) {
 			properties.bound_all_parameters = false;
 			continue;
 		}
-		parameter_data->value = Value(parameter_data->return_type);
-		value_map[parameter_index] = parameter_data;
+		param->SetValue(Value(param->return_type));
+		value_map[identifier] = param;
 	}
 }
 
@@ -151,7 +153,8 @@ static bool OperatorSupportsSerialization(LogicalOperator &op) {
 	return op.SupportSerialization();
 }
 
-void Planner::VerifyPlan(ClientContext &context, unique_ptr<LogicalOperator> &op, bound_parameter_map_t *map) {
+void Planner::VerifyPlan(ClientContext &context, unique_ptr<LogicalOperator> &op,
+                         optional_ptr<bound_parameter_map_t> map) {
 #ifdef DUCKDB_ALTERNATIVE_VERIFY
 	// if alternate verification is enabled we run the original operator
 	return;
@@ -164,8 +167,16 @@ void Planner::VerifyPlan(ClientContext &context, unique_ptr<LogicalOperator> &op
 		return;
 	}
 
+	// format (de)serialization of this operator
+	try {
+		auto blob = BinarySerializer::Serialize(*op, true);
+		bound_parameter_map_t parameters;
+		auto result = BinaryDeserializer::Deserialize<LogicalOperator>(context, parameters, blob.data(), blob.size());
+	} catch (SerializationException &ex) {
+		// pass
+	}
+
 	BufferedSerializer serializer;
-	serializer.is_query_plan = true;
 	try {
 		op->Serialize(serializer);
 	} catch (NotImplementedException &ex) {