duckdb 0.7.2-dev457.0 → 0.7.2-dev586.0

package/src/duckdb/src/main/client_context.cpp

@@ -1,52 +1,52 @@
 #include "duckdb/main/client_context.hpp"
 
-#include "duckdb/main/client_context_file_opener.hpp"
-#include "duckdb/main/query_profiler.hpp"
-#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/scalar_function_catalog_entry.hpp"
+#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_search_path.hpp"
+#include "duckdb/common/file_system.hpp"
+#include "duckdb/common/http_state.hpp"
+#include "duckdb/common/preserved_error.hpp"
+#include "duckdb/common/progress_bar/progress_bar.hpp"
 #include "duckdb/common/serializer/buffered_deserializer.hpp"
+#include "duckdb/common/serializer/buffered_file_writer.hpp"
 #include "duckdb/common/serializer/buffered_serializer.hpp"
+#include "duckdb/common/types/column_data_collection.hpp"
+#include "duckdb/execution/column_binding_resolver.hpp"
+#include "duckdb/execution/operator/helper/physical_result_collector.hpp"
 #include "duckdb/execution/physical_plan_generator.hpp"
+#include "duckdb/main/appender.hpp"
+#include "duckdb/main/attached_database.hpp"
+#include "duckdb/main/client_context_file_opener.hpp"
+#include "duckdb/main/client_data.hpp"
 #include "duckdb/main/database.hpp"
+#include "duckdb/main/database_manager.hpp"
+#include "duckdb/main/error_manager.hpp"
 #include "duckdb/main/materialized_query_result.hpp"
-#include "duckdb/main/client_data.hpp"
+#include "duckdb/main/query_profiler.hpp"
 #include "duckdb/main/query_result.hpp"
+#include "duckdb/main/relation.hpp"
 #include "duckdb/main/stream_query_result.hpp"
 #include "duckdb/optimizer/optimizer.hpp"
-#include "duckdb/parser/parser.hpp"
+#include "duckdb/parallel/task_scheduler.hpp"
 #include "duckdb/parser/expression/constant_expression.hpp"
 #include "duckdb/parser/expression/parameter_expression.hpp"
 #include "duckdb/parser/parsed_data/create_function_info.hpp"
+#include "duckdb/parser/parsed_expression_iterator.hpp"
+#include "duckdb/parser/parser.hpp"
+#include "duckdb/parser/query_node/select_node.hpp"
 #include "duckdb/parser/statement/drop_statement.hpp"
+#include "duckdb/parser/statement/execute_statement.hpp"
 #include "duckdb/parser/statement/explain_statement.hpp"
+#include "duckdb/parser/statement/prepare_statement.hpp"
+#include "duckdb/parser/statement/relation_statement.hpp"
 #include "duckdb/parser/statement/select_statement.hpp"
 #include "duckdb/planner/operator/logical_execute.hpp"
 #include "duckdb/planner/planner.hpp"
-#include "duckdb/transaction/transaction_manager.hpp"
-#include "duckdb/transaction/transaction.hpp"
-#include "duckdb/storage/data_table.hpp"
-#include "duckdb/main/appender.hpp"
-#include "duckdb/main/relation.hpp"
-#include "duckdb/parser/statement/relation_statement.hpp"
-#include "duckdb/parallel/task_scheduler.hpp"
-#include "duckdb/common/serializer/buffered_file_writer.hpp"
 #include "duckdb/planner/pragma_handler.hpp"
-#include "duckdb/common/file_system.hpp"
-#include "duckdb/execution/column_binding_resolver.hpp"
-#include "duckdb/execution/operator/helper/physical_result_collector.hpp"
-#include "duckdb/parser/query_node/select_node.hpp"
-#include "duckdb/parser/parsed_expression_iterator.hpp"
-#include "duckdb/parser/statement/prepare_statement.hpp"
-#include "duckdb/parser/statement/execute_statement.hpp"
-#include "duckdb/common/types/column_data_collection.hpp"
-#include "duckdb/common/preserved_error.hpp"
-#include "duckdb/common/progress_bar/progress_bar.hpp"
-#include "duckdb/main/error_manager.hpp"
-#include "duckdb/main/database_manager.hpp"
+#include "duckdb/storage/data_table.hpp"
 #include "duckdb/transaction/meta_transaction.hpp"
-#include "duckdb/common/http_stats.hpp"
-#include "duckdb/main/attached_database.hpp"
+#include "duckdb/transaction/transaction.hpp"
+#include "duckdb/transaction/transaction_manager.hpp"
 
 namespace duckdb {
 
@@ -156,8 +156,8 @@ void ClientContext::BeginQueryInternal(ClientContextLock &lock, const string &qu
 PreservedError ClientContext::EndQueryInternal(ClientContextLock &lock, bool success, bool invalidate_transaction) {
 	client_data->profiler->EndQuery();
 
-	if (client_data->http_stats) {
-		client_data->http_stats->Reset();
+	if (client_data->http_state) {
+		client_data->http_state->Reset();
 	}
 
 	// Notify any registered state of query end
@@ -717,9 +717,7 @@ unique_ptr<PendingQueryResult> ClientContext::PendingStatementOrPreparedStatemen
 	auto &profiler = QueryProfiler::Get(*this);
 	profiler.StartQuery(query, IsExplainAnalyze(statement ? statement.get() : prepared->unbound_statement.get()));
 
-	if (IsExplainAnalyze(statement ? statement.get() : prepared->unbound_statement.get())) {
-		client_data->http_stats = make_unique<HTTPStats>();
-	}
+	client_data->http_state = make_unique<HTTPState>();
 
 	bool invalidate_query = true;
 	try {

package/src/duckdb/src/main/client_data.cpp

@@ -1,15 +1,16 @@
 #include "duckdb/main/client_data.hpp"
-#include "duckdb/main/client_context.hpp"
-#include "duckdb/main/client_context_file_opener.hpp"
-#include "duckdb/main/query_profiler.hpp"
-#include "duckdb/common/http_stats.hpp"
-#include "duckdb/common/random_engine.hpp"
-#include "duckdb/catalog/catalog_search_path.hpp"
+
 #include "duckdb/catalog/catalog.hpp"
+#include "duckdb/catalog/catalog_search_path.hpp"
+#include "duckdb/common/http_state.hpp"
+#include "duckdb/common/random_engine.hpp"
 #include "duckdb/common/serializer/buffered_file_writer.hpp"
 #include "duckdb/main/attached_database.hpp"
+#include "duckdb/main/client_context.hpp"
+#include "duckdb/main/client_context_file_opener.hpp"
 #include "duckdb/main/database.hpp"
 #include "duckdb/main/database_manager.hpp"
+#include "duckdb/main/query_profiler.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/main/database.cpp

@@ -115,6 +115,15 @@ ConnectionManager &ConnectionManager::Get(DatabaseInstance &db) {
 	return db.GetConnectionManager();
 }
 
+ClientContext *ConnectionManager::GetConnection(DatabaseInstance *db) {
+	for (auto &conn : connections) {
+		if (conn.first->db.get() == db) {
+			return conn.first;
+		}
+	}
+	return nullptr;
+}
+
 ConnectionManager &ConnectionManager::Get(ClientContext &context) {
 	return ConnectionManager::Get(DatabaseInstance::GetDatabase(context));
 }

package/src/duckdb/src/main/query_profiler.cpp

@@ -1,21 +1,23 @@
 #include "duckdb/main/query_profiler.hpp"
-#include "duckdb/common/to_string.hpp"
+
 #include "duckdb/common/fstream.hpp"
+#include "duckdb/common/http_state.hpp"
+#include "duckdb/common/limits.hpp"
 #include "duckdb/common/printer.hpp"
 #include "duckdb/common/string_util.hpp"
-#include "duckdb/execution/physical_operator.hpp"
-#include "duckdb/execution/operator/join/physical_delim_join.hpp"
-#include "duckdb/execution/operator/helper/physical_execute.hpp"
-#include "duckdb/common/http_stats.hpp"
+#include "duckdb/common/to_string.hpp"
 #include "duckdb/common/tree_renderer.hpp"
-#include "duckdb/common/limits.hpp"
 #include "duckdb/execution/expression_executor.hpp"
-#include "duckdb/planner/expression/bound_function_expression.hpp"
+#include "duckdb/execution/operator/helper/physical_execute.hpp"
+#include "duckdb/execution/operator/join/physical_delim_join.hpp"
+#include "duckdb/execution/physical_operator.hpp"
 #include "duckdb/main/client_config.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/client_data.hpp"
-#include <utility>
+#include "duckdb/planner/expression/bound_function_expression.hpp"
+
 #include <algorithm>
+#include <utility>
 
 namespace duckdb {
 
@@ -377,15 +379,15 @@ void QueryProfiler::QueryTreeToStream(std::ostream &ss) const {
 		return;
 	}
 
-	if (context.client_data->http_stats && !context.client_data->http_stats->IsEmpty()) {
+	if (context.client_data->http_state && !context.client_data->http_state->IsEmpty()) {
 		string read =
-		    "in: " + StringUtil::BytesToHumanReadableString(context.client_data->http_stats->total_bytes_received);
+		    "in: " + StringUtil::BytesToHumanReadableString(context.client_data->http_state->total_bytes_received);
 		string written =
-		    "out: " + StringUtil::BytesToHumanReadableString(context.client_data->http_stats->total_bytes_sent);
-		string head = "#HEAD: " + to_string(context.client_data->http_stats->head_count);
-		string get = "#GET: " + to_string(context.client_data->http_stats->get_count);
-		string put = "#PUT: " + to_string(context.client_data->http_stats->put_count);
-		string post = "#POST: " + to_string(context.client_data->http_stats->post_count);
+		    "out: " + StringUtil::BytesToHumanReadableString(context.client_data->http_state->total_bytes_sent);
+		string head = "#HEAD: " + to_string(context.client_data->http_state->head_count);
+		string get = "#GET: " + to_string(context.client_data->http_state->get_count);
+		string put = "#PUT: " + to_string(context.client_data->http_state->put_count);
+		string post = "#POST: " + to_string(context.client_data->http_state->post_count);
 
 		constexpr idx_t TOTAL_BOX_WIDTH = 39;
 		ss << "┌─────────────────────────────────────┐\n";

package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp

@@ -35,7 +35,7 @@ unique_ptr<Expression> RegexOptimizationRule::Apply(LogicalOperator &op, vector<
 
 	auto constant_value = ExpressionExecutor::EvaluateScalar(GetContext(), *constant_expr);
 	D_ASSERT(constant_value.type() == constant_expr->return_type);
-	auto &patt_str = StringValue::Get(constant_value);
+	auto patt_str = StringValue::Get(constant_value);
 
 	duckdb_re2::RE2 pattern(patt_str);
 	if (!pattern.ok()) {
@@ -47,7 +47,14 @@ unique_ptr<Expression> RegexOptimizationRule::Apply(LogicalOperator &op, vector<
 		auto contains = make_unique<BoundFunctionExpression>(root->return_type, ContainsFun::GetFunction(),
 		                                                     std::move(root->children), nullptr);
 
-		contains->children[1] = make_unique<BoundConstantExpression>(Value(patt_str));
+		string min;
+		string max;
+		pattern.PossibleMatchRange(&min, &max, patt_str.size());
+		if (min == max) {
+			contains->children[1] = make_unique<BoundConstantExpression>(Value(std::move(min)));
+		} else {
+			contains->children[1] = make_unique<BoundConstantExpression>(Value(std::move(patt_str)));
+		}
 		return std::move(contains);
 	}
 	return nullptr;

package/src/duckdb/src/parser/transform/statement/transform_rename.cpp

@@ -6,7 +6,9 @@ namespace duckdb {
 unique_ptr<AlterStatement> Transformer::TransformRename(duckdb_libpgquery::PGNode *node) {
 	auto stmt = reinterpret_cast<duckdb_libpgquery::PGRenameStmt *>(node);
 	D_ASSERT(stmt);
-	D_ASSERT(stmt->relation);
+	if (!stmt->relation) {
+		throw NotImplementedException("Altering schemas is not yet supported");
+	}
 
 	unique_ptr<AlterInfo> info;
 
@@ -17,8 +19,6 @@ unique_ptr<AlterStatement> Transformer::TransformRename(duckdb_libpgquery::PGNod
 	if (stmt->relation->relname) {
 		data.name = stmt->relation->relname;
 	}
-	if (stmt->relation->schemaname) {
-	}
 	// first we check the type of ALTER
 	switch (stmt->renameType) {
 	case duckdb_libpgquery::PG_OBJECT_COLUMN: {
@@ -36,7 +36,6 @@ unique_ptr<AlterStatement> Transformer::TransformRename(duckdb_libpgquery::PGNod
 		info = make_unique<RenameTableInfo>(std::move(data), new_name);
 		break;
 	}
-
 	case duckdb_libpgquery::PG_OBJECT_VIEW: {
 		// change view name
 		string new_name = stmt->newname;

package/src/duckdb/src/planner/binder/tableref/bind_table_function.cpp

@@ -126,9 +126,18 @@ Binder::BindTableFunctionInternal(TableFunction &table_function, const string &f
 	unique_ptr<FunctionData> bind_data;
 	vector<LogicalType> return_types;
 	vector<string> return_names;
-	if (table_function.bind) {
+	if (table_function.bind || table_function.bind_replace) {
 		TableFunctionBindInput bind_input(parameters, named_parameters, input_table_types, input_table_names,
 		                                  table_function.function_info.get());
+		if (table_function.bind_replace) {
+			auto new_plan = table_function.bind_replace(context, bind_input);
+			if (new_plan != nullptr) {
+				return CreatePlan(*Bind(*new_plan));
+			} else if (!table_function.bind) {
+				throw BinderException("Failed to bind \"%s\": nullptr returned from bind_replace without bind function",
+				                      table_function.name);
+			}
+		}
 		bind_data = table_function.bind(context, bind_input, return_types, return_names);
 		if (table_function.name == "pandas_scan" || table_function.name == "arrow_scan") {
 			auto arrow_bind = (PyTableFunctionData *)bind_data.get();
@@ -153,6 +162,7 @@ Binder::BindTableFunctionInternal(TableFunction &table_function, const string &f
 			return_names[i] = "C" + to_string(i);
 		}
 	}
+
 	auto get = make_unique<LogicalGet>(bind_index, table_function, std::move(bind_data), return_types, return_names);
 	get->parameters = parameters;
 	get->named_parameters = named_parameters;

package/src/duckdb/src/planner/logical_operator.cpp

@@ -128,6 +128,8 @@ void LogicalOperator::Verify(ClientContext &context) {
 			continue;
 		}
 		BufferedSerializer serializer;
+		// We are serializing a query plan
+		serializer.is_query_plan = true;
 		try {
 			expressions[expr_idx]->Serialize(serializer);
 		} catch (NotImplementedException &ex) {
@@ -136,7 +138,7 @@ void LogicalOperator::Verify(ClientContext &context) {
 		}
 
 		auto data = serializer.GetData();
-		auto deserializer = BufferedDeserializer(data.data.get(), data.size);
+		auto deserializer = BufferedContextDeserializer(context, data.data.get(), data.size);
 
 		PlanDeserializationState state(context);
 		auto deserialized_expression = Expression::Deserialize(deserializer, state);
@@ -371,7 +373,7 @@ unique_ptr<LogicalOperator> LogicalOperator::Copy(ClientContext &context) const
 		                              std::string(ex.what()));
 	}
 	auto data = logical_op_serializer.GetData();
-	auto logical_op_deserializer = BufferedDeserializer(data.data.get(), data.size);
+	auto logical_op_deserializer = BufferedContextDeserializer(context, data.data.get(), data.size);
 	PlanDeserializationState state(context);
 	auto op_copy = LogicalOperator::Deserialize(logical_op_deserializer, state);
 	return op_copy;

package/src/duckdb/src/planner/planner.cpp

@@ -184,6 +184,7 @@ void Planner::VerifyPlan(ClientContext &context, unique_ptr<LogicalOperator> &op
 	}
 
 	BufferedSerializer serializer;
+	serializer.is_query_plan = true;
 	try {
 		op->Serialize(serializer);
 	} catch (NotImplementedException &ex) {
@@ -191,7 +192,7 @@ void Planner::VerifyPlan(ClientContext &context, unique_ptr<LogicalOperator> &op
 		return;
 	}
 	auto data = serializer.GetData();
-	auto deserializer = BufferedDeserializer(data.data.get(), data.size);
+	auto deserializer = BufferedContextDeserializer(context, data.data.get(), data.size);
 
 	PlanDeserializationState state(context);
 	auto new_plan = LogicalOperator::Deserialize(deserializer, state);

package/src/duckdb/src/storage/checkpoint_manager.cpp

@@ -128,6 +128,8 @@ void SingleFileCheckpointReader::LoadFromStorage() {
 	con.BeginTransaction();
 	// create the MetaBlockReader to read from the storage
 	MetaBlockReader reader(block_manager, meta_block);
+	reader.SetCatalog(&catalog.GetAttached().GetCatalog());
+	reader.SetContext(con.context.get());
 	LoadCheckpoint(*con.context, reader);
 	con.Commit();
 }
@@ -395,13 +397,16 @@ void CheckpointReader::ReadIndex(ClientContext &context, MetaBlockReader &reader
 //===--------------------------------------------------------------------===//
 // Custom Types
 //===--------------------------------------------------------------------===//
-void CheckpointWriter::WriteType(TypeCatalogEntry &table) {
-	table.Serialize(GetMetaBlockWriter());
+void CheckpointWriter::WriteType(TypeCatalogEntry &type) {
+	type.Serialize(GetMetaBlockWriter());
 }
 
 void CheckpointReader::ReadType(ClientContext &context, MetaBlockReader &reader) {
 	auto info = TypeCatalogEntry::Deserialize(reader);
-	catalog.CreateType(context, info.get());
+	auto catalog_entry = (TypeCatalogEntry *)catalog.CreateType(context, info.get());
+	if (info->type.id() == LogicalTypeId::ENUM) {
+		EnumType::SetCatalog(info->type, catalog_entry);
+	}
 }
 
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/storage/meta_block_reader.cpp

@@ -1,5 +1,7 @@
 #include "duckdb/storage/meta_block_reader.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
+#include "duckdb/main/connection_manager.hpp"
+#include "duckdb/main/database.hpp"
 
 #include <cstring>
 
@@ -34,6 +36,16 @@ void MetaBlockReader::ReadData(data_ptr_t buffer, idx_t read_size) {
 	offset += read_size;
 }
 
+ClientContext &MetaBlockReader::GetContext() {
+	if (!context) {
+		throw InternalException("Meta Block Reader is missing context");
+	}
+	return *context;
+}
+Catalog *MetaBlockReader::GetCatalog() {
+	return catalog;
+}
+
 void MetaBlockReader::ReadNewBlock(block_id_t id) {
 	auto &buffer_manager = block_manager.buffer_manager;
 
@@ -52,4 +64,14 @@ void MetaBlockReader::ReadNewBlock(block_id_t id) {
 	offset = sizeof(block_id_t);
 }
 
+void MetaBlockReader::SetCatalog(Catalog *catalog_p) {
+	D_ASSERT(!catalog);
+	catalog = catalog_p;
+}
+
+void MetaBlockReader::SetContext(ClientContext *context_p) {
+	D_ASSERT(!context);
+	context = context_p;
+}
+
 } // namespace duckdb

package/src/duckdb/src/storage/storage_info.cpp

@@ -2,7 +2,7 @@
 
 namespace duckdb {
 
-const uint64_t VERSION_NUMBER = 46;
+const uint64_t VERSION_NUMBER = 47;
 
 struct StorageVersionInfo {
 	const char *version_name;

package/src/duckdb/src/storage/wal_replay.cpp

@@ -24,17 +24,19 @@
 namespace duckdb {
 
 bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
-	auto initial_reader = make_unique<BufferedFileReader>(FileSystem::Get(database), path.c_str());
+	Connection con(database.GetDatabase());
+	auto initial_reader = make_unique<BufferedFileReader>(FileSystem::Get(database), path.c_str(), con.context.get());
 	if (initial_reader->Finished()) {
 		// WAL is empty
 		return false;
 	}
-	Connection con(database.GetDatabase());
+
 	con.BeginTransaction();
 
 	// first deserialize the WAL to look for a checkpoint flag
 	// if there is a checkpoint flag, we might have already flushed the contents of the WAL to disk
 	ReplayState checkpoint_state(database, *con.context, *initial_reader);
+	initial_reader->catalog = &checkpoint_state.catalog;
 	checkpoint_state.deserialize_only = true;
 	try {
 		while (true) {
@@ -70,7 +72,8 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
 	}
 
 	// we need to recover from the WAL: actually set up the replay state
-	BufferedFileReader reader(FileSystem::Get(database), path.c_str());
+	BufferedFileReader reader(FileSystem::Get(database), path.c_str(), con.context.get());
+	reader.catalog = &checkpoint_state.catalog;
 	ReplayState state(database, *con.context, reader);
 
 	// replay the WAL
@@ -192,6 +195,7 @@ void ReplayState::ReplayEntry(WALType entry_type) {
 // Replay Table
 //===--------------------------------------------------------------------===//
 void ReplayState::ReplayCreateTable() {
+
 	auto info = TableCatalogEntry::Deserialize(source, context);
 	if (deserialize_only) {
 		return;
@@ -278,10 +282,9 @@ void ReplayState::ReplayDropSchema() {
 //===--------------------------------------------------------------------===//
 void ReplayState::ReplayCreateType() {
 	auto info = TypeCatalogEntry::Deserialize(source);
-	if (deserialize_only) {
+	if (Catalog::TypeExists(context, info->catalog, info->schema, info->name)) {
 		return;
 	}
-
 	catalog.CreateType(context, info.get());
 }
 

package/src/duckdb/src/storage/write_ahead_log.cpp

@@ -281,12 +281,12 @@ void WriteAheadLog::WriteUpdate(DataChunk &chunk, const vector<column_t> &column
 //===--------------------------------------------------------------------===//
 // Write ALTER Statement
 //===--------------------------------------------------------------------===//
-void WriteAheadLog::WriteAlter(AlterInfo &info) {
+void WriteAheadLog::WriteAlter(data_ptr_t ptr, idx_t data_size) {
 	if (skip_writing) {
 		return;
 	}
 	writer->Write<WALType>(WALType::ALTER_INFO);
-	info.Serialize(*writer);
+	writer->WriteData(ptr, data_size);
 }
 
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/transaction/commit_state.cpp

@@ -46,12 +46,16 @@ void CommitState::WriteCatalogEntry(CatalogEntry *entry, data_ptr_t dataptr) {
 			// ALTER TABLE statement, read the extra data after the entry
 			auto extra_data_size = Load<idx_t>(dataptr);
 			auto extra_data = (data_ptr_t)(dataptr + sizeof(idx_t));
-			// deserialize it
+
 			BufferedDeserializer source(extra_data, extra_data_size);
-			auto info = AlterInfo::Deserialize(source);
-			// write the alter table in the log
-			table_entry->CommitAlter(*info);
-			log->WriteAlter(*info);
+			string column_name = source.Read<string>();
+
+			if (!column_name.empty()) {
+				// write the alter table in the log
+				table_entry->CommitAlter(column_name);
+			}
+
+			log->WriteAlter(source.ptr, source.endptr - source.ptr);
 		} else {
 			// CREATE TABLE statement
 			log->WriteCreateTable((TableCatalogEntry *)parent);
@@ -71,9 +75,9 @@ void CommitState::WriteCatalogEntry(CatalogEntry *entry, data_ptr_t dataptr) {
 			auto extra_data = (data_ptr_t)(dataptr + sizeof(idx_t));
 			// deserialize it
 			BufferedDeserializer source(extra_data, extra_data_size);
-			auto info = AlterInfo::Deserialize(source);
+			string column_name = source.Read<string>();
 			// write the alter table in the log
-			log->WriteAlter(*info);
+			log->WriteAlter(source.ptr, source.endptr - source.ptr);
 		} else {
 			log->WriteCreateView((ViewCatalogEntry *)parent);
 		}