duckdb 0.8.1-dev96.0 → 0.8.1

package/src/duckdb/src/main/config.cpp

@@ -82,6 +82,7 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_GLOBAL(ForceBitpackingModeSetting),
                                                  DUCKDB_LOCAL(HomeDirectorySetting),
                                                  DUCKDB_LOCAL(LogQueryPathSetting),
+                                                 DUCKDB_GLOBAL(LockConfigurationSetting),
                                                  DUCKDB_GLOBAL(ImmediateTransactionModeSetting),
                                                  DUCKDB_LOCAL(IntegerDivisionSetting),
                                                  DUCKDB_LOCAL(MaximumExpressionDepthSetting),
@@ -104,6 +105,7 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_GLOBAL(TempDirectorySetting),
                                                  DUCKDB_GLOBAL(ThreadsSetting),
                                                  DUCKDB_GLOBAL(UsernameSetting),
+                                                 DUCKDB_GLOBAL(ExportLargeBufferArrow),
                                                  DUCKDB_GLOBAL_ALIAS("user", UsernameSetting),
                                                  DUCKDB_GLOBAL_ALIAS("wal_autocheckpoint", CheckpointThresholdSetting),
                                                  DUCKDB_GLOBAL_ALIAS("worker_threads", ThreadsSetting),

package/src/duckdb/src/main/database.cpp

@@ -171,6 +171,7 @@ void DatabaseInstance::CreateMainDatabase() {
 	}
 
 	// initialize the database
+	initial_database->SetInitialDatabase();
 	initial_database->Initialize();
 }
 

package/src/duckdb/src/main/database_manager.cpp

@@ -43,6 +43,11 @@ void DatabaseManager::AddDatabase(ClientContext &context, unique_ptr<AttachedDat
 }
 
 void DatabaseManager::DetachDatabase(ClientContext &context, const string &name, OnEntryNotFound if_not_found) {
+	if (GetDefaultDatabase(context) == name) {
+		throw BinderException("Cannot detach database \"%s\" because it is the default database. Select a different "
+		                      "database using `USE` to allow detaching this database",
+		                      name);
+	}
 	if (!databases->DropEntry(context, name, false, true)) {
 		if (if_not_found == OnEntryNotFound::THROW_EXCEPTION) {
 			throw BinderException("Failed to detach database with name \"%s\": database not found", name);
@@ -82,6 +87,22 @@ const string &DatabaseManager::GetDefaultDatabase(ClientContext &context) {
 	return default_entry.catalog;
 }
 
+// LCOV_EXCL_START
+void DatabaseManager::SetDefaultDatabase(ClientContext &context, const string &new_value) {
+	auto db_entry = GetDatabase(context, new_value);
+
+	if (!db_entry) {
+		throw InternalException("Database \"%s\" not found", new_value);
+	} else if (db_entry->IsTemporary()) {
+		throw InternalException("Cannot set the default database to a temporary database");
+	} else if (db_entry->IsSystem()) {
+		throw InternalException("Cannot set the default database to a system database");
+	}
+
+	default_database = new_value;
+}
+// LCOV_EXCL_STOP
+
 vector<reference<AttachedDatabase>> DatabaseManager::GetDatabases(ClientContext &context) {
 	vector<reference<AttachedDatabase>> result;
 	databases->Scan(context, [&](CatalogEntry &entry) { result.push_back(entry.Cast<AttachedDatabase>()); });

package/src/duckdb/src/main/query_result.cpp

@@ -3,7 +3,6 @@
 #include "duckdb/common/vector.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/common/box_renderer.hpp"
-
 namespace duckdb {
 
 BaseQueryResult::BaseQueryResult(QueryResultType type, StatementType statement_type, StatementProperties properties_p,
@@ -71,7 +70,8 @@ idx_t CurrentChunk::RemainingSize() {
 	return data_chunk->size() - position;
 }
 
-QueryResult::QueryResult(QueryResultType type, PreservedError error) : BaseQueryResult(type, std::move(error)) {
+QueryResult::QueryResult(QueryResultType type, PreservedError error)
+    : BaseQueryResult(type, std::move(error)), client_properties("UTC", ArrowOffsetSize::REGULAR) {
 }
 
 QueryResult::~QueryResult() {
@@ -164,6 +164,10 @@ string QueryResult::HeaderToString() {
 	return result;
 }
 
+ArrowOptions QueryResult::GetArrowOptions(QueryResult &query_result) {
+	return {query_result.client_properties.arrow_offset_size, query_result.client_properties.time_zone};
+}
+
 string QueryResult::GetConfigTimezone(QueryResult &query_result) {
 	return query_result.client_properties.time_zone;
 }

package/src/duckdb/src/main/settings/settings.cpp

@@ -151,7 +151,7 @@ void OrderedAggregateThreshold::ResetLocal(ClientContext &context) {
 
 void OrderedAggregateThreshold::SetLocal(ClientContext &context, const Value &input) {
 	const auto param = input.GetValue<uint64_t>();
-	if (!param) {
+	if (param <= 0) {
 		throw ParserException("Invalid option for PRAGMA ordered_aggregate_threshold, value must be positive");
 	}
 	ClientConfig::GetConfig(context).ordered_aggregate_threshold = param;
@@ -189,7 +189,7 @@ Value DebugWindowMode::GetSetting(ClientContext &context) {
 // Debug AsOf Join
 //===--------------------------------------------------------------------===//
 void DebugAsOfIEJoin::ResetLocal(ClientContext &context) {
-	ClientConfig::GetConfig(context).force_no_cross_product = ClientConfig().force_asof_iejoin;
+	ClientConfig::GetConfig(context).force_asof_iejoin = ClientConfig().force_asof_iejoin;
 }
 
 void DebugAsOfIEJoin::SetLocal(ClientContext &context, const Value &input) {
@@ -433,7 +433,6 @@ Value EnableHTTPMetadataCacheSetting::GetSetting(ClientContext &context) {
 //===--------------------------------------------------------------------===//
 // Enable Profiling
 //===--------------------------------------------------------------------===//
-
 void EnableProfilingSetting::ResetLocal(ClientContext &context) {
 	auto &config = ClientConfig::GetConfig(context);
 	config.profiler_print_format = ClientConfig().profiler_print_format;
@@ -718,10 +717,10 @@ Value IntegerDivisionSetting::GetSetting(ClientContext &context) {
 	auto &config = ClientConfig::GetConfig(context);
 	return Value(config.integer_division);
 }
+
 //===--------------------------------------------------------------------===//
 // Log Query Path
 //===--------------------------------------------------------------------===//
-
 void LogQueryPathSetting::ResetLocal(ClientContext &context) {
 	auto &client_data = ClientData::Get(context);
 	// TODO: verify that this does the right thing
@@ -745,6 +744,23 @@ Value LogQueryPathSetting::GetSetting(ClientContext &context) {
 	return client_data.log_query_writer ? Value(client_data.log_query_writer->path) : Value();
 }
 
+//===--------------------------------------------------------------------===//
+// Lock Configuration
+//===--------------------------------------------------------------------===//
+void LockConfigurationSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
+	auto new_value = input.GetValue<bool>();
+	config.options.lock_configuration = new_value;
+}
+
+void LockConfigurationSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) {
+	config.options.lock_configuration = DBConfig().options.lock_configuration;
+}
+
+Value LockConfigurationSetting::GetSetting(ClientContext &context) {
+	auto &config = DBConfig::GetConfig(context);
+	return Value::BOOLEAN(config.options.lock_configuration);
+}
+
 //===--------------------------------------------------------------------===//
 // Immediate Transaction Mode
 //===--------------------------------------------------------------------===//
@@ -879,6 +895,25 @@ Value PreserveInsertionOrder::GetSetting(ClientContext &context) {
 	return Value::BOOLEAN(config.options.preserve_insertion_order);
 }
 
+//===--------------------------------------------------------------------===//
+// ExportLargeBufferArrow
+//===--------------------------------------------------------------------===//
+void ExportLargeBufferArrow::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
+	auto export_large_buffers_arrow = input.GetValue<bool>();
+
+	config.options.arrow_offset_size = export_large_buffers_arrow ? ArrowOffsetSize::LARGE : ArrowOffsetSize::REGULAR;
+}
+
+void ExportLargeBufferArrow::ResetGlobal(DatabaseInstance *db, DBConfig &config) {
+	config.options.arrow_offset_size = DBConfig().options.arrow_offset_size;
+}
+
+Value ExportLargeBufferArrow::GetSetting(ClientContext &context) {
+	auto &config = DBConfig::GetConfig(context);
+	bool export_large_buffers_arrow = config.options.arrow_offset_size == ArrowOffsetSize::LARGE;
+	return Value::BOOLEAN(export_large_buffers_arrow);
+}
+
 //===--------------------------------------------------------------------===//
 // Profiler History Size
 //===--------------------------------------------------------------------===//
@@ -988,7 +1023,7 @@ void SchemaSetting::ResetLocal(ClientContext &context) {
 void SchemaSetting::SetLocal(ClientContext &context, const Value &input) {
 	auto parameter = input.ToString();
 	auto &client_data = ClientData::Get(context);
-	client_data.catalog_search_path->Set(CatalogSearchEntry::Parse(parameter), true);
+	client_data.catalog_search_path->Set(CatalogSearchEntry::Parse(parameter), CatalogSetPathType::SET_SCHEMA);
 }
 
 Value SchemaSetting::GetSetting(ClientContext &context) {
@@ -1008,7 +1043,7 @@ void SearchPathSetting::ResetLocal(ClientContext &context) {
 void SearchPathSetting::SetLocal(ClientContext &context, const Value &input) {
 	auto parameter = input.ToString();
 	auto &client_data = ClientData::Get(context);
-	client_data.catalog_search_path->Set(CatalogSearchEntry::ParseList(parameter), false);
+	client_data.catalog_search_path->Set(CatalogSearchEntry::ParseList(parameter), CatalogSetPathType::SET_SCHEMAS);
 }
 
 Value SearchPathSetting::GetSetting(ClientContext &context) {

package/src/duckdb/src/optimizer/unnest_rewriter.cpp

@@ -26,11 +26,7 @@ void UnnestRewriterPlanUpdater::VisitExpression(unique_ptr<Expression> *expressi
 		for (idx_t i = 0; i < replace_bindings.size(); i++) {
 			if (bound_column_ref.binding == replace_bindings[i].old_binding) {
 				bound_column_ref.binding = replace_bindings[i].new_binding;
-			}
-			// previously pointing to the LOGICAL_DELIM_GET
-			if (bound_column_ref.binding.table_index == replace_bindings[i].old_binding.table_index &&
-			    replace_bindings[i].old_binding.column_index == DConstants::INVALID_INDEX) {
-				bound_column_ref.binding = replace_bindings[i].new_binding;
+				break;
 			}
 		}
 	}
@@ -49,6 +45,7 @@ unique_ptr<LogicalOperator> UnnestRewriter::Optimize(unique_ptr<LogicalOperator>
 
 		// rearrange the logical operators
 		if (RewriteCandidate(candidate)) {
+			updater.overwritten_tbl_idx = overwritten_tbl_idx;
 			// update the bindings of the BOUND_UNNEST expression
 			UpdateBoundUnnestBindings(updater, candidate);
 			// update the sequence of LOGICAL_PROJECTION(s)
@@ -106,7 +103,6 @@ void UnnestRewriter::FindCandidates(unique_ptr<LogicalOperator> *op_ptr,
 	if (curr_op->get()->type == LogicalOperatorType::LOGICAL_UNNEST) {
 		candidates.push_back(op_ptr);
 	}
-	return;
 }
 
 bool UnnestRewriter::RewriteCandidate(unique_ptr<LogicalOperator> *candidate) {
@@ -147,6 +143,11 @@ bool UnnestRewriter::RewriteCandidate(unique_ptr<LogicalOperator> *candidate) {
 	auto &unnest = curr_op->get()->Cast<LogicalUnnest>();
 	D_ASSERT(unnest.children[0]->type == LogicalOperatorType::LOGICAL_DELIM_GET);
 	overwritten_tbl_idx = unnest.children[0]->Cast<LogicalDelimGet>().table_index;
+
+	D_ASSERT(!unnest.children.empty());
+	auto &delim_get = unnest.children[0]->Cast<LogicalDelimGet>();
+	D_ASSERT(delim_get.chunk_types.size() > 1);
+	distinct_unnest_count = delim_get.chunk_types.size();
 	unnest.children[0] = std::move(lhs_op);
 
 	// replace the LOGICAL_DELIM_JOIN with its RHS child operator
@@ -168,10 +169,11 @@ void UnnestRewriter::UpdateRHSBindings(unique_ptr<LogicalOperator> *plan_ptr, un
 		D_ASSERT(curr_op->get()->type == LogicalOperatorType::LOGICAL_PROJECTION);
 		auto &proj = curr_op->get()->Cast<LogicalProjection>();
 
-		// pop the two last expressions from all projections (delim_idx and UNNEST column)
-		D_ASSERT(proj.expressions.size() > 2);
-		proj.expressions.pop_back();
-		proj.expressions.pop_back();
+		// pop the unnest columns and the delim index
+		D_ASSERT(proj.expressions.size() > distinct_unnest_count);
+		for (idx_t i = 0; i < distinct_unnest_count; i++) {
+			proj.expressions.pop_back();
+		}
 
 		// store all shifted current bindings
 		idx_t tbl_idx = proj.table_index;
@@ -263,14 +265,23 @@ void UnnestRewriter::UpdateBoundUnnestBindings(UnnestRewriterPlanUpdater &update
 	auto &unnest = curr_op->get()->Cast<LogicalUnnest>();
 
 	D_ASSERT(unnest.children.size() == 1);
-	auto unnest_child_cols = unnest.children[0]->GetColumnBindings();
-	for (idx_t delim_col_idx = 0; delim_col_idx < delim_columns.size(); delim_col_idx++) {
-		for (idx_t child_col_idx = 0; child_col_idx < unnest_child_cols.size(); child_col_idx++) {
-			if (delim_columns[delim_col_idx].table_index == unnest_child_cols[child_col_idx].table_index) {
-				ColumnBinding old_binding(overwritten_tbl_idx, DConstants::INVALID_INDEX);
-				updater.replace_bindings.emplace_back(old_binding, delim_columns[delim_col_idx]);
+	auto unnest_cols = unnest.children[0]->GetColumnBindings();
+
+	for (idx_t i = 0; i < delim_columns.size(); i++) {
+		auto delim_binding = delim_columns[i];
+
+		auto unnest_it = unnest_cols.begin();
+		while (unnest_it != unnest_cols.end()) {
+			auto unnest_binding = *unnest_it;
+
+			if (delim_binding.table_index == unnest_binding.table_index) {
+				unnest_binding.table_index = overwritten_tbl_idx;
+				unnest_binding.column_index++;
+				updater.replace_bindings.emplace_back(unnest_binding, delim_binding);
+				unnest_cols.erase(unnest_it);
 				break;
 			}
+			unnest_it++;
 		}
 	}
 

package/src/duckdb/src/parallel/executor.cpp

@@ -95,26 +95,50 @@ void Executor::SchedulePipeline(const shared_ptr<MetaPipeline> &meta_pipeline, S
 
 		// create events/stack for this pipeline
 		auto pipeline_event = make_shared<PipelineEvent>(pipeline);
-		optional_ptr<Event> pipeline_finish_event_ptr;
-		if (meta_pipeline->HasFinishEvent(pipeline.get())) {
+
+		auto finish_group = meta_pipeline->GetFinishGroup(pipeline.get());
+		if (finish_group) {
+			// this pipeline is part of a finish group
+			const auto group_entry = event_map.find(*finish_group.get());
+			D_ASSERT(group_entry != event_map.end());
+			auto &group_stack = group_entry->second;
+			PipelineEventStack pipeline_stack(base_stack.pipeline_initialize_event, *pipeline_event,
+			                                  group_stack.pipeline_finish_event, base_stack.pipeline_complete_event);
+
+			// dependencies: base_finish -> pipeline_event -> group_finish
+			pipeline_stack.pipeline_event.AddDependency(base_stack.pipeline_event);
+			group_stack.pipeline_finish_event.AddDependency(pipeline_stack.pipeline_event);
+
+			// add pipeline stack to event map
+			event_map.insert(make_pair(reference<Pipeline>(*pipeline), pipeline_stack));
+		} else if (meta_pipeline->HasFinishEvent(pipeline.get())) {
 			// this pipeline has its own finish event (despite going into the same sink - Finalize twice!)
 			auto pipeline_finish_event = make_shared<PipelineFinishEvent>(pipeline);
-			pipeline_finish_event_ptr = pipeline_finish_event.get();
+			PipelineEventStack pipeline_stack(base_stack.pipeline_initialize_event, *pipeline_event,
+			                                  *pipeline_finish_event, base_stack.pipeline_complete_event);
 			events.push_back(std::move(pipeline_finish_event));
-			base_stack.pipeline_complete_event.AddDependency(*pipeline_finish_event_ptr);
+
+			// dependencies: base_finish -> pipeline_event -> pipeline_finish -> base_complete
+			pipeline_stack.pipeline_event.AddDependency(base_stack.pipeline_finish_event);
+			pipeline_stack.pipeline_finish_event.AddDependency(pipeline_stack.pipeline_event);
+			base_stack.pipeline_complete_event.AddDependency(pipeline_stack.pipeline_finish_event);
+
+			// add pipeline stack to event map
+			event_map.insert(make_pair(reference<Pipeline>(*pipeline), pipeline_stack));
+
 		} else {
-			pipeline_finish_event_ptr = &base_stack.pipeline_finish_event;
-		}
-		PipelineEventStack pipeline_stack(base_stack.pipeline_initialize_event, *pipeline_event,
-		                                  *pipeline_finish_event_ptr, base_stack.pipeline_complete_event);
-		events.push_back(std::move(pipeline_event));
+			// no additional finish event
+			PipelineEventStack pipeline_stack(base_stack.pipeline_initialize_event, *pipeline_event,
+			                                  base_stack.pipeline_finish_event, base_stack.pipeline_complete_event);
 
-		// dependencies: base_initialize -> pipeline_event -> base_finish
-		pipeline_stack.pipeline_event.AddDependency(base_stack.pipeline_initialize_event);
-		pipeline_stack.pipeline_finish_event.AddDependency(pipeline_stack.pipeline_event);
+			// dependencies: base_initialize -> pipeline_event -> base_finish
+			pipeline_stack.pipeline_event.AddDependency(base_stack.pipeline_initialize_event);
+			base_stack.pipeline_finish_event.AddDependency(pipeline_stack.pipeline_event);
 
-		// add pipeline stack to event map
-		event_map.insert(make_pair(reference<Pipeline>(*pipeline), pipeline_stack));
+			// add pipeline stack to event map
+			event_map.insert(make_pair(reference<Pipeline>(*pipeline), pipeline_stack));
+		}
+		events.push_back(std::move(pipeline_event));
 	}
 
 	// add base stack to the event data too

package/src/duckdb/src/parallel/meta_pipeline.cpp

@@ -70,7 +70,6 @@ void MetaPipeline::AssignNextBatchIndex(Pipeline *pipeline) {
 void MetaPipeline::Build(PhysicalOperator &op) {
 	D_ASSERT(pipelines.size() == 1);
 	D_ASSERT(children.empty());
-	D_ASSERT(final_pipelines.empty());
 	op.BuildPipelines(*pipelines.back(), *this);
 }
 
@@ -125,13 +124,28 @@ void MetaPipeline::AddDependenciesFrom(Pipeline *dependant, Pipeline *start, boo
 }
 
 void MetaPipeline::AddFinishEvent(Pipeline *pipeline) {
+	D_ASSERT(finish_pipelines.find(pipeline) == finish_pipelines.end());
 	finish_pipelines.insert(pipeline);
+
+	// add all pipelines that were added since 'pipeline' was added (including 'pipeline') to the finish group
+	auto it = pipelines.begin();
+	for (; it->get() != pipeline; it++) {
+	}
+	it++;
+	for (; it != pipelines.end(); it++) {
+		finish_map.emplace(it->get(), pipeline);
+	}
 }
 
-bool MetaPipeline::HasFinishEvent(Pipeline *pipeline) {
+bool MetaPipeline::HasFinishEvent(Pipeline *pipeline) const {
 	return finish_pipelines.find(pipeline) != finish_pipelines.end();
 }
 
+optional_ptr<Pipeline> MetaPipeline::GetFinishGroup(Pipeline *pipeline) const {
+	auto it = finish_map.find(pipeline);
+	return it == finish_map.end() ? nullptr : it->second;
+}
+
 Pipeline *MetaPipeline::CreateUnionPipeline(Pipeline &current, bool order_matters) {
 	// create the union pipeline (batch index 0, should be set correctly afterwards)
 	auto union_pipeline = CreatePipeline();
@@ -162,7 +176,7 @@ void MetaPipeline::CreateChildPipeline(Pipeline &current, PhysicalOperator &op,
 	auto child_pipeline = pipelines.back().get();
 	child_pipeline->base_batch_index = current.base_batch_index;
 
-	// child pipeline has a depency (within this MetaPipeline on all pipelines that were scheduled
+	// child pipeline has a dependency (within this MetaPipeline on all pipelines that were scheduled
 	// between 'current' and now (including 'current') - set them up
 	dependencies[child_pipeline].push_back(&current);
 	AddDependenciesFrom(child_pipeline, last_pipeline, false);

package/src/duckdb/src/parser/column_definition.cpp

@@ -50,6 +50,7 @@ void ColumnDefinition::Serialize(Serializer &serializer) const {
 		writer.WriteOptional(default_value);
 	}
 	writer.WriteField<TableColumnType>(category);
+	writer.WriteField<duckdb::CompressionType>(compression_type);
 	writer.Finalize();
 }
 
@@ -59,16 +60,12 @@ ColumnDefinition ColumnDefinition::Deserialize(Deserializer &source) {
 	auto column_type = reader.ReadRequiredSerializable<LogicalType, LogicalType>();
 	auto expression = reader.ReadOptional<ParsedExpression>(nullptr);
 	auto category = reader.ReadField<TableColumnType>(TableColumnType::STANDARD);
+	auto compression_type = reader.ReadField<duckdb::CompressionType>(duckdb::CompressionType::COMPRESSION_AUTO);
 	reader.Finalize();
 
-	switch (category) {
-	case TableColumnType::STANDARD:
-		return ColumnDefinition(column_name, column_type, std::move(expression), TableColumnType::STANDARD);
-	case TableColumnType::GENERATED:
-		return ColumnDefinition(column_name, column_type, std::move(expression), TableColumnType::GENERATED);
-	default:
-		throw NotImplementedException("Type not implemented for TableColumnType");
-	}
+	ColumnDefinition result(column_name, column_type, std::move(expression), category);
+	result.compression_type = compression_type;
+	return result;
 }
 
 const unique_ptr<ParsedExpression> &ColumnDefinition::DefaultValue() const {

package/src/duckdb/src/parser/parsed_data/create_info.cpp

@@ -5,7 +5,6 @@
 #include "duckdb/parser/parsed_data/create_table_info.hpp"
 #include "duckdb/parser/parsed_data/create_view_info.hpp"
 #include "duckdb/parser/parsed_data/create_sequence_info.hpp"
-#include "duckdb/parser/parsed_data/create_database_info.hpp"
 #include "duckdb/parser/parsed_data/create_type_info.hpp"
 #include "duckdb/parser/parsed_data/alter_info.hpp"
 #include "duckdb/parser/parsed_data/create_macro_info.hpp"
@@ -43,8 +42,6 @@ unique_ptr<CreateInfo> CreateInfo::Deserialize(Deserializer &deserializer) {
 		return CreateSchemaInfo::Deserialize(deserializer);
 	case CatalogType::VIEW_ENTRY:
 		return CreateViewInfo::Deserialize(deserializer);
-	case CatalogType::DATABASE_ENTRY:
-		return CreateDatabaseInfo::Deserialize(deserializer);
 	case CatalogType::TYPE_ENTRY:
 		return CreateTypeInfo::Deserialize(deserializer);
 	case CatalogType::MACRO_ENTRY:

package/src/duckdb/src/parser/parser.cpp

@@ -1,18 +1,17 @@
 #include "duckdb/parser/parser.hpp"
 
-#include "duckdb/parser/transformer.hpp"
 #include "duckdb/parser/parsed_data/create_table_info.hpp"
+#include "duckdb/parser/parser_extension.hpp"
+#include "duckdb/parser/query_error_context.hpp"
+#include "duckdb/parser/query_node/select_node.hpp"
 #include "duckdb/parser/statement/create_statement.hpp"
 #include "duckdb/parser/statement/extension_statement.hpp"
 #include "duckdb/parser/statement/select_statement.hpp"
 #include "duckdb/parser/statement/update_statement.hpp"
-#include "duckdb/parser/query_node/select_node.hpp"
 #include "duckdb/parser/tableref/expressionlistref.hpp"
-#include "postgres_parser.hpp"
-#include "duckdb/parser/query_error_context.hpp"
-#include "duckdb/parser/parser_extension.hpp"
-
+#include "duckdb/parser/transformer.hpp"
 #include "parser/parser.hpp"
+#include "postgres_parser.hpp"
 
 namespace duckdb {
 
@@ -45,7 +44,7 @@ static bool ReplaceUnicodeSpaces(const string &query, string &new_query, vector<
 // This function strips unicode space characters from the query and replaces them with regular spaces
 // It returns true if any unicode space characters were found and stripped
 // See here for a list of unicode space characters - https://jkorpela.fi/chars/spaces.html
-static bool StripUnicodeSpaces(const string &query_str, string &new_query) {
+bool Parser::StripUnicodeSpaces(const string &query_str, string &new_query) {
 	const idx_t NBSP_LEN = 2;
 	const idx_t USP_LEN = 3;
 	idx_t pos = 0;
@@ -124,6 +123,29 @@ end:
 	return ReplaceUnicodeSpaces(query_str, new_query, unicode_spaces);
 }
 
+vector<string> SplitQueryStringIntoStatements(const string &query) {
+	// Break sql string down into sql statements using the tokenizer
+	vector<string> query_statements;
+	auto tokens = Parser::Tokenize(query);
+	auto next_statement_start = 0;
+	for (idx_t i = 1; i < tokens.size(); ++i) {
+		auto &t_prev = tokens[i - 1];
+		auto &t = tokens[i];
+		if (t_prev.type == SimplifiedTokenType::SIMPLIFIED_TOKEN_OPERATOR) {
+			// LCOV_EXCL_START
+			for (idx_t c = t_prev.start; c <= t.start; ++c) {
+				if (query.c_str()[c] == ';') {
+					query_statements.emplace_back(query.substr(next_statement_start, t.start - next_statement_start));
+					next_statement_start = tokens[i].start;
+				}
+			}
+			// LCOV_EXCL_STOP
+		}
+	}
+	query_statements.emplace_back(query.substr(next_statement_start, query.size() - next_statement_start));
+	return query_statements;
+}
+
 void Parser::ParseQuery(const string &query) {
 	Transformer transformer(options);
 	string parser_error;
@@ -138,39 +160,77 @@ void Parser::ParseQuery(const string &query) {
 	}
 	{
 		PostgresParser::SetPreserveIdentifierCase(options.preserve_identifier_case);
-		PostgresParser parser;
-		parser.Parse(query);
-		if (parser.success) {
-			if (!parser.parse_tree) {
-				// empty statement
-				return;
-			}
-
-			// if it succeeded, we transform the Postgres parse tree into a list of
-			// SQLStatements
-			transformer.TransformParseTree(parser.parse_tree, statements);
-		} else {
-			parser_error = QueryErrorContext::Format(query, parser.error_message, parser.error_location - 1);
-		}
-	}
-	if (!parser_error.empty()) {
-		if (options.extensions) {
-			for (auto &ext : *options.extensions) {
-				D_ASSERT(ext.parse_function);
-				auto result = ext.parse_function(ext.parser_info.get(), query);
-				if (result.type == ParserExtensionResultType::PARSE_SUCCESSFUL) {
-					auto statement = make_uniq<ExtensionStatement>(ext, std::move(result.parse_data));
-					statement->stmt_length = query.size();
-					statement->stmt_location = 0;
-					statements.push_back(std::move(statement));
+		bool parsing_succeed = false;
+		// Creating a new scope to prevent multiple PostgresParser destructors being called
+		// which led to some memory issues
+		{
+			PostgresParser parser;
+			parser.Parse(query);
+			if (parser.success) {
+				if (!parser.parse_tree) {
+					// empty statement
 					return;
 				}
-				if (result.type == ParserExtensionResultType::DISPLAY_EXTENSION_ERROR) {
-					throw ParserException(result.error);
+
+				// if it succeeded, we transform the Postgres parse tree into a list of
+				// SQLStatements
+				transformer.TransformParseTree(parser.parse_tree, statements);
+				parsing_succeed = true;
+			} else {
+				parser_error = QueryErrorContext::Format(query, parser.error_message, parser.error_location - 1);
+			}
+		}
+		// If DuckDB fails to parse the entire sql string, break the string down into individual statements
+		// using ';' as the delimiter so that parser extensions can parse the statement
+		if (parsing_succeed) {
+			// no-op
+			// return here would require refactoring into another function. o.w. will just no-op in order to run wrap up
+			// code at the end of this function
+		} else if (!options.extensions || options.extensions->empty()) {
+			throw ParserException(parser_error);
+		} else {
+			// split sql string into statements and re-parse using extension
+			auto query_statements = SplitQueryStringIntoStatements(query);
+			for (auto const &query_statement : query_statements) {
+				PostgresParser another_parser;
+				another_parser.Parse(query_statement);
+				// LCOV_EXCL_START
+				// first see if DuckDB can parse this individual query statement
+				if (another_parser.success) {
+					if (!another_parser.parse_tree) {
+						// empty statement
+						continue;
+					}
+					transformer.TransformParseTree(another_parser.parse_tree, statements);
+				} else {
+					// let extensions parse the statement which DuckDB failed to parse
+					bool parsed_single_statement = false;
+					for (auto &ext : *options.extensions) {
+						D_ASSERT(!parsed_single_statement);
+						D_ASSERT(ext.parse_function);
+						auto result = ext.parse_function(ext.parser_info.get(), query_statement);
+						if (result.type == ParserExtensionResultType::PARSE_SUCCESSFUL) {
+							auto statement = make_uniq<ExtensionStatement>(ext, std::move(result.parse_data));
+							statement->stmt_length = query_statement.size();
+							statement->stmt_location = 0;
+							statements.push_back(std::move(statement));
+							parsed_single_statement = true;
+							break;
+						} else if (result.type == ParserExtensionResultType::DISPLAY_EXTENSION_ERROR) {
+							throw ParserException(result.error);
+						} else {
+							// We move to the next one!
+						}
+					}
+					if (!parsed_single_statement) {
+						parser_error = QueryErrorContext::Format(query, another_parser.error_message,
+						                                         another_parser.error_location - 1);
+						throw ParserException(parser_error);
+					}
 				}
+				// LCOV_EXCL_STOP
 			}
 		}
-		throw ParserException(parser_error);
 	}
 	if (!statements.empty()) {
 		auto &last_statement = statements.back();

package/src/duckdb/src/parser/transform/expression/transform_function.cpp

@@ -309,6 +309,7 @@ unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::P
 			order_bys.reset();   // NOLINT
 			filter_expr.reset(); // NOLINT
 			children.clear();    // NOLINT
+			root.agg_distinct = false;
 			children.emplace_back(std::move(unordered));
 			children.emplace_back(std::move(sense));
 			children.emplace_back(std::move(nulls));

package/src/duckdb/src/parser/transform/expression/transform_operator.cpp

@@ -82,6 +82,9 @@ unique_ptr<ParsedExpression> Transformer::TransformAExprInternal(duckdb_libpgque
 		subquery_expr->child = std::move(left_expr);
 		subquery_expr->comparison_type = OperatorToExpressionType(name);
 		subquery_expr->query_location = root.location;
+		if (subquery_expr->comparison_type == ExpressionType::INVALID) {
+			throw ParserException("Unsupported comparison \"%s\" for ANY/ALL subquery", name);
+		}
 
 		if (root.kind == duckdb_libpgquery::PG_AEXPR_OP_ALL) {
 			// ALL sublink is equivalent to NOT(ANY) with inverted comparison

package/src/duckdb/src/parser/transform/helpers/nodetype_to_string.cpp

@@ -816,8 +816,6 @@ std::string Transformer::NodetypeToString(duckdb_libpgquery::PGNodeTag type) { /
 		return "T_PGAttachStmt";
 	case duckdb_libpgquery::T_PGUseStmt:
 		return "T_PGUseStmt";
-	case duckdb_libpgquery::T_PGCreateDatabaseStmt:
-		return "T_PGCreateDatabaseStmt";
 	default:
 		return "(UNKNOWN)";
 	}

package/src/duckdb/src/parser/transform/statement/transform_drop.cpp

@@ -34,9 +34,6 @@ unique_ptr<SQLStatement> Transformer::TransformDrop(duckdb_libpgquery::PGDropStm
 	case duckdb_libpgquery::PG_OBJECT_TYPE:
 		info.type = CatalogType::TYPE_ENTRY;
 		break;
-	case duckdb_libpgquery::PG_OBJECT_DATABASE:
-		info.type = CatalogType::DATABASE_ENTRY;
-		break;
 	default:
 		throw NotImplementedException("Cannot drop this type yet");
 	}