duckdb 0.7.2-dev1637.0 → 0.7.2-dev1684.0

package/src/duckdb/src/include/duckdb/execution/operator/order/physical_order.hpp

@@ -41,7 +41,7 @@ public:
 	                    LocalSourceState &lstate) const override;
 
 	bool IsSource() const override {
-		return false;
+		return true;
 	}
 
 	bool ParallelSource() const override {
@@ -52,6 +52,10 @@ public:
 		return true;
 	}
 
+	OrderPreservationType SourceOrder() const override {
+		return OrderPreservationType::FIXED_ORDER;
+	}
+
 public:
 	// Sink interface
 	unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
@@ -68,6 +72,9 @@ public:
 	bool ParallelSink() const override {
 		return true;
 	}
+	bool SinkOrderDependent() const override {
+		return false;
+	}
 
 public:
 	string ParamsToString() const override;

package/src/duckdb/src/include/duckdb/execution/operator/order/physical_top_n.hpp

@@ -34,6 +34,13 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+	OrderPreservationType SourceOrder() const override {
+		return OrderPreservationType::FIXED_ORDER;
+	}
+
 public:
 	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,
 	                    DataChunk &input) const override;

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_batch_insert.hpp

@@ -46,6 +46,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink interface
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_copy_to_file.hpp

@@ -42,6 +42,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink interface
 	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate,
@@ -56,7 +60,7 @@ public:
 		return true;
 	}
 
-	bool IsOrderDependent() const override {
+	bool SinkOrderDependent() const override {
 		return true;
 	}
 

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_delete.hpp

@@ -36,6 +36,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink interface
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_export.hpp

@@ -41,6 +41,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink interface
 	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate,

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_insert.hpp

@@ -80,6 +80,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink interface
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
@@ -98,6 +102,10 @@ public:
 		return parallel;
 	}
 
+	bool SinkOrderDependent() const override {
+		return true;
+	}
+
 public:
 	static void GetInsertInfo(const BoundCreateTableInfo &info, vector<LogicalType> &insert_types,
 	                          vector<unique_ptr<Expression>> &bound_defaults);

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_update.hpp

@@ -39,6 +39,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink interface
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;

package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp

@@ -33,6 +33,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	void BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) override;
 };

package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_dummy_scan.hpp

@@ -25,5 +25,9 @@ public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_empty_result.hpp

@@ -24,5 +24,9 @@ public:
 public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_positional_scan.hpp

@@ -39,6 +39,10 @@ public:
 	             LocalSourceState &lstate) const override;
 
 	double GetProgress(ClientContext &context, GlobalSourceState &gstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp

@@ -60,6 +60,9 @@ public:
 	idx_t GetBatchIndex(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	                    LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
 	bool ParallelSource() const override {
 		return true;
 	}

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_alter.hpp

@@ -31,6 +31,10 @@ public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_attach.hpp

@@ -31,6 +31,10 @@ public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_function.hpp

@@ -31,6 +31,10 @@ public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_index.hpp

@@ -43,6 +43,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	//! Sink interface, thread-local sink states
 	unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_schema.hpp

@@ -31,6 +31,10 @@ public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_sequence.hpp

@@ -31,6 +31,10 @@ public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_table.hpp

@@ -32,5 +32,9 @@ public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_type.hpp

@@ -29,6 +29,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink interface
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_view.hpp

@@ -31,6 +31,10 @@ public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_detach.hpp

@@ -30,6 +30,10 @@ public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_drop.hpp

@@ -31,6 +31,10 @@ public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp

@@ -33,6 +33,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink interface
 	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,

package/src/duckdb/src/include/duckdb/execution/physical_operator.hpp

@@ -16,6 +16,7 @@
 #include "duckdb/execution/execution_context.hpp"
 #include "duckdb/optimizer/join_order/join_node.hpp"
 #include "duckdb/execution/physical_operator_states.hpp"
+#include "duckdb/common/enums/order_preservation_type.hpp"
 
 namespace duckdb {
 class Event;
@@ -74,12 +75,6 @@ public:
 
 	virtual void Verify();
 
-	//! Whether or not the operator depends on the order of the input chunks
-	//! If this is set to true, we cannot do things like caching intermediate vectors
-	virtual bool IsOrderDependent() const {
-		return false;
-	}
-
 public:
 	// Operator interface
 	virtual unique_ptr<OperatorState> GetOperatorState(ExecutionContext &context) const;
@@ -97,6 +92,11 @@ public:
 		return false;
 	}
 
+	//! The influence the operator has on order (insertion order means no influence)
+	virtual OrderPreservationType OperatorOrder() const {
+		return OrderPreservationType::INSERTION_ORDER;
+	}
+
 public:
 	// Source interface
 	virtual unique_ptr<LocalSourceState> GetLocalSourceState(ExecutionContext &context,
@@ -119,8 +119,9 @@ public:
 		return false;
 	}
 
-	virtual bool IsOrderPreserving() const {
-		return true;
+	//! The type of order emitted by the operator (as a source)
+	virtual OrderPreservationType SourceOrder() const {
+		return OrderPreservationType::INSERTION_ORDER;
 	}
 
 	//! Returns the current progress percentage, or a negative value if progress bars are not supported
@@ -161,11 +162,16 @@ public:
 		return false;
 	}
 
+	//! Whether or not the sink operator depends on the order of the input chunks
+	//! If this is set to true, we cannot do things like caching intermediate vectors
+	virtual bool SinkOrderDependent() const {
+		return false;
+	}
+
 public:
 	// Pipeline construction
 	virtual vector<const PhysicalOperator *> GetSources() const;
 	bool AllSourcesSupportBatchIndex() const;
-	virtual bool AllOperatorsPreserveOrder() const;
 
 	virtual void BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline);
 

package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp

@@ -90,7 +90,7 @@ public:
 		return source;
 	}
 
-	//! Returns whether any of the operators in the pipeline care about preserving insertion order
+	//! Returns whether any of the operators in the pipeline care about preserving order
 	bool IsOrderDependent() const;
 
 private:

package/src/duckdb/src/include/duckdb/planner/pragma_handler.hpp

@@ -29,8 +29,9 @@ private:
 	ClientContext &context;
 
 private:
-	//! Handles a pragma statement, (potentially) returning a new statement to replace the current one
-	string HandlePragma(SQLStatement *statement);
+	//! Handles a pragma statement, returns whether the statement was expanded, if it was expanded the 'resulting_query'
+	//! contains the statement(s) to replace the current one
+	bool HandlePragma(SQLStatement *statement, string &resulting_query);
 
 	void HandlePragmaStatementsInternal(vector<unique_ptr<SQLStatement>> &statements);
 };

package/src/duckdb/src/main/extension/extension_helper.cpp

@@ -78,6 +78,13 @@
 #include "inet-extension.hpp"
 #endif
 
+#if defined(BUILD_AUTOCOMPLETE_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#define AUTOCOMPLETE_STATICALLY_LOADED true
+#include "sql_auto_complete-extension.hpp"
+#else
+#define AUTOCOMPLETE_STATICALLY_LOADED false
+#endif
+
 // Load the generated header file containing our list of extension headers
 #if defined(OOTE_HEADERS_AVAILABLE) && OOTE_HEADERS_AVAILABLE
 #include "extension_oote_loader.hpp"
@@ -97,6 +104,7 @@ static DefaultExtension internal_extensions[] = {
     {"httpfs", "Adds support for reading and writing files over a HTTP(S) connection", HTTPFS_STATICALLY_LOADED},
     {"json", "Adds support for JSON operations", JSON_STATICALLY_LOADED},
     {"jemalloc", "Overwrites system allocator with JEMalloc", JEMALLOC_STATICALLY_LOADED},
+    {"autocomplete", "Add supports for autocomplete in the shell", AUTOCOMPLETE_STATICALLY_LOADED},
     {"motherduck", "Enables motherduck integration with the system", false},
     {"sqlite_scanner", "Adds support for reading SQLite database files", false},
     {"postgres_scanner", "Adds support for reading from a Postgres database", false},
@@ -134,8 +142,8 @@ bool ExtensionHelper::AllowAutoInstall(const string &extension) {
 // Load Statically Compiled Extension
 //===--------------------------------------------------------------------===//
 void ExtensionHelper::LoadAllExtensions(DuckDB &db) {
-	unordered_set<string> extensions {"parquet",    "icu",  "tpch",  "tpcds",    "fts",  "httpfs",
-	                                  "visualizer", "json", "excel", "sqlsmith", "inet", "jemalloc"};
+	unordered_set<string> extensions {"parquet", "icu",   "tpch",     "tpcds", "fts",      "httpfs",      "visualizer",
+	                                  "json",    "excel", "sqlsmith", "inet",  "jemalloc", "autocomplete"};
 	for (auto &ext : extensions) {
 		LoadExtensionInternal(db, ext, true);
 	}
@@ -244,6 +252,13 @@ ExtensionLoadResult ExtensionHelper::LoadExtensionInternal(DuckDB &db, const std
 #else
 		// jemalloc extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
+#endif
+	} else if (extension == "autocomplete") {
+#if defined(BUILD_AUTOCOMPLETE_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+		db.LoadExtension<SQLAutoCompleteExtension>();
+#else
+		// autocomplete extension required but not build: skip this test
+		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "inet") {
 #if defined(BUILD_INET_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)

package/src/duckdb/src/main/query_profiler.cpp

@@ -690,7 +690,7 @@ ExpressionExecutorInfo::ExpressionExecutorInfo(ExpressionExecutor &executor, con
 ExpressionRootInfo::ExpressionRootInfo(ExpressionExecutorState &state, string name)
     : current_count(state.profiler.current_count), sample_count(state.profiler.sample_count),
       sample_tuples_count(state.profiler.sample_tuples_count), tuples_count(state.profiler.tuples_count),
-      name(state.name), time(state.profiler.time) {
+      name("expression"), time(state.profiler.time) {
 	// Use the name of expression-tree as extra-info
 	extra_info = std::move(name);
 	auto expression_info_p = make_uniq<ExpressionInfo>();

package/src/duckdb/src/parallel/pipeline.cpp

@@ -98,20 +98,29 @@ bool Pipeline::ScheduleParallel(shared_ptr<Event> &event) {
 
 bool Pipeline::IsOrderDependent() const {
 	auto &config = DBConfig::GetConfig(executor.context);
-	if (!config.options.preserve_insertion_order) {
-		return false;
-	}
-	if (sink && sink->IsOrderDependent()) {
-		return true;
-	}
-	if (source && source->IsOrderDependent()) {
-		return true;
+	if (source) {
+		auto source_order = source->SourceOrder();
+		if (source_order == OrderPreservationType::FIXED_ORDER) {
+			return true;
+		}
+		if (source_order == OrderPreservationType::NO_ORDER) {
+			return false;
+		}
 	}
 	for (auto &op : operators) {
-		if (op->IsOrderDependent()) {
+		if (op->OperatorOrder() == OrderPreservationType::NO_ORDER) {
+			return false;
+		}
+		if (op->OperatorOrder() == OrderPreservationType::FIXED_ORDER) {
 			return true;
 		}
 	}
+	if (!config.options.preserve_insertion_order) {
+		return false;
+	}
+	if (sink && sink->SinkOrderDependent()) {
+		return true;
+	}
 	return false;
 }
 
@@ -148,6 +157,9 @@ bool Pipeline::LaunchScanTasks(shared_ptr<Event> &event, idx_t max_threads) {
 
 void Pipeline::ResetSink() {
 	if (sink) {
+		if (!sink->IsSink()) {
+			throw InternalException("Sink of pipeline does not have IsSink set");
+		}
 		lock_guard<mutex> guard(sink->lock);
 		if (!sink->sink_state) {
 			sink->sink_state = sink->GetGlobalSinkState(GetClientContext());
@@ -172,6 +184,9 @@ void Pipeline::Reset() {
 }
 
 void Pipeline::ResetSource(bool force) {
+	if (source && !source->IsSource()) {
+		throw InternalException("Source of pipeline does not have IsSource set");
+	}
 	if (force || !source_state) {
 		source_state = source->GetGlobalSourceState(GetClientContext());
 	}

package/src/duckdb/src/planner/pragma_handler.cpp

@@ -32,8 +32,9 @@ void PragmaHandler::HandlePragmaStatementsInternal(vector<unique_ptr<SQLStatemen
 		if (statements[i]->type == StatementType::PRAGMA_STATEMENT) {
 			// PRAGMA statement: check if we need to replace it by a new set of statements
 			PragmaHandler handler(context);
-			auto new_query = handler.HandlePragma(statements[i].get());
-			if (!new_query.empty()) {
+			string new_query;
+			bool expanded = handler.HandlePragma(statements[i].get(), new_query);
+			if (expanded) {
 				// this PRAGMA statement gets replaced by a new query string
 				// push the new query string through the parser again and add it to the transformer
 				Parser parser(context.GetParserOptions());
@@ -67,7 +68,7 @@ void PragmaHandler::HandlePragmaStatements(ClientContextLock &lock, vector<uniqu
 	context.RunFunctionInTransactionInternal(lock, [&]() { HandlePragmaStatementsInternal(statements); });
 }
 
-string PragmaHandler::HandlePragma(SQLStatement *statement) { // PragmaInfo &info
+bool PragmaHandler::HandlePragma(SQLStatement *statement, string &resulting_query) { // PragmaInfo &info
 	auto info = *(statement->Cast<PragmaStatement>()).info;
 	auto entry =
 	    Catalog::GetEntry<PragmaFunctionCatalogEntry>(context, INVALID_CATALOG, DEFAULT_SCHEMA, info.name, false);
@@ -84,9 +85,10 @@ string PragmaHandler::HandlePragma(SQLStatement *statement) { // PragmaInfo &inf
 		Binder::BindNamedParameters(bound_function.named_parameters, info.named_parameters, error_context,
 		                            bound_function.name);
 		FunctionParameters parameters {info.parameters, info.named_parameters};
-		return bound_function.query(context, parameters);
+		resulting_query = bound_function.query(context, parameters);
+		return true;
 	}
-	return string();
+	return false;
 }
 
 } // namespace duckdb