duckdb 0.7.2-dev1637.0 → 0.7.2-dev1671.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.7.2-dev1637.0",
+  "version": "0.7.2-dev1671.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/execution/operator/aggregate/physical_hash_aggregate.cpp

@@ -843,7 +843,7 @@ public:
 			auto &grouping_gstate = ht_state.grouping_states[sidx];
 			count += grouping.table_data.Size(*grouping_gstate.table_state);
 		}
-		return MaxValue<idx_t>(1, count / RowGroup::ROW_GROUP_SIZE);
+		return MaxValue<idx_t>(1, count / STANDARD_VECTOR_SIZE);
 	}
 };
 

package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp

@@ -166,6 +166,16 @@ public:
 	}
 };
 
+bool PhysicalUngroupedAggregate::SinkOrderDependent() const {
+	for (auto &expr : aggregates) {
+		auto &aggr = expr->Cast<BoundAggregateExpression>();
+		if (aggr.function.order_dependent == AggregateOrderDependent::ORDER_DEPENDENT) {
+			return true;
+		}
+	}
+	return false;
+}
+
 unique_ptr<GlobalSinkState> PhysicalUngroupedAggregate::GetGlobalSinkState(ClientContext &context) const {
 	return make_uniq<UngroupedAggregateGlobalState>(*this, context);
 }

package/src/duckdb/src/execution/operator/helper/physical_execute.cpp

@@ -12,11 +12,6 @@ vector<PhysicalOperator *> PhysicalExecute::GetChildren() const {
 	return {plan};
 }
 
-bool PhysicalExecute::AllOperatorsPreserveOrder() const {
-	D_ASSERT(plan);
-	return plan->AllOperatorsPreserveOrder();
-}
-
 void PhysicalExecute::BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) {
 	// EXECUTE statement: build pipeline on child
 	meta_pipeline.Build(plan);

package/src/duckdb/src/execution/operator/helper/physical_materialized_collector.cpp

@@ -75,4 +75,8 @@ bool PhysicalMaterializedCollector::ParallelSink() const {
 	return parallel;
 }
 
+bool PhysicalMaterializedCollector::SinkOrderDependent() const {
+	return true;
+}
+
 } // namespace duckdb

package/src/duckdb/src/execution/operator/helper/physical_result_collector.cpp

@@ -35,11 +35,6 @@ vector<PhysicalOperator *> PhysicalResultCollector::GetChildren() const {
 	return {plan};
 }
 
-bool PhysicalResultCollector::AllOperatorsPreserveOrder() const {
-	D_ASSERT(plan);
-	return plan->AllOperatorsPreserveOrder();
-}
-
 void PhysicalResultCollector::BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) {
 	// operator is a sink, build a pipeline
 	sink_state.reset();

package/src/duckdb/src/execution/operator/helper/physical_streaming_limit.cpp

@@ -60,8 +60,8 @@ OperatorResultType PhysicalStreamingLimit::Execute(ExecutionContext &context, Da
 	return OperatorResultType::NEED_MORE_INPUT;
 }
 
-bool PhysicalStreamingLimit::IsOrderDependent() const {
-	return !parallel;
+OrderPreservationType PhysicalStreamingLimit::OperatorOrder() const {
+	return OrderPreservationType::FIXED_ORDER;
 }
 
 bool PhysicalStreamingLimit::ParallelOperator() const {

package/src/duckdb/src/execution/operator/set/physical_union.cpp

@@ -22,8 +22,19 @@ void PhysicalUnion::BuildPipelines(Pipeline &current, MetaPipeline &meta_pipelin
 
 	// order matters if any of the downstream operators are order dependent,
 	// or if the sink preserves order, but does not support batch indices to do so
-	auto snk = meta_pipeline.GetSink();
-	bool order_matters = current.IsOrderDependent() || (snk && snk->IsOrderPreserving() && !snk->RequiresBatchIndex());
+	auto sink = meta_pipeline.GetSink();
+	bool order_matters = false;
+	if (current.IsOrderDependent()) {
+		order_matters = true;
+	}
+	if (sink) {
+		if (sink->SinkOrderDependent() && !sink->RequiresBatchIndex()) {
+			order_matters = true;
+		}
+		if (!sink->ParallelSink()) {
+			order_matters = true;
+		}
+	}
 
 	// create a union pipeline that is identical to 'current'
 	auto union_pipeline = meta_pipeline.CreateUnionPipeline(current, order_matters);

package/src/duckdb/src/execution/physical_operator.cpp

@@ -184,21 +184,6 @@ bool PhysicalOperator::AllSourcesSupportBatchIndex() const {
 	return true;
 }
 
-bool PhysicalOperator::AllOperatorsPreserveOrder() const {
-	if (type == PhysicalOperatorType::ORDER_BY) {
-		return true;
-	}
-	if (!IsOrderPreserving()) {
-		return false;
-	}
-	for (auto &child : children) {
-		if (!child->AllOperatorsPreserveOrder()) {
-			return false;
-		}
-	}
-	return true;
-}
-
 void PhysicalOperator::Verify() {
 #ifdef DEBUG
 	auto sources = GetSources();

package/src/duckdb/src/execution/physical_plan/plan_insert.cpp

@@ -10,14 +10,34 @@
 
 namespace duckdb {
 
+static OrderPreservationType OrderPreservationRecursive(PhysicalOperator &op) {
+	if (op.IsSource()) {
+		return op.SourceOrder();
+	}
+	for (auto &child : op.children) {
+		auto child_preservation = OrderPreservationRecursive(*child);
+		if (child_preservation != OrderPreservationType::INSERTION_ORDER) {
+			return child_preservation;
+		}
+	}
+	return OrderPreservationType::INSERTION_ORDER;
+}
+
 bool PhysicalPlanGenerator::PreserveInsertionOrder(ClientContext &context, PhysicalOperator &plan) {
 	auto &config = DBConfig::GetConfig(context);
-	if (!config.options.preserve_insertion_order) {
-		// preserving insertion order is disabled by config
+
+	auto preservation_type = OrderPreservationRecursive(plan);
+	if (preservation_type == OrderPreservationType::FIXED_ORDER) {
+		// always need to maintain preservation order
+		return true;
+	}
+	if (preservation_type == OrderPreservationType::NO_ORDER) {
+		// never need to preserve order
 		return false;
 	}
-	if (!plan.AllOperatorsPreserveOrder()) {
-		// the plan has no order defined: no need to preserve insertion order
+	// preserve insertion order - check flags
+	if (!config.options.preserve_insertion_order) {
+		// preserving insertion order is disabled by config
 		return false;
 	}
 	return true;

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.7.2-dev1637"
+#define DUCKDB_VERSION "0.7.2-dev1671"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "bacede9532"
+#define DUCKDB_SOURCE_ID "846a32b4c8"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/common/enums/order_preservation_type.hpp

@@ -0,0 +1,24 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// src/include/duckdb/common/enums/order_preservation_type.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/constants.hpp"
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// Order Preservation Type
+//===--------------------------------------------------------------------===//
+enum class OrderPreservationType : uint8_t {
+	NO_ORDER,        // the operator makes no guarantees on order preservation (i.e. it might re-order the entire input)
+	INSERTION_ORDER, // the operator maintains the order of the child operators
+	FIXED_ORDER      // the operator outputs rows in a fixed order that must be maintained (e.g. ORDER BY)
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_hash_aggregate.hpp

@@ -93,12 +93,15 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
 	bool ParallelSource() const override {
 		return true;
 	}
 
-	bool IsOrderPreserving() const override {
-		return false;
+	OrderPreservationType SourceOrder() const override {
+		return OrderPreservationType::NO_ORDER;
 	}
 
 public:
@@ -122,6 +125,10 @@ public:
 		return true;
 	}
 
+	bool SinkOrderDependent() const override {
+		return false;
+	}
+
 public:
 	string ParamsToString() const override;
 	//! Toggle multi-scan capability on a hash table, which prevents the scan of the aggregate from being destructive

package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_perfecthash_aggregate.hpp

@@ -37,6 +37,13 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+	OrderPreservationType SourceOrder() const override {
+		return OrderPreservationType::NO_ORDER;
+	}
+
 public:
 	// Sink interface
 	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,
@@ -59,7 +66,7 @@ public:
 		return true;
 	}
 
-	bool IsOrderPreserving() const override {
+	bool SinkOrderDependent() const override {
 		return false;
 	}
 

package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_streaming_window.hpp

@@ -33,8 +33,8 @@ public:
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
 	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
-	bool IsOrderDependent() const override {
-		return true;
+	OrderPreservationType OperatorOrder() const override {
+		return OrderPreservationType::FIXED_ORDER;
 	}
 
 	string ParamsToString() const override;

package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_ungrouped_aggregate.hpp

@@ -39,6 +39,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink interface
 	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,
@@ -60,6 +64,8 @@ public:
 		return true;
 	}
 
+	bool SinkOrderDependent() const override;
+
 private:
 	//! Finalize the distinct aggregates
 	SinkFinalizeType FinalizeDistinct(Pipeline &pipeline, Event &event, ClientContext &context,

package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_window.hpp

@@ -38,12 +38,15 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
 	bool ParallelSource() const override {
 		return true;
 	}
 
-	bool IsOrderPreserving() const override {
-		return true;
+	OrderPreservationType SourceOrder() const override {
+		return OrderPreservationType::NO_ORDER;
 	}
 
 public:
@@ -65,7 +68,7 @@ public:
 		return !is_order_dependent;
 	}
 
-	bool IsOrderDependent() const override {
+	bool SinkOrderDependent() const override {
 		return is_order_dependent;
 	}
 

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_execute.hpp

@@ -28,7 +28,6 @@ public:
 	vector<PhysicalOperator *> GetChildren() const override;
 
 public:
-	bool AllOperatorsPreserveOrder() const override;
 	void BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) override;
 };
 

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_explain_analyze.hpp

@@ -28,6 +28,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink Interface
 	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_limit.hpp

@@ -28,7 +28,7 @@ public:
 	unique_ptr<Expression> offset_expression;
 
 public:
-	bool IsOrderDependent() const override {
+	bool SinkOrderDependent() const override {
 		return true;
 	}
 
@@ -38,6 +38,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink Interface
 	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_limit_percent.hpp

@@ -33,7 +33,7 @@ public:
 	unique_ptr<Expression> offset_expression;
 
 public:
-	bool IsOrderDependent() const override {
+	bool SinkOrderDependent() const override {
 		return true;
 	}
 
@@ -43,6 +43,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink Interface
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_load.hpp

@@ -30,6 +30,10 @@ public:
 	// Source interface
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_materialized_collector.hpp

@@ -31,6 +31,7 @@ public:
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
 
 	bool ParallelSink() const override;
+	bool SinkOrderDependent() const override;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_pragma.hpp

@@ -34,6 +34,10 @@ public:
 	// Source interface
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_prepare.hpp

@@ -31,6 +31,10 @@ public:
 	// Source interface
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_reservoir_sample.hpp

@@ -28,6 +28,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink interface
 	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_reset.hpp

@@ -33,6 +33,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	const std::string name;
 	const SetScope scope;

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_result_collector.hpp

@@ -37,8 +37,11 @@ public:
 
 public:
 	vector<PhysicalOperator *> GetChildren() const override;
-	bool AllOperatorsPreserveOrder() const override;
 	void BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_set.hpp

@@ -33,6 +33,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 	static void SetExtensionVariable(ClientContext &context, ExtensionOption &extension_option, const string &name,
 	                                 SetScope scope, const Value &value);
 

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_streaming_limit.hpp

@@ -35,7 +35,7 @@ public:
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
 	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
-	bool IsOrderDependent() const override;
+	OrderPreservationType OperatorOrder() const override;
 	bool ParallelOperator() const override;
 };
 

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_transaction.hpp

@@ -29,6 +29,10 @@ public:
 public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_vacuum.hpp

@@ -29,6 +29,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink interface
 	unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;

package/src/duckdb/src/include/duckdb/execution/operator/join/physical_cross_product.hpp

@@ -26,19 +26,17 @@ public:
 	// Operator Interface
 	unique_ptr<OperatorState> GetOperatorState(ExecutionContext &context) const override;
 
+	OrderPreservationType OperatorOrder() const override {
+		return OrderPreservationType::NO_ORDER;
+	}
 	bool ParallelOperator() const override {
 		return true;
 	}
 
 protected:
-	// CachingOperator Interface
 	OperatorResultType ExecuteInternal(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
 	                                   GlobalOperatorState &gstate, OperatorState &state) const override;
 
-	bool IsOrderPreserving() const override {
-		return false;
-	}
-
 public:
 	// Sink Interface
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
@@ -51,6 +49,9 @@ public:
 	bool ParallelSink() const override {
 		return true;
 	}
+	bool SinkOrderDependent() const override {
+		return false;
+	}
 
 public:
 	void BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) override;

package/src/duckdb/src/include/duckdb/execution/operator/join/physical_delim_join.hpp

@@ -46,7 +46,10 @@ public:
 	bool ParallelSink() const override {
 		return true;
 	}
-	bool IsOrderPreserving() const override {
+	OrderPreservationType SourceOrder() const override {
+		return OrderPreservationType::NO_ORDER;
+	}
+	bool SinkOrderDependent() const override {
 		return false;
 	}
 	string ParamsToString() const override;

package/src/duckdb/src/include/duckdb/execution/operator/join/physical_index_join.hpp

@@ -55,6 +55,9 @@ public:
 public:
 	unique_ptr<OperatorState> GetOperatorState(ExecutionContext &context) const override;
 
+	OrderPreservationType OperatorOrder() const override {
+		return OrderPreservationType::NO_ORDER;
+	}
 	bool ParallelOperator() const override {
 		return true;
 	}
@@ -63,10 +66,6 @@ protected:
 	OperatorResultType ExecuteInternal(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
 	                                   GlobalOperatorState &gstate, OperatorState &state) const override;
 
-	bool IsOrderPreserving() const override {
-		return false;
-	}
-
 public:
 	void BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) override;
 	vector<const PhysicalOperator *> GetSources() const override;

package/src/duckdb/src/include/duckdb/execution/operator/join/physical_join.hpp

@@ -37,7 +37,13 @@ public:
 	void BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) override;
 	vector<const PhysicalOperator *> GetSources() const override;
 
-	bool IsOrderPreserving() const override {
+	OrderPreservationType SourceOrder() const override {
+		return OrderPreservationType::NO_ORDER;
+	}
+	OrderPreservationType OperatorOrder() const override {
+		return OrderPreservationType::NO_ORDER;
+	}
+	bool SinkOrderDependent() const override {
 		return false;
 	}
 };

package/src/duckdb/src/include/duckdb/execution/operator/order/physical_order.hpp

@@ -41,7 +41,7 @@ public:
 	                    LocalSourceState &lstate) const override;
 
 	bool IsSource() const override {
-		return false;
+		return true;
 	}
 
 	bool ParallelSource() const override {
@@ -52,6 +52,10 @@ public:
 		return true;
 	}
 
+	OrderPreservationType SourceOrder() const override {
+		return OrderPreservationType::FIXED_ORDER;
+	}
+
 public:
 	// Sink interface
 	unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
@@ -68,6 +72,9 @@ public:
 	bool ParallelSink() const override {
 		return true;
 	}
+	bool SinkOrderDependent() const override {
+		return false;
+	}
 
 public:
 	string ParamsToString() const override;

package/src/duckdb/src/include/duckdb/execution/operator/order/physical_top_n.hpp

@@ -34,6 +34,13 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+	OrderPreservationType SourceOrder() const override {
+		return OrderPreservationType::FIXED_ORDER;
+	}
+
 public:
 	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,
 	                    DataChunk &input) const override;

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_batch_insert.hpp

@@ -46,6 +46,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink interface
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_copy_to_file.hpp

@@ -42,6 +42,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink interface
 	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate,
@@ -56,7 +60,7 @@ public:
 		return true;
 	}
 
-	bool IsOrderDependent() const override {
+	bool SinkOrderDependent() const override {
 		return true;
 	}
 

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_delete.hpp

@@ -36,6 +36,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink interface
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_export.hpp

@@ -41,6 +41,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink interface
 	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate,

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_insert.hpp

@@ -80,6 +80,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink interface
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
@@ -98,6 +102,10 @@ public:
 		return parallel;
 	}
 
+	bool SinkOrderDependent() const override {
+		return true;
+	}
+
 public:
 	static void GetInsertInfo(const BoundCreateTableInfo &info, vector<LogicalType> &insert_types,
 	                          vector<unique_ptr<Expression>> &bound_defaults);

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_update.hpp

@@ -39,6 +39,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink interface
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;

package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp

@@ -33,6 +33,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	void BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) override;
 };

package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_dummy_scan.hpp

@@ -25,5 +25,9 @@ public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_empty_result.hpp

@@ -24,5 +24,9 @@ public:
 public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_positional_scan.hpp

@@ -39,6 +39,10 @@ public:
 	             LocalSourceState &lstate) const override;
 
 	double GetProgress(ClientContext &context, GlobalSourceState &gstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp

@@ -60,6 +60,9 @@ public:
 	idx_t GetBatchIndex(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	                    LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
 	bool ParallelSource() const override {
 		return true;
 	}

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_alter.hpp

@@ -31,6 +31,10 @@ public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_attach.hpp

@@ -31,6 +31,10 @@ public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_function.hpp

@@ -31,6 +31,10 @@ public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_index.hpp

@@ -43,6 +43,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	//! Sink interface, thread-local sink states
 	unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_schema.hpp

@@ -31,6 +31,10 @@ public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_sequence.hpp

@@ -31,6 +31,10 @@ public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_table.hpp

@@ -32,5 +32,9 @@ public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_type.hpp

@@ -29,6 +29,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink interface
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_view.hpp

@@ -31,6 +31,10 @@ public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_detach.hpp

@@ -30,6 +30,10 @@ public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_drop.hpp

@@ -31,6 +31,10 @@ public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp

@@ -33,6 +33,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink interface
 	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,

package/src/duckdb/src/include/duckdb/execution/physical_operator.hpp

@@ -16,6 +16,7 @@
 #include "duckdb/execution/execution_context.hpp"
 #include "duckdb/optimizer/join_order/join_node.hpp"
 #include "duckdb/execution/physical_operator_states.hpp"
+#include "duckdb/common/enums/order_preservation_type.hpp"
 
 namespace duckdb {
 class Event;
@@ -74,12 +75,6 @@ public:
 
 	virtual void Verify();
 
-	//! Whether or not the operator depends on the order of the input chunks
-	//! If this is set to true, we cannot do things like caching intermediate vectors
-	virtual bool IsOrderDependent() const {
-		return false;
-	}
-
 public:
 	// Operator interface
 	virtual unique_ptr<OperatorState> GetOperatorState(ExecutionContext &context) const;
@@ -97,6 +92,11 @@ public:
 		return false;
 	}
 
+	//! The influence the operator has on order (insertion order means no influence)
+	virtual OrderPreservationType OperatorOrder() const {
+		return OrderPreservationType::INSERTION_ORDER;
+	}
+
 public:
 	// Source interface
 	virtual unique_ptr<LocalSourceState> GetLocalSourceState(ExecutionContext &context,
@@ -119,8 +119,9 @@ public:
 		return false;
 	}
 
-	virtual bool IsOrderPreserving() const {
-		return true;
+	//! The type of order emitted by the operator (as a source)
+	virtual OrderPreservationType SourceOrder() const {
+		return OrderPreservationType::INSERTION_ORDER;
 	}
 
 	//! Returns the current progress percentage, or a negative value if progress bars are not supported
@@ -161,11 +162,16 @@ public:
 		return false;
 	}
 
+	//! Whether or not the sink operator depends on the order of the input chunks
+	//! If this is set to true, we cannot do things like caching intermediate vectors
+	virtual bool SinkOrderDependent() const {
+		return false;
+	}
+
 public:
 	// Pipeline construction
 	virtual vector<const PhysicalOperator *> GetSources() const;
 	bool AllSourcesSupportBatchIndex() const;
-	virtual bool AllOperatorsPreserveOrder() const;
 
 	virtual void BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline);
 

package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp

@@ -90,7 +90,7 @@ public:
 		return source;
 	}
 
-	//! Returns whether any of the operators in the pipeline care about preserving insertion order
+	//! Returns whether any of the operators in the pipeline care about preserving order
 	bool IsOrderDependent() const;
 
 private:

package/src/duckdb/src/main/extension/extension_helper.cpp

@@ -78,6 +78,13 @@
 #include "inet-extension.hpp"
 #endif
 
+#if defined(BUILD_AUTOCOMPLETE_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#define AUTOCOMPLETE_STATICALLY_LOADED true
+#include "sql_auto_complete-extension.hpp"
+#else
+#define AUTOCOMPLETE_STATICALLY_LOADED false
+#endif
+
 // Load the generated header file containing our list of extension headers
 #if defined(OOTE_HEADERS_AVAILABLE) && OOTE_HEADERS_AVAILABLE
 #include "extension_oote_loader.hpp"
@@ -97,6 +104,7 @@ static DefaultExtension internal_extensions[] = {
     {"httpfs", "Adds support for reading and writing files over a HTTP(S) connection", HTTPFS_STATICALLY_LOADED},
     {"json", "Adds support for JSON operations", JSON_STATICALLY_LOADED},
     {"jemalloc", "Overwrites system allocator with JEMalloc", JEMALLOC_STATICALLY_LOADED},
+    {"autocomplete", "Add supports for autocomplete in the shell", AUTOCOMPLETE_STATICALLY_LOADED},
     {"motherduck", "Enables motherduck integration with the system", false},
     {"sqlite_scanner", "Adds support for reading SQLite database files", false},
     {"postgres_scanner", "Adds support for reading from a Postgres database", false},
@@ -134,8 +142,8 @@ bool ExtensionHelper::AllowAutoInstall(const string &extension) {
 // Load Statically Compiled Extension
 //===--------------------------------------------------------------------===//
 void ExtensionHelper::LoadAllExtensions(DuckDB &db) {
-	unordered_set<string> extensions {"parquet",    "icu",  "tpch",  "tpcds",    "fts",  "httpfs",
-	                                  "visualizer", "json", "excel", "sqlsmith", "inet", "jemalloc"};
+	unordered_set<string> extensions {"parquet", "icu",   "tpch",     "tpcds", "fts",      "httpfs",      "visualizer",
+	                                  "json",    "excel", "sqlsmith", "inet",  "jemalloc", "autocomplete"};
 	for (auto &ext : extensions) {
 		LoadExtensionInternal(db, ext, true);
 	}
@@ -244,6 +252,13 @@ ExtensionLoadResult ExtensionHelper::LoadExtensionInternal(DuckDB &db, const std
 #else
 		// jemalloc extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
+#endif
+	} else if (extension == "autocomplete") {
+#if defined(BUILD_AUTOCOMPLETE_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+		db.LoadExtension<SQLAutoCompleteExtension>();
+#else
+		// autocomplete extension required but not build: skip this test
+		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "inet") {
 #if defined(BUILD_INET_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)

package/src/duckdb/src/parallel/pipeline.cpp

@@ -98,20 +98,29 @@ bool Pipeline::ScheduleParallel(shared_ptr<Event> &event) {
 
 bool Pipeline::IsOrderDependent() const {
 	auto &config = DBConfig::GetConfig(executor.context);
-	if (!config.options.preserve_insertion_order) {
-		return false;
-	}
-	if (sink && sink->IsOrderDependent()) {
-		return true;
-	}
-	if (source && source->IsOrderDependent()) {
-		return true;
+	if (source) {
+		auto source_order = source->SourceOrder();
+		if (source_order == OrderPreservationType::FIXED_ORDER) {
+			return true;
+		}
+		if (source_order == OrderPreservationType::NO_ORDER) {
+			return false;
+		}
 	}
 	for (auto &op : operators) {
-		if (op->IsOrderDependent()) {
+		if (op->OperatorOrder() == OrderPreservationType::NO_ORDER) {
+			return false;
+		}
+		if (op->OperatorOrder() == OrderPreservationType::FIXED_ORDER) {
 			return true;
 		}
 	}
+	if (!config.options.preserve_insertion_order) {
+		return false;
+	}
+	if (sink && sink->SinkOrderDependent()) {
+		return true;
+	}
 	return false;
 }
 
@@ -148,6 +157,9 @@ bool Pipeline::LaunchScanTasks(shared_ptr<Event> &event, idx_t max_threads) {
 
 void Pipeline::ResetSink() {
 	if (sink) {
+		if (!sink->IsSink()) {
+			throw InternalException("Sink of pipeline does not have IsSink set");
+		}
 		lock_guard<mutex> guard(sink->lock);
 		if (!sink->sink_state) {
 			sink->sink_state = sink->GetGlobalSinkState(GetClientContext());
@@ -172,6 +184,9 @@ void Pipeline::Reset() {
 }
 
 void Pipeline::ResetSource(bool force) {
+	if (source && !source->IsSource()) {
+		throw InternalException("Source of pipeline does not have IsSource set");
+	}
 	if (force || !source_state) {
 		source_state = source->GetGlobalSourceState(GetClientContext());
 	}