duckdb 0.7.2-dev1637.0 → 0.7.2-dev1684.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.7.2-dev1637.0",
+  "version": "0.7.2-dev1684.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/execution/expression_executor.cpp

@@ -56,7 +56,7 @@ Allocator &ExpressionExecutor::GetAllocator() {
 
 void ExpressionExecutor::AddExpression(const Expression &expr) {
 	expressions.push_back(&expr);
-	auto state = make_uniq<ExpressionExecutorState>(expr.ToString());
+	auto state = make_uniq<ExpressionExecutorState>();
 	Initialize(expr, *state);
 	state->Verify();
 	states.push_back(std::move(state));

package/src/duckdb/src/execution/expression_executor_state.cpp

@@ -31,11 +31,10 @@ ClientContext &ExpressionState::GetContext() {
 	return root.executor->GetContext();
 }
 
-ExpressionState::ExpressionState(const Expression &expr, ExpressionExecutorState &root)
-    : expr(expr), root(root), name(expr.ToString()) {
+ExpressionState::ExpressionState(const Expression &expr, ExpressionExecutorState &root) : expr(expr), root(root) {
 }
 
-ExpressionExecutorState::ExpressionExecutorState(const string &name) : profiler(), name(name) {
+ExpressionExecutorState::ExpressionExecutorState() : profiler() {
 }
 
 void ExpressionState::Verify(ExpressionExecutorState &root_executor) {

package/src/duckdb/src/execution/operator/aggregate/physical_hash_aggregate.cpp

@@ -843,7 +843,7 @@ public:
 			auto &grouping_gstate = ht_state.grouping_states[sidx];
 			count += grouping.table_data.Size(*grouping_gstate.table_state);
 		}
-		return MaxValue<idx_t>(1, count / RowGroup::ROW_GROUP_SIZE);
+		return MaxValue<idx_t>(1, count / STANDARD_VECTOR_SIZE);
 	}
 };
 

package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp

@@ -166,6 +166,16 @@ public:
 	}
 };
 
+bool PhysicalUngroupedAggregate::SinkOrderDependent() const {
+	for (auto &expr : aggregates) {
+		auto &aggr = expr->Cast<BoundAggregateExpression>();
+		if (aggr.function.order_dependent == AggregateOrderDependent::ORDER_DEPENDENT) {
+			return true;
+		}
+	}
+	return false;
+}
+
 unique_ptr<GlobalSinkState> PhysicalUngroupedAggregate::GetGlobalSinkState(ClientContext &context) const {
 	return make_uniq<UngroupedAggregateGlobalState>(*this, context);
 }

package/src/duckdb/src/execution/operator/helper/physical_execute.cpp

@@ -12,11 +12,6 @@ vector<PhysicalOperator *> PhysicalExecute::GetChildren() const {
 	return {plan};
 }
 
-bool PhysicalExecute::AllOperatorsPreserveOrder() const {
-	D_ASSERT(plan);
-	return plan->AllOperatorsPreserveOrder();
-}
-
 void PhysicalExecute::BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) {
 	// EXECUTE statement: build pipeline on child
 	meta_pipeline.Build(plan);

package/src/duckdb/src/execution/operator/helper/physical_materialized_collector.cpp

@@ -75,4 +75,8 @@ bool PhysicalMaterializedCollector::ParallelSink() const {
 	return parallel;
 }
 
+bool PhysicalMaterializedCollector::SinkOrderDependent() const {
+	return true;
+}
+
 } // namespace duckdb

package/src/duckdb/src/execution/operator/helper/physical_result_collector.cpp

@@ -35,11 +35,6 @@ vector<PhysicalOperator *> PhysicalResultCollector::GetChildren() const {
 	return {plan};
 }
 
-bool PhysicalResultCollector::AllOperatorsPreserveOrder() const {
-	D_ASSERT(plan);
-	return plan->AllOperatorsPreserveOrder();
-}
-
 void PhysicalResultCollector::BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) {
 	// operator is a sink, build a pipeline
 	sink_state.reset();

package/src/duckdb/src/execution/operator/helper/physical_streaming_limit.cpp

@@ -60,8 +60,8 @@ OperatorResultType PhysicalStreamingLimit::Execute(ExecutionContext &context, Da
 	return OperatorResultType::NEED_MORE_INPUT;
 }
 
-bool PhysicalStreamingLimit::IsOrderDependent() const {
-	return !parallel;
+OrderPreservationType PhysicalStreamingLimit::OperatorOrder() const {
+	return OrderPreservationType::FIXED_ORDER;
 }
 
 bool PhysicalStreamingLimit::ParallelOperator() const {

package/src/duckdb/src/execution/operator/set/physical_union.cpp

@@ -22,8 +22,19 @@ void PhysicalUnion::BuildPipelines(Pipeline &current, MetaPipeline &meta_pipelin
 
 	// order matters if any of the downstream operators are order dependent,
 	// or if the sink preserves order, but does not support batch indices to do so
-	auto snk = meta_pipeline.GetSink();
-	bool order_matters = current.IsOrderDependent() || (snk && snk->IsOrderPreserving() && !snk->RequiresBatchIndex());
+	auto sink = meta_pipeline.GetSink();
+	bool order_matters = false;
+	if (current.IsOrderDependent()) {
+		order_matters = true;
+	}
+	if (sink) {
+		if (sink->SinkOrderDependent() && !sink->RequiresBatchIndex()) {
+			order_matters = true;
+		}
+		if (!sink->ParallelSink()) {
+			order_matters = true;
+		}
+	}
 
 	// create a union pipeline that is identical to 'current'
 	auto union_pipeline = meta_pipeline.CreateUnionPipeline(current, order_matters);

package/src/duckdb/src/execution/physical_operator.cpp

@@ -184,21 +184,6 @@ bool PhysicalOperator::AllSourcesSupportBatchIndex() const {
 	return true;
 }
 
-bool PhysicalOperator::AllOperatorsPreserveOrder() const {
-	if (type == PhysicalOperatorType::ORDER_BY) {
-		return true;
-	}
-	if (!IsOrderPreserving()) {
-		return false;
-	}
-	for (auto &child : children) {
-		if (!child->AllOperatorsPreserveOrder()) {
-			return false;
-		}
-	}
-	return true;
-}
-
 void PhysicalOperator::Verify() {
 #ifdef DEBUG
 	auto sources = GetSources();

package/src/duckdb/src/execution/physical_plan/plan_insert.cpp

@@ -10,14 +10,34 @@
 
 namespace duckdb {
 
+static OrderPreservationType OrderPreservationRecursive(PhysicalOperator &op) {
+	if (op.IsSource()) {
+		return op.SourceOrder();
+	}
+	for (auto &child : op.children) {
+		auto child_preservation = OrderPreservationRecursive(*child);
+		if (child_preservation != OrderPreservationType::INSERTION_ORDER) {
+			return child_preservation;
+		}
+	}
+	return OrderPreservationType::INSERTION_ORDER;
+}
+
 bool PhysicalPlanGenerator::PreserveInsertionOrder(ClientContext &context, PhysicalOperator &plan) {
 	auto &config = DBConfig::GetConfig(context);
-	if (!config.options.preserve_insertion_order) {
-		// preserving insertion order is disabled by config
+
+	auto preservation_type = OrderPreservationRecursive(plan);
+	if (preservation_type == OrderPreservationType::FIXED_ORDER) {
+		// always need to maintain preservation order
+		return true;
+	}
+	if (preservation_type == OrderPreservationType::NO_ORDER) {
+		// never need to preserve order
 		return false;
 	}
-	if (!plan.AllOperatorsPreserveOrder()) {
-		// the plan has no order defined: no need to preserve insertion order
+	// preserve insertion order - check flags
+	if (!config.options.preserve_insertion_order) {
+		// preserving insertion order is disabled by config
 		return false;
 	}
 	return true;

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.7.2-dev1637"
+#define DUCKDB_VERSION "0.7.2-dev1684"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "bacede9532"
+#define DUCKDB_SOURCE_ID "754ff32005"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/common/enums/order_preservation_type.hpp

@@ -0,0 +1,24 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// src/include/duckdb/common/enums/order_preservation_type.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/constants.hpp"
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// Order Preservation Type
+//===--------------------------------------------------------------------===//
+enum class OrderPreservationType : uint8_t {
+	NO_ORDER,        // the operator makes no guarantees on order preservation (i.e. it might re-order the entire input)
+	INSERTION_ORDER, // the operator maintains the order of the child operators
+	FIXED_ORDER      // the operator outputs rows in a fixed order that must be maintained (e.g. ORDER BY)
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/expression_executor_state.hpp

@@ -29,7 +29,6 @@ struct ExpressionState {
 	vector<unique_ptr<ExpressionState>> child_states;
 	vector<LogicalType> types;
 	DataChunk intermediate_chunk;
-	string name;
 	CycleCounter profiler;
 
 public:
@@ -55,12 +54,11 @@ public:
 };
 
 struct ExpressionExecutorState {
-	explicit ExpressionExecutorState(const string &name);
+	ExpressionExecutorState();
 
 	unique_ptr<ExpressionState> root_state;
 	ExpressionExecutor *executor = nullptr;
 	CycleCounter profiler;
-	string name;
 
 	void Verify();
 };

package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_hash_aggregate.hpp

@@ -93,12 +93,15 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
 	bool ParallelSource() const override {
 		return true;
 	}
 
-	bool IsOrderPreserving() const override {
-		return false;
+	OrderPreservationType SourceOrder() const override {
+		return OrderPreservationType::NO_ORDER;
 	}
 
 public:
@@ -122,6 +125,10 @@ public:
 		return true;
 	}
 
+	bool SinkOrderDependent() const override {
+		return false;
+	}
+
 public:
 	string ParamsToString() const override;
 	//! Toggle multi-scan capability on a hash table, which prevents the scan of the aggregate from being destructive

package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_perfecthash_aggregate.hpp

@@ -37,6 +37,13 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+	OrderPreservationType SourceOrder() const override {
+		return OrderPreservationType::NO_ORDER;
+	}
+
 public:
 	// Sink interface
 	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,
@@ -59,7 +66,7 @@ public:
 		return true;
 	}
 
-	bool IsOrderPreserving() const override {
+	bool SinkOrderDependent() const override {
 		return false;
 	}
 

package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_streaming_window.hpp

@@ -33,8 +33,8 @@ public:
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
 	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
-	bool IsOrderDependent() const override {
-		return true;
+	OrderPreservationType OperatorOrder() const override {
+		return OrderPreservationType::FIXED_ORDER;
 	}
 
 	string ParamsToString() const override;

package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_ungrouped_aggregate.hpp

@@ -39,6 +39,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink interface
 	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,
@@ -60,6 +64,8 @@ public:
 		return true;
 	}
 
+	bool SinkOrderDependent() const override;
+
 private:
 	//! Finalize the distinct aggregates
 	SinkFinalizeType FinalizeDistinct(Pipeline &pipeline, Event &event, ClientContext &context,

package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_window.hpp

@@ -38,12 +38,15 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
 	bool ParallelSource() const override {
 		return true;
 	}
 
-	bool IsOrderPreserving() const override {
-		return true;
+	OrderPreservationType SourceOrder() const override {
+		return OrderPreservationType::NO_ORDER;
 	}
 
 public:
@@ -65,7 +68,7 @@ public:
 		return !is_order_dependent;
 	}
 
-	bool IsOrderDependent() const override {
+	bool SinkOrderDependent() const override {
 		return is_order_dependent;
 	}
 

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_execute.hpp

@@ -28,7 +28,6 @@ public:
 	vector<PhysicalOperator *> GetChildren() const override;
 
 public:
-	bool AllOperatorsPreserveOrder() const override;
 	void BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) override;
 };
 

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_explain_analyze.hpp

@@ -28,6 +28,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink Interface
 	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_limit.hpp

@@ -28,7 +28,7 @@ public:
 	unique_ptr<Expression> offset_expression;
 
 public:
-	bool IsOrderDependent() const override {
+	bool SinkOrderDependent() const override {
 		return true;
 	}
 
@@ -38,6 +38,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink Interface
 	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_limit_percent.hpp

@@ -33,7 +33,7 @@ public:
 	unique_ptr<Expression> offset_expression;
 
 public:
-	bool IsOrderDependent() const override {
+	bool SinkOrderDependent() const override {
 		return true;
 	}
 
@@ -43,6 +43,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink Interface
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_load.hpp

@@ -30,6 +30,10 @@ public:
 	// Source interface
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_materialized_collector.hpp

@@ -31,6 +31,7 @@ public:
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
 
 	bool ParallelSink() const override;
+	bool SinkOrderDependent() const override;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_pragma.hpp

@@ -34,6 +34,10 @@ public:
 	// Source interface
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_prepare.hpp

@@ -31,6 +31,10 @@ public:
 	// Source interface
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_reservoir_sample.hpp

@@ -28,6 +28,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink interface
 	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_reset.hpp

@@ -33,6 +33,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	const std::string name;
 	const SetScope scope;

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_result_collector.hpp

@@ -37,8 +37,11 @@ public:
 
 public:
 	vector<PhysicalOperator *> GetChildren() const override;
-	bool AllOperatorsPreserveOrder() const override;
 	void BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_set.hpp

@@ -33,6 +33,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 	static void SetExtensionVariable(ClientContext &context, ExtensionOption &extension_option, const string &name,
 	                                 SetScope scope, const Value &value);
 

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_streaming_limit.hpp

@@ -35,7 +35,7 @@ public:
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
 	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
-	bool IsOrderDependent() const override;
+	OrderPreservationType OperatorOrder() const override;
 	bool ParallelOperator() const override;
 };
 

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_transaction.hpp

@@ -29,6 +29,10 @@ public:
 public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_vacuum.hpp

@@ -29,6 +29,10 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	bool IsSource() const override {
+		return true;
+	}
+
 public:
 	// Sink interface
 	unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;

package/src/duckdb/src/include/duckdb/execution/operator/join/physical_cross_product.hpp

@@ -26,19 +26,17 @@ public:
 	// Operator Interface
 	unique_ptr<OperatorState> GetOperatorState(ExecutionContext &context) const override;
 
+	OrderPreservationType OperatorOrder() const override {
+		return OrderPreservationType::NO_ORDER;
+	}
 	bool ParallelOperator() const override {
 		return true;
 	}
 
 protected:
-	// CachingOperator Interface
 	OperatorResultType ExecuteInternal(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
 	                                   GlobalOperatorState &gstate, OperatorState &state) const override;
 
-	bool IsOrderPreserving() const override {
-		return false;
-	}
-
 public:
 	// Sink Interface
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
@@ -51,6 +49,9 @@ public:
 	bool ParallelSink() const override {
 		return true;
 	}
+	bool SinkOrderDependent() const override {
+		return false;
+	}
 
 public:
 	void BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) override;

package/src/duckdb/src/include/duckdb/execution/operator/join/physical_delim_join.hpp

@@ -46,7 +46,10 @@ public:
 	bool ParallelSink() const override {
 		return true;
 	}
-	bool IsOrderPreserving() const override {
+	OrderPreservationType SourceOrder() const override {
+		return OrderPreservationType::NO_ORDER;
+	}
+	bool SinkOrderDependent() const override {
 		return false;
 	}
 	string ParamsToString() const override;

package/src/duckdb/src/include/duckdb/execution/operator/join/physical_index_join.hpp

@@ -55,6 +55,9 @@ public:
 public:
 	unique_ptr<OperatorState> GetOperatorState(ExecutionContext &context) const override;
 
+	OrderPreservationType OperatorOrder() const override {
+		return OrderPreservationType::NO_ORDER;
+	}
 	bool ParallelOperator() const override {
 		return true;
 	}
@@ -63,10 +66,6 @@ protected:
 	OperatorResultType ExecuteInternal(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
 	                                   GlobalOperatorState &gstate, OperatorState &state) const override;
 
-	bool IsOrderPreserving() const override {
-		return false;
-	}
-
 public:
 	void BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) override;
 	vector<const PhysicalOperator *> GetSources() const override;

package/src/duckdb/src/include/duckdb/execution/operator/join/physical_join.hpp

@@ -37,7 +37,13 @@ public:
 	void BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) override;
 	vector<const PhysicalOperator *> GetSources() const override;
 
-	bool IsOrderPreserving() const override {
+	OrderPreservationType SourceOrder() const override {
+		return OrderPreservationType::NO_ORDER;
+	}
+	OrderPreservationType OperatorOrder() const override {
+		return OrderPreservationType::NO_ORDER;
+	}
+	bool SinkOrderDependent() const override {
 		return false;
 	}
 };