duckdb 0.3.2-dev330.0 → 0.3.2-dev341.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.2-dev330.0",
+  "version": "0.3.2-dev341.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -49529,7 +49529,9 @@ idx_t GroupedAggregateHashTable::Scan(idx_t &scan_position, DataChunk &result) {
 }
 
 void GroupedAggregateHashTable::Finalize() {
-	D_ASSERT(!is_finalized);
+	if (is_finalized) {
+		return;
+	}
 
 	// early release hashes, not needed for partition/scan
 	hashes_hdl.reset();
@@ -65967,18 +65969,11 @@ public:
 	vector<vector<unique_ptr<Expression>>> expressions;
 
 public:
-	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
-	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
-	             LocalSourceState &lstate) const override;
-
-public:
-	// Sink interface
-	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate,
-	                    DataChunk &input) const override;
-
-	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
+	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
+	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
+	                           OperatorState &state) const override;
 
-	bool IsSink() const override {
+	bool ParallelOperator() const override {
 		return true;
 	}
 
@@ -65994,7 +65989,7 @@ public:
 
 namespace duckdb {
 
-class ExpressionScanState : public GlobalSourceState {
+class ExpressionScanState : public OperatorState {
 public:
 	explicit ExpressionScanState(const PhysicalExpressionScan &op) : expression_index(0) {
 		temp_chunk.Initialize(op.GetTypes());
@@ -66006,16 +66001,26 @@ public:
 	DataChunk temp_chunk;
 };
 
-class ExpressionSinkState : public GlobalSinkState {
-public:
-	ExpressionSinkState() {
-	}
+unique_ptr<OperatorState> PhysicalExpressionScan::GetOperatorState(ClientContext &context) const {
+	return make_unique<ExpressionScanState>(*this);
+}
 
-	DataChunk child_chunk;
-};
+OperatorResultType PhysicalExpressionScan::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
+                                                   OperatorState &state_p) const {
+	auto &state = (ExpressionScanState &)state_p;
 
-unique_ptr<GlobalSourceState> PhysicalExpressionScan::GetGlobalSourceState(ClientContext &context) const {
-	return make_unique<ExpressionScanState>(*this);
+	for (; chunk.size() + input.size() <= STANDARD_VECTOR_SIZE && state.expression_index < expressions.size();
+	     state.expression_index++) {
+		state.temp_chunk.Reset();
+		EvaluateExpression(state.expression_index, &input, state.temp_chunk);
+		chunk.Append(state.temp_chunk);
+	}
+	if (state.expression_index < expressions.size()) {
+		return OperatorResultType::HAVE_MORE_OUTPUT;
+	} else {
+		state.expression_index = 0;
+		return OperatorResultType::NEED_MORE_INPUT;
+	}
 }
 
 void PhysicalExpressionScan::EvaluateExpression(idx_t expression_idx, DataChunk *child_chunk, DataChunk &result) const {
@@ -66028,20 +66033,6 @@ void PhysicalExpressionScan::EvaluateExpression(idx_t expression_idx, DataChunk
 	}
 }
 
-void PhysicalExpressionScan::GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate_p,
-                                     LocalSourceState &lstate) const {
-	D_ASSERT(sink_state);
-	auto &state = (ExpressionScanState &)gstate_p;
-	auto &gstate = (ExpressionSinkState &)*sink_state;
-
-	for (; chunk.size() < STANDARD_VECTOR_SIZE && state.expression_index < expressions.size();
-	     state.expression_index++) {
-		state.temp_chunk.Reset();
-		EvaluateExpression(state.expression_index, &gstate.child_chunk, state.temp_chunk);
-		chunk.Append(state.temp_chunk);
-	}
-}
-
 bool PhysicalExpressionScan::IsFoldable() const {
 	for (auto &expr_list : expressions) {
 		for (auto &expr : expr_list) {
@@ -66053,24 +66044,6 @@ bool PhysicalExpressionScan::IsFoldable() const {
 	return true;
 }
 
-SinkResultType PhysicalExpressionScan::Sink(ExecutionContext &context, GlobalSinkState &gstate_p,
-                                            LocalSinkState &lstate, DataChunk &input) const {
-	auto &gstate = (ExpressionSinkState &)gstate_p;
-
-	D_ASSERT(children.size() == 1);
-	D_ASSERT(gstate.child_chunk.size() == 0);
-	if (input.size() != 1) {
-		throw InternalException("Expected expression scan child to have exactly one element");
-	}
-	gstate.child_chunk.Move(input);
-	gstate.child_chunk.Verify();
-	return SinkResultType::FINISHED;
-}
-
-unique_ptr<GlobalSinkState> PhysicalExpressionScan::GetGlobalSinkState(ClientContext &context) const {
-	return make_unique<ExpressionSinkState>();
-}
-
 } // namespace duckdb
 
 
@@ -84826,14 +84799,7 @@ static void LeastGreatestFunction(DataChunk &args, ExpressionState &state, Vecto
 	}
 	auto result_type = VectorType::CONSTANT_VECTOR;
 	for (idx_t col_idx = 0; col_idx < args.ColumnCount(); col_idx++) {
-		if (args.data[col_idx].GetVectorType() == VectorType::CONSTANT_VECTOR) {
-			if (ConstantVector::IsNull(args.data[col_idx])) {
-				// constant NULL: result is constant NULL
-				result.SetVectorType(VectorType::CONSTANT_VECTOR);
-				ConstantVector::SetNull(result, true);
-				return;
-			}
-		} else {
+		if (args.data[col_idx].GetVectorType() != VectorType::CONSTANT_VECTOR) {
 			// non-constant input: result is not a constant vector
 			result_type = VectorType::FLAT_VECTOR;
 		}
@@ -84843,60 +84809,67 @@ static void LeastGreatestFunction(DataChunk &args, ExpressionState &state, Vecto
 		}
 	}
 
-	// we start off performing a binary operation between the first two inputs, where we store the lowest (or highest)
-	// directly in the result
-	BinaryExecutor::ExecuteGeneric<T, T, T, BinarySingleArgumentOperatorWrapper, LeastOperator<OP>, bool>(
-	    args.data[0], args.data[1], result, args.size(), false);
-
-	// now we loop over the other columns and compare it to the stored result
 	auto result_data = FlatVector::GetData<T>(result);
 	auto &result_mask = FlatVector::Validity(result);
-	SelectionVector rsel;
-	idx_t rcount = 0;
-	// create a selection vector from the mask
-	rsel.Initialize();
-	for (idx_t i = 0; i < args.size(); i++) {
-		if (result_mask.RowIsValid(i)) {
-			rsel.set_index(rcount++, i);
+	// copy over the first column
+	bool result_has_value[STANDARD_VECTOR_SIZE];
+	{
+		VectorData vdata;
+		args.data[0].Orrify(args.size(), vdata);
+		auto input_data = (T *)vdata.data;
+		for (idx_t i = 0; i < args.size(); i++) {
+			auto vindex = vdata.sel->get_index(i);
+			if (vdata.validity.RowIsValid(vindex)) {
+				result_data[i] = input_data[vindex];
+				result_has_value[i] = true;
+			} else {
+				result_has_value[i] = false;
+			}
 		}
 	}
-	for (idx_t col_idx = 2; col_idx < args.ColumnCount(); col_idx++) {
+	// now handle the remainder of the columns
+	for (idx_t col_idx = 1; col_idx < args.ColumnCount(); col_idx++) {
+		if (args.data[col_idx].GetVectorType() == VectorType::CONSTANT_VECTOR &&
+		    ConstantVector::IsNull(args.data[col_idx])) {
+			// ignore null vector
+			continue;
+		}
+
 		VectorData vdata;
 		args.data[col_idx].Orrify(args.size(), vdata);
 
 		auto input_data = (T *)vdata.data;
 		if (!vdata.validity.AllValid()) {
 			// potential new null entries: have to check the null mask
-			idx_t new_count = 0;
-			for (idx_t i = 0; i < rcount; i++) {
-				auto rindex = rsel.get_index(i);
-				auto vindex = vdata.sel->get_index(rindex);
-				if (!vdata.validity.RowIsValid(vindex)) {
-					// new null entry: set nullmask
-					result_mask.SetInvalid(rindex);
-				} else {
+			for (idx_t i = 0; i < args.size(); i++) {
+				auto vindex = vdata.sel->get_index(i);
+				if (vdata.validity.RowIsValid(vindex)) {
 					// not a null entry: perform the operation and add to new set
 					auto ivalue = input_data[vindex];
-					if (OP::template Operation<T>(ivalue, result_data[rindex])) {
-						result_data[rindex] = ivalue;
+					if (!result_has_value[i] || OP::template Operation<T>(ivalue, result_data[i])) {
+						result_has_value[i] = true;
+						result_data[i] = ivalue;
 					}
-					rsel.set_index(new_count++, rindex);
 				}
 			}
-			rcount = new_count;
 		} else {
 			// no new null entries: only need to perform the operation
-			for (idx_t i = 0; i < rcount; i++) {
-				auto rindex = rsel.get_index(i);
-				auto vindex = vdata.sel->get_index(rindex);
+			for (idx_t i = 0; i < args.size(); i++) {
+				auto vindex = vdata.sel->get_index(i);
 
 				auto ivalue = input_data[vindex];
-				if (OP::template Operation<T>(ivalue, result_data[rindex])) {
-					result_data[rindex] = ivalue;
+				if (!result_has_value[i] || OP::template Operation<T>(ivalue, result_data[i])) {
+					result_has_value[i] = true;
+					result_data[i] = ivalue;
 				}
 			}
 		}
 	}
+	for (idx_t i = 0; i < args.size(); i++) {
+		if (!result_has_value[i]) {
+			result_mask.SetInvalid(i);
+		}
+	}
 	result.SetVectorType(result_type);
 }
 
@@ -117164,6 +117137,8 @@ struct CSEReplacementState {
 	column_binding_map_t<idx_t> column_map;
 	//! The set of expressions of the resulting projection
 	vector<unique_ptr<Expression>> expressions;
+	//! Cached expressions that are kept around so the expression_map always contains valid expressions
+	vector<unique_ptr<Expression>> cached_expressions;
 };
 
 void CommonSubExpressionOptimizer::VisitOperator(LogicalOperator &op) {
@@ -117239,6 +117214,8 @@ void CommonSubExpressionOptimizer::PerformCSEReplacement(unique_ptr<Expression>
 				// has not been pushed yet: push it
 				node.column_index = state.expressions.size();
 				state.expressions.push_back(move(*expr_ptr));
+			} else {
+				state.cached_expressions.push_back(move(*expr_ptr));
 			}
 			// replace the original expression with a bound column ref
 			*expr_ptr = make_unique<BoundColumnRefExpression>(alias, type,
@@ -126069,7 +126046,6 @@ void Executor::BuildPipelines(PhysicalOperator *op, Pipeline *current) {
 		case PhysicalOperatorType::TOP_N:
 		case PhysicalOperatorType::COPY_TO_FILE:
 		case PhysicalOperatorType::LIMIT:
-		case PhysicalOperatorType::EXPRESSION_SCAN:
 		case PhysicalOperatorType::EXPLAIN_ANALYZE:
 			D_ASSERT(op->children.size() == 1);
 			// single operator:
@@ -154822,6 +154798,15 @@ void ExpressionIterator::EnumerateTableRefChildren(BoundTableRef &ref,
 		EnumerateTableRefChildren(*bound_crossproduct.right, callback);
 		break;
 	}
+	case TableReferenceType::EXPRESSION_LIST: {
+		auto &bound_expr_list = (BoundExpressionListRef &)ref;
+		for (auto &expr_list : bound_expr_list.values) {
+			for (auto &expr : expr_list) {
+				EnumerateExpression(expr, callback);
+			}
+		}
+		break;
+	}
 	case TableReferenceType::JOIN: {
 		auto &bound_join = (BoundJoinRef &)ref;
 		EnumerateExpression(bound_join.condition, callback);
@@ -156577,6 +156562,29 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 	case LogicalOperatorType::LOGICAL_DISTINCT:
 		plan->children[0] = PushDownDependentJoin(move(plan->children[0]));
 		return plan;
+	case LogicalOperatorType::LOGICAL_EXPRESSION_GET: {
+		// expression get
+		// first we flatten the dependent join in the child
+		plan->children[0] = PushDownDependentJoinInternal(move(plan->children[0]));
+		// then we replace any correlated expressions with the corresponding entry in the correlated_map
+		RewriteCorrelatedExpressions rewriter(base_binding, correlated_map);
+		rewriter.VisitOperator(*plan);
+		// now we add all the correlated columns to each of the expressions of the expression scan
+		auto expr_get = (LogicalExpressionGet *)plan.get();
+		for (idx_t i = 0; i < correlated_columns.size(); i++) {
+			for (auto &expr_list : expr_get->expressions) {
+				auto colref = make_unique<BoundColumnRefExpression>(
+				    correlated_columns[i].type, ColumnBinding(base_binding.table_index, base_binding.column_index + i));
+				expr_list.push_back(move(colref));
+			}
+			expr_get->expr_types.push_back(correlated_columns[i].type);
+		}
+
+		base_binding.table_index = expr_get->table_index;
+		this->delim_offset = base_binding.column_index = expr_get->expr_types.size() - correlated_columns.size();
+		this->data_offset = 0;
+		return plan;
+	}
 	case LogicalOperatorType::LOGICAL_ORDER_BY:
 		throw ParserException("ORDER BY not supported in correlated subquery");
 	default:

package/src/duckdb.hpp

@@ -10,8 +10,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 #pragma once
 #define DUCKDB_AMALGAMATION 1
-#define DUCKDB_SOURCE_ID "ae8562793"
-#define DUCKDB_VERSION "0.3.2-dev330"
+#define DUCKDB_SOURCE_ID "32060df04"
+#define DUCKDB_VERSION "0.3.2-dev341"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //