duckdb 0.3.1-dev360.0 → 0.3.1-dev399.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.1-dev360.0",
+  "version": "0.3.1-dev399.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -4922,6 +4922,8 @@ public:
 	void ExecuteExpression(idx_t expr_idx, Vector &result);
 	//! Evaluate a scalar expression and fold it into a single value
 	static Value EvaluateScalar(const Expression &expr);
+	//! Try to evaluate a scalar expression and fold it into a single value, returns false if an exception is thrown
+	static bool TryEvaluateScalar(const Expression &expr, Value &result);
 
 	//! Initialize the state of a given expression
 	static unique_ptr<ExpressionState> InitializeState(const Expression &expr, ExpressionExecutorState &state);
@@ -5005,6 +5007,8 @@ protected:
 	//! Verify that the output of a step in the ExpressionExecutor is correct
 	void Verify(const Expression &expr, Vector &result, idx_t count);
 
+	void FillSwitch(Vector &vector, Vector &result, SelectionVector &sel, sel_t count);
+
 private:
 	//! The states of the expression executor; this holds any intermediates and temporary states of expressions
 	vector<unique_ptr<ExpressionExecutorState>> states;
@@ -34023,10 +34027,11 @@ static void QuicksortInPlace(ChunkCollection *sort_by, vector<OrderType> &desc,
 }
 
 void ChunkCollection::Sort(vector<OrderType> &desc, vector<OrderByNullType> &null_order, idx_t result[]) {
-	D_ASSERT(result);
 	if (count == 0) {
 		return;
 	}
+	D_ASSERT(result);
+
 	// start off with an initial quicksort
 	int64_t part = QuicksortInitial(this, desc, null_order, result);
 
@@ -48392,9 +48397,6 @@ idx_t ExpressionExecutor::Select(const BoundBetweenExpression &expr, ExpressionS
 
 namespace duckdb {
 
-void Case(Vector &res_true, Vector &res_false, Vector &result, SelectionVector &tside, idx_t tcount,
-          SelectionVector &fside, idx_t fcount);
-
 struct CaseExpressionState : public ExpressionState {
 	CaseExpressionState(const Expression &expr, ExpressionExecutorState &root)
 	    : ExpressionState(expr, root), true_sel(STANDARD_VECTOR_SIZE), false_sel(STANDARD_VECTOR_SIZE) {
@@ -48442,7 +48444,8 @@ void ExpressionExecutor::Execute(const BoundCaseExpression &expr, ExpressionStat
 		Execute(*expr.result_if_true, res_true_state, &true_sel, tcount, res_true);
 		Execute(*expr.result_if_false, res_false_state, &false_sel, fcount, res_false);
 
-		Case(res_true, res_false, result, true_sel, tcount, false_sel, fcount);
+		FillSwitch(res_true, result, true_sel, tcount);
+		FillSwitch(res_false, result, false_sel, fcount);
 		if (sel) {
 			result.Slice(*sel, count);
 		}
@@ -48500,111 +48503,77 @@ void ValidityFillLoop(Vector &vector, Vector &result, SelectionVector &sel, sel_
 	}
 }
 
-template <class T>
-void TemplatedCaseLoop(Vector &res_true, Vector &res_false, Vector &result, SelectionVector &tside, idx_t tcount,
-                       SelectionVector &fside, idx_t fcount) {
-	TemplatedFillLoop<T>(res_true, result, tside, tcount);
-	TemplatedFillLoop<T>(res_false, result, fside, fcount);
-}
-
-void ValidityCaseLoop(Vector &res_true, Vector &res_false, Vector &result, SelectionVector &tside, idx_t tcount,
-                      SelectionVector &fside, idx_t fcount) {
-	ValidityFillLoop(res_true, result, tside, tcount);
-	ValidityFillLoop(res_false, result, fside, fcount);
-}
-
-void Case(Vector &res_true, Vector &res_false, Vector &result, SelectionVector &tside, idx_t tcount,
-          SelectionVector &fside, idx_t fcount) {
-	D_ASSERT(res_true.GetType() == res_false.GetType() && res_true.GetType() == result.GetType());
-
+void ExpressionExecutor::FillSwitch(Vector &vector, Vector &result, SelectionVector &sel, sel_t count) {
 	switch (result.GetType().InternalType()) {
 	case PhysicalType::BOOL:
 	case PhysicalType::INT8:
-		TemplatedCaseLoop<int8_t>(res_true, res_false, result, tside, tcount, fside, fcount);
+		TemplatedFillLoop<int8_t>(vector, result, sel, count);
 		break;
 	case PhysicalType::INT16:
-		TemplatedCaseLoop<int16_t>(res_true, res_false, result, tside, tcount, fside, fcount);
+		TemplatedFillLoop<int16_t>(vector, result, sel, count);
 		break;
 	case PhysicalType::INT32:
-		TemplatedCaseLoop<int32_t>(res_true, res_false, result, tside, tcount, fside, fcount);
+		TemplatedFillLoop<int32_t>(vector, result, sel, count);
 		break;
 	case PhysicalType::INT64:
-		TemplatedCaseLoop<int64_t>(res_true, res_false, result, tside, tcount, fside, fcount);
+		TemplatedFillLoop<int64_t>(vector, result, sel, count);
 		break;
 	case PhysicalType::UINT8:
-		TemplatedCaseLoop<uint8_t>(res_true, res_false, result, tside, tcount, fside, fcount);
+		TemplatedFillLoop<uint8_t>(vector, result, sel, count);
 		break;
 	case PhysicalType::UINT16:
-		TemplatedCaseLoop<uint16_t>(res_true, res_false, result, tside, tcount, fside, fcount);
+		TemplatedFillLoop<uint16_t>(vector, result, sel, count);
 		break;
 	case PhysicalType::UINT32:
-		TemplatedCaseLoop<uint32_t>(res_true, res_false, result, tside, tcount, fside, fcount);
+		TemplatedFillLoop<uint32_t>(vector, result, sel, count);
 		break;
 	case PhysicalType::UINT64:
-		TemplatedCaseLoop<uint64_t>(res_true, res_false, result, tside, tcount, fside, fcount);
+		TemplatedFillLoop<uint64_t>(vector, result, sel, count);
 		break;
 	case PhysicalType::INT128:
-		TemplatedCaseLoop<hugeint_t>(res_true, res_false, result, tside, tcount, fside, fcount);
+		TemplatedFillLoop<hugeint_t>(vector, result, sel, count);
 		break;
 	case PhysicalType::FLOAT:
-		TemplatedCaseLoop<float>(res_true, res_false, result, tside, tcount, fside, fcount);
+		TemplatedFillLoop<float>(vector, result, sel, count);
 		break;
 	case PhysicalType::DOUBLE:
-		TemplatedCaseLoop<double>(res_true, res_false, result, tside, tcount, fside, fcount);
+		TemplatedFillLoop<double>(vector, result, sel, count);
 		break;
 	case PhysicalType::INTERVAL:
-		TemplatedCaseLoop<interval_t>(res_true, res_false, result, tside, tcount, fside, fcount);
+		TemplatedFillLoop<interval_t>(vector, result, sel, count);
 		break;
 	case PhysicalType::VARCHAR:
-		TemplatedCaseLoop<string_t>(res_true, res_false, result, tside, tcount, fside, fcount);
-		StringVector::AddHeapReference(result, res_true);
-		StringVector::AddHeapReference(result, res_false);
+		TemplatedFillLoop<string_t>(vector, result, sel, count);
+		StringVector::AddHeapReference(result, vector);
 		break;
 	case PhysicalType::STRUCT: {
-		auto &res_true_entries = StructVector::GetEntries(res_true);
-		auto &res_false_entries = StructVector::GetEntries(res_false);
+		auto &vector_entries = StructVector::GetEntries(vector);
 		auto &result_entries = StructVector::GetEntries(result);
-		D_ASSERT(res_true_entries.size() == res_false_entries.size() &&
-		         res_true_entries.size() == result_entries.size());
-		ValidityCaseLoop(res_true, res_false, result, tside, tcount, fside, fcount);
-		for (idx_t i = 0; i < res_true_entries.size(); i++) {
-			Case(*res_true_entries[i], *res_false_entries[i], *result_entries[i], tside, tcount, fside, fcount);
+		ValidityFillLoop(vector, result, sel, count);
+		D_ASSERT(vector_entries.size() == result_entries.size());
+		for (idx_t i = 0; i < vector_entries.size(); i++) {
+			FillSwitch(*vector_entries[i], *result_entries[i], sel, count);
 		}
 		break;
 	}
 	case PhysicalType::LIST: {
-		idx_t offset = 0;
-
-		auto &true_child = ListVector::GetEntry(res_true);
-		offset += ListVector::GetListSize(res_true);
-		ListVector::Append(result, true_child, ListVector::GetListSize(res_true));
-
-		auto &false_child = ListVector::GetEntry(res_false);
-		ListVector::Append(result, false_child, ListVector::GetListSize(res_false));
+		idx_t offset = ListVector::GetListSize(result);
+		auto &list_child = ListVector::GetEntry(vector);
+		ListVector::Append(result, list_child, ListVector::GetListSize(vector));
 
 		// all the false offsets need to be incremented by true_child.count
-		TemplatedFillLoop<list_entry_t>(res_true, result, tside, tcount);
-
-		// FIXME the nullmask here is likely borked
-		// TODO uuugly
-		VectorData fdata;
-		res_false.Orrify(fcount, fdata);
-
-		auto data = (list_entry_t *)fdata.data;
-		auto res = FlatVector::GetData<list_entry_t>(result);
-		auto &mask = FlatVector::Validity(result);
+		TemplatedFillLoop<list_entry_t>(vector, result, sel, count);
+		if (offset == 0) {
+			break;
+		}
 
-		for (idx_t i = 0; i < fcount; i++) {
-			auto fidx = fdata.sel->get_index(i);
-			auto res_idx = fside.get_index(i);
-			auto list_entry = data[fidx];
-			list_entry.offset += offset;
-			res[res_idx] = list_entry;
-			mask.Set(res_idx, fdata.validity.RowIsValid(fidx));
+		auto result_data = FlatVector::GetData<list_entry_t>(result);
+		for (idx_t i = 0; i < count; i++) {
+			auto result_idx = sel.get_index(i);
+			result_data[result_idx].offset += offset;
 		}
 
-		result.Verify(tside, tcount);
-		result.Verify(fside, fcount);
+		result.Verify(sel, count);
 		break;
 	}
 	default:
@@ -49218,6 +49187,57 @@ void ExpressionExecutor::Execute(const BoundOperatorExpression &expr, Expression
 			// directly use the result
 			result.Reference(intermediate);
 		}
+	} else if (expr.type == ExpressionType::OPERATOR_COALESCE) {
+		SelectionVector sel_a(count);
+		SelectionVector sel_b(count);
+		SelectionVector slice_sel(count);
+		SelectionVector result_sel(count);
+		SelectionVector *next_sel = &sel_a;
+		const SelectionVector *current_sel = sel;
+		idx_t remaining_count = count;
+		idx_t next_count;
+		for (idx_t child = 0; child < expr.children.size(); child++) {
+			Vector vector_to_check(expr.children[child]->return_type);
+			Execute(*expr.children[child], state->child_states[child].get(), current_sel, remaining_count,
+			        vector_to_check);
+
+			VectorData vdata;
+			vector_to_check.Orrify(remaining_count, vdata);
+
+			idx_t result_count = 0;
+			next_count = 0;
+			for (idx_t i = 0; i < remaining_count; i++) {
+				auto base_idx = current_sel ? current_sel->get_index(i) : i;
+				auto idx = vdata.sel->get_index(i);
+				if (vdata.validity.RowIsValid(idx)) {
+					slice_sel.set_index(result_count, i);
+					result_sel.set_index(result_count++, base_idx);
+				} else {
+					next_sel->set_index(next_count++, base_idx);
+				}
+			}
+			if (result_count > 0) {
+				vector_to_check.Slice(slice_sel, result_count);
+				FillSwitch(vector_to_check, result, result_sel, result_count);
+			}
+			current_sel = next_sel;
+			next_sel = next_sel == &sel_a ? &sel_b : &sel_a;
+			remaining_count = next_count;
+			if (next_count == 0) {
+				break;
+			}
+		}
+		if (remaining_count > 0) {
+			auto &result_mask = FlatVector::Validity(result);
+			for (idx_t i = 0; i < remaining_count; i++) {
+				result_mask.SetInvalid(current_sel->get_index(i));
+			}
+		}
+		if (sel) {
+			result.Slice(*sel, count);
+		} else if (count == 1) {
+			result.SetVectorType(VectorType::CONSTANT_VECTOR);
+		}
 	} else if (expr.children.size() == 1) {
 		state->intermediate_chunk.Reset();
 		auto &child = state->intermediate_chunk.data[0];
@@ -49382,6 +49402,15 @@ Value ExpressionExecutor::EvaluateScalar(const Expression &expr) {
 	return result_value;
 }
 
+bool ExpressionExecutor::TryEvaluateScalar(const Expression &expr, Value &result) {
+	try {
+		result = EvaluateScalar(expr);
+		return true;
+	} catch (...) {
+		return false;
+	}
+}
+
 void ExpressionExecutor::Verify(const Expression &expr, Vector &vector, idx_t count) {
 	D_ASSERT(expr.return_type.id() == vector.GetType().id());
 	vector.Verify(count);
@@ -77156,9 +77185,11 @@ struct SortedAggregateFunction {
 
 			// Apply the sort before delegating the chunks
 			const auto agg_count = state->ordering.Count();
-			reordering.resize(agg_count);
-			state->ordering.Sort(order_bind->order_sense, order_bind->null_order, reordering.data());
-			state->arguments.Reorder(reordering.data());
+			if (agg_count > 0) {
+				reordering.resize(agg_count);
+				state->ordering.Sort(order_bind->order_sense, order_bind->null_order, reordering.data());
+				state->arguments.Reorder(reordering.data());
+			}
 
 			for (auto &chunk : state->arguments.Chunks()) {
 				// These are all simple updates, so use it if available
@@ -87128,7 +87159,9 @@ struct ChrOperator {
 	static inline TR Operation(const TA &input) {
 		char c[5] = {'\0', '\0', '\0', '\0', '\0'};
 		int utf8_bytes = 4;
-		Utf8Proc::CodepointToUtf8(input, utf8_bytes, &c[0]);
+		if (input < 0 || !Utf8Proc::CodepointToUtf8(input, utf8_bytes, &c[0])) {
+			throw InvalidInputException("Invalid UTF8 Codepoint %d", input);
+		}
 		return string_t(&c[0]);
 	}
 };
@@ -88869,7 +88902,9 @@ unique_ptr<FunctionData> BindPrintfFunction(ClientContext &context, ScalarFuncti
 template <class FORMAT_FUN, class CTX>
 static void PrintfFunction(DataChunk &args, ExpressionState &state, Vector &result) {
 	auto &format_string = args.data[0];
+	auto &result_validity = FlatVector::Validity(result);
 	result.SetVectorType(VectorType::CONSTANT_VECTOR);
+	result_validity.Initialize(args.size());
 	for (idx_t i = 0; i < args.ColumnCount(); i++) {
 		switch (args.data[i].GetVectorType()) {
 		case VectorType::CONSTANT_VECTOR:
@@ -88884,7 +88919,7 @@ static void PrintfFunction(DataChunk &args, ExpressionState &state, Vector &resu
 			// FLAT VECTOR, we can directly OR the nullmask
 			args.data[i].Normalify(args.size());
 			result.SetVectorType(VectorType::FLAT_VECTOR);
-			FlatVector::Validity(result).Combine(FlatVector::Validity(args.data[i]), args.size());
+			result_validity.Combine(FlatVector::Validity(args.data[i]), args.size());
 			break;
 		}
 	}
@@ -108069,7 +108104,7 @@ unique_ptr<Expression> InClauseRewriter::VisitReplace(BoundOperatorExpression &e
 	DataChunk chunk;
 	chunk.Initialize(types);
 	for (idx_t i = 1; i < expr.children.size(); i++) {
-		// reoslve this expression to a constant
+		// resolve this expression to a constant
 		auto value = ExpressionExecutor::EvaluateScalar(*expr.children[i]);
 		idx_t index = chunk.size();
 		chunk.SetCardinality(chunk.size() + 1);
@@ -111479,7 +111514,10 @@ unique_ptr<Expression> ComparisonSimplificationRule::Apply(LogicalOperator &op,
 	// the constant_expr is a scalar expression that we have to fold
 	// use an ExpressionExecutor to execute the expression
 	D_ASSERT(constant_expr->IsFoldable());
-	auto constant_value = ExpressionExecutor::EvaluateScalar(*constant_expr);
+	Value constant_value;
+	if (!ExpressionExecutor::TryEvaluateScalar(*constant_expr, constant_value)) {
+		return nullptr;
+	}
 	if (constant_value.is_null && !(expr->type == ExpressionType::COMPARE_NOT_DISTINCT_FROM ||
 	                                expr->type == ExpressionType::COMPARE_DISTINCT_FROM)) {
 		// comparison with constant NULL, return NULL
@@ -111549,7 +111587,11 @@ unique_ptr<Expression> ConjunctionSimplificationRule::Apply(LogicalOperator &op,
 	// the constant_expr is a scalar expression that we have to fold
 	// use an ExpressionExecutor to execute the expression
 	D_ASSERT(constant_expr->IsFoldable());
-	auto constant_value = ExpressionExecutor::EvaluateScalar(*constant_expr).CastAs(LogicalType::BOOLEAN);
+	Value constant_value;
+	if (!ExpressionExecutor::TryEvaluateScalar(*constant_expr, constant_value)) {
+		return nullptr;
+	}
+	constant_value = constant_value.CastAs(LogicalType::BOOLEAN);
 	if (constant_value.is_null) {
 		// we can't simplify conjunctions with a constant NULL
 		return nullptr;
@@ -111608,7 +111650,11 @@ unique_ptr<Expression> ConstantFoldingRule::Apply(LogicalOperator &op, vector<Ex
 	D_ASSERT(root->IsFoldable() && root->type != ExpressionType::VALUE_CONSTANT);
 
 	// use an ExpressionExecutor to execute the expression
-	auto result_value = ExpressionExecutor::EvaluateScalar(*root);
+
+	Value result_value;
+	if (!ExpressionExecutor::TryEvaluateScalar(*root, result_value)) {
+		return nullptr;
+	}
 	D_ASSERT(result_value.type().InternalType() == root->return_type.InternalType());
 	// now get the value from the result vector and insert it back into the plan as a constant expression
 	return make_unique<BoundConstantExpression>(result_value);
@@ -112800,6 +112846,42 @@ unique_ptr<BaseStatistics> StatisticsPropagator::PropagateExpression(BoundOperat
 		return nullptr;
 	}
 	switch (expr.type) {
+	case ExpressionType::OPERATOR_COALESCE:
+		// COALESCE, merge stats of all children
+		for (idx_t i = 0; i < expr.children.size(); i++) {
+			D_ASSERT(child_stats[i]);
+			if (!child_stats[i]->CanHaveNoNull()) {
+				// this child is always NULL, we can remove it from the coalesce
+				// UNLESS there is only one node remaining
+				if (expr.children.size() > 1) {
+					expr.children.erase(expr.children.begin() + i);
+					child_stats.erase(child_stats.begin() + i);
+					i--;
+				}
+			} else if (!child_stats[i]->CanHaveNull()) {
+				// coalesce child cannot have NULL entries
+				// this is the last coalesce node that influences the result
+				// we can erase any children after this node
+				if (i + 1 < expr.children.size()) {
+					expr.children.erase(expr.children.begin() + i + 1, expr.children.end());
+					child_stats.erase(child_stats.begin() + i + 1, child_stats.end());
+				}
+				break;
+			}
+		}
+		D_ASSERT(!expr.children.empty());
+		D_ASSERT(expr.children.size() == child_stats.size());
+		if (expr.children.size() == 1) {
+			// coalesce of one entry: simply return that entry
+			*expr_ptr = move(expr.children[0]);
+		} else {
+			// coalesce of multiple entries
+			// merge the stats
+			for (idx_t i = 1; i < expr.children.size(); i++) {
+				child_stats[0]->Merge(*child_stats[i]);
+			}
+		}
+		return move(child_stats[0]);
 	case ExpressionType::OPERATOR_IS_NULL:
 		if (!child_stats[0]->CanHaveNull()) {
 			// child has no null values: x IS NULL will always be false
@@ -135334,7 +135416,7 @@ static LogicalType ResolveNotType(OperatorExpression &op, vector<BoundExpression
 
 static LogicalType ResolveInType(OperatorExpression &op, vector<BoundExpression *> &children) {
 	if (children.empty()) {
-		return LogicalType::BOOLEAN;
+		throw InternalException("IN requires at least a single child node");
 	}
 	// get the maximum type from the children
 	LogicalType max_type = children[0]->expr->return_type;
@@ -135360,8 +135442,11 @@ static LogicalType ResolveOperatorType(OperatorExpression &op, vector<BoundExpre
 		return LogicalType::BOOLEAN;
 	case ExpressionType::COMPARE_IN:
 	case ExpressionType::COMPARE_NOT_IN:
-	case ExpressionType::OPERATOR_COALESCE:
 		return ResolveInType(op, children);
+	case ExpressionType::OPERATOR_COALESCE: {
+		ResolveInType(op, children);
+		return children[0]->expr->return_type;
+	}
 	case ExpressionType::OPERATOR_NOT:
 		return ResolveNotType(op, children);
 	default:
@@ -135424,23 +135509,11 @@ BindResult ExpressionBinder::BindExpression(OperatorExpression &op, idx_t depth)
 	LogicalType result_type = ResolveOperatorType(op, children);
 	if (op.type == ExpressionType::OPERATOR_COALESCE) {
 		if (children.empty()) {
-			return BindResult("COALESCE needs at least one child");
-		}
-		unique_ptr<Expression> current_node;
-		for (size_t i = children.size(); i > 0; i--) {
-			auto child = move(children[i - 1]->expr);
-			if (!current_node) {
-				// no node yet: simply move the child
-				current_node = move(child);
-			} else {
-				// create a case statement
-				auto check =
-				    make_unique<BoundOperatorExpression>(ExpressionType::OPERATOR_IS_NOT_NULL, LogicalType::BOOLEAN);
-				check->children.push_back(child->Copy());
-				current_node = make_unique<BoundCaseExpression>(move(check), move(child), move(current_node));
-			}
+			throw BinderException("COALESCE needs at least one child");
+		}
+		if (children.size() == 1) {
+			return BindResult(move(children[0]->expr));
 		}
-		return BindResult(move(current_node));
 	}
 
 	auto result = make_unique<BoundOperatorExpression>(op.type, result_type);

package/src/duckdb.hpp

@@ -10,8 +10,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 #pragma once
 #define DUCKDB_AMALGAMATION 1
-#define DUCKDB_SOURCE_ID "88f04111c"
-#define DUCKDB_VERSION "0.3.1-dev360"
+#define DUCKDB_SOURCE_ID "a45a86a8a"
+#define DUCKDB_VERSION "0.3.1-dev399"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -2132,6 +2132,9 @@ public:
 	sel_t *data() {
 		return sel_vector;
 	}
+	const sel_t *data() const {
+		return sel_vector;
+	}
 	buffer_ptr<SelectionData> sel_data() {
 		return selection_data;
 	}