duckdb 0.3.2-dev658.0 → 0.3.2-dev679.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.2-dev658.0",
+  "version": "0.3.2-dev679.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -8540,6 +8540,8 @@ string PhysicalOperatorToString(PhysicalOperatorType type) {
 		return "TOP_N";
 	case PhysicalOperatorType::WINDOW:
 		return "WINDOW";
+	case PhysicalOperatorType::STREAMING_WINDOW:
+		return "STREAMING_WINDOW";
 	case PhysicalOperatorType::UNNEST:
 		return "UNNEST";
 	case PhysicalOperatorType::SIMPLE_AGGREGATE:
@@ -24915,8 +24917,8 @@ static void GatherNestedVector(Vector &rows, const SelectionVector &row_sel, Vec
 	auto ptrs = FlatVector::GetData<data_ptr_t>(rows);
 
 	// Build the gather locations
-	data_ptr_t data_locations[STANDARD_VECTOR_SIZE];
-	data_ptr_t mask_locations[STANDARD_VECTOR_SIZE];
+	auto data_locations = unique_ptr<data_ptr_t[]>(new data_ptr_t[count]);
+	auto mask_locations = unique_ptr<data_ptr_t[]>(new data_ptr_t[count]);
 	for (idx_t i = 0; i < count; i++) {
 		auto row_idx = row_sel.get_index(i);
 		mask_locations[i] = ptrs[row_idx];
@@ -24924,7 +24926,7 @@ static void GatherNestedVector(Vector &rows, const SelectionVector &row_sel, Vec
 	}
 
 	// Deserialise into the selected locations
-	RowOperations::HeapGather(col, count, col_sel, col_no, data_locations, mask_locations);
+	RowOperations::HeapGather(col, count, col_sel, col_no, data_locations.get(), mask_locations.get());
 }
 
 void RowOperations::Gather(Vector &rows, const SelectionVector &row_sel, Vector &col, const SelectionVector &col_sel,
@@ -37647,6 +37649,19 @@ T Value::GetValueInternal() const {
 		return Cast::Operation<interval_t, T>(value_.interval);
 	case LogicalTypeId::DECIMAL:
 		return CastAs(LogicalType::DOUBLE).GetValueInternal<T>();
+	case LogicalTypeId::ENUM: {
+		switch (type_.InternalType()) {
+		case PhysicalType::UINT8:
+			return Cast::Operation<uint8_t, T>(value_.utinyint);
+		case PhysicalType::UINT16:
+			return Cast::Operation<uint16_t, T>(value_.usmallint);
+		case PhysicalType::UINT32:
+			return Cast::Operation<uint32_t, T>(value_.uinteger);
+		default:
+			throw InternalException("Invalid Internal Type for ENUMs");
+		}
+	}
+
 	default:
 		throw NotImplementedException("Unimplemented type \"%s\" for GetValue()", type_.ToString());
 	}
@@ -52678,6 +52693,149 @@ string PhysicalSimpleAggregate::ParamsToString() const {
 
 
 
+namespace duckdb {
+
+//! PhysicalStreamingWindow implements streaming window functions (i.e. with an empty OVER clause)
+class PhysicalStreamingWindow : public PhysicalOperator {
+public:
+	PhysicalStreamingWindow(vector<LogicalType> types, vector<unique_ptr<Expression>> select_list,
+	                        idx_t estimated_cardinality,
+	                        PhysicalOperatorType type = PhysicalOperatorType::STREAMING_WINDOW);
+
+	//! The projection list of the WINDOW statement
+	vector<unique_ptr<Expression>> select_list;
+
+public:
+	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
+
+	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
+	                           OperatorState &state) const override;
+
+	string ParamsToString() const override;
+};
+
+} // namespace duckdb
+
+
+
+
+
+
+
+namespace duckdb {
+
+PhysicalStreamingWindow::PhysicalStreamingWindow(vector<LogicalType> types, vector<unique_ptr<Expression>> select_list,
+                                                 idx_t estimated_cardinality, PhysicalOperatorType type)
+    : PhysicalOperator(type, move(types), estimated_cardinality), select_list(move(select_list)) {
+}
+
+class StreamingWindowState : public OperatorState {
+public:
+	StreamingWindowState() : initialized(false), row_number(1) {
+	}
+
+	void Initialize(DataChunk &input, const vector<unique_ptr<Expression>> &expressions) {
+		for (idx_t expr_idx = 0; expr_idx < expressions.size(); expr_idx++) {
+			auto &expr = *expressions[expr_idx];
+			switch (expr.GetExpressionType()) {
+			case ExpressionType::WINDOW_FIRST_VALUE: {
+				auto &wexpr = (BoundWindowExpression &)expr;
+				auto &ref = (BoundReferenceExpression &)*wexpr.children[0];
+				const_vectors.push_back(make_unique<Vector>(input.data[ref.index].GetValue(0)));
+				break;
+			}
+			case ExpressionType::WINDOW_PERCENT_RANK: {
+				const_vectors.push_back(make_unique<Vector>(Value((double)0)));
+				break;
+			}
+			case ExpressionType::WINDOW_RANK:
+			case ExpressionType::WINDOW_RANK_DENSE: {
+				const_vectors.push_back(make_unique<Vector>(Value((int64_t)1)));
+				break;
+			}
+			default:
+				const_vectors.push_back(nullptr);
+			}
+		}
+		initialized = true;
+	}
+
+public:
+	bool initialized;
+	int64_t row_number;
+	vector<unique_ptr<Vector>> const_vectors;
+};
+
+unique_ptr<OperatorState> PhysicalStreamingWindow::GetOperatorState(ClientContext &context) const {
+	return make_unique<StreamingWindowState>();
+}
+
+OperatorResultType PhysicalStreamingWindow::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
+                                                    OperatorState &state_p) const {
+	auto &state = (StreamingWindowState &)state_p;
+	if (!state.initialized) {
+		state.Initialize(input, select_list);
+	}
+	// Put payload columns in place
+	for (idx_t col_idx = 0; col_idx < input.data.size(); col_idx++) {
+		chunk.data[col_idx].Reference(input.data[col_idx]);
+	}
+	// Compute window function
+	const idx_t count = input.size();
+	for (idx_t expr_idx = 0; expr_idx < select_list.size(); expr_idx++) {
+		idx_t col_idx = input.data.size() + expr_idx;
+		auto &expr = *select_list[expr_idx];
+		switch (expr.GetExpressionType()) {
+		case ExpressionType::WINDOW_FIRST_VALUE:
+		case ExpressionType::WINDOW_PERCENT_RANK:
+		case ExpressionType::WINDOW_RANK:
+		case ExpressionType::WINDOW_RANK_DENSE: {
+			// Reference constant vector
+			chunk.data[col_idx].Reference(*state.const_vectors[expr_idx]);
+			break;
+		}
+		case ExpressionType::WINDOW_ROW_NUMBER: {
+			// Set row numbers
+			auto rdata = FlatVector::GetData<int64_t>(chunk.data[col_idx]);
+			for (idx_t i = 0; i < count; i++) {
+				rdata[i] = state.row_number + i;
+			}
+			break;
+		}
+		default:
+			throw NotImplementedException("%s for StreamingWindow", ExpressionTypeToString(expr.GetExpressionType()));
+		}
+	}
+	state.row_number += count;
+	chunk.SetCardinality(count);
+	return OperatorResultType::NEED_MORE_INPUT;
+}
+
+string PhysicalStreamingWindow::ParamsToString() const {
+	string result;
+	for (idx_t i = 0; i < select_list.size(); i++) {
+		if (i > 0) {
+			result += "\n";
+		}
+		result += select_list[i]->GetName();
+	}
+	return result;
+}
+
+} // namespace duckdb
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/aggregate/physical_window.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+
 
 namespace duckdb {
 
@@ -68396,6 +68554,9 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalUpdate &op
 
 
 
+
+
+
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -68429,12 +68590,28 @@ protected:
 } // namespace duckdb
 
 
-
-
 #include <numeric>
 
 namespace duckdb {
 
+static bool IsStreamingWindow(unique_ptr<Expression> &expr) {
+	auto wexpr = reinterpret_cast<BoundWindowExpression *>(expr.get());
+	if (!wexpr->partitions.empty() || !wexpr->orders.empty() || wexpr->ignore_nulls) {
+		return false;
+	}
+	switch (wexpr->type) {
+	// TODO: add more expression types here?
+	case ExpressionType::WINDOW_FIRST_VALUE:
+	case ExpressionType::WINDOW_PERCENT_RANK:
+	case ExpressionType::WINDOW_RANK:
+	case ExpressionType::WINDOW_RANK_DENSE:
+	case ExpressionType::WINDOW_ROW_NUMBER:
+		return true;
+	default:
+		return false;
+	}
+}
+
 unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalWindow &op) {
 	D_ASSERT(op.children.size() == 1);
 
@@ -68450,11 +68627,23 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalWindow &op
 	const auto output_idx = types.size() - op.expressions.size();
 	types.resize(output_idx);
 
+	// Identify streaming windows
+	vector<idx_t> blocking_windows;
+	vector<idx_t> streaming_windows;
+	for (idx_t expr_idx = 0; expr_idx < op.expressions.size(); expr_idx++) {
+		if (IsStreamingWindow(op.expressions[expr_idx])) {
+			streaming_windows.push_back(expr_idx);
+		} else {
+			blocking_windows.push_back(expr_idx);
+		}
+	}
+
 	// Process the window functions by sharing the partition/order definitions
 	vector<idx_t> evaluation_order;
-	vector<idx_t> remaining(op.expressions.size());
-	std::iota(remaining.begin(), remaining.end(), 0);
-	while (!remaining.empty()) {
+	while (!blocking_windows.empty() || !streaming_windows.empty()) {
+		const bool process_streaming = blocking_windows.empty();
+		auto &remaining = process_streaming ? streaming_windows : blocking_windows;
+
 		// Find all functions that share the partitioning of the first remaining expression
 		const auto over_idx = remaining[0];
 		auto over_expr = reinterpret_cast<BoundWindowExpression *>(op.expressions[over_idx].get());
@@ -68480,7 +68669,12 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalWindow &op
 		}
 
 		// Chain the new window operator on top of the plan
-		auto window = make_unique<PhysicalWindow>(types, move(select_list), op.estimated_cardinality);
+		unique_ptr<PhysicalOperator> window;
+		if (process_streaming) {
+			window = make_unique<PhysicalStreamingWindow>(types, move(select_list), op.estimated_cardinality);
+		} else {
+			window = make_unique<PhysicalWindow>(types, move(select_list), op.estimated_cardinality);
+		}
 		window->children.push_back(move(plan));
 		plan = move(window);
 
@@ -77539,6 +77733,7 @@ void BuiltinFunctions::Initialize() {
 	RegisterRegressiveAggregates();
 
 	RegisterDateFunctions();
+	RegisterEnumFunctions();
 	RegisterGenericFunctions();
 	RegisterMathFunctions();
 	RegisterOperators();
@@ -82043,6 +82238,163 @@ void BuiltinFunctions::RegisterDateFunctions() {
 	Register<ToIntervalFun>();
 }
 
+} // namespace duckdb
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/function/scalar/enum_functions.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+
+namespace duckdb {
+
+struct EnumFirst {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct EnumLast {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct EnumRange {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct EnumRangeBoundary {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+} // namespace duckdb
+
+
+namespace duckdb {
+
+static void EnumFirstFunction(DataChunk &input, ExpressionState &state, Vector &result) {
+	D_ASSERT(input.GetTypes().size() == 1);
+	auto &enum_vector = EnumType::GetValuesInsertOrder(input.GetTypes()[0]);
+	auto val = Value(enum_vector.GetValue(0));
+	result.Reference(val);
+}
+
+static void EnumLastFunction(DataChunk &input, ExpressionState &state, Vector &result) {
+	D_ASSERT(input.GetTypes().size() == 1);
+	auto enum_size = EnumType::GetSize(input.GetTypes()[0]);
+	auto &enum_vector = EnumType::GetValuesInsertOrder(input.GetTypes()[0]);
+	auto val = Value(enum_vector.GetValue(enum_size - 1));
+	result.Reference(val);
+}
+
+static void EnumRangeFunction(DataChunk &input, ExpressionState &state, Vector &result) {
+	D_ASSERT(input.GetTypes().size() == 1);
+	auto enum_size = EnumType::GetSize(input.GetTypes()[0]);
+	auto &enum_vector = EnumType::GetValuesInsertOrder(input.GetTypes()[0]);
+	vector<Value> enum_values;
+	for (idx_t i = 0; i < enum_size; i++) {
+		enum_values.emplace_back(enum_vector.GetValue(i));
+	}
+	auto val = Value::LIST(enum_values);
+	result.Reference(val);
+}
+
+static void EnumRangeBoundaryFunction(DataChunk &input, ExpressionState &state, Vector &result) {
+	D_ASSERT(input.GetTypes().size() == 2);
+	idx_t start, end;
+	auto first_param = input.GetValue(0, 0);
+	auto second_param = input.GetValue(1, 0);
+
+	auto &enum_vector = first_param.is_null ? EnumType::GetValuesInsertOrder(input.GetTypes()[1])
+	                                        : EnumType::GetValuesInsertOrder(input.GetTypes()[0]);
+
+	if (first_param.is_null) {
+		start = 0;
+	} else {
+		start = first_param.GetValue<uint32_t>();
+	}
+	if (second_param.is_null) {
+		end = EnumType::GetSize(input.GetTypes()[0]);
+	} else {
+		end = second_param.GetValue<uint32_t>() + 1;
+	}
+	vector<Value> enum_values;
+	for (idx_t i = start; i < end; i++) {
+		enum_values.emplace_back(enum_vector.GetValue(i));
+	}
+	Value val;
+	if (enum_values.empty()) {
+		val = Value::EMPTYLIST(LogicalType::VARCHAR);
+	} else {
+		val = Value::LIST(enum_values);
+	}
+	result.Reference(val);
+}
+
+unique_ptr<FunctionData> BindEnumFunction(ClientContext &context, ScalarFunction &bound_function,
+                                          vector<unique_ptr<Expression>> &arguments) {
+	if (arguments[0]->return_type.id() != LogicalTypeId::ENUM) {
+		throw BinderException("This function needs an ENUM as an argument");
+	}
+	return nullptr;
+}
+
+unique_ptr<FunctionData> BindEnumRangeBoundaryFunction(ClientContext &context, ScalarFunction &bound_function,
+                                                       vector<unique_ptr<Expression>> &arguments) {
+	if (arguments[0]->return_type.id() != LogicalTypeId::ENUM && arguments[0]->return_type != LogicalType::SQLNULL) {
+		throw BinderException("This function needs an ENUM as an argument");
+	}
+	if (arguments[1]->return_type.id() != LogicalTypeId::ENUM && arguments[1]->return_type != LogicalType::SQLNULL) {
+		throw BinderException("This function needs an ENUM as an argument");
+	}
+	if (arguments[0]->return_type == LogicalType::SQLNULL && arguments[1]->return_type == LogicalType::SQLNULL) {
+		throw BinderException("This function needs an ENUM as an argument");
+	}
+	if (arguments[0]->return_type.id() == LogicalTypeId::ENUM &&
+	    arguments[1]->return_type.id() == LogicalTypeId::ENUM &&
+	    arguments[0]->return_type != arguments[1]->return_type) {
+
+		throw BinderException("The parameters need to link to ONLY one enum OR be NULL ");
+	}
+	return nullptr;
+}
+
+void EnumFirst::RegisterFunction(BuiltinFunctions &set) {
+	set.AddFunction(ScalarFunction("enum_first", {LogicalType::ANY}, LogicalType::VARCHAR, EnumFirstFunction, false,
+	                               BindEnumFunction));
+}
+
+void EnumLast::RegisterFunction(BuiltinFunctions &set) {
+	set.AddFunction(ScalarFunction("enum_last", {LogicalType::ANY}, LogicalType::VARCHAR, EnumLastFunction, false,
+	                               BindEnumFunction));
+}
+
+void EnumRange::RegisterFunction(BuiltinFunctions &set) {
+	set.AddFunction(ScalarFunction("enum_range", {LogicalType::ANY}, LogicalType::LIST(LogicalType::VARCHAR),
+	                               EnumRangeFunction, false, BindEnumFunction));
+}
+
+void EnumRangeBoundary::RegisterFunction(BuiltinFunctions &set) {
+	set.AddFunction(ScalarFunction("enum_range_boundary", {LogicalType::ANY, LogicalType::ANY},
+	                               LogicalType::LIST(LogicalType::VARCHAR), EnumRangeBoundaryFunction, false,
+	                               BindEnumRangeBoundaryFunction));
+}
+
+} // namespace duckdb
+
+
+namespace duckdb {
+
+void BuiltinFunctions::RegisterEnumFunctions() {
+	Register<EnumFirst>();
+	Register<EnumLast>();
+	Register<EnumRange>();
+	Register<EnumRangeBoundary>();
+}
+
 } // namespace duckdb
 //===----------------------------------------------------------------------===//
 //                         DuckDB
@@ -145303,8 +145655,10 @@ BindResult ExpressionBinder::BindExpression(BetweenExpression &expr, idx_t depth
 		lower.expr = PushCollation(context, move(lower.expr), collation, false);
 		upper.expr = PushCollation(context, move(upper.expr), collation, false);
 	}
-	if (!input.expr->HasSideEffects() && !input.expr->HasParameter()) {
-		// the expression does not have side effects: create two comparisons
+	if (!input.expr->HasSideEffects() && !input.expr->HasParameter() && !input.expr->HasSubquery()) {
+		// the expression does not have side effects and can be copied: create two comparisons
+		// the reason we do this is that individual comparisons are easier to handle in optimizers
+		// if both comparisons remain they will be folded together again into a single BETWEEN in the optimizer
 		auto left_compare = make_unique<BoundComparisonExpression>(ExpressionType::COMPARE_GREATERTHANOREQUALTO,
 		                                                           input.expr->Copy(), move(lower.expr));
 		auto right_compare = make_unique<BoundComparisonExpression>(ExpressionType::COMPARE_LESSTHANOREQUALTO,

package/src/duckdb.hpp

@@ -10,8 +10,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 #pragma once
 #define DUCKDB_AMALGAMATION 1
-#define DUCKDB_SOURCE_ID "0f8960ac3"
-#define DUCKDB_VERSION "0.3.2-dev658"
+#define DUCKDB_SOURCE_ID "000fbffa3"
+#define DUCKDB_VERSION "0.3.2-dev679"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -5382,11 +5382,11 @@ public:
 	}
 
 	//! Returns the type of the expression
-	ExpressionType GetExpressionType() {
+	ExpressionType GetExpressionType() const {
 		return type;
 	}
 	//! Returns the class of the expression
-	ExpressionClass GetExpressionClass() {
+	ExpressionClass GetExpressionClass() const {
 		return expression_class;
 	}
 
@@ -5742,6 +5742,7 @@ private:
 
 	// scalar functions
 	void RegisterDateFunctions();
+	void RegisterEnumFunctions();
 	void RegisterGenericFunctions();
 	void RegisterMathFunctions();
 	void RegisterOperators();
@@ -9186,6 +9187,7 @@ enum class PhysicalOperatorType : uint8_t {
 	COPY_TO_FILE,
 	RESERVOIR_SAMPLE,
 	STREAMING_SAMPLE,
+	STREAMING_WINDOW,
 	// -----------------------------
 	// Scans
 	// -----------------------------