duckdb 0.3.2-dev668.0 → 0.3.2-dev685.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.2-dev668.0",
+  "version": "0.3.2-dev685.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -8540,6 +8540,8 @@ string PhysicalOperatorToString(PhysicalOperatorType type) {
 		return "TOP_N";
 	case PhysicalOperatorType::WINDOW:
 		return "WINDOW";
+	case PhysicalOperatorType::STREAMING_WINDOW:
+		return "STREAMING_WINDOW";
 	case PhysicalOperatorType::UNNEST:
 		return "UNNEST";
 	case PhysicalOperatorType::SIMPLE_AGGREGATE:
@@ -35802,6 +35804,15 @@ dtime_t Interval::Add(dtime_t left, interval_t right, date_t &date) {
 	return left;
 }
 
+timestamp_t Interval::Add(timestamp_t left, interval_t right) {
+	date_t date;
+	dtime_t time;
+	Timestamp::Convert(left, date, time);
+	auto new_date = Interval::Add(date, right);
+	auto new_time = Interval::Add(time, right, new_date);
+	return Timestamp::FromDatetime(new_date, new_time);
+}
+
 } // namespace duckdb
 
 
@@ -52691,6 +52702,149 @@ string PhysicalSimpleAggregate::ParamsToString() const {
 
 
 
+namespace duckdb {
+
+//! PhysicalStreamingWindow implements streaming window functions (i.e. with an empty OVER clause)
+class PhysicalStreamingWindow : public PhysicalOperator {
+public:
+	PhysicalStreamingWindow(vector<LogicalType> types, vector<unique_ptr<Expression>> select_list,
+	                        idx_t estimated_cardinality,
+	                        PhysicalOperatorType type = PhysicalOperatorType::STREAMING_WINDOW);
+
+	//! The projection list of the WINDOW statement
+	vector<unique_ptr<Expression>> select_list;
+
+public:
+	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
+
+	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
+	                           OperatorState &state) const override;
+
+	string ParamsToString() const override;
+};
+
+} // namespace duckdb
+
+
+
+
+
+
+
+namespace duckdb {
+
+PhysicalStreamingWindow::PhysicalStreamingWindow(vector<LogicalType> types, vector<unique_ptr<Expression>> select_list,
+                                                 idx_t estimated_cardinality, PhysicalOperatorType type)
+    : PhysicalOperator(type, move(types), estimated_cardinality), select_list(move(select_list)) {
+}
+
+class StreamingWindowState : public OperatorState {
+public:
+	StreamingWindowState() : initialized(false), row_number(1) {
+	}
+
+	void Initialize(DataChunk &input, const vector<unique_ptr<Expression>> &expressions) {
+		for (idx_t expr_idx = 0; expr_idx < expressions.size(); expr_idx++) {
+			auto &expr = *expressions[expr_idx];
+			switch (expr.GetExpressionType()) {
+			case ExpressionType::WINDOW_FIRST_VALUE: {
+				auto &wexpr = (BoundWindowExpression &)expr;
+				auto &ref = (BoundReferenceExpression &)*wexpr.children[0];
+				const_vectors.push_back(make_unique<Vector>(input.data[ref.index].GetValue(0)));
+				break;
+			}
+			case ExpressionType::WINDOW_PERCENT_RANK: {
+				const_vectors.push_back(make_unique<Vector>(Value((double)0)));
+				break;
+			}
+			case ExpressionType::WINDOW_RANK:
+			case ExpressionType::WINDOW_RANK_DENSE: {
+				const_vectors.push_back(make_unique<Vector>(Value((int64_t)1)));
+				break;
+			}
+			default:
+				const_vectors.push_back(nullptr);
+			}
+		}
+		initialized = true;
+	}
+
+public:
+	bool initialized;
+	int64_t row_number;
+	vector<unique_ptr<Vector>> const_vectors;
+};
+
+unique_ptr<OperatorState> PhysicalStreamingWindow::GetOperatorState(ClientContext &context) const {
+	return make_unique<StreamingWindowState>();
+}
+
+OperatorResultType PhysicalStreamingWindow::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
+                                                    OperatorState &state_p) const {
+	auto &state = (StreamingWindowState &)state_p;
+	if (!state.initialized) {
+		state.Initialize(input, select_list);
+	}
+	// Put payload columns in place
+	for (idx_t col_idx = 0; col_idx < input.data.size(); col_idx++) {
+		chunk.data[col_idx].Reference(input.data[col_idx]);
+	}
+	// Compute window function
+	const idx_t count = input.size();
+	for (idx_t expr_idx = 0; expr_idx < select_list.size(); expr_idx++) {
+		idx_t col_idx = input.data.size() + expr_idx;
+		auto &expr = *select_list[expr_idx];
+		switch (expr.GetExpressionType()) {
+		case ExpressionType::WINDOW_FIRST_VALUE:
+		case ExpressionType::WINDOW_PERCENT_RANK:
+		case ExpressionType::WINDOW_RANK:
+		case ExpressionType::WINDOW_RANK_DENSE: {
+			// Reference constant vector
+			chunk.data[col_idx].Reference(*state.const_vectors[expr_idx]);
+			break;
+		}
+		case ExpressionType::WINDOW_ROW_NUMBER: {
+			// Set row numbers
+			auto rdata = FlatVector::GetData<int64_t>(chunk.data[col_idx]);
+			for (idx_t i = 0; i < count; i++) {
+				rdata[i] = state.row_number + i;
+			}
+			break;
+		}
+		default:
+			throw NotImplementedException("%s for StreamingWindow", ExpressionTypeToString(expr.GetExpressionType()));
+		}
+	}
+	state.row_number += count;
+	chunk.SetCardinality(count);
+	return OperatorResultType::NEED_MORE_INPUT;
+}
+
+string PhysicalStreamingWindow::ParamsToString() const {
+	string result;
+	for (idx_t i = 0; i < select_list.size(); i++) {
+		if (i > 0) {
+			result += "\n";
+		}
+		result += select_list[i]->GetName();
+	}
+	return result;
+}
+
+} // namespace duckdb
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/aggregate/physical_window.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+
 
 namespace duckdb {
 
@@ -68409,6 +68563,9 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalUpdate &op
 
 
 
+
+
+
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -68442,12 +68599,28 @@ protected:
 } // namespace duckdb
 
 
-
-
 #include <numeric>
 
 namespace duckdb {
 
+static bool IsStreamingWindow(unique_ptr<Expression> &expr) {
+	auto wexpr = reinterpret_cast<BoundWindowExpression *>(expr.get());
+	if (!wexpr->partitions.empty() || !wexpr->orders.empty() || wexpr->ignore_nulls) {
+		return false;
+	}
+	switch (wexpr->type) {
+	// TODO: add more expression types here?
+	case ExpressionType::WINDOW_FIRST_VALUE:
+	case ExpressionType::WINDOW_PERCENT_RANK:
+	case ExpressionType::WINDOW_RANK:
+	case ExpressionType::WINDOW_RANK_DENSE:
+	case ExpressionType::WINDOW_ROW_NUMBER:
+		return true;
+	default:
+		return false;
+	}
+}
+
 unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalWindow &op) {
 	D_ASSERT(op.children.size() == 1);
 
@@ -68463,11 +68636,23 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalWindow &op
 	const auto output_idx = types.size() - op.expressions.size();
 	types.resize(output_idx);
 
+	// Identify streaming windows
+	vector<idx_t> blocking_windows;
+	vector<idx_t> streaming_windows;
+	for (idx_t expr_idx = 0; expr_idx < op.expressions.size(); expr_idx++) {
+		if (IsStreamingWindow(op.expressions[expr_idx])) {
+			streaming_windows.push_back(expr_idx);
+		} else {
+			blocking_windows.push_back(expr_idx);
+		}
+	}
+
 	// Process the window functions by sharing the partition/order definitions
 	vector<idx_t> evaluation_order;
-	vector<idx_t> remaining(op.expressions.size());
-	std::iota(remaining.begin(), remaining.end(), 0);
-	while (!remaining.empty()) {
+	while (!blocking_windows.empty() || !streaming_windows.empty()) {
+		const bool process_streaming = blocking_windows.empty();
+		auto &remaining = process_streaming ? streaming_windows : blocking_windows;
+
 		// Find all functions that share the partitioning of the first remaining expression
 		const auto over_idx = remaining[0];
 		auto over_expr = reinterpret_cast<BoundWindowExpression *>(op.expressions[over_idx].get());
@@ -68493,7 +68678,12 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalWindow &op
 		}
 
 		// Chain the new window operator on top of the plan
-		auto window = make_unique<PhysicalWindow>(types, move(select_list), op.estimated_cardinality);
+		unique_ptr<PhysicalOperator> window;
+		if (process_streaming) {
+			window = make_unique<PhysicalStreamingWindow>(types, move(select_list), op.estimated_cardinality);
+		} else {
+			window = make_unique<PhysicalWindow>(types, move(select_list), op.estimated_cardinality);
+		}
 		window->children.push_back(move(plan));
 		plan = move(window);
 
@@ -83341,6 +83531,95 @@ void ListValueFun::RegisterFunction(BuiltinFunctions &set) {
 
 namespace duckdb {
 
+struct NumericRangeInfo {
+	using TYPE = int64_t;
+	using INCREMENT_TYPE = int64_t;
+
+	static int64_t DefaultStart() {
+		return 0;
+	}
+	static int64_t DefaultIncrement() {
+		return 1;
+	}
+
+	static uint64_t ListLength(int64_t start_value, int64_t end_value, int64_t increment_value, bool inclusive_bound) {
+		if (increment_value == 0) {
+			return 0;
+		}
+		if (start_value > end_value && increment_value > 0) {
+			return 0;
+		}
+		if (start_value < end_value && increment_value < 0) {
+			return 0;
+		}
+		int64_t total_diff = AbsValue(end_value - start_value);
+		int64_t increment = AbsValue(increment_value);
+		int64_t total_values = total_diff / increment;
+		if (total_diff % increment == 0) {
+			if (inclusive_bound) {
+				total_values++;
+			}
+		} else {
+			total_values++;
+		}
+		return total_values;
+	}
+
+	static void Increment(int64_t &input, int64_t increment) {
+		input += increment;
+	}
+};
+struct TimestampRangeInfo {
+	using TYPE = timestamp_t;
+	using INCREMENT_TYPE = interval_t;
+
+	static timestamp_t DefaultStart() {
+		throw InternalException("Default start not implemented for timestamp range");
+	}
+	static interval_t DefaultIncrement() {
+		throw InternalException("Default increment not implemented for timestamp range");
+	}
+	static uint64_t ListLength(timestamp_t start_value, timestamp_t end_value, interval_t increment_value,
+	                           bool inclusive_bound) {
+		bool is_positive = increment_value.months > 0 || increment_value.days > 0 || increment_value.micros > 0;
+		bool is_negative = increment_value.months < 0 || increment_value.days < 0 || increment_value.micros < 0;
+		if (!is_negative && !is_positive) {
+			// interval is 0: no result
+			return 0;
+		}
+		if (is_negative && is_positive) {
+			// we don't allow a mix of
+			throw InvalidInputException("Interval with mix of negative/positive entries not supported");
+		}
+		if (start_value > end_value && is_positive) {
+			return 0;
+		}
+		if (start_value < end_value && is_negative) {
+			return 0;
+		}
+		int64_t total_values = 0;
+		if (is_negative) {
+			// negative interval, start_value is going down
+			while (inclusive_bound ? start_value >= end_value : start_value > end_value) {
+				start_value = Interval::Add(start_value, increment_value);
+				total_values++;
+			}
+		} else {
+			// positive interval, start_value is going up
+			while (inclusive_bound ? start_value <= end_value : start_value < end_value) {
+				start_value = Interval::Add(start_value, increment_value);
+				total_values++;
+			}
+		}
+		return total_values;
+	}
+
+	static void Increment(timestamp_t &input, interval_t increment) {
+		input = Interval::Add(input, increment);
+	}
+};
+
+template <class OP, bool INCLUSIVE_BOUND>
 class RangeInfoStruct {
 public:
 	explicit RangeInfoStruct(DataChunk &args_p) : args(args_p) {
@@ -83372,64 +83651,46 @@ public:
 		return true;
 	}
 
-	int64_t StartListValue(idx_t row_idx) {
+	typename OP::TYPE StartListValue(idx_t row_idx) {
 		if (args.ColumnCount() == 1) {
-			return 0;
+			return OP::DefaultStart();
 		} else {
-			auto data = (int64_t *)vdata[0].data;
+			auto data = (typename OP::TYPE *)vdata[0].data;
 			auto idx = vdata[0].sel->get_index(row_idx);
 			return data[idx];
 		}
 	}
 
-	int64_t EndListValue(idx_t row_idx) {
+	typename OP::TYPE EndListValue(idx_t row_idx) {
 		idx_t vdata_idx = args.ColumnCount() == 1 ? 0 : 1;
-		auto data = (int64_t *)vdata[vdata_idx].data;
+		auto data = (typename OP::TYPE *)vdata[vdata_idx].data;
 		auto idx = vdata[vdata_idx].sel->get_index(row_idx);
 		return data[idx];
 	}
 
-	int64_t ListIncrementValue(idx_t row_idx) {
+	typename OP::INCREMENT_TYPE ListIncrementValue(idx_t row_idx) {
 		if (args.ColumnCount() < 3) {
-			return 1;
+			return OP::DefaultIncrement();
 		} else {
-			auto data = (int64_t *)vdata[2].data;
+			auto data = (typename OP::INCREMENT_TYPE *)vdata[2].data;
 			auto idx = vdata[2].sel->get_index(row_idx);
 			return data[idx];
 		}
 	}
 
-	void GetListValues(idx_t row_idx, int64_t &start_value, int64_t &end_value, int64_t &increment_value) {
+	void GetListValues(idx_t row_idx, typename OP::TYPE &start_value, typename OP::TYPE &end_value,
+	                   typename OP::INCREMENT_TYPE &increment_value) {
 		start_value = StartListValue(row_idx);
 		end_value = EndListValue(row_idx);
 		increment_value = ListIncrementValue(row_idx);
 	}
 
-	uint64_t ListLength(idx_t row_idx, bool inclusive_bound) {
-		int64_t start_value;
-		int64_t end_value;
-		int64_t increment_value;
+	uint64_t ListLength(idx_t row_idx) {
+		typename OP::TYPE start_value;
+		typename OP::TYPE end_value;
+		typename OP::INCREMENT_TYPE increment_value;
 		GetListValues(row_idx, start_value, end_value, increment_value);
-		if (increment_value == 0) {
-			return 0;
-		}
-		if (start_value > end_value && increment_value > 0) {
-			return 0;
-		}
-		if (start_value < end_value && increment_value < 0) {
-			return 0;
-		}
-		int64_t total_diff = AbsValue(end_value - start_value);
-		int64_t increment = AbsValue(increment_value);
-		int64_t total_values = total_diff / increment;
-		if (total_diff % increment == 0) {
-			if (inclusive_bound) {
-				total_values++;
-			}
-		} else {
-			total_values++;
-		}
-		return total_values;
+		return OP::ListLength(start_value, end_value, increment_value, INCLUSIVE_BOUND);
 	}
 
 private:
@@ -83437,11 +83698,11 @@ private:
 	VectorData vdata[3];
 };
 
-template <bool INCLUSIVE_BOUND>
+template <class OP, bool INCLUSIVE_BOUND>
 static void ListRangeFunction(DataChunk &args, ExpressionState &state, Vector &result) {
 	D_ASSERT(result.GetType().id() == LogicalTypeId::LIST);
 
-	RangeInfoStruct info(args);
+	RangeInfoStruct<OP, INCLUSIVE_BOUND> info(args);
 	idx_t args_size = 1;
 	auto result_type = VectorType::CONSTANT_VECTOR;
 	for (idx_t i = 0; i < args.ColumnCount(); i++) {
@@ -83461,23 +83722,23 @@ static void ListRangeFunction(DataChunk &args, ExpressionState &state, Vector &r
 			list_data[i].length = 0;
 		} else {
 			list_data[i].offset = total_size;
-			list_data[i].length = info.ListLength(i, INCLUSIVE_BOUND);
+			list_data[i].length = info.ListLength(i);
 			total_size += list_data[i].length;
 		}
 	}
 
 	// now construct the child vector of the list
 	ListVector::Reserve(result, total_size);
-	auto range_data = FlatVector::GetData<int64_t>(ListVector::GetEntry(result));
+	auto range_data = FlatVector::GetData<typename OP::TYPE>(ListVector::GetEntry(result));
 	idx_t total_idx = 0;
 	for (idx_t i = 0; i < args_size; i++) {
-		int64_t start_value = info.StartListValue(i);
-		int64_t increment = info.ListIncrementValue(i);
+		typename OP::TYPE start_value = info.StartListValue(i);
+		typename OP::INCREMENT_TYPE increment = info.ListIncrementValue(i);
 
-		int64_t range_value = start_value;
+		typename OP::TYPE range_value = start_value;
 		for (idx_t range_idx = 0; range_idx < list_data[i].length; range_idx++) {
 			range_data[total_idx++] = range_value;
-			range_value += increment;
+			OP::Increment(range_value, increment);
 		}
 	}
 
@@ -83490,21 +83751,31 @@ static void ListRangeFunction(DataChunk &args, ExpressionState &state, Vector &r
 void ListRangeFun::RegisterFunction(BuiltinFunctions &set) {
 	// the arguments and return types are actually set in the binder function
 	ScalarFunctionSet range_set("range");
-	range_set.AddFunction(
-	    ScalarFunction({LogicalType::BIGINT}, LogicalType::LIST(LogicalType::BIGINT), ListRangeFunction<false>));
+	range_set.AddFunction(ScalarFunction({LogicalType::BIGINT}, LogicalType::LIST(LogicalType::BIGINT),
+	                                     ListRangeFunction<NumericRangeInfo, false>));
 	range_set.AddFunction(ScalarFunction({LogicalType::BIGINT, LogicalType::BIGINT},
-	                                     LogicalType::LIST(LogicalType::BIGINT), ListRangeFunction<false>));
+	                                     LogicalType::LIST(LogicalType::BIGINT),
+	                                     ListRangeFunction<NumericRangeInfo, false>));
 	range_set.AddFunction(ScalarFunction({LogicalType::BIGINT, LogicalType::BIGINT, LogicalType::BIGINT},
-	                                     LogicalType::LIST(LogicalType::BIGINT), ListRangeFunction<false>));
+	                                     LogicalType::LIST(LogicalType::BIGINT),
+	                                     ListRangeFunction<NumericRangeInfo, false>));
+	range_set.AddFunction(ScalarFunction({LogicalType::TIMESTAMP, LogicalType::TIMESTAMP, LogicalType::INTERVAL},
+	                                     LogicalType::LIST(LogicalType::TIMESTAMP),
+	                                     ListRangeFunction<TimestampRangeInfo, false>));
 	set.AddFunction(range_set);
 
 	ScalarFunctionSet generate_series("generate_series");
-	generate_series.AddFunction(
-	    ScalarFunction({LogicalType::BIGINT}, LogicalType::LIST(LogicalType::BIGINT), ListRangeFunction<true>));
+	generate_series.AddFunction(ScalarFunction({LogicalType::BIGINT}, LogicalType::LIST(LogicalType::BIGINT),
+	                                           ListRangeFunction<NumericRangeInfo, true>));
 	generate_series.AddFunction(ScalarFunction({LogicalType::BIGINT, LogicalType::BIGINT},
-	                                           LogicalType::LIST(LogicalType::BIGINT), ListRangeFunction<true>));
+	                                           LogicalType::LIST(LogicalType::BIGINT),
+	                                           ListRangeFunction<NumericRangeInfo, true>));
 	generate_series.AddFunction(ScalarFunction({LogicalType::BIGINT, LogicalType::BIGINT, LogicalType::BIGINT},
-	                                           LogicalType::LIST(LogicalType::BIGINT), ListRangeFunction<true>));
+	                                           LogicalType::LIST(LogicalType::BIGINT),
+	                                           ListRangeFunction<NumericRangeInfo, true>));
+	generate_series.AddFunction(ScalarFunction({LogicalType::TIMESTAMP, LogicalType::TIMESTAMP, LogicalType::INTERVAL},
+	                                           LogicalType::LIST(LogicalType::TIMESTAMP),
+	                                           ListRangeFunction<TimestampRangeInfo, true>));
 	set.AddFunction(generate_series);
 }
 
@@ -85066,12 +85337,7 @@ date_t AddOperator::Operation(interval_t left, date_t right) {
 
 template <>
 timestamp_t AddOperator::Operation(timestamp_t left, interval_t right) {
-	date_t date;
-	dtime_t time;
-	Timestamp::Convert(left, date, time);
-	auto new_date = Interval::Add(date, right);
-	auto new_time = Interval::Add(time, right, new_date);
-	return Timestamp::FromDatetime(new_date, new_time);
+	return Interval::Add(left, right);
 }
 
 template <>

package/src/duckdb.hpp

@@ -10,8 +10,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 #pragma once
 #define DUCKDB_AMALGAMATION 1
-#define DUCKDB_SOURCE_ID "a5e6d9bd4"
-#define DUCKDB_VERSION "0.3.2-dev668"
+#define DUCKDB_SOURCE_ID "3e3b464ce"
+#define DUCKDB_VERSION "0.3.2-dev685"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -5382,11 +5382,11 @@ public:
 	}
 
 	//! Returns the type of the expression
-	ExpressionType GetExpressionType() {
+	ExpressionType GetExpressionType() const {
 		return type;
 	}
 	//! Returns the class of the expression
-	ExpressionClass GetExpressionClass() {
+	ExpressionClass GetExpressionClass() const {
 		return expression_class;
 	}
 
@@ -6076,6 +6076,8 @@ public:
 
 	//! Add an interval to a date
 	static date_t Add(date_t left, interval_t right);
+	//! Add an interval to a timestamp
+	static timestamp_t Add(timestamp_t left, interval_t right);
 	//! Add an interval to a time. In case the time overflows or underflows, modify the date by the overflow.
 	//! For example if we go from 23:00 to 02:00, we add a day to the date
 	static dtime_t Add(dtime_t left, interval_t right, date_t &date);
@@ -9187,6 +9189,7 @@ enum class PhysicalOperatorType : uint8_t {
 	COPY_TO_FILE,
 	RESERVOIR_SAMPLE,
 	STREAMING_SAMPLE,
+	STREAMING_WINDOW,
 	// -----------------------------
 	// Scans
 	// -----------------------------