duckdb 0.3.3-dev1387.0 → 0.3.3-dev1397.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.3-dev1387.0",
+  "version": "0.3.3-dev1397.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -53138,10 +53138,11 @@ public:
 	vector<unique_ptr<Expression>> select_list;
 
 public:
+	unique_ptr<GlobalOperatorState> GetGlobalOperatorState(ClientContext &context) const override;
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	string ParamsToString() const override;
 };
@@ -53161,9 +53162,18 @@ PhysicalStreamingWindow::PhysicalStreamingWindow(vector<LogicalType> types, vect
     : PhysicalOperator(type, move(types), estimated_cardinality), select_list(move(select_list)) {
 }
 
+class StreamingWindowGlobalState : public GlobalOperatorState {
+public:
+	StreamingWindowGlobalState() : row_number(1) {
+	}
+
+	//! The next row number.
+	std::atomic<int64_t> row_number;
+};
+
 class StreamingWindowState : public OperatorState {
 public:
-	StreamingWindowState() : initialized(false), row_number(1) {
+	StreamingWindowState() : initialized(false) {
 	}
 
 	void Initialize(DataChunk &input, const vector<unique_ptr<Expression>> &expressions) {
@@ -53194,16 +53204,20 @@ public:
 
 public:
 	bool initialized;
-	int64_t row_number;
 	vector<unique_ptr<Vector>> const_vectors;
 };
 
+unique_ptr<GlobalOperatorState> PhysicalStreamingWindow::GetGlobalOperatorState(ClientContext &context) const {
+	return make_unique<StreamingWindowGlobalState>();
+}
+
 unique_ptr<OperatorState> PhysicalStreamingWindow::GetOperatorState(ClientContext &context) const {
 	return make_unique<StreamingWindowState>();
 }
 
 OperatorResultType PhysicalStreamingWindow::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                                    OperatorState &state_p) const {
+                                                    GlobalOperatorState &gstate_p, OperatorState &state_p) const {
+	auto &gstate = (StreamingWindowGlobalState &)gstate_p;
 	auto &state = (StreamingWindowState &)state_p;
 	if (!state.initialized) {
 		state.Initialize(input, select_list);
@@ -53230,7 +53244,7 @@ OperatorResultType PhysicalStreamingWindow::Execute(ExecutionContext &context, D
 			// Set row numbers
 			auto rdata = FlatVector::GetData<int64_t>(chunk.data[col_idx]);
 			for (idx_t i = 0; i < count; i++) {
-				rdata[i] = state.row_number + i;
+				rdata[i] = gstate.row_number + i;
 			}
 			break;
 		}
@@ -53238,7 +53252,7 @@ OperatorResultType PhysicalStreamingWindow::Execute(ExecutionContext &context, D
 			throw NotImplementedException("%s for StreamingWindow", ExpressionTypeToString(expr.GetExpressionType()));
 		}
 	}
-	state.row_number += count;
+	gstate.row_number += count;
 	chunk.SetCardinality(count);
 	return OperatorResultType::NEED_MORE_INPUT;
 }
@@ -54828,7 +54842,7 @@ public:
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	bool ParallelOperator() const override {
 		return true;
@@ -54881,7 +54895,7 @@ unique_ptr<OperatorState> PhysicalFilter::GetOperatorState(ClientContext &contex
 }
 
 OperatorResultType PhysicalFilter::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                           OperatorState &state_p) const {
+                                           GlobalOperatorState &gstate, OperatorState &state_p) const {
 	auto &state = (FilterState &)state_p;
 	idx_t result_count = state.executor.SelectExpression(input, state.sel);
 	if (result_count == input.size()) {
@@ -56051,7 +56065,7 @@ public:
 	// Operator interface
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	bool ParallelOperator() const override {
 		return true;
@@ -56120,7 +56134,7 @@ unique_ptr<OperatorState> PhysicalStreamingSample::GetOperatorState(ClientContex
 }
 
 OperatorResultType PhysicalStreamingSample::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                                    OperatorState &state) const {
+                                                    GlobalOperatorState &gstate, OperatorState &state) const {
 	switch (method) {
 	case SampleMethod::BERNOULLI_SAMPLE:
 		BernoulliSample(input, chunk, state);
@@ -56450,7 +56464,7 @@ public:
 	// Operator Interface
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	bool ParallelOperator() const override {
 		return true;
@@ -56784,7 +56798,7 @@ public:
 	// Operator Interface
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	bool ParallelOperator() const override {
 		return true;
@@ -56923,7 +56937,7 @@ unique_ptr<OperatorState> PhysicalBlockwiseNLJoin::GetOperatorState(ClientContex
 }
 
 OperatorResultType PhysicalBlockwiseNLJoin::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                                    OperatorState &state_p) const {
+                                                    GlobalOperatorState &gstate_p, OperatorState &state_p) const {
 	D_ASSERT(input.size() > 0);
 	auto &state = (BlockwiseNLJoinState &)state_p;
 	auto &gstate = (BlockwiseNLJoinGlobalState &)*sink_state;
@@ -57186,7 +57200,7 @@ public:
 	// Operator Interface
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	bool ParallelOperator() const override {
 		return true;
@@ -57264,7 +57278,7 @@ unique_ptr<OperatorState> PhysicalCrossProduct::GetOperatorState(ClientContext &
 }
 
 OperatorResultType PhysicalCrossProduct::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                                 OperatorState &state_p) const {
+                                                 GlobalOperatorState &gstate, OperatorState &state_p) const {
 	auto &state = (CrossProductOperatorState &)state_p;
 	auto &sink = (CrossProductGlobalState &)*sink_state;
 	auto &right_collection = sink.rhs_materialized;
@@ -57767,7 +57781,7 @@ unique_ptr<OperatorState> PhysicalHashJoin::GetOperatorState(ClientContext &cont
 }
 
 OperatorResultType PhysicalHashJoin::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                             OperatorState &state_p) const {
+                                             GlobalOperatorState &gstate, OperatorState &state_p) const {
 	auto &state = (PhysicalHashJoinState &)state_p;
 	auto &sink = (HashJoinGlobalState &)*sink_state;
 	D_ASSERT(sink.finalized);
@@ -57868,7 +57882,7 @@ public:
 public:
 	// Operator Interface
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 public:
 	// Source interface
@@ -59079,7 +59093,7 @@ void PhysicalIEJoin::ResolveComplexJoin(ExecutionContext &context, DataChunk &ch
 }
 
 OperatorResultType PhysicalIEJoin::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                           OperatorState &state) const {
+                                           GlobalOperatorState &gstate, OperatorState &state) const {
 	return OperatorResultType::FINISHED;
 }
 
@@ -59366,7 +59380,7 @@ public:
 public:
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	bool ParallelOperator() const override {
 		return true;
@@ -59652,7 +59666,7 @@ void PhysicalIndexJoin::GetRHSMatches(ExecutionContext &context, DataChunk &inpu
 }
 
 OperatorResultType PhysicalIndexJoin::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                              OperatorState &state_p) const {
+                                              GlobalOperatorState &gstate, OperatorState &state_p) const {
 	auto &state = (IndexJoinOperatorState &)state_p;
 
 	state.result_size = 0;
@@ -59729,7 +59743,7 @@ public:
 	// Operator Interface
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	bool ParallelOperator() const override {
 		return true;
@@ -60017,7 +60031,7 @@ unique_ptr<OperatorState> PhysicalNestedLoopJoin::GetOperatorState(ClientContext
 }
 
 OperatorResultType PhysicalNestedLoopJoin::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                                   OperatorState &state_p) const {
+                                                   GlobalOperatorState &gstate_p, OperatorState &state_p) const {
 	auto &gstate = (NestedLoopJoinGlobalState &)*sink_state;
 
 	if (gstate.right_chunks.Count() == 0) {
@@ -60239,7 +60253,7 @@ public:
 	// Operator Interface
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	bool ParallelOperator() const override {
 		return true;
@@ -61227,7 +61241,7 @@ OperatorResultType PhysicalPiecewiseMergeJoin::ResolveComplexJoin(ExecutionConte
 }
 
 OperatorResultType PhysicalPiecewiseMergeJoin::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                                       OperatorState &state) const {
+                                                       GlobalOperatorState &gstate_p, OperatorState &state) const {
 	auto &gstate = (MergeJoinGlobalState &)*sink_state;
 
 	if (gstate.Count() == 0) {
@@ -64966,7 +64980,7 @@ public:
 public:
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	bool ParallelOperator() const override {
 		return true;
@@ -65002,7 +65016,7 @@ PhysicalProjection::PhysicalProjection(vector<LogicalType> types, vector<unique_
 }
 
 OperatorResultType PhysicalProjection::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                               OperatorState &state_p) const {
+                                               GlobalOperatorState &gstate, OperatorState &state_p) const {
 	auto &state = (ProjectionState &)state_p;
 	state.executor.Execute(input, chunk);
 	return OperatorResultType::NEED_MORE_INPUT;
@@ -65047,7 +65061,7 @@ public:
 public:
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	bool ParallelOperator() const override {
 		return true;
@@ -65088,7 +65102,7 @@ unique_ptr<OperatorState> PhysicalTableInOutFunction::GetOperatorState(ClientCon
 }
 
 OperatorResultType PhysicalTableInOutFunction::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                                       OperatorState &state_p) const {
+                                                       GlobalOperatorState &gstate, OperatorState &state_p) const {
 	auto &state = (TableInOutFunctionState &)state_p;
 
 	if (!state.initialized) {
@@ -65131,7 +65145,7 @@ public:
 public:
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	bool ParallelOperator() const override {
 		return true;
@@ -65291,7 +65305,7 @@ unique_ptr<OperatorState> PhysicalUnnest::GetOperatorState(ClientContext &contex
 }
 
 OperatorResultType PhysicalUnnest::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                           OperatorState &state_p) const {
+                                           GlobalOperatorState &gstate, OperatorState &state_p) const {
 	auto &state = (UnnestOperatorState &)state_p;
 	do {
 		if (state.first_fetch) {
@@ -65579,7 +65593,7 @@ public:
 public:
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	bool ParallelOperator() const override {
 		return true;
@@ -65614,7 +65628,7 @@ unique_ptr<OperatorState> PhysicalExpressionScan::GetOperatorState(ClientContext
 }
 
 OperatorResultType PhysicalExpressionScan::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                                   OperatorState &state_p) const {
+                                                   GlobalOperatorState &gstate, OperatorState &state_p) const {
 	auto &state = (ExpressionScanState &)state_p;
 
 	for (; chunk.size() + input.size() <= STANDARD_VECTOR_SIZE && state.expression_index < expressions.size();
@@ -66761,6 +66775,11 @@ void PhysicalRecursiveCTE::ExecuteRecursivePipelines(ExecutionContext &context)
 			// reset the sink state for any intermediate sinks
 			sink->sink_state = sink->GetGlobalSinkState(context.client);
 		}
+		for (auto &op : pipeline->GetOperators()) {
+			if (op) {
+				op->op_state = op->GetGlobalOperatorState(context.client);
+			}
+		}
 		pipeline->Reset();
 	}
 	auto &executor = pipelines[0]->executor;
@@ -67315,8 +67334,12 @@ unique_ptr<OperatorState> PhysicalOperator::GetOperatorState(ClientContext &cont
 	return make_unique<OperatorState>();
 }
 
+unique_ptr<GlobalOperatorState> PhysicalOperator::GetGlobalOperatorState(ClientContext &context) const {
+	return make_unique<GlobalOperatorState>();
+}
+
 OperatorResultType PhysicalOperator::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                             OperatorState &state) const {
+                                             GlobalOperatorState &gstate, OperatorState &state) const {
 	throw InternalException("Calling Execute on a node that is not an operator!");
 }
 // LCOV_EXCL_STOP
@@ -74864,9 +74887,9 @@ struct VectorMinMaxBase {
 			const auto sidx = sdata.sel->get_index(i);
 			auto state = states[sidx];
 			if (!state->value) {
-				Assign(state, input, idx);
+				Assign(state, input, i);
 			} else {
-				OP::template Execute(state, input, idx, count);
+				OP::template Execute(state, input, i, count);
 			}
 		}
 	}
@@ -74885,10 +74908,19 @@ struct VectorMinMaxBase {
 	template <class T, class STATE>
 	static void Finalize(Vector &result, FunctionData *, STATE *state, T *target, ValidityMask &mask, idx_t idx) {
 		if (!state->value) {
-			// we need to use FlatVector::SetNull here
+			// we need to use SetNull here
 			// since for STRUCT columns only setting the validity mask of the struct is incorrect
 			// as for a struct column, we need to also set ALL child columns to NULL
-			FlatVector::SetNull(result, idx, true);
+			switch (result.GetVectorType()) {
+			case VectorType::FLAT_VECTOR:
+				FlatVector::SetNull(result, idx, true);
+				break;
+			case VectorType::CONSTANT_VECTOR:
+				ConstantVector::SetNull(result, true);
+				break;
+			default:
+				throw InternalException("Invalid result vector type for nested min/max");
+			}
 		} else {
 			VectorOperations::Copy(*state->value, result, 1, 0, idx);
 		}
@@ -104615,8 +104647,12 @@ void duckdb_destroy_value(duckdb_value *value) {
 	}
 }
 
+duckdb_value duckdb_create_varchar_length(const char *text, idx_t length) {
+	return (duckdb_value) new duckdb::Value(std::string(text, length));
+}
+
 duckdb_value duckdb_create_varchar(const char *text) {
-	return (duckdb_value) new duckdb::Value(text);
+	return duckdb_create_varchar_length(text, strlen(text));
 }
 
 duckdb_value duckdb_create_int64(int64_t input) {
@@ -131644,6 +131680,7 @@ void Executor::AddChildPipeline(Pipeline *current) {
 
 void Executor::BuildPipelines(PhysicalOperator *op, Pipeline *current) {
 	D_ASSERT(current);
+	op->op_state.reset();
 	if (op->IsSink()) {
 		// operator is a sink, build a pipeline
 		op->sink_state.reset();
@@ -132146,6 +132183,13 @@ void Pipeline::Reset() {
 	if (sink && !sink->sink_state) {
 		sink->sink_state = sink->GetGlobalSinkState(GetClientContext());
 	}
+
+	for (auto &op : operators) {
+		if (op && !op->op_state) {
+			op->op_state = op->GetGlobalOperatorState(GetClientContext());
+		}
+	}
+
 	ResetSource();
 }
 
@@ -132514,7 +132558,7 @@ OperatorResultType PipelineExecutor::Execute(DataChunk &input, DataChunk &result
 			// if current_idx > source_idx, we pass the previous' operators output through the Execute of the current
 			// operator
 			StartOperator(current_operator);
-			auto result = current_operator->Execute(context, prev_chunk, current_chunk,
+			auto result = current_operator->Execute(context, prev_chunk, current_chunk, *current_operator->op_state,
 			                                        *intermediate_states[current_intermediate - 1]);
 			EndOperator(current_operator, &current_chunk);
 			if (result == OperatorResultType::HAVE_MORE_OUTPUT) {

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "a69a377f6"
-#define DUCKDB_VERSION "v0.3.3-dev1387"
+#define DUCKDB_SOURCE_ID "c3ba7e5b0"
+#define DUCKDB_VERSION "v0.3.3-dev1397"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -10172,6 +10172,12 @@ public:
 	}
 };
 
+class GlobalOperatorState {
+public:
+	virtual ~GlobalOperatorState() {
+	}
+};
+
 class GlobalSinkState {
 public:
 	GlobalSinkState() : state(SinkFinalizeType::READY) {
@@ -10225,6 +10231,8 @@ public:
 	idx_t estimated_cardinality;
 	//! The global sink state of this operator
 	unique_ptr<GlobalSinkState> sink_state;
+	//! The global state of this operator
+	unique_ptr<GlobalOperatorState> op_state;
 
 public:
 	virtual string GetName() const;
@@ -10246,8 +10254,9 @@ public:
 public:
 	// Operator interface
 	virtual unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const;
+	virtual unique_ptr<GlobalOperatorState> GetGlobalOperatorState(ClientContext &context) const;
 	virtual OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                                   OperatorState &state) const;
+	                                   GlobalOperatorState &gstate, OperatorState &state) const;
 
 	virtual bool ParallelOperator() const {
 		return false;
@@ -10630,7 +10639,7 @@ class Pipeline : public std::enable_shared_from_this<Pipeline> {
 	friend class PipelineFinishEvent;
 
 public:
-	Pipeline(Executor &execution_context);
+	explicit Pipeline(Executor &execution_context);
 
 	Executor &executor;
 
@@ -15887,13 +15896,22 @@ Destroys the value and de-allocates all memory allocated for that type.
 DUCKDB_API void duckdb_destroy_value(duckdb_value *value);
 
 /*!
-Creates a value from a varchar
+Creates a value from a null-terminated string
 
-* value: The varchar value
+* value: The null-terminated string
 * returns: The value. This must be destroyed with `duckdb_destroy_value`.
 */
 DUCKDB_API duckdb_value duckdb_create_varchar(const char *text);
 
+/*!
+Creates a value from a string
+
+* value: The text
+* length: The length of the text
+* returns: The value. This must be destroyed with `duckdb_destroy_value`.
+*/
+DUCKDB_API duckdb_value duckdb_create_varchar_length(const char *text, idx_t length);
+
 /*!
 Creates a value from an int64