duckdb 0.3.3-dev1383.0 → 0.3.3-dev1393.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.3-dev1383.0",
+  "version": "0.3.3-dev1393.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -53138,10 +53138,11 @@ public:
 	vector<unique_ptr<Expression>> select_list;
 
 public:
+	unique_ptr<GlobalOperatorState> GetGlobalOperatorState(ClientContext &context) const override;
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	string ParamsToString() const override;
 };
@@ -53161,9 +53162,18 @@ PhysicalStreamingWindow::PhysicalStreamingWindow(vector<LogicalType> types, vect
     : PhysicalOperator(type, move(types), estimated_cardinality), select_list(move(select_list)) {
 }
 
+class StreamingWindowGlobalState : public GlobalOperatorState {
+public:
+	StreamingWindowGlobalState() : row_number(1) {
+	}
+
+	//! The next row number.
+	std::atomic<int64_t> row_number;
+};
+
 class StreamingWindowState : public OperatorState {
 public:
-	StreamingWindowState() : initialized(false), row_number(1) {
+	StreamingWindowState() : initialized(false) {
 	}
 
 	void Initialize(DataChunk &input, const vector<unique_ptr<Expression>> &expressions) {
@@ -53194,16 +53204,20 @@ public:
 
 public:
 	bool initialized;
-	int64_t row_number;
 	vector<unique_ptr<Vector>> const_vectors;
 };
 
+unique_ptr<GlobalOperatorState> PhysicalStreamingWindow::GetGlobalOperatorState(ClientContext &context) const {
+	return make_unique<StreamingWindowGlobalState>();
+}
+
 unique_ptr<OperatorState> PhysicalStreamingWindow::GetOperatorState(ClientContext &context) const {
 	return make_unique<StreamingWindowState>();
 }
 
 OperatorResultType PhysicalStreamingWindow::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                                    OperatorState &state_p) const {
+                                                    GlobalOperatorState &gstate_p, OperatorState &state_p) const {
+	auto &gstate = (StreamingWindowGlobalState &)gstate_p;
 	auto &state = (StreamingWindowState &)state_p;
 	if (!state.initialized) {
 		state.Initialize(input, select_list);
@@ -53230,7 +53244,7 @@ OperatorResultType PhysicalStreamingWindow::Execute(ExecutionContext &context, D
 			// Set row numbers
 			auto rdata = FlatVector::GetData<int64_t>(chunk.data[col_idx]);
 			for (idx_t i = 0; i < count; i++) {
-				rdata[i] = state.row_number + i;
+				rdata[i] = gstate.row_number + i;
 			}
 			break;
 		}
@@ -53238,7 +53252,7 @@ OperatorResultType PhysicalStreamingWindow::Execute(ExecutionContext &context, D
 			throw NotImplementedException("%s for StreamingWindow", ExpressionTypeToString(expr.GetExpressionType()));
 		}
 	}
-	state.row_number += count;
+	gstate.row_number += count;
 	chunk.SetCardinality(count);
 	return OperatorResultType::NEED_MORE_INPUT;
 }
@@ -54828,7 +54842,7 @@ public:
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	bool ParallelOperator() const override {
 		return true;
@@ -54881,7 +54895,7 @@ unique_ptr<OperatorState> PhysicalFilter::GetOperatorState(ClientContext &contex
 }
 
 OperatorResultType PhysicalFilter::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                           OperatorState &state_p) const {
+                                           GlobalOperatorState &gstate, OperatorState &state_p) const {
 	auto &state = (FilterState &)state_p;
 	idx_t result_count = state.executor.SelectExpression(input, state.sel);
 	if (result_count == input.size()) {
@@ -56051,7 +56065,7 @@ public:
 	// Operator interface
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	bool ParallelOperator() const override {
 		return true;
@@ -56120,7 +56134,7 @@ unique_ptr<OperatorState> PhysicalStreamingSample::GetOperatorState(ClientContex
 }
 
 OperatorResultType PhysicalStreamingSample::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                                    OperatorState &state) const {
+                                                    GlobalOperatorState &gstate, OperatorState &state) const {
 	switch (method) {
 	case SampleMethod::BERNOULLI_SAMPLE:
 		BernoulliSample(input, chunk, state);
@@ -56450,7 +56464,7 @@ public:
 	// Operator Interface
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	bool ParallelOperator() const override {
 		return true;
@@ -56784,7 +56798,7 @@ public:
 	// Operator Interface
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	bool ParallelOperator() const override {
 		return true;
@@ -56923,7 +56937,7 @@ unique_ptr<OperatorState> PhysicalBlockwiseNLJoin::GetOperatorState(ClientContex
 }
 
 OperatorResultType PhysicalBlockwiseNLJoin::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                                    OperatorState &state_p) const {
+                                                    GlobalOperatorState &gstate_p, OperatorState &state_p) const {
 	D_ASSERT(input.size() > 0);
 	auto &state = (BlockwiseNLJoinState &)state_p;
 	auto &gstate = (BlockwiseNLJoinGlobalState &)*sink_state;
@@ -57186,7 +57200,7 @@ public:
 	// Operator Interface
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	bool ParallelOperator() const override {
 		return true;
@@ -57264,7 +57278,7 @@ unique_ptr<OperatorState> PhysicalCrossProduct::GetOperatorState(ClientContext &
 }
 
 OperatorResultType PhysicalCrossProduct::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                                 OperatorState &state_p) const {
+                                                 GlobalOperatorState &gstate, OperatorState &state_p) const {
 	auto &state = (CrossProductOperatorState &)state_p;
 	auto &sink = (CrossProductGlobalState &)*sink_state;
 	auto &right_collection = sink.rhs_materialized;
@@ -57767,7 +57781,7 @@ unique_ptr<OperatorState> PhysicalHashJoin::GetOperatorState(ClientContext &cont
 }
 
 OperatorResultType PhysicalHashJoin::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                             OperatorState &state_p) const {
+                                             GlobalOperatorState &gstate, OperatorState &state_p) const {
 	auto &state = (PhysicalHashJoinState &)state_p;
 	auto &sink = (HashJoinGlobalState &)*sink_state;
 	D_ASSERT(sink.finalized);
@@ -57868,7 +57882,7 @@ public:
 public:
 	// Operator Interface
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 public:
 	// Source interface
@@ -59079,7 +59093,7 @@ void PhysicalIEJoin::ResolveComplexJoin(ExecutionContext &context, DataChunk &ch
 }
 
 OperatorResultType PhysicalIEJoin::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                           OperatorState &state) const {
+                                           GlobalOperatorState &gstate, OperatorState &state) const {
 	return OperatorResultType::FINISHED;
 }
 
@@ -59366,7 +59380,7 @@ public:
 public:
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	bool ParallelOperator() const override {
 		return true;
@@ -59652,7 +59666,7 @@ void PhysicalIndexJoin::GetRHSMatches(ExecutionContext &context, DataChunk &inpu
 }
 
 OperatorResultType PhysicalIndexJoin::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                              OperatorState &state_p) const {
+                                              GlobalOperatorState &gstate, OperatorState &state_p) const {
 	auto &state = (IndexJoinOperatorState &)state_p;
 
 	state.result_size = 0;
@@ -59729,7 +59743,7 @@ public:
 	// Operator Interface
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	bool ParallelOperator() const override {
 		return true;
@@ -60017,7 +60031,7 @@ unique_ptr<OperatorState> PhysicalNestedLoopJoin::GetOperatorState(ClientContext
 }
 
 OperatorResultType PhysicalNestedLoopJoin::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                                   OperatorState &state_p) const {
+                                                   GlobalOperatorState &gstate_p, OperatorState &state_p) const {
 	auto &gstate = (NestedLoopJoinGlobalState &)*sink_state;
 
 	if (gstate.right_chunks.Count() == 0) {
@@ -60239,7 +60253,7 @@ public:
 	// Operator Interface
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	bool ParallelOperator() const override {
 		return true;
@@ -61227,7 +61241,7 @@ OperatorResultType PhysicalPiecewiseMergeJoin::ResolveComplexJoin(ExecutionConte
 }
 
 OperatorResultType PhysicalPiecewiseMergeJoin::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                                       OperatorState &state) const {
+                                                       GlobalOperatorState &gstate_p, OperatorState &state) const {
 	auto &gstate = (MergeJoinGlobalState &)*sink_state;
 
 	if (gstate.Count() == 0) {
@@ -64966,7 +64980,7 @@ public:
 public:
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	bool ParallelOperator() const override {
 		return true;
@@ -65002,7 +65016,7 @@ PhysicalProjection::PhysicalProjection(vector<LogicalType> types, vector<unique_
 }
 
 OperatorResultType PhysicalProjection::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                               OperatorState &state_p) const {
+                                               GlobalOperatorState &gstate, OperatorState &state_p) const {
 	auto &state = (ProjectionState &)state_p;
 	state.executor.Execute(input, chunk);
 	return OperatorResultType::NEED_MORE_INPUT;
@@ -65047,7 +65061,7 @@ public:
 public:
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	bool ParallelOperator() const override {
 		return true;
@@ -65088,7 +65102,7 @@ unique_ptr<OperatorState> PhysicalTableInOutFunction::GetOperatorState(ClientCon
 }
 
 OperatorResultType PhysicalTableInOutFunction::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                                       OperatorState &state_p) const {
+                                                       GlobalOperatorState &gstate, OperatorState &state_p) const {
 	auto &state = (TableInOutFunctionState &)state_p;
 
 	if (!state.initialized) {
@@ -65131,7 +65145,7 @@ public:
 public:
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	bool ParallelOperator() const override {
 		return true;
@@ -65291,7 +65305,7 @@ unique_ptr<OperatorState> PhysicalUnnest::GetOperatorState(ClientContext &contex
 }
 
 OperatorResultType PhysicalUnnest::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                           OperatorState &state_p) const {
+                                           GlobalOperatorState &gstate, OperatorState &state_p) const {
 	auto &state = (UnnestOperatorState &)state_p;
 	do {
 		if (state.first_fetch) {
@@ -65579,7 +65593,7 @@ public:
 public:
 	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                           OperatorState &state) const override;
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
 	bool ParallelOperator() const override {
 		return true;
@@ -65614,7 +65628,7 @@ unique_ptr<OperatorState> PhysicalExpressionScan::GetOperatorState(ClientContext
 }
 
 OperatorResultType PhysicalExpressionScan::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                                   OperatorState &state_p) const {
+                                                   GlobalOperatorState &gstate, OperatorState &state_p) const {
 	auto &state = (ExpressionScanState &)state_p;
 
 	for (; chunk.size() + input.size() <= STANDARD_VECTOR_SIZE && state.expression_index < expressions.size();
@@ -66761,6 +66775,11 @@ void PhysicalRecursiveCTE::ExecuteRecursivePipelines(ExecutionContext &context)
 			// reset the sink state for any intermediate sinks
 			sink->sink_state = sink->GetGlobalSinkState(context.client);
 		}
+		for (auto &op : pipeline->GetOperators()) {
+			if (op) {
+				op->op_state = op->GetGlobalOperatorState(context.client);
+			}
+		}
 		pipeline->Reset();
 	}
 	auto &executor = pipelines[0]->executor;
@@ -67315,8 +67334,12 @@ unique_ptr<OperatorState> PhysicalOperator::GetOperatorState(ClientContext &cont
 	return make_unique<OperatorState>();
 }
 
+unique_ptr<GlobalOperatorState> PhysicalOperator::GetGlobalOperatorState(ClientContext &context) const {
+	return make_unique<GlobalOperatorState>();
+}
+
 OperatorResultType PhysicalOperator::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-                                             OperatorState &state) const {
+                                             GlobalOperatorState &gstate, OperatorState &state) const {
 	throw InternalException("Calling Execute on a node that is not an operator!");
 }
 // LCOV_EXCL_STOP
@@ -131644,6 +131667,7 @@ void Executor::AddChildPipeline(Pipeline *current) {
 
 void Executor::BuildPipelines(PhysicalOperator *op, Pipeline *current) {
 	D_ASSERT(current);
+	op->op_state.reset();
 	if (op->IsSink()) {
 		// operator is a sink, build a pipeline
 		op->sink_state.reset();
@@ -132146,6 +132170,13 @@ void Pipeline::Reset() {
 	if (sink && !sink->sink_state) {
 		sink->sink_state = sink->GetGlobalSinkState(GetClientContext());
 	}
+
+	for (auto &op : operators) {
+		if (op && !op->op_state) {
+			op->op_state = op->GetGlobalOperatorState(GetClientContext());
+		}
+	}
+
 	ResetSource();
 }
 
@@ -132514,7 +132545,7 @@ OperatorResultType PipelineExecutor::Execute(DataChunk &input, DataChunk &result
 			// if current_idx > source_idx, we pass the previous' operators output through the Execute of the current
 			// operator
 			StartOperator(current_operator);
-			auto result = current_operator->Execute(context, prev_chunk, current_chunk,
+			auto result = current_operator->Execute(context, prev_chunk, current_chunk, *current_operator->op_state,
 			                                        *intermediate_states[current_intermediate - 1]);
 			EndOperator(current_operator, &current_chunk);
 			if (result == OperatorResultType::HAVE_MORE_OUTPUT) {
@@ -153253,6 +153284,9 @@ namespace duckdb {
 
 BindResult ExpressionBinder::BindExpression(ParameterExpression &expr, idx_t depth) {
 	auto bound_parameter = make_unique<BoundParameterExpression>(expr.parameter_nr);
+	if (!binder.parameters) {
+		throw std::runtime_error("Unexpected prepared parameter. This type of statement can't be prepared!");
+	}
 	binder.parameters->push_back(bound_parameter.get());
 	return BindResult(move(bound_parameter));
 }
@@ -171297,7 +171331,7 @@ void LocalTableStorage::InitializeScan(LocalScanState &state, TableFilterSet *ta
 		// nothing to scan
 		return;
 	}
-	state.SetStorage(this);
+	state.SetStorage(shared_from_this());
 
 	state.chunk_index = 0;
 	state.max_index = collection.ChunkCount() - 1;
@@ -171321,12 +171355,12 @@ LocalScanState::~LocalScanState() {
 	SetStorage(nullptr);
 }
 
-void LocalScanState::SetStorage(LocalTableStorage *new_storage) {
-	if (storage != nullptr) {
+void LocalScanState::SetStorage(shared_ptr<LocalTableStorage> new_storage) {
+	if (storage) {
 		D_ASSERT(storage->active_scans > 0);
 		storage->active_scans--;
 	}
-	storage = new_storage;
+	storage = move(new_storage);
 	if (storage) {
 		storage->active_scans++;
 	}
@@ -171440,7 +171474,7 @@ void LocalStorage::Append(DataTable *table, DataChunk &chunk) {
 	auto entry = table_storage.find(table);
 	LocalTableStorage *storage;
 	if (entry == table_storage.end()) {
-		auto new_storage = make_unique<LocalTableStorage>(*table);
+		auto new_storage = make_shared<LocalTableStorage>(*table);
 		storage = new_storage.get();
 		table_storage.insert(make_pair(table, move(new_storage)));
 	} else {

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "3093e74b1"
-#define DUCKDB_VERSION "v0.3.3-dev1383"
+#define DUCKDB_SOURCE_ID "dece0efa7"
+#define DUCKDB_VERSION "v0.3.3-dev1393"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -10172,6 +10172,12 @@ public:
 	}
 };
 
+class GlobalOperatorState {
+public:
+	virtual ~GlobalOperatorState() {
+	}
+};
+
 class GlobalSinkState {
 public:
 	GlobalSinkState() : state(SinkFinalizeType::READY) {
@@ -10225,6 +10231,8 @@ public:
 	idx_t estimated_cardinality;
 	//! The global sink state of this operator
 	unique_ptr<GlobalSinkState> sink_state;
+	//! The global state of this operator
+	unique_ptr<GlobalOperatorState> op_state;
 
 public:
 	virtual string GetName() const;
@@ -10246,8 +10254,9 @@ public:
 public:
 	// Operator interface
 	virtual unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const;
+	virtual unique_ptr<GlobalOperatorState> GetGlobalOperatorState(ClientContext &context) const;
 	virtual OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
-	                                   OperatorState &state) const;
+	                                   GlobalOperatorState &gstate, OperatorState &state) const;
 
 	virtual bool ParallelOperator() const {
 		return false;
@@ -10630,7 +10639,7 @@ class Pipeline : public std::enable_shared_from_this<Pipeline> {
 	friend class PipelineFinishEvent;
 
 public:
-	Pipeline(Executor &execution_context);
+	explicit Pipeline(Executor &execution_context);
 
 	Executor &executor;
 
@@ -14435,9 +14444,9 @@ struct ColumnFetchState {
 struct LocalScanState {
 	~LocalScanState();
 
-	void SetStorage(LocalTableStorage *storage);
+	void SetStorage(shared_ptr<LocalTableStorage> storage);
 	LocalTableStorage *GetStorage() {
-		return storage;
+		return storage.get();
 	}
 
 	idx_t chunk_index;
@@ -14446,7 +14455,7 @@ struct LocalScanState {
 	TableFilterSet *table_filters;
 
 private:
-	LocalTableStorage *storage = nullptr;
+	shared_ptr<LocalTableStorage> storage;
 };
 
 class RowGroupScanState {
@@ -14507,7 +14516,7 @@ class DataTable;
 class WriteAheadLog;
 struct TableAppendState;
 
-class LocalTableStorage {
+class LocalTableStorage : public std::enable_shared_from_this<LocalTableStorage> {
 public:
 	explicit LocalTableStorage(DataTable &table);
 	~LocalTableStorage();
@@ -14590,7 +14599,7 @@ private:
 
 private:
 	Transaction &transaction;
-	unordered_map<DataTable *, unique_ptr<LocalTableStorage>> table_storage;
+	unordered_map<DataTable *, shared_ptr<LocalTableStorage>> table_storage;
 
 	void Flush(DataTable &table, LocalTableStorage &storage);
 };