duckdb 0.3.3-dev865.0 → 0.3.3-dev918.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.3-dev865.0",
+  "version": "0.3.3-dev918.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -53208,7 +53208,7 @@ private:
 };
 
 template <typename T, typename OP>
-struct OperationCompare : public std::binary_function<T, T, bool> {
+struct OperationCompare : public std::function<bool(T, T)> {
 	inline bool operator()(const T &lhs, const T &val) const {
 		return OP::template Operation(lhs, val);
 	}
@@ -62217,7 +62217,7 @@ namespace duckdb {
 class PhysicalInsert : public PhysicalOperator {
 public:
 	PhysicalInsert(vector<LogicalType> types, TableCatalogEntry *table, vector<idx_t> column_index_map,
-	               vector<unique_ptr<Expression>> bound_defaults, idx_t estimated_cardinality);
+	               vector<unique_ptr<Expression>> bound_defaults, idx_t estimated_cardinality, bool return_chunk);
 
 	//! The map from insert column index to table column index
 	vector<idx_t> column_index_map;
@@ -62225,6 +62225,8 @@ public:
 	TableCatalogEntry *table;
 	//! The default expressions of the columns for which no value is provided
 	vector<unique_ptr<Expression>> bound_defaults;
+	//! If the returning statement is present, return the whole chunk
+	bool return_chunk;
 
 public:
 	// Source interface
@@ -62266,11 +62268,13 @@ namespace duckdb {
 //===--------------------------------------------------------------------===//
 class InsertGlobalState : public GlobalSinkState {
 public:
-	InsertGlobalState() : insert_count(0) {
+	InsertGlobalState() : insert_count(0), returned_chunk_count(0) {
 	}
 
 	mutex lock;
 	idx_t insert_count;
+	ChunkCollection return_chunk_collection;
+	idx_t returned_chunk_count;
 };
 
 class InsertLocalState : public LocalSinkState {
@@ -62285,9 +62289,11 @@ public:
 };
 
 PhysicalInsert::PhysicalInsert(vector<LogicalType> types, TableCatalogEntry *table, vector<idx_t> column_index_map,
-                               vector<unique_ptr<Expression>> bound_defaults, idx_t estimated_cardinality)
+                               vector<unique_ptr<Expression>> bound_defaults, idx_t estimated_cardinality,
+                               bool return_chunk)
     : PhysicalOperator(PhysicalOperatorType::INSERT, move(types), estimated_cardinality),
-      column_index_map(std::move(column_index_map)), table(table), bound_defaults(move(bound_defaults)) {
+      column_index_map(std::move(column_index_map)), table(table), bound_defaults(move(bound_defaults)),
+      return_chunk(return_chunk) {
 }
 
 SinkResultType PhysicalInsert::Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,
@@ -62300,6 +62306,7 @@ SinkResultType PhysicalInsert::Sink(ExecutionContext &context, GlobalSinkState &
 
 	istate.insert_chunk.Reset();
 	istate.insert_chunk.SetCardinality(chunk);
+
 	if (!column_index_map.empty()) {
 		// columns specified by the user, use column_index_map
 		for (idx_t i = 0; i < table->columns.size(); i++) {
@@ -62323,6 +62330,11 @@ SinkResultType PhysicalInsert::Sink(ExecutionContext &context, GlobalSinkState &
 
 	lock_guard<mutex> glock(gstate.lock);
 	table->storage->Append(*table, context.client, istate.insert_chunk);
+
+	if (return_chunk) {
+		gstate.return_chunk_collection.Append(istate.insert_chunk);
+	}
+
 	gstate.insert_count += chunk.size();
 	return SinkResultType::NEED_MORE_INPUT;
 }
@@ -62360,14 +62372,27 @@ unique_ptr<GlobalSourceState> PhysicalInsert::GetGlobalSourceState(ClientContext
 void PhysicalInsert::GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
                              LocalSourceState &lstate) const {
 	auto &state = (InsertSourceState &)gstate;
-	auto &g = (InsertGlobalState &)*sink_state;
+	auto &insert_gstate = (InsertGlobalState &)*sink_state;
 	if (state.finished) {
 		return;
 	}
+	if (!return_chunk) {
+		chunk.SetCardinality(1);
+		chunk.SetValue(0, 0, Value::BIGINT(insert_gstate.insert_count));
+		state.finished = true;
+	}
 
-	chunk.SetCardinality(1);
-	chunk.SetValue(0, 0, Value::BIGINT(g.insert_count));
-	state.finished = true;
+	idx_t chunk_return = insert_gstate.returned_chunk_count;
+	if (chunk_return >= insert_gstate.return_chunk_collection.Chunks().size()) {
+		return;
+	}
+
+	chunk.Reference(insert_gstate.return_chunk_collection.GetChunk(chunk_return));
+	chunk.SetCardinality((insert_gstate.return_chunk_collection.GetChunk(chunk_return)).size());
+	insert_gstate.returned_chunk_count += 1;
+	if (insert_gstate.returned_chunk_count >= insert_gstate.return_chunk_collection.Chunks().size()) {
+		state.finished = true;
+	}
 }
 
 } // namespace duckdb
@@ -67549,7 +67574,7 @@ namespace duckdb {
 class LogicalInsert : public LogicalOperator {
 public:
 	explicit LogicalInsert(TableCatalogEntry *table)
-	    : LogicalOperator(LogicalOperatorType::LOGICAL_INSERT), table(table) {
+	    : LogicalOperator(LogicalOperatorType::LOGICAL_INSERT), table(table), table_index(0), return_chunk(false) {
 	}
 
 	vector<vector<unique_ptr<Expression>>> insert_values;
@@ -67559,12 +67584,26 @@ public:
 	vector<LogicalType> expected_types;
 	//! The base table to insert into
 	TableCatalogEntry *table;
+	idx_t table_index;
+	//! if returning option is used, return actual chunk to projection
+	bool return_chunk;
 	//! The default statements used by the table
 	vector<unique_ptr<Expression>> bound_defaults;
 
 protected:
+	vector<ColumnBinding> GetColumnBindings() override {
+		if (return_chunk) {
+			return GenerateColumnBindings(table_index, table->columns.size());
+		}
+		return {ColumnBinding(0, 0)};
+	}
+
 	void ResolveTypes() override {
-		types.emplace_back(LogicalType::BIGINT);
+		if (return_chunk) {
+			types = table->GetTypes();
+		} else {
+			types.emplace_back(LogicalType::BIGINT);
+		}
 	}
 };
 } // namespace duckdb
@@ -67581,7 +67620,7 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalInsert &op
 
 	dependencies.insert(op.table);
 	auto insert = make_unique<PhysicalInsert>(op.types, op.table, op.column_index_map, move(op.bound_defaults),
-	                                          op.estimated_cardinality);
+	                                          op.estimated_cardinality, op.return_chunk);
 	if (plan) {
 		insert->children.push_back(move(plan));
 	}
@@ -104935,21 +104974,26 @@ public:
 
 	void AddConnection(ClientContext &context) {
 		lock_guard<mutex> lock(connections_lock);
-		connections.push_back(weak_ptr<ClientContext>(context.shared_from_this()));
+		connections.insert(make_pair(&context, weak_ptr<ClientContext>(context.shared_from_this())));
+	}
+
+	void RemoveConnection(ClientContext &context) {
+		lock_guard<mutex> lock(connections_lock);
+		connections.erase(&context);
 	}
 
 	vector<shared_ptr<ClientContext>> GetConnectionList() {
 		vector<shared_ptr<ClientContext>> result;
-		for (size_t i = 0; i < connections.size(); i++) {
-			auto connection = connections[i].lock();
+		for (auto &it : connections) {
+			auto connection = it.second.lock();
 			if (!connection) {
-				connections.erase(connections.begin() + i);
-				i--;
+				connections.erase(it.first);
 				continue;
 			} else {
 				result.push_back(move(connection));
 			}
 		}
+
 		return result;
 	}
 
@@ -104958,7 +105002,7 @@ public:
 
 public:
 	mutex connections_lock;
-	vector<weak_ptr<ClientContext>> connections;
+	unordered_map<ClientContext *, weak_ptr<ClientContext>> connections;
 };
 
 } // namespace duckdb
@@ -104977,6 +105021,10 @@ Connection::Connection(DatabaseInstance &database) : context(make_shared<ClientC
 Connection::Connection(DuckDB &database) : Connection(*database.instance) {
 }
 
+Connection::~Connection() {
+	ConnectionManager::Get(*context->db).RemoveConnection(*context);
+}
+
 string Connection::GetProfilingInformation(ProfilerPrintFormat format) {
 	auto &profiler = QueryProfiler::Get(*context);
 	if (format == ProfilerPrintFormat::JSON) {
@@ -116169,6 +116217,9 @@ public:
 	//! Schema name to insert to
 	string schema;
 
+	//! keep track of optional returningList if statement contains a RETURNING keyword
+	vector<unique_ptr<ParsedExpression>> returning_list;
+
 protected:
 	InsertStatement(const InsertStatement &other);
 
@@ -120942,6 +120993,8 @@ private:
 	unique_ptr<LogicalOperator> PushdownSetOperation(unique_ptr<LogicalOperator> op);
 	//! Push down a LogicalGet op
 	unique_ptr<LogicalOperator> PushdownGet(unique_ptr<LogicalOperator> op);
+	//! Push down a LogicalLimit op
+	unique_ptr<LogicalOperator> PushdownLimit(unique_ptr<LogicalOperator> op);
 	// Pushdown an inner join
 	unique_ptr<LogicalOperator> PushdownInnerJoin(unique_ptr<LogicalOperator> op, unordered_set<idx_t> &left_bindings,
 	                                              unordered_set<idx_t> &right_bindings);
@@ -121007,6 +121060,8 @@ unique_ptr<LogicalOperator> FilterPushdown::Rewrite(unique_ptr<LogicalOperator>
 	}
 	case LogicalOperatorType::LOGICAL_GET:
 		return PushdownGet(move(op));
+	case LogicalOperatorType::LOGICAL_LIMIT:
+		return PushdownLimit(move(op));
 	default:
 		return FinishPushdown(move(op));
 	}
@@ -123937,6 +123992,25 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownLeftJoin(unique_ptr<LogicalO
 
 
 
+
+
+namespace duckdb {
+
+unique_ptr<LogicalOperator> FilterPushdown::PushdownLimit(unique_ptr<LogicalOperator> op) {
+	auto &limit = (LogicalLimit &)*op;
+
+	if (!limit.limit && limit.limit_val == 0) {
+		return make_unique<LogicalEmptyResult>(move(op));
+	}
+
+	return FinishPushdown(move(op));
+}
+
+} // namespace duckdb
+
+
+
+
 namespace duckdb {
 
 using Filter = FilterPushdown::Filter;
@@ -124445,6 +124519,15 @@ void RemoveUnusedColumns::VisitOperator(LogicalOperator &op) {
 		remove.VisitOperator(*op.children[0]);
 		return;
 	}
+	case LogicalOperatorType::LOGICAL_INSERT: {
+		//! When RETURNING is used, a PROJECTION is the top level operator
+		//! We still need to project all values from the insert so the projection
+		//! on top of the INSERT can select from only that table values being inserted
+		RemoveUnusedColumns remove(binder, context, true);
+		remove.VisitOperatorExpressions(op);
+		remove.VisitOperator(*op.children[0]);
+		return;
+	}
 	case LogicalOperatorType::LOGICAL_GET:
 		LogicalOperatorVisitor::VisitOperatorExpressions(op);
 		if (!everything_referenced) {
@@ -146299,6 +146382,11 @@ unique_ptr<InsertStatement> Transformer::TransformInsert(duckdb_libpgquery::PGNo
 			result->columns.emplace_back(target->name);
 		}
 	}
+
+	// Grab and transform the returning columns from the parser.
+	if (stmt->returningList) {
+		Transformer::TransformExpressionList(*(stmt->returningList), result->returning_list);
+	}
 	result->select_statement = TransformSelect(stmt->selectStmt, false);
 
 	auto qname = TransformQualifiedName(stmt->relation);
@@ -152271,6 +152359,35 @@ protected:
 
 
 
+
+
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/planner/expression_binder/returning_binder.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+namespace duckdb {
+
+//! The RETURNING binder is responsible for binding expressions within the RETURNING statement
+class ReturningBinder : public ExpressionBinder {
+public:
+	ReturningBinder(Binder &binder, ClientContext &context);
+
+protected:
+	BindResult BindExpression(unique_ptr<ParsedExpression> *expr_ptr, idx_t depth,
+	                          bool root_expression = false) override;
+};
+
+} // namespace duckdb
+
+
 namespace duckdb {
 
 static void CheckInsertColumnCountMismatch(int64_t expected_columns, int64_t result_columns, bool columns_provided,
@@ -152378,18 +152495,73 @@ BoundStatement Binder::Bind(InsertStatement &stmt) {
 		}
 	}
 
-	// insert from select statement
 	// parse select statement and add to logical plan
 	auto root_select = Bind(*stmt.select_statement);
 	CheckInsertColumnCountMismatch(expected_columns, root_select.types.size(), !stmt.columns.empty(),
 	                               table->name.c_str());
 
 	auto root = CastLogicalOperatorToTypes(root_select.types, insert->expected_types, move(root_select.plan));
+
 	insert->AddChild(move(root));
 
-	result.plan = move(insert);
-	this->allow_stream_result = false;
-	return result;
+	if (!stmt.returning_list.empty()) {
+		insert->return_chunk = true;
+		// clear result type and names because returning list will update them
+		result.types.clear();
+		result.names.clear();
+
+		auto insert_table_index = GenerateTableIndex();
+		insert->table_index = insert_table_index;
+
+		auto binder = Binder::CreateBinder(context);
+		auto types = table->GetTypes();
+		vector<std::string> names;
+		for (auto &col : table->columns) {
+			names.push_back(col.name);
+		}
+
+		binder->bind_context.AddGenericBinding(insert->table_index, stmt.table, names, types);
+		ReturningBinder returning_binder(*binder, context);
+
+		unique_ptr<LogicalOperator> insert_as_logicaloperator = move(insert);
+
+		vector<unique_ptr<Expression>> projection_expressions;
+		LogicalType result_type;
+		for (auto &returning_expr : stmt.returning_list) {
+			auto expr_type = returning_expr->GetExpressionType();
+			if (expr_type == ExpressionType::STAR) {
+				auto generated_star_list = vector<unique_ptr<ParsedExpression>>();
+				binder->bind_context.GenerateAllColumnExpressions((StarExpression &)*returning_expr,
+				                                                  generated_star_list);
+
+				for (auto &star_column : generated_star_list) {
+					auto star_expr = returning_binder.Bind(star_column, &result_type);
+					result.types.push_back(result_type);
+					result.names.push_back(star_expr->GetName());
+					projection_expressions.push_back(move(star_expr));
+				}
+			} else {
+				auto expr = returning_binder.Bind(returning_expr, &result_type);
+				result.names.push_back(expr->GetName());
+				result.types.push_back(result_type);
+				projection_expressions.push_back(move(expr));
+			}
+		}
+
+		auto projection = make_unique<LogicalProjection>(GenerateTableIndex(), move(projection_expressions));
+		projection->AddChild(move(insert_as_logicaloperator));
+		result.plan = move(projection);
+		D_ASSERT(result.types.size() == result.names.size());
+		this->allow_stream_result = true;
+		return result;
+	} else {
+		insert->table_index = 0;
+		insert->return_chunk = false;
+		D_ASSERT(result.types.size() == result.names.size());
+		result.plan = move(insert);
+		this->allow_stream_result = false;
+		return result;
+	}
 }
 
 } // namespace duckdb
@@ -156138,6 +156310,28 @@ string RelationBinder::UnsupportedAggregateMessage() {
 
 
 
+namespace duckdb {
+
+ReturningBinder::ReturningBinder(Binder &binder, ClientContext &context) : ExpressionBinder(binder, context) {
+}
+
+BindResult ReturningBinder::BindExpression(unique_ptr<ParsedExpression> *expr_ptr, idx_t depth, bool root_expression) {
+	auto &expr = **expr_ptr;
+	switch (expr.GetExpressionClass()) {
+	case ExpressionClass::SUBQUERY:
+		return BindResult("SUBQUERY is not supported in returning statements");
+	case ExpressionClass::BOUND_SUBQUERY:
+		return BindResult("BOUND SUBQUERY is not supported in returning statements");
+	default:
+		return ExpressionBinder::BindExpression(expr_ptr, depth);
+	}
+}
+
+} // namespace duckdb
+
+
+
+
 
 
 

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "a004ed237"
-#define DUCKDB_VERSION "v0.3.3-dev865"
+#define DUCKDB_SOURCE_ID "3111480a9"
+#define DUCKDB_VERSION "v0.3.3-dev918"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -16647,6 +16647,7 @@ class Connection {
 public:
 	DUCKDB_API explicit Connection(DuckDB &database);
 	DUCKDB_API explicit Connection(DatabaseInstance &database);
+	DUCKDB_API ~Connection();
 
 	shared_ptr<ClientContext> context;
 	warning_callback warning_cb;
@@ -16828,6 +16829,7 @@ private:
 };
 
 } // namespace duckdb
+
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //