duckdb 0.5.1-dev138.0 → 0.5.1-dev149.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.5.1-dev138.0",
+  "version": "0.5.1-dev149.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -3763,6 +3763,19 @@ idx_t TableCatalogEntry::StandardColumnCount() const {
 	return count;
 }
 
+unique_ptr<BaseStatistics> TableCatalogEntry::GetStatistics(ClientContext &context, column_t column_id) {
+	if (column_id == COLUMN_IDENTIFIER_ROW_ID) {
+		return nullptr;
+	}
+	if (column_id >= columns.size()) {
+		throw InternalException("TableCatalogEntry::GetStatistics column_id out of range");
+	}
+	if (columns[column_id].Generated()) {
+		return nullptr;
+	}
+	return storage->GetStatistics(context, columns[column_id].StorageOid());
+}
+
 unique_ptr<CatalogEntry> TableCatalogEntry::AlterEntry(ClientContext &context, AlterInfo *info) {
 	D_ASSERT(!internal);
 	if (info->type != AlterType::ALTER_TABLE) {
@@ -4052,7 +4065,9 @@ unique_ptr<CatalogEntry> TableCatalogEntry::SetDefault(ClientContext &context, S
 		auto copy = columns[i].Copy();
 		if (default_idx == i) {
 			// set the default value of this column
-			D_ASSERT(!copy.Generated()); // Shouldnt reach here - DEFAULT value isn't supported for Generated Columns
+			if (copy.Generated()) {
+				throw BinderException("Cannot SET DEFAULT for generated column \"%s\"", columns[i].Name());
+			}
 			copy.SetDefaultValue(info.expression ? info.expression->Copy() : nullptr);
 		}
 		create_info->columns.push_back(move(copy));
@@ -22539,14 +22554,16 @@ struct IntervalToStringCast {
 			if (micros < 0) {
 				// negative time: append negative sign
 				buffer[length++] = '-';
+			} else {
 				micros = -micros;
 			}
-			int64_t hour = micros / Interval::MICROS_PER_HOUR;
-			micros -= hour * Interval::MICROS_PER_HOUR;
-			int64_t min = micros / Interval::MICROS_PER_MINUTE;
-			micros -= min * Interval::MICROS_PER_MINUTE;
-			int64_t sec = micros / Interval::MICROS_PER_SEC;
-			micros -= sec * Interval::MICROS_PER_SEC;
+			int64_t hour = -(micros / Interval::MICROS_PER_HOUR);
+			micros += hour * Interval::MICROS_PER_HOUR;
+			int64_t min = -(micros / Interval::MICROS_PER_MINUTE);
+			micros += min * Interval::MICROS_PER_MINUTE;
+			int64_t sec = -(micros / Interval::MICROS_PER_SEC);
+			micros += sec * Interval::MICROS_PER_SEC;
+			micros = -micros;
 
 			if (hour < 10) {
 				buffer[length++] = '0';
@@ -80801,6 +80818,7 @@ void PhysicalRecursiveCTE::ExecuteRecursivePipelines(ExecutionContext &context)
 void PhysicalRecursiveCTE::BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) {
 	op_state.reset();
 	sink_state.reset();
+	pipelines.clear();
 
 	// recursive CTE
 	state.SetPipelineSource(current, this);
@@ -121654,8 +121672,7 @@ static unique_ptr<BaseStatistics> TableScanStatistics(ClientContext &context, co
 		// we don't emit any statistics for tables that have outstanding transaction-local data
 		return nullptr;
 	}
-	auto storage_idx = GetStorageIndex(*bind_data.table, column_id);
-	return bind_data.table->storage->GetStatistics(context, storage_idx);
+	return bind_data.table->GetStatistics(context, column_id);
 }
 
 static void TableScanFunc(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) {
@@ -142557,9 +142574,7 @@ void CardinalityEstimator::UpdateTotalDomains(JoinNode *node, LogicalOperator *o
 			// Get HLL stats here
 			auto actual_binding = relation_column_to_original_column[key];
 
-			// sometimes base stats is null (test_709.test) returns null for base stats while
-			// there is still a catalog table. Anybody know anything about this?
-			auto base_stats = catalog_table->storage->GetStatistics(context, actual_binding.column_index);
+			auto base_stats = catalog_table->GetStatistics(context, actual_binding.column_index);
 			if (base_stats) {
 				count = base_stats->GetDistinctCount();
 			}
@@ -147919,21 +147934,35 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownAggregate(unique_ptr<Logical
 	FilterPushdown child_pushdown(optimizer);
 	for (idx_t i = 0; i < filters.size(); i++) {
 		auto &f = *filters[i];
-		// check if any aggregate or GROUPING functions are in the set
-		if (f.bindings.find(aggr.aggregate_index) == f.bindings.end() &&
-		    f.bindings.find(aggr.groupings_index) == f.bindings.end()) {
-			// no aggregate! we can push this down
-			// rewrite any group bindings within the filter
-			f.filter = ReplaceGroupBindings(aggr, move(f.filter));
-			// add the filter to the child node
-			if (child_pushdown.AddFilter(move(f.filter)) == FilterResult::UNSATISFIABLE) {
-				// filter statically evaluates to false, strip tree
-				return make_unique<LogicalEmptyResult>(move(op));
+		if (f.bindings.find(aggr.aggregate_index) != f.bindings.end()) {
+			// filter on aggregate: cannot pushdown
+			continue;
+		}
+		if (f.bindings.find(aggr.groupings_index) != f.bindings.end()) {
+			// filter on GROUPINGS function: cannot pushdown
+			continue;
+		}
+		// if there are any empty grouping sets, we cannot push down filters
+		bool has_empty_grouping_sets = false;
+		for (auto &grp : aggr.grouping_sets) {
+			if (grp.empty()) {
+				has_empty_grouping_sets = true;
 			}
-			// erase the filter from here
-			filters.erase(filters.begin() + i);
-			i--;
 		}
+		if (has_empty_grouping_sets) {
+			continue;
+		}
+		// no aggregate! we can push this down
+		// rewrite any group bindings within the filter
+		f.filter = ReplaceGroupBindings(aggr, move(f.filter));
+		// add the filter to the child node
+		if (child_pushdown.AddFilter(move(f.filter)) == FilterResult::UNSATISFIABLE) {
+			// filter statically evaluates to false, strip tree
+			return make_unique<LogicalEmptyResult>(move(op));
+		}
+		// erase the filter from here
+		filters.erase(filters.begin() + i);
+		i--;
 	}
 	child_pushdown.GenerateFilters();
 
@@ -177191,6 +177220,7 @@ protected:
 	BindResult BindExpression(unique_ptr<ParsedExpression> *expr, idx_t depth, bool root_expression = false) override;
 
 	string UnsupportedAggregateMessage() override;
+	bool CanContainSubqueries() override;
 };
 
 } // namespace duckdb
@@ -185053,6 +185083,10 @@ string ConstantBinder::UnsupportedAggregateMessage() {
 	return clause + " cannot contain aggregates!";
 }
 
+bool ConstantBinder::CanContainSubqueries() {
+	return false;
+}
+
 } // namespace duckdb
 
 
@@ -185854,6 +185888,9 @@ unique_ptr<Expression> ExpressionBinder::Bind(unique_ptr<ParsedExpression> &expr
 	// bind the main expression
 	auto error_msg = Bind(&expr, 0, root_expression);
 	if (!error_msg.empty()) {
+		if (!CanContainSubqueries()) {
+			throw BinderException(error_msg);
+		}
 		// failed to bind: try to bind correlated columns in the expression (if any)
 		bool success = BindCorrelatedColumns(expr);
 		if (!success) {
@@ -185911,6 +185948,10 @@ string ExpressionBinder::Bind(unique_ptr<ParsedExpression> *expr, idx_t depth, b
 	return string();
 }
 
+bool ExpressionBinder::CanContainSubqueries() {
+	return true;
+}
+
 } // namespace duckdb
 
 
@@ -198329,6 +198370,9 @@ unique_ptr<BaseStatistics> DataTable::GetStatistics(ClientContext &context, colu
 		return nullptr;
 	}
 	lock_guard<mutex> stats_guard(stats_lock);
+	if (column_id >= column_stats.size()) {
+		throw InternalException("Call to GetStatistics is out of range");
+	}
 	return column_stats[column_id]->stats->Copy();
 }
 
@@ -201143,6 +201187,7 @@ idx_t ChunkVectorInfo::Delete(Transaction &transaction, row_t rows[], idx_t coun
 		}
 		// after verifying that there are no conflicts we mark the tuple as deleted
 		deleted[rows[i]] = transaction.transaction_id;
+		rows[deleted_tuples] = rows[i];
 		deleted_tuples++;
 	}
 	return deleted_tuples;
@@ -204049,9 +204094,15 @@ void VersionDeleteState::Flush() {
 		return;
 	}
 	// delete in the current info
-	delete_count += current_info->Delete(transaction, rows, count);
-	// now push the delete into the undo buffer
-	transaction.PushDelete(table, current_info, rows, count, base_row + chunk_row);
+	// it is possible for delete statements to delete the same tuple multiple times when combined with a USING clause
+	// in the current_info->Delete, we check which tuples are actually deleted (excluding duplicate deletions)
+	// this is returned in the actual_delete_count
+	auto actual_delete_count = current_info->Delete(transaction, rows, count);
+	delete_count += actual_delete_count;
+	if (actual_delete_count > 0) {
+		// now push the delete into the undo buffer, but only if any deletes were actually performed
+		transaction.PushDelete(table, current_info, rows, actual_delete_count, base_row + chunk_row);
+	}
 	count = 0;
 }
 
@@ -206887,6 +206938,7 @@ void CleanupState::CleanupUpdate(UpdateInfo *info) {
 
 void CleanupState::CleanupDelete(DeleteInfo *info) {
 	auto version_table = info->table;
+	D_ASSERT(version_table->info->cardinality >= info->count);
 	version_table->info->cardinality -= info->count;
 	if (version_table->info->indexes.Empty()) {
 		// this table has no indexes: no cleanup to be done

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "3ccb7e6d9"
-#define DUCKDB_VERSION "v0.5.1-dev138"
+#define DUCKDB_SOURCE_ID "bb671e06a"
+#define DUCKDB_VERSION "v0.5.1-dev149"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -9935,6 +9935,9 @@ public:
 	vector<LogicalType> GetTypes();
 	string ToSQL() override;
 
+	//! Get statistics of a column (physical or virtual) within the table
+	unique_ptr<BaseStatistics> GetStatistics(ClientContext &context, column_t column_id);
+
 	//! Serialize the meta information of the TableCatalogEntry a serializer
 	virtual void Serialize(Serializer &serializer);
 	//! Deserializes to a CreateTableInfo
@@ -15987,6 +15990,7 @@ protected:
 
 	virtual string UnsupportedAggregateMessage();
 	virtual string UnsupportedUnnestMessage();
+	virtual bool CanContainSubqueries();
 
 	Binder &binder;
 	ClientContext &context;
@@ -22751,6 +22755,11 @@ public:
 	void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) override;
 
 	void Append(idx_t start, idx_t end, transaction_t commit_id);
+	//! Performs a delete in the ChunkVectorInfo - returns how many tuples were actually deleted
+	//! The number of rows that were actually deleted might be lower than the input count
+	//! In case we delete rows that were already deleted
+	//! Note that "rows" is written to to reflect the row ids that were actually deleted
+	//! i.e. after calling this function, rows will hold [0..actual_delete_count] row ids of the actually deleted tuples
 	idx_t Delete(Transaction &transaction, row_t rows[], idx_t count);
 	void CommitDelete(transaction_t commit_id, row_t rows[], idx_t count);
 
@@ -24382,7 +24391,9 @@ public:
 		this->is_root = true;
 	}
 
+	//! Get statistics of a physical column within the table
 	unique_ptr<BaseStatistics> GetStatistics(ClientContext &context, column_t column_id);
+	//! Sets statistics of a physical column within the table
 	void SetStatistics(column_t column_id, const std::function<void(BaseStatistics &)> &set_fun);
 
 	//! Checkpoint the table to the specified table data writer