duckdb 0.7.2-dev865.0 → 0.7.2-dev886.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.7.2-dev865.0",
+  "version": "0.7.2-dev886.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/catalog/default/default_functions.cpp

@@ -131,6 +131,9 @@ static DefaultMacro internal_macros[] = {
 	// nested list aggregates
 	{DEFAULT_SCHEMA, "list_histogram", {"l", nullptr}, "list_aggr(l, 'histogram')"},
 
+	// date functions
+	{DEFAULT_SCHEMA, "date_add", {"date", "interval", nullptr}, "date + interval"},
+
 	{nullptr, nullptr, {nullptr}, nullptr}
 	};
 

package/src/duckdb/src/catalog/duck_catalog.cpp

@@ -46,32 +46,59 @@ bool DuckCatalog::IsDuckCatalog() {
 //===--------------------------------------------------------------------===//
 // Schema
 //===--------------------------------------------------------------------===//
-CatalogEntry *DuckCatalog::CreateSchema(CatalogTransaction transaction, CreateSchemaInfo *info) {
-	D_ASSERT(!info->schema.empty());
+CatalogEntry *DuckCatalog::CreateSchemaInternal(CatalogTransaction transaction, CreateSchemaInfo *info) {
 	DependencyList dependencies;
 	auto entry = make_unique<DuckSchemaEntry>(this, info->schema, info->internal);
 	auto result = entry.get();
 	if (!schemas->CreateEntry(transaction, info->schema, std::move(entry), dependencies)) {
-		if (info->on_conflict == OnCreateConflict::ERROR_ON_CONFLICT) {
+		return nullptr;
+	}
+	return (CatalogEntry *)result;
+}
+
+CatalogEntry *DuckCatalog::CreateSchema(CatalogTransaction transaction, CreateSchemaInfo *info) {
+	D_ASSERT(!info->schema.empty());
+	auto result = CreateSchemaInternal(transaction, info);
+	if (!result) {
+		switch (info->on_conflict) {
+		case OnCreateConflict::ERROR_ON_CONFLICT:
 			throw CatalogException("Schema with name %s already exists!", info->schema);
-		} else {
-			D_ASSERT(info->on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT);
+		case OnCreateConflict::REPLACE_ON_CONFLICT: {
+			DropInfo drop_info;
+			drop_info.type = CatalogType::SCHEMA_ENTRY;
+			drop_info.catalog = info->catalog;
+			drop_info.name = info->schema;
+			DropSchema(transaction, &drop_info);
+			result = CreateSchemaInternal(transaction, info);
+			if (!result) {
+				throw InternalException("Failed to create schema entry in CREATE_OR_REPLACE");
+			}
+			break;
+		}
+		case OnCreateConflict::IGNORE_ON_CONFLICT:
+			break;
+		default:
+			throw InternalException("Unsupported OnCreateConflict for CreateSchema");
 		}
 		return nullptr;
 	}
 	return result;
 }
 
-void DuckCatalog::DropSchema(ClientContext &context, DropInfo *info) {
+void DuckCatalog::DropSchema(CatalogTransaction transaction, DropInfo *info) {
 	D_ASSERT(!info->name.empty());
 	ModifyCatalog();
-	if (!schemas->DropEntry(GetCatalogTransaction(context), info->name, info->cascade)) {
+	if (!schemas->DropEntry(transaction, info->name, info->cascade)) {
 		if (!info->if_exists) {
 			throw CatalogException("Schema with name \"%s\" does not exist!", info->name);
 		}
 	}
 }
 
+void DuckCatalog::DropSchema(ClientContext &context, DropInfo *info) {
+	DropSchema(GetCatalogTransaction(context), info);
+}
+
 void DuckCatalog::ScanSchemas(ClientContext &context, std::function<void(CatalogEntry *)> callback) {
 	schemas->Scan(GetCatalogTransaction(context), [&](CatalogEntry *entry) { callback(entry); });
 }

package/src/duckdb/src/common/box_renderer.cpp

@@ -197,6 +197,60 @@ list<ColumnDataCollection> BoxRenderer::FetchRenderCollections(ClientContext &co
 	return collections;
 }
 
+list<ColumnDataCollection> BoxRenderer::PivotCollections(ClientContext &context, list<ColumnDataCollection> input,
+                                                         vector<string> &column_names,
+                                                         vector<LogicalType> &result_types, idx_t row_count) {
+	auto &top = input.front();
+	auto &bottom = input.back();
+
+	vector<LogicalType> varchar_types;
+	vector<string> new_names;
+	new_names.emplace_back("Column");
+	new_names.emplace_back("Type");
+	varchar_types.emplace_back(LogicalType::VARCHAR);
+	varchar_types.emplace_back(LogicalType::VARCHAR);
+	for (idx_t r = 0; r < top.Count(); r++) {
+		new_names.emplace_back("Row " + to_string(r + 1));
+		varchar_types.emplace_back(LogicalType::VARCHAR);
+	}
+	for (idx_t r = 0; r < bottom.Count(); r++) {
+		auto row_index = row_count - bottom.Count() + r + 1;
+		new_names.emplace_back("Row " + to_string(row_index));
+		varchar_types.emplace_back(LogicalType::VARCHAR);
+	}
+	//
+	DataChunk row_chunk;
+	row_chunk.Initialize(Allocator::DefaultAllocator(), varchar_types);
+	std::list<ColumnDataCollection> result;
+	result.emplace_back(context, varchar_types);
+	result.emplace_back(context, varchar_types);
+	auto &res_coll = result.front();
+	ColumnDataAppendState append_state;
+	res_coll.InitializeAppend(append_state);
+	for (idx_t c = 0; c < top.ColumnCount(); c++) {
+		vector<column_t> column_ids {c};
+		auto row_index = row_chunk.size();
+		idx_t current_index = 0;
+		row_chunk.SetValue(current_index++, row_index, column_names[c]);
+		row_chunk.SetValue(current_index++, row_index, RenderType(result_types[c]));
+		for (auto &collection : input) {
+			for (auto &chunk : collection.Chunks(column_ids)) {
+				for (idx_t r = 0; r < chunk.size(); r++) {
+					row_chunk.SetValue(current_index++, row_index, chunk.GetValue(0, r));
+				}
+			}
+		}
+		row_chunk.SetCardinality(row_chunk.size() + 1);
+		if (row_chunk.size() == STANDARD_VECTOR_SIZE || c + 1 == top.ColumnCount()) {
+			res_coll.Append(append_state, row_chunk);
+			row_chunk.Reset();
+		}
+	}
+	column_names = std::move(new_names);
+	result_types = std::move(varchar_types);
+	return result;
+}
+
 string ConvertRenderValue(const string &input) {
 	return StringUtil::Replace(StringUtil::Replace(input, "\n", "\\n"), string("\0", 1), "\\0");
 }
@@ -213,11 +267,10 @@ string BoxRenderer::GetRenderValue(ColumnDataRowCollection &rows, idx_t c, idx_t
 	}
 }
 
-vector<idx_t> BoxRenderer::ComputeRenderWidths(const vector<string> &names, const ColumnDataCollection &result,
+vector<idx_t> BoxRenderer::ComputeRenderWidths(const vector<string> &names, const vector<LogicalType> &result_types,
                                                list<ColumnDataCollection> &collections, idx_t min_width,
                                                idx_t max_width, vector<idx_t> &column_map, idx_t &total_length) {
-	auto column_count = result.ColumnCount();
-	auto &result_types = result.Types();
+	auto column_count = result_types.size();
 
 	vector<idx_t> widths;
 	widths.reserve(column_count);
@@ -357,13 +410,15 @@ void BoxRenderer::RenderHeader(const vector<string> &names, const vector<Logical
 	ss << std::endl;
 
 	// render the types
-	for (idx_t c = 0; c < column_count; c++) {
-		auto column_idx = column_map[c];
-		auto type = column_idx == SPLIT_COLUMN ? "" : RenderType(result_types[column_idx]);
-		RenderValue(ss, type, widths[c]);
+	if (config.render_mode == RenderMode::ROWS) {
+		for (idx_t c = 0; c < column_count; c++) {
+			auto column_idx = column_map[c];
+			auto type = column_idx == SPLIT_COLUMN ? "" : RenderType(result_types[column_idx]);
+			RenderValue(ss, type, widths[c]);
+		}
+		ss << config.VERTICAL;
+		ss << std::endl;
 	}
-	ss << config.VERTICAL;
-	ss << std::endl;
 
 	// render the line under the header
 	ss << config.LMIDDLE;
@@ -390,12 +445,14 @@ void BoxRenderer::RenderValues(const list<ColumnDataCollection> &collections, co
 	auto column_count = column_map.size();
 
 	vector<ValueRenderAlignment> alignments;
-	for (idx_t c = 0; c < column_count; c++) {
-		auto column_idx = column_map[c];
-		if (column_idx == SPLIT_COLUMN) {
-			alignments.push_back(ValueRenderAlignment::MIDDLE);
-		} else {
-			alignments.push_back(TypeAlignment(result_types[column_idx]));
+	if (config.render_mode == RenderMode::ROWS) {
+		for (idx_t c = 0; c < column_count; c++) {
+			auto column_idx = column_map[c];
+			if (column_idx == SPLIT_COLUMN) {
+				alignments.push_back(ValueRenderAlignment::MIDDLE);
+			} else {
+				alignments.push_back(TypeAlignment(result_types[column_idx]));
+			}
 		}
 	}
 
@@ -409,13 +466,28 @@ void BoxRenderer::RenderValues(const list<ColumnDataCollection> &collections, co
 			} else {
 				str = GetRenderValue(rows, column_idx, r);
 			}
-			RenderValue(ss, str, widths[c], alignments[c]);
+			ValueRenderAlignment alignment;
+			if (config.render_mode == RenderMode::ROWS) {
+				alignment = alignments[c];
+			} else {
+				if (c < 2) {
+					alignment = ValueRenderAlignment::LEFT;
+				} else if (c == SPLIT_COLUMN) {
+					alignment = ValueRenderAlignment::MIDDLE;
+				} else {
+					alignment = ValueRenderAlignment::RIGHT;
+				}
+			}
+			RenderValue(ss, str, widths[c], alignment);
 		}
 		ss << config.VERTICAL;
 		ss << std::endl;
 	}
 
 	if (bottom_rows > 0) {
+		if (config.render_mode == RenderMode::COLUMNS) {
+			throw InternalException("Columns render mode does not support bottom rows");
+		}
 		// render the bottom rows
 		// first render the divider
 		auto brows = bottom_collection.GetRows();
@@ -607,15 +679,19 @@ void BoxRenderer::Render(ClientContext &context, const vector<string> &names, co
 
 	// fetch the top and bottom render collections from the result
 	auto collections = FetchRenderCollections(context, result, top_rows, bottom_rows);
-
-	auto &result_types = result.Types();
+	auto column_names = names;
+	auto result_types = result.Types();
+	if (config.render_mode == RenderMode::COLUMNS) {
+		collections = PivotCollections(context, std::move(collections), column_names, result_types, row_count);
+	}
 
 	// for each column, figure out the width
 	// start off by figuring out the name of the header by looking at the column name and column type
 	idx_t min_width = has_hidden_rows || row_count == 0 ? minimum_row_length : 0;
 	vector<idx_t> column_map;
 	idx_t total_length;
-	auto widths = ComputeRenderWidths(names, result, collections, min_width, max_width, column_map, total_length);
+	auto widths =
+	    ComputeRenderWidths(column_names, result_types, collections, min_width, max_width, column_map, total_length);
 
 	// render boundaries for the individual columns
 	vector<idx_t> boundaries;
@@ -631,7 +707,7 @@ void BoxRenderer::Render(ClientContext &context, const vector<string> &names, co
 
 	// now begin rendering
 	// first render the header
-	RenderHeader(names, result_types, column_map, widths, boundaries, total_length, row_count > 0, ss);
+	RenderHeader(column_names, result_types, column_map, widths, boundaries, total_length, row_count > 0, ss);
 
 	// render the values, if there are any
 	RenderValues(collections, column_map, widths, result_types, ss);
@@ -649,10 +725,20 @@ void BoxRenderer::Render(ClientContext &context, const vector<string> &names, co
 		}
 	}
 	idx_t column_count = column_map.size();
-	if (has_hidden_columns) {
-		column_count--;
-		column_count_str += " (" + to_string(column_count) + " shown)";
+	if (config.render_mode == RenderMode::COLUMNS) {
+		if (has_hidden_columns) {
+			has_hidden_rows = true;
+			shown_str = " (" + to_string(column_count - 3) + " shown)";
+		} else {
+			shown_str = string();
+		}
+	} else {
+		if (has_hidden_columns) {
+			column_count--;
+			column_count_str += " (" + to_string(column_count) + " shown)";
+		}
 	}
+
 	RenderRowCount(std::move(row_count_str), std::move(shown_str), column_count_str, boundaries, has_hidden_rows,
 	               has_hidden_columns, total_length, row_count, column_count, minimum_row_length, ss);
 }

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.7.2-dev865"
+#define DUCKDB_VERSION "0.7.2-dev886"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "61325fdd83"
+#define DUCKDB_SOURCE_ID "7d73c6e6a8"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/catalog/duck_catalog.hpp

@@ -59,8 +59,9 @@ public:
 	DUCKDB_API string GetDBPath() override;
 
 private:
+	DUCKDB_API void DropSchema(CatalogTransaction transaction, DropInfo *info);
 	DUCKDB_API void DropSchema(ClientContext &context, DropInfo *info) override;
-
+	CatalogEntry *CreateSchemaInternal(CatalogTransaction transaction, CreateSchemaInfo *info);
 	void Verify() override;
 
 private:

package/src/duckdb/src/include/duckdb/common/box_renderer.hpp

@@ -18,6 +18,7 @@ class ColumnDataCollection;
 class ColumnDataRowCollection;
 
 enum class ValueRenderAlignment { LEFT, MIDDLE, RIGHT };
+enum class RenderMode { ROWS, COLUMNS };
 
 struct BoxRendererConfig {
 	// a max_width of 0 means we default to the terminal width
@@ -30,7 +31,10 @@ struct BoxRendererConfig {
 	// the max col width determines the maximum size of a single column
 	// note that the max col width is only used if the result does not fit on the screen
 	idx_t max_col_width = 20;
+	//! how to render NULL values
 	string null_value = "NULL";
+	//! Whether or not to render row-wise or column-wise
+	RenderMode render_mode = RenderMode::ROWS;
 
 #ifndef DUCKDB_ASCII_TREE_RENDERER
 	const char *LTCORNER = "\342\224\214"; // "┌";
@@ -94,10 +98,12 @@ private:
 	string RenderType(const LogicalType &type);
 	ValueRenderAlignment TypeAlignment(const LogicalType &type);
 	string GetRenderValue(ColumnDataRowCollection &rows, idx_t c, idx_t r);
-
 	list<ColumnDataCollection> FetchRenderCollections(ClientContext &context, const ColumnDataCollection &result,
 	                                                  idx_t top_rows, idx_t bottom_rows);
-	vector<idx_t> ComputeRenderWidths(const vector<string> &names, const ColumnDataCollection &result,
+	list<ColumnDataCollection> PivotCollections(ClientContext &context, list<ColumnDataCollection> input,
+	                                            vector<string> &column_names, vector<LogicalType> &result_types,
+	                                            idx_t row_count);
+	vector<idx_t> ComputeRenderWidths(const vector<string> &names, const vector<LogicalType> &result_types,
 	                                  list<ColumnDataCollection> &collections, idx_t min_width, idx_t max_width,
 	                                  vector<idx_t> &column_map, idx_t &total_length);
 	void RenderHeader(const vector<string> &names, const vector<LogicalType> &result_types,

package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp

@@ -67,6 +67,8 @@ private:
 	unique_ptr<LogicalOperator> PushdownSingleJoin(unique_ptr<LogicalOperator> op, unordered_set<idx_t> &left_bindings,
 	                                               unordered_set<idx_t> &right_bindings);
 
+	//! Push any remaining filters into a LogicalFilter at this level
+	unique_ptr<LogicalOperator> PushFinalFilters(unique_ptr<LogicalOperator> op);
 	// Finish pushing down at this operator, creating a LogicalFilter to store any of the stored filters and recursively
 	// pushing down into its children (if any)
 	unique_ptr<LogicalOperator> FinishPushdown(unique_ptr<LogicalOperator> op);

package/src/duckdb/src/optimizer/filter_pushdown.cpp

@@ -105,13 +105,7 @@ void FilterPushdown::GenerateFilters() {
 	});
 }
 
-unique_ptr<LogicalOperator> FilterPushdown::FinishPushdown(unique_ptr<LogicalOperator> op) {
-	// unhandled type, first perform filter pushdown in its children
-	for (auto &child : op->children) {
-		FilterPushdown pushdown(optimizer);
-		child = pushdown.Rewrite(std::move(child));
-	}
-	// now push any existing filters
+unique_ptr<LogicalOperator> FilterPushdown::PushFinalFilters(unique_ptr<LogicalOperator> op) {
 	if (filters.empty()) {
 		// no filters to push
 		return op;
@@ -124,6 +118,16 @@ unique_ptr<LogicalOperator> FilterPushdown::FinishPushdown(unique_ptr<LogicalOpe
 	return std::move(filter);
 }
 
+unique_ptr<LogicalOperator> FilterPushdown::FinishPushdown(unique_ptr<LogicalOperator> op) {
+	// unhandled type, first perform filter pushdown in its children
+	for (auto &child : op->children) {
+		FilterPushdown pushdown(optimizer);
+		child = pushdown.Rewrite(std::move(child));
+	}
+	// now push any existing filters
+	return PushFinalFilters(std::move(op));
+}
+
 void FilterPushdown::Filter::ExtractBindings() {
 	bindings.clear();
 	LogicalJoin::GetExpressionBindings(*filter, bindings);

package/src/duckdb/src/optimizer/pushdown/pushdown_left_join.cpp

@@ -122,16 +122,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownLeftJoin(unique_ptr<LogicalO
 	right_pushdown.GenerateFilters();
 	op->children[0] = left_pushdown.Rewrite(std::move(op->children[0]));
 	op->children[1] = right_pushdown.Rewrite(std::move(op->children[1]));
-	if (filters.empty()) {
-		// no filters to push
-		return op;
-	}
-	auto filter = make_unique<LogicalFilter>();
-	for (auto &f : filters) {
-		filter->expressions.push_back(std::move(f->filter));
-	}
-	filter->children.push_back(std::move(op));
-	return std::move(filter);
+	return PushFinalFilters(std::move(op));
 }
 
 } // namespace duckdb

package/src/duckdb/src/optimizer/pushdown/pushdown_mark_join.cpp

@@ -77,7 +77,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownMarkJoin(unique_ptr<LogicalO
 	}
 	op->children[0] = left_pushdown.Rewrite(std::move(op->children[0]));
 	op->children[1] = right_pushdown.Rewrite(std::move(op->children[1]));
-	return FinishPushdown(std::move(op));
+	return PushFinalFilters(std::move(op));
 }
 
 } // namespace duckdb

package/src/duckdb/src/optimizer/pushdown/pushdown_single_join.cpp

@@ -23,7 +23,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownSingleJoin(unique_ptr<Logica
 	}
 	op->children[0] = left_pushdown.Rewrite(std::move(op->children[0]));
 	op->children[1] = right_pushdown.Rewrite(std::move(op->children[1]));
-	return FinishPushdown(std::move(op));
+	return PushFinalFilters(std::move(op));
 }
 
 } // namespace duckdb

package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp

@@ -10,6 +10,7 @@
 #include "duckdb/parser/statement/drop_statement.hpp"
 #include "duckdb/parser/parsed_data/drop_info.hpp"
 #include "duckdb/parser/expression/cast_expression.hpp"
+#include "duckdb/parser/expression/function_expression.hpp"
 #include "duckdb/parser/result_modifier.hpp"
 #include "duckdb/parser/tableref/subqueryref.hpp"
 
@@ -99,15 +100,35 @@ unique_ptr<SQLStatement> Transformer::CreatePivotStatement(unique_ptr<SQLStateme
 unique_ptr<QueryNode> Transformer::TransformPivotStatement(duckdb_libpgquery::PGSelectStmt *stmt) {
 	auto pivot = stmt->pivot;
 	auto source = TransformTableRefNode(pivot->source);
-	auto columns = TransformPivotList(pivot->columns);
 
 	auto select_node = make_unique<SelectNode>();
 	// handle the CTEs
 	if (stmt->withClause) {
 		TransformCTE(reinterpret_cast<duckdb_libpgquery::PGWithClause *>(stmt->withClause), select_node->cte_map);
 	}
+	if (!pivot->columns) {
+		// no pivot columns - not actually a pivot
+		select_node->from_table = std::move(source);
+		if (pivot->groups) {
+			auto groups = TransformStringList(pivot->groups);
+			GroupingSet set;
+			for (idx_t gr = 0; gr < groups.size(); gr++) {
+				auto &group = groups[gr];
+				auto colref = make_unique<ColumnRefExpression>(group);
+				select_node->select_list.push_back(colref->Copy());
+				select_node->groups.group_expressions.push_back(std::move(colref));
+				set.insert(gr);
+			}
+			select_node->groups.grouping_sets.push_back(std::move(set));
+		}
+		if (pivot->aggrs) {
+			TransformExpressionList(*pivot->aggrs, select_node->select_list);
+		}
+		return std::move(select_node);
+	}
 
 	// generate CREATE TYPE statements for each of the columns that do not have an IN list
+	auto columns = TransformPivotList(pivot->columns);
 	auto pivot_idx = PivotEntryCount();
 	for (idx_t c = 0; c < columns.size(); c++) {
 		auto &col = columns[c];
@@ -131,13 +152,17 @@ unique_ptr<QueryNode> Transformer::TransformPivotStatement(duckdb_libpgquery::PG
 
 	auto pivot_ref = make_unique<PivotRef>();
 	pivot_ref->source = std::move(source);
-	if (pivot->aggrs) {
-		TransformExpressionList(*pivot->aggrs, pivot_ref->aggregates);
+	if (pivot->unpivots) {
+		pivot_ref->unpivot_names = TransformStringList(pivot->unpivots);
 	} else {
-		if (!pivot->unpivots) {
-			throw InternalException("No unpivots and no aggrs");
+		if (pivot->aggrs) {
+			TransformExpressionList(*pivot->aggrs, pivot_ref->aggregates);
+		} else {
+			// pivot but no aggregates specified - push a count star
+			vector<unique_ptr<ParsedExpression>> children;
+			auto function = make_unique<FunctionExpression>("count_star", std::move(children));
+			pivot_ref->aggregates.push_back(std::move(function));
 		}
-		pivot_ref->unpivot_names = TransformStringList(pivot->unpivots);
 	}
 	if (pivot->groups) {
 		pivot_ref->groups = TransformStringList(pivot->groups);

package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp

@@ -56,7 +56,7 @@ static void ConstructPivots(PivotRef &ref, idx_t pivot_idx, vector<unique_ptr<Pa
 				auto &function = (FunctionExpression &)*copy;
 				// add the filter and alias to the aggregate function
 				function.filter = expr->Copy();
-				if (ref.aggregates.size() > 1) {
+				if (ref.aggregates.size() > 1 || !function.alias.empty()) {
 					// if there are multiple aggregates specified we add the name of the aggregate as well
 					function.alias = name + "_" + function.GetName();
 				} else {