duckdb 0.7.2-dev0.0 → 0.7.2-dev1138.0

package/src/duckdb/src/catalog/catalog_entry/type_catalog_entry.cpp

@@ -23,7 +23,13 @@ void TypeCatalogEntry::Serialize(Serializer &serializer) {
 	FieldWriter writer(serializer);
 	writer.WriteString(schema->name);
 	writer.WriteString(name);
-	writer.WriteSerializable(user_type);
+	if (user_type.id() == LogicalTypeId::ENUM) {
+		// We have to serialize Enum Values
+		writer.AddField();
+		user_type.SerializeEnumType(writer.GetSerializer());
+	} else {
+		writer.WriteSerializable(user_type);
+	}
 	writer.Finalize();
 }
 
@@ -43,7 +49,7 @@ string TypeCatalogEntry::ToSQL() {
 	std::stringstream ss;
 	switch (user_type.id()) {
 	case (LogicalTypeId::ENUM): {
-		Vector values_insert_order(EnumType::GetValuesInsertOrder(user_type));
+		auto &values_insert_order = EnumType::GetValuesInsertOrder(user_type);
 		idx_t size = EnumType::GetSize(user_type);
 		ss << "CREATE TYPE ";
 		ss << KeywordHelper::WriteOptionallyQuoted(name);

package/src/duckdb/src/catalog/catalog_set.cpp

@@ -258,6 +258,7 @@ bool CatalogSet::AlterEntry(CatalogTransaction transaction, const string &name,
 
 	// serialize the AlterInfo into a temporary buffer
 	BufferedSerializer serializer;
+	serializer.WriteString(alter_info->GetColumnName());
 	alter_info->Serialize(serializer);
 	BinaryData serialized_alter = serializer.GetData();
 

package/src/duckdb/src/catalog/default/default_functions.cpp

@@ -131,6 +131,9 @@ static DefaultMacro internal_macros[] = {
 	// nested list aggregates
 	{DEFAULT_SCHEMA, "list_histogram", {"l", nullptr}, "list_aggr(l, 'histogram')"},
 
+	// date functions
+	{DEFAULT_SCHEMA, "date_add", {"date", "interval", nullptr}, "date + interval"},
+
 	{nullptr, nullptr, {nullptr}, nullptr}
 	};
 

package/src/duckdb/src/catalog/dependency_list.cpp

@@ -1,5 +1,6 @@
 #include "duckdb/catalog/dependency_list.hpp"
 #include "duckdb/catalog/catalog_entry.hpp"
+#include "duckdb/catalog/catalog.hpp"
 
 namespace duckdb {
 
@@ -10,4 +11,15 @@ void DependencyList::AddDependency(CatalogEntry *entry) {
 	set.insert(entry);
 }
 
+void DependencyList::VerifyDependencies(Catalog *catalog, const string &name) {
+	for (auto &dep : set) {
+		if (dep->catalog != catalog) {
+			throw DependencyException(
+			    "Error adding dependency for object \"%s\" - dependency \"%s\" is in catalog "
+			    "\"%s\", which does not match the catalog \"%s\".\nCross catalog dependencies are not supported.",
+			    name, dep->name, dep->catalog->GetName(), catalog->GetName());
+		}
+	}
+}
+
 } // namespace duckdb

package/src/duckdb/src/catalog/duck_catalog.cpp

@@ -46,32 +46,59 @@ bool DuckCatalog::IsDuckCatalog() {
 //===--------------------------------------------------------------------===//
 // Schema
 //===--------------------------------------------------------------------===//
-CatalogEntry *DuckCatalog::CreateSchema(CatalogTransaction transaction, CreateSchemaInfo *info) {
-	D_ASSERT(!info->schema.empty());
+CatalogEntry *DuckCatalog::CreateSchemaInternal(CatalogTransaction transaction, CreateSchemaInfo *info) {
 	DependencyList dependencies;
 	auto entry = make_unique<DuckSchemaEntry>(this, info->schema, info->internal);
 	auto result = entry.get();
 	if (!schemas->CreateEntry(transaction, info->schema, std::move(entry), dependencies)) {
-		if (info->on_conflict == OnCreateConflict::ERROR_ON_CONFLICT) {
+		return nullptr;
+	}
+	return (CatalogEntry *)result;
+}
+
+CatalogEntry *DuckCatalog::CreateSchema(CatalogTransaction transaction, CreateSchemaInfo *info) {
+	D_ASSERT(!info->schema.empty());
+	auto result = CreateSchemaInternal(transaction, info);
+	if (!result) {
+		switch (info->on_conflict) {
+		case OnCreateConflict::ERROR_ON_CONFLICT:
 			throw CatalogException("Schema with name %s already exists!", info->schema);
-		} else {
-			D_ASSERT(info->on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT);
+		case OnCreateConflict::REPLACE_ON_CONFLICT: {
+			DropInfo drop_info;
+			drop_info.type = CatalogType::SCHEMA_ENTRY;
+			drop_info.catalog = info->catalog;
+			drop_info.name = info->schema;
+			DropSchema(transaction, &drop_info);
+			result = CreateSchemaInternal(transaction, info);
+			if (!result) {
+				throw InternalException("Failed to create schema entry in CREATE_OR_REPLACE");
+			}
+			break;
+		}
+		case OnCreateConflict::IGNORE_ON_CONFLICT:
+			break;
+		default:
+			throw InternalException("Unsupported OnCreateConflict for CreateSchema");
 		}
 		return nullptr;
 	}
 	return result;
 }
 
-void DuckCatalog::DropSchema(ClientContext &context, DropInfo *info) {
+void DuckCatalog::DropSchema(CatalogTransaction transaction, DropInfo *info) {
 	D_ASSERT(!info->name.empty());
 	ModifyCatalog();
-	if (!schemas->DropEntry(GetCatalogTransaction(context), info->name, info->cascade)) {
+	if (!schemas->DropEntry(transaction, info->name, info->cascade)) {
 		if (!info->if_exists) {
 			throw CatalogException("Schema with name \"%s\" does not exist!", info->name);
 		}
 	}
 }
 
+void DuckCatalog::DropSchema(ClientContext &context, DropInfo *info) {
+	DropSchema(GetCatalogTransaction(context), info);
+}
+
 void DuckCatalog::ScanSchemas(ClientContext &context, std::function<void(CatalogEntry *)> callback) {
 	schemas->Scan(GetCatalogTransaction(context), [&](CatalogEntry *entry) { callback(entry); });
 }

package/src/duckdb/src/common/arrow/arrow_appender.cpp

@@ -4,6 +4,7 @@
 #include "duckdb/common/array.hpp"
 #include "duckdb/common/types/interval.hpp"
 #include "duckdb/common/types/uuid.hpp"
+#include "duckdb/function/table/arrow.hpp"
 
 namespace duckdb {
 
@@ -127,7 +128,11 @@ struct ArrowScalarConverter {
 struct ArrowIntervalConverter {
 	template <class TGT, class SRC>
 	static TGT Operation(SRC input) {
-		return Interval::GetMilli(input);
+		ArrowInterval result;
+		result.months = input.months;
+		result.days = input.days;
+		result.nanoseconds = input.micros * Interval::NANOS_PER_MICRO;
+		return result;
 	}
 
 	static bool SkipNulls() {
@@ -136,7 +141,6 @@ struct ArrowIntervalConverter {
 
 	template <class TGT>
 	static void SetNull(TGT &value) {
-		value = 0;
 	}
 };
 
@@ -185,11 +189,51 @@ struct ArrowScalarData : public ArrowScalarBaseData<TGT, SRC, OP> {
 //===--------------------------------------------------------------------===//
 template <class TGT>
 struct ArrowEnumData : public ArrowScalarBaseData<TGT> {
+	static idx_t GetLength(string_t input) {
+		return input.GetSize();
+	}
+	static void WriteData(data_ptr_t target, string_t input) {
+		memcpy(target, input.GetDataUnsafe(), input.GetSize());
+	}
+	static void EnumAppendVector(ArrowAppendData &append_data, const Vector &input, idx_t size) {
+		D_ASSERT(input.GetVectorType() == VectorType::FLAT_VECTOR);
+
+		// resize the validity mask and set up the validity buffer for iteration
+		ResizeValidity(append_data.validity, append_data.row_count + size);
+
+		// resize the offset buffer - the offset buffer holds the offsets into the child array
+		append_data.main_buffer.resize(append_data.main_buffer.size() + sizeof(uint32_t) * (size + 1));
+		auto data = (string_t *)FlatVector::GetData<string_t>(input);
+		auto offset_data = (uint32_t *)append_data.main_buffer.data();
+		if (append_data.row_count == 0) {
+			// first entry
+			offset_data[0] = 0;
+		}
+		// now append the string data to the auxiliary buffer
+		// the auxiliary buffer's length depends on the string lengths, so we resize as required
+		auto last_offset = offset_data[append_data.row_count];
+		for (idx_t i = 0; i < size; i++) {
+			auto offset_idx = append_data.row_count + i + 1;
+
+			auto string_length = GetLength(data[i]);
+
+			// append the offset data
+			auto current_offset = last_offset + string_length;
+			offset_data[offset_idx] = current_offset;
+
+			// resize the string buffer if required, and write the string data
+			append_data.aux_buffer.resize(current_offset);
+			WriteData(append_data.aux_buffer.data() + last_offset, data[i]);
+
+			last_offset = current_offset;
+		}
+		append_data.row_count += size;
+	}
 	static void Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity) {
 		result.main_buffer.reserve(capacity * sizeof(TGT));
 		// construct the enum child data
 		auto enum_data = InitializeArrowChild(LogicalType::VARCHAR, EnumType::GetSize(type));
-		enum_data->append_vector(*enum_data, EnumType::GetValuesInsertOrder(type), EnumType::GetSize(type));
+		EnumAppendVector(*enum_data, EnumType::GetValuesInsertOrder(type), EnumType::GetSize(type));
 		result.child_data.push_back(std::move(enum_data));
 	}
 
@@ -629,7 +673,7 @@ static void InitializeFunctionPointers(ArrowAppendData &append_data, const Logic
 		}
 		break;
 	case LogicalTypeId::INTERVAL:
-		InitializeFunctionPointers<ArrowScalarData<int64_t, interval_t, ArrowIntervalConverter>>(append_data);
+		InitializeFunctionPointers<ArrowScalarData<ArrowInterval, interval_t, ArrowIntervalConverter>>(append_data);
 		break;
 	case LogicalTypeId::STRUCT:
 		InitializeFunctionPointers<ArrowStructData>(append_data);

package/src/duckdb/src/common/arrow/arrow_converter.cpp

@@ -150,7 +150,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 		child.format = "tsm:";
 		break;
 	case LogicalTypeId::INTERVAL:
-		child.format = "tDm";
+		child.format = "tin";
 		break;
 	case LogicalTypeId::DECIMAL: {
 		uint8_t width, scale;

package/src/duckdb/src/common/box_renderer.cpp

@@ -197,6 +197,60 @@ list<ColumnDataCollection> BoxRenderer::FetchRenderCollections(ClientContext &co
 	return collections;
 }
 
+list<ColumnDataCollection> BoxRenderer::PivotCollections(ClientContext &context, list<ColumnDataCollection> input,
+                                                         vector<string> &column_names,
+                                                         vector<LogicalType> &result_types, idx_t row_count) {
+	auto &top = input.front();
+	auto &bottom = input.back();
+
+	vector<LogicalType> varchar_types;
+	vector<string> new_names;
+	new_names.emplace_back("Column");
+	new_names.emplace_back("Type");
+	varchar_types.emplace_back(LogicalType::VARCHAR);
+	varchar_types.emplace_back(LogicalType::VARCHAR);
+	for (idx_t r = 0; r < top.Count(); r++) {
+		new_names.emplace_back("Row " + to_string(r + 1));
+		varchar_types.emplace_back(LogicalType::VARCHAR);
+	}
+	for (idx_t r = 0; r < bottom.Count(); r++) {
+		auto row_index = row_count - bottom.Count() + r + 1;
+		new_names.emplace_back("Row " + to_string(row_index));
+		varchar_types.emplace_back(LogicalType::VARCHAR);
+	}
+	//
+	DataChunk row_chunk;
+	row_chunk.Initialize(Allocator::DefaultAllocator(), varchar_types);
+	std::list<ColumnDataCollection> result;
+	result.emplace_back(context, varchar_types);
+	result.emplace_back(context, varchar_types);
+	auto &res_coll = result.front();
+	ColumnDataAppendState append_state;
+	res_coll.InitializeAppend(append_state);
+	for (idx_t c = 0; c < top.ColumnCount(); c++) {
+		vector<column_t> column_ids {c};
+		auto row_index = row_chunk.size();
+		idx_t current_index = 0;
+		row_chunk.SetValue(current_index++, row_index, column_names[c]);
+		row_chunk.SetValue(current_index++, row_index, RenderType(result_types[c]));
+		for (auto &collection : input) {
+			for (auto &chunk : collection.Chunks(column_ids)) {
+				for (idx_t r = 0; r < chunk.size(); r++) {
+					row_chunk.SetValue(current_index++, row_index, chunk.GetValue(0, r));
+				}
+			}
+		}
+		row_chunk.SetCardinality(row_chunk.size() + 1);
+		if (row_chunk.size() == STANDARD_VECTOR_SIZE || c + 1 == top.ColumnCount()) {
+			res_coll.Append(append_state, row_chunk);
+			row_chunk.Reset();
+		}
+	}
+	column_names = std::move(new_names);
+	result_types = std::move(varchar_types);
+	return result;
+}
+
 string ConvertRenderValue(const string &input) {
 	return StringUtil::Replace(StringUtil::Replace(input, "\n", "\\n"), string("\0", 1), "\\0");
 }
@@ -213,11 +267,10 @@ string BoxRenderer::GetRenderValue(ColumnDataRowCollection &rows, idx_t c, idx_t
 	}
 }
 
-vector<idx_t> BoxRenderer::ComputeRenderWidths(const vector<string> &names, const ColumnDataCollection &result,
+vector<idx_t> BoxRenderer::ComputeRenderWidths(const vector<string> &names, const vector<LogicalType> &result_types,
                                                list<ColumnDataCollection> &collections, idx_t min_width,
                                                idx_t max_width, vector<idx_t> &column_map, idx_t &total_length) {
-	auto column_count = result.ColumnCount();
-	auto &result_types = result.Types();
+	auto column_count = result_types.size();
 
 	vector<idx_t> widths;
 	widths.reserve(column_count);
@@ -357,13 +410,15 @@ void BoxRenderer::RenderHeader(const vector<string> &names, const vector<Logical
 	ss << std::endl;
 
 	// render the types
-	for (idx_t c = 0; c < column_count; c++) {
-		auto column_idx = column_map[c];
-		auto type = column_idx == SPLIT_COLUMN ? "" : RenderType(result_types[column_idx]);
-		RenderValue(ss, type, widths[c]);
+	if (config.render_mode == RenderMode::ROWS) {
+		for (idx_t c = 0; c < column_count; c++) {
+			auto column_idx = column_map[c];
+			auto type = column_idx == SPLIT_COLUMN ? "" : RenderType(result_types[column_idx]);
+			RenderValue(ss, type, widths[c]);
+		}
+		ss << config.VERTICAL;
+		ss << std::endl;
 	}
-	ss << config.VERTICAL;
-	ss << std::endl;
 
 	// render the line under the header
 	ss << config.LMIDDLE;
@@ -390,12 +445,14 @@ void BoxRenderer::RenderValues(const list<ColumnDataCollection> &collections, co
 	auto column_count = column_map.size();
 
 	vector<ValueRenderAlignment> alignments;
-	for (idx_t c = 0; c < column_count; c++) {
-		auto column_idx = column_map[c];
-		if (column_idx == SPLIT_COLUMN) {
-			alignments.push_back(ValueRenderAlignment::MIDDLE);
-		} else {
-			alignments.push_back(TypeAlignment(result_types[column_idx]));
+	if (config.render_mode == RenderMode::ROWS) {
+		for (idx_t c = 0; c < column_count; c++) {
+			auto column_idx = column_map[c];
+			if (column_idx == SPLIT_COLUMN) {
+				alignments.push_back(ValueRenderAlignment::MIDDLE);
+			} else {
+				alignments.push_back(TypeAlignment(result_types[column_idx]));
+			}
 		}
 	}
 
@@ -409,13 +466,28 @@ void BoxRenderer::RenderValues(const list<ColumnDataCollection> &collections, co
 			} else {
 				str = GetRenderValue(rows, column_idx, r);
 			}
-			RenderValue(ss, str, widths[c], alignments[c]);
+			ValueRenderAlignment alignment;
+			if (config.render_mode == RenderMode::ROWS) {
+				alignment = alignments[c];
+			} else {
+				if (c < 2) {
+					alignment = ValueRenderAlignment::LEFT;
+				} else if (c == SPLIT_COLUMN) {
+					alignment = ValueRenderAlignment::MIDDLE;
+				} else {
+					alignment = ValueRenderAlignment::RIGHT;
+				}
+			}
+			RenderValue(ss, str, widths[c], alignment);
 		}
 		ss << config.VERTICAL;
 		ss << std::endl;
 	}
 
 	if (bottom_rows > 0) {
+		if (config.render_mode == RenderMode::COLUMNS) {
+			throw InternalException("Columns render mode does not support bottom rows");
+		}
 		// render the bottom rows
 		// first render the divider
 		auto brows = bottom_collection.GetRows();
@@ -607,15 +679,19 @@ void BoxRenderer::Render(ClientContext &context, const vector<string> &names, co
 
 	// fetch the top and bottom render collections from the result
 	auto collections = FetchRenderCollections(context, result, top_rows, bottom_rows);
-
-	auto &result_types = result.Types();
+	auto column_names = names;
+	auto result_types = result.Types();
+	if (config.render_mode == RenderMode::COLUMNS) {
+		collections = PivotCollections(context, std::move(collections), column_names, result_types, row_count);
+	}
 
 	// for each column, figure out the width
 	// start off by figuring out the name of the header by looking at the column name and column type
 	idx_t min_width = has_hidden_rows || row_count == 0 ? minimum_row_length : 0;
 	vector<idx_t> column_map;
 	idx_t total_length;
-	auto widths = ComputeRenderWidths(names, result, collections, min_width, max_width, column_map, total_length);
+	auto widths =
+	    ComputeRenderWidths(column_names, result_types, collections, min_width, max_width, column_map, total_length);
 
 	// render boundaries for the individual columns
 	vector<idx_t> boundaries;
@@ -631,7 +707,7 @@ void BoxRenderer::Render(ClientContext &context, const vector<string> &names, co
 
 	// now begin rendering
 	// first render the header
-	RenderHeader(names, result_types, column_map, widths, boundaries, total_length, row_count > 0, ss);
+	RenderHeader(column_names, result_types, column_map, widths, boundaries, total_length, row_count > 0, ss);
 
 	// render the values, if there are any
 	RenderValues(collections, column_map, widths, result_types, ss);
@@ -649,10 +725,20 @@ void BoxRenderer::Render(ClientContext &context, const vector<string> &names, co
 		}
 	}
 	idx_t column_count = column_map.size();
-	if (has_hidden_columns) {
-		column_count--;
-		column_count_str += " (" + to_string(column_count) + " shown)";
+	if (config.render_mode == RenderMode::COLUMNS) {
+		if (has_hidden_columns) {
+			has_hidden_rows = true;
+			shown_str = " (" + to_string(column_count - 3) + " shown)";
+		} else {
+			shown_str = string();
+		}
+	} else {
+		if (has_hidden_columns) {
+			column_count--;
+			column_count_str += " (" + to_string(column_count) + " shown)";
+		}
 	}
+
 	RenderRowCount(std::move(row_count_str), std::move(shown_str), column_count_str, boundaries, has_hidden_rows,
 	               has_hidden_columns, total_length, row_count, column_count, minimum_row_length, ss);
 }