duckdb 0.7.1-dev2.0 → 0.7.1-dev284.0

package/src/duckdb/src/catalog/catalog.cpp

@@ -2,6 +2,7 @@
 
 #include "duckdb/catalog/catalog_search_path.hpp"
 #include "duckdb/catalog/catalog_entry/list.hpp"
+#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_set.hpp"
 #include "duckdb/catalog/default/default_schemas.hpp"
 #include "duckdb/catalog/catalog_entry/type_catalog_entry.hpp"
@@ -251,6 +252,20 @@ CatalogEntry *Catalog::CreateCollation(CatalogTransaction transaction, SchemaCat
 	return schema->CreateCollation(transaction, info);
 }
 
+//===--------------------------------------------------------------------===//
+// Index
+//===--------------------------------------------------------------------===//
+CatalogEntry *Catalog::CreateIndex(CatalogTransaction transaction, CreateIndexInfo *info) {
+	auto &context = transaction.GetContext();
+	return CreateIndex(context, info);
+}
+
+CatalogEntry *Catalog::CreateIndex(ClientContext &context, CreateIndexInfo *info) {
+	auto schema = GetSchema(context, info->schema);
+	auto table = GetEntry<TableCatalogEntry>(context, schema->name, info->table->table_name);
+	return schema->CreateIndex(context, info, table);
+}
+
 //===--------------------------------------------------------------------===//
 // Lookup Structures
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/catalog/catalog_entry/index_catalog_entry.cpp

@@ -19,10 +19,11 @@ string IndexCatalogEntry::ToSQL() {
 	return sql;
 }
 
-void IndexCatalogEntry::Serialize(duckdb::MetaBlockWriter &serializer) {
-	// Here we serialize the index metadata in the following order:
-	// schema name, table name, index name, sql, index type, index constraint type, expression list.
-	// column_ids, unbound_expression
+void IndexCatalogEntry::Serialize(Serializer &serializer) {
+	// here we serialize the index metadata in the following order:
+	// schema name, table name, index name, sql, index type, index constraint type, expression list, parsed expressions,
+	// column IDs
+
 	FieldWriter writer(serializer);
 	writer.WriteString(GetSchemaName());
 	writer.WriteString(GetTableName());
@@ -37,9 +38,9 @@ void IndexCatalogEntry::Serialize(duckdb::MetaBlockWriter &serializer) {
 }
 
 unique_ptr<CreateIndexInfo> IndexCatalogEntry::Deserialize(Deserializer &source, ClientContext &context) {
-	// Here we deserialize the index metadata in the following order:
-	// root block, root offset, schema name, table name, index name, sql, index type, index constraint type, expression
-	// list.
+	// here we deserialize the index metadata in the following order:
+	// schema name, table schema name, table name, index name, sql, index type, index constraint type, expression list,
+	// parsed expression list, column IDs
 
 	auto create_index_info = make_unique<CreateIndexInfo>();
 

package/src/duckdb/src/common/enums/logical_operator_type.cpp

@@ -100,6 +100,8 @@ string LogicalOperatorToString(LogicalOperatorType type) {
 		return "CREATE_SCHEMA";
 	case LogicalOperatorType::LOGICAL_ATTACH:
 		return "ATTACH";
+	case LogicalOperatorType::LOGICAL_DETACH:
+		return "ATTACH";
 	case LogicalOperatorType::LOGICAL_DROP:
 		return "DROP";
 	case LogicalOperatorType::LOGICAL_PRAGMA:

package/src/duckdb/src/common/enums/physical_operator_type.cpp

@@ -133,6 +133,8 @@ string PhysicalOperatorToString(PhysicalOperatorType type) {
 		return "CREATE_TYPE";
 	case PhysicalOperatorType::ATTACH:
 		return "ATTACH";
+	case PhysicalOperatorType::DETACH:
+		return "DETACH";
 	case PhysicalOperatorType::RESULT_COLLECTOR:
 		return "RESULT_COLLECTOR";
 	case PhysicalOperatorType::EXTENSION:

package/src/duckdb/src/common/enums/statement_type.cpp

@@ -57,6 +57,8 @@ string StatementTypeToString(StatementType type) {
 		return "LOGICAL_PLAN";
 	case StatementType::ATTACH_STATEMENT:
 		return "ATTACH";
+	case StatementType::DETACH_STATEMENT:
+		return "DETACH";
 	case StatementType::INVALID_STATEMENT:
 		break;
 	}

package/src/duckdb/src/common/file_system.cpp

@@ -335,6 +335,20 @@ bool FileSystem::CanHandleFile(const string &fpath) {
 	throw NotImplementedException("%s: CanHandleFile is not implemented!", GetName());
 }
 
+IOException FileSystem::MissingFileException(const string &file_path, ClientContext &context) {
+	const string prefixes[] = {"http://", "https://", "s3://"};
+	for (auto &prefix : prefixes) {
+		if (StringUtil::StartsWith(file_path, prefix)) {
+			if (!context.db->LoadedExtensions().count("httpfs")) {
+				return MissingExtensionException("No files found that match the pattern \"%s\", because the httpfs "
+				                                 "extension is not loaded. Try loading the extension: LOAD HTTPFS",
+				                                 file_path);
+			}
+		}
+	}
+	return IOException("No files found that match the pattern \"%s\"", file_path);
+}
+
 void FileSystem::Seek(FileHandle &handle, idx_t location) {
 	throw NotImplementedException("%s: Seek is not implemented!", GetName());
 }

package/src/duckdb/src/common/hive_partitioning.cpp

@@ -150,6 +150,7 @@ HivePartitionedColumnData::HivePartitionedColumnData(const HivePartitionedColumn
 void HivePartitionedColumnData::ComputePartitionIndices(PartitionedColumnDataAppendState &state, DataChunk &input) {
 	Vector hashes(LogicalType::HASH, input.size());
 	input.Hash(group_by_columns, hashes);
+	hashes.Flatten(input.size());
 
 	for (idx_t i = 0; i < input.size(); i++) {
 		HivePartitionKey key;

package/src/duckdb/src/common/operator/cast_operators.cpp

@@ -946,13 +946,13 @@ static bool IntegerCastLoop(const char *buf, idx_t len, T &result, bool strict)
 					ExponentData exponent {0, false};
 					int negative = buf[pos] == '-';
 					if (negative) {
-						if (!IntegerCastLoop<ExponentData, true, false, IntegerCastOperation>(buf + pos, len - pos,
-						                                                                      exponent, strict)) {
+						if (!IntegerCastLoop<ExponentData, true, false, IntegerCastOperation, decimal_separator>(
+						        buf + pos, len - pos, exponent, strict)) {
 							return false;
 						}
 					} else {
-						if (!IntegerCastLoop<ExponentData, false, false, IntegerCastOperation>(buf + pos, len - pos,
-						                                                                       exponent, strict)) {
+						if (!IntegerCastLoop<ExponentData, false, false, IntegerCastOperation, decimal_separator>(
+						        buf + pos, len - pos, exponent, strict)) {
 							return false;
 						}
 					}
@@ -1554,16 +1554,22 @@ dtime_t Cast::Operation(string_t input) {
 //===--------------------------------------------------------------------===//
 template <>
 bool TryCastErrorMessage::Operation(string_t input, timestamp_t &result, string *error_message, bool strict) {
-	if (!TryCast::Operation<string_t, timestamp_t>(input, result, strict)) {
+	auto cast_result = Timestamp::TryConvertTimestamp(input.GetDataUnsafe(), input.GetSize(), result);
+	if (cast_result == TimestampCastResult::SUCCESS) {
+		return true;
+	}
+	if (cast_result == TimestampCastResult::ERROR_INCORRECT_FORMAT) {
 		HandleCastError::AssignError(Timestamp::ConversionError(input), error_message);
-		return false;
+	} else {
+		HandleCastError::AssignError(Timestamp::UnsupportedTimezoneError(input), error_message);
 	}
-	return true;
+	return false;
 }
 
 template <>
 bool TryCast::Operation(string_t input, timestamp_t &result, bool strict) {
-	return Timestamp::TryConvertTimestamp(input.GetDataUnsafe(), input.GetSize(), result);
+	return Timestamp::TryConvertTimestamp(input.GetDataUnsafe(), input.GetSize(), result) ==
+	       TimestampCastResult::SUCCESS;
 }
 
 template <>

package/src/duckdb/src/common/printer.cpp

@@ -65,7 +65,7 @@ idx_t Printer::TerminalWidth() {
 	int columns, rows;
 
 	GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &csbi);
-	rows = csbi.srWindow.Bottom - csbi.srWindow.Top + 1;
+	rows = csbi.srWindow.Right - csbi.srWindow.Left + 1;
 	return rows;
 #else
 	struct winsize w;

package/src/duckdb/src/common/types/time.cpp

@@ -115,7 +115,7 @@ bool Time::TryConvertTime(const char *buf, idx_t len, idx_t &pos, dtime_t &resul
 		if (!strict) {
 			// last chance, check if we can parse as timestamp
 			timestamp_t timestamp;
-			if (Timestamp::TryConvertTimestamp(buf, len, timestamp)) {
+			if (Timestamp::TryConvertTimestamp(buf, len, timestamp) == TimestampCastResult::SUCCESS) {
 				if (!Timestamp::IsFinite(timestamp)) {
 					return false;
 				}

package/src/duckdb/src/common/types/timestamp.cpp

@@ -88,11 +88,27 @@ bool Timestamp::TryConvertTimestampTZ(const char *str, idx_t len, timestamp_t &r
 	return true;
 }
 
-bool Timestamp::TryConvertTimestamp(const char *str, idx_t len, timestamp_t &result) {
+TimestampCastResult Timestamp::TryConvertTimestamp(const char *str, idx_t len, timestamp_t &result) {
 	string_t tz(nullptr, 0);
 	bool has_offset = false;
 	// We don't understand TZ without an extension, so fail if one was provided.
-	return TryConvertTimestampTZ(str, len, result, has_offset, tz) && !tz.GetSize();
+	auto success = TryConvertTimestampTZ(str, len, result, has_offset, tz);
+	if (!success) {
+		return TimestampCastResult::ERROR_INCORRECT_FORMAT;
+	}
+	if (tz.GetSize() == 0) {
+		// no timezone provided - success!
+		return TimestampCastResult::SUCCESS;
+	}
+	if (tz.GetSize() == 3) {
+		// we can ONLY handle UTC without ICU being loaded
+		auto tz_ptr = tz.GetDataUnsafe();
+		if ((tz_ptr[0] == 'u' || tz_ptr[0] == 'U') && (tz_ptr[1] == 't' || tz_ptr[1] == 'T') &&
+		    (tz_ptr[2] == 'c' || tz_ptr[2] == 'C')) {
+			return TimestampCastResult::SUCCESS;
+		}
+	}
+	return TimestampCastResult::ERROR_NON_UTC_TIMEZONE;
 }
 
 string Timestamp::ConversionError(const string &str) {
@@ -101,16 +117,31 @@ string Timestamp::ConversionError(const string &str) {
 	                          str);
 }
 
+string Timestamp::UnsupportedTimezoneError(const string &str) {
+	return StringUtil::Format("timestamp field value \"%s\" has a timestamp that is not UTC.\nUse the TIMESTAMPTZ type "
+	                          "with the ICU extension loaded to handle non-UTC timestamps.",
+	                          str);
+}
+
 string Timestamp::ConversionError(string_t str) {
 	return Timestamp::ConversionError(str.GetString());
 }
 
+string Timestamp::UnsupportedTimezoneError(string_t str) {
+	return Timestamp::UnsupportedTimezoneError(str.GetString());
+}
+
 timestamp_t Timestamp::FromCString(const char *str, idx_t len) {
 	timestamp_t result;
-	if (!Timestamp::TryConvertTimestamp(str, len, result)) {
+	auto cast_result = Timestamp::TryConvertTimestamp(str, len, result);
+	if (cast_result == TimestampCastResult::SUCCESS) {
+		return result;
+	}
+	if (cast_result == TimestampCastResult::ERROR_NON_UTC_TIMEZONE) {
+		throw ConversionException(Timestamp::UnsupportedTimezoneError(string(str, len)));
+	} else {
 		throw ConversionException(Timestamp::ConversionError(string(str, len)));
 	}
-	return result;
 }
 
 bool Timestamp::TryParseUTCOffset(const char *str, idx_t &pos, idx_t len, int &hour_offset, int &minute_offset) {

package/src/duckdb/src/common/types.cpp

@@ -504,11 +504,32 @@ LogicalType TransformStringToLogicalType(const string &str) {
 	return Parser::ParseColumnList("dummy " + str).GetColumn(LogicalIndex(0)).Type();
 }
 
+LogicalType GetUserTypeRecursive(const LogicalType &type, ClientContext &context) {
+	if (type.id() == LogicalTypeId::USER && type.HasAlias()) {
+		return Catalog::GetSystemCatalog(context).GetType(context, SYSTEM_CATALOG, DEFAULT_SCHEMA, type.GetAlias());
+	}
+	// Look for LogicalTypeId::USER in nested types
+	if (type.id() == LogicalTypeId::STRUCT) {
+		child_list_t<LogicalType> children;
+		children.reserve(StructType::GetChildCount(type));
+		for (auto &child : StructType::GetChildTypes(type)) {
+			children.emplace_back(child.first, GetUserTypeRecursive(child.second, context));
+		}
+		return LogicalType::STRUCT(std::move(children));
+	}
+	if (type.id() == LogicalTypeId::LIST) {
+		return LogicalType::LIST(GetUserTypeRecursive(ListType::GetChildType(type), context));
+	}
+	if (type.id() == LogicalTypeId::MAP) {
+		return LogicalType::MAP(GetUserTypeRecursive(MapType::KeyType(type), context),
+		                        GetUserTypeRecursive(MapType::ValueType(type), context));
+	}
+	// Not LogicalTypeId::USER or a nested type
+	return type;
+}
+
 LogicalType TransformStringToLogicalType(const string &str, ClientContext &context) {
-	auto type = TransformStringToLogicalType(str);
-	return type.id() == LogicalTypeId::USER
-	           ? Catalog::GetSystemCatalog(context).GetType(context, SYSTEM_CATALOG, DEFAULT_SCHEMA, str)
-	           : type;
+	return GetUserTypeRecursive(TransformStringToLogicalType(str), context);
 }
 
 bool LogicalType::IsIntegral() const {
@@ -888,18 +909,23 @@ void LogicalType::SetAlias(string alias) {
 }
 
 string LogicalType::GetAlias() const {
-	if (!type_info_) {
-		return string();
-	} else {
+	if (id() == LogicalTypeId::USER) {
+		return UserType::GetTypeName(*this);
+	}
+	if (type_info_) {
 		return type_info_->alias;
 	}
+	return string();
 }
 
 bool LogicalType::HasAlias() const {
-	if (!type_info_) {
-		return false;
+	if (id() == LogicalTypeId::USER) {
+		return !UserType::GetTypeName(*this).empty();
+	}
+	if (type_info_ && !type_info_->alias.empty()) {
+		return true;
 	}
-	return !type_info_->alias.empty();
+	return false;
 }
 
 void LogicalType::SetCatalog(LogicalType &type, TypeCatalogEntry *catalog_entry) {

package/src/duckdb/src/execution/column_binding_resolver.cpp

@@ -65,9 +65,12 @@ void ColumnBindingResolver::VisitOperator(LogicalOperator &op) {
 		// ON CONFLICT DO UPDATE clause
 		auto &insert_op = (LogicalInsert &)op;
 		if (insert_op.action_type != OnConflictAction::THROW) {
+			// Get the bindings from the children
 			VisitOperatorChildren(op);
-			auto dummy_bindings = LogicalOperator::GenerateColumnBindings(
-			    insert_op.excluded_table_index, insert_op.table->GetColumns().PhysicalColumnCount());
+			auto column_count = insert_op.table->GetColumns().PhysicalColumnCount();
+			auto dummy_bindings = LogicalOperator::GenerateColumnBindings(insert_op.excluded_table_index, column_count);
+			// Now insert our dummy bindings at the start of the bindings,
+			// so the first 'column_count' indices of the chunk are reserved for our 'excluded' columns
 			bindings.insert(bindings.begin(), dummy_bindings.begin(), dummy_bindings.end());
 			if (insert_op.on_conflict_condition) {
 				VisitExpression(&insert_op.on_conflict_condition);