duckdb 0.7.1-dev7.0 → 0.7.1

package/src/duckdb/src/common/bind_helpers.cpp

@@ -4,6 +4,7 @@
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/types/value.hpp"
 #include "duckdb/common/case_insensitive_map.hpp"
+#include <numeric>
 
 namespace duckdb {
 
@@ -64,4 +65,58 @@ vector<bool> ParseColumnList(const Value &value, vector<string> &names, const st
 	return ParseColumnList(children, names, loption);
 }
 
+vector<idx_t> ParseColumnsOrdered(const vector<Value> &set, vector<string> &names, const string &loption) {
+	vector<idx_t> result;
+
+	if (set.empty()) {
+		throw BinderException("\"%s\" expects a column list or * as parameter", loption);
+	}
+
+	// Maps option to bool indicating if its found and the index in the original set
+	case_insensitive_map_t<std::pair<bool, idx_t>> option_map;
+	for (idx_t i = 0; i < set.size(); i++) {
+		option_map[set[i].ToString()] = {false, i};
+	}
+	result.resize(option_map.size());
+
+	for (idx_t i = 0; i < names.size(); i++) {
+		auto entry = option_map.find(names[i]);
+		if (entry != option_map.end()) {
+			result[entry->second.second] = i;
+			entry->second.first = true;
+		}
+	}
+	for (auto &entry : option_map) {
+		if (!entry.second.first) {
+			throw BinderException("\"%s\" expected to find %s, but it was not found in the table", loption,
+			                      entry.first.c_str());
+		}
+	}
+	return result;
+}
+
+vector<idx_t> ParseColumnsOrdered(const Value &value, vector<string> &names, const string &loption) {
+	vector<idx_t> result;
+
+	// Only accept a list of arguments
+	if (value.type().id() != LogicalTypeId::LIST) {
+		// Support a single argument if it's '*'
+		if (value.type().id() == LogicalTypeId::VARCHAR && value.GetValue<string>() == "*") {
+			result.resize(names.size(), 0);
+			std::iota(std::begin(result), std::end(result), 0);
+			return result;
+		}
+		throw BinderException("\"%s\" expects a column list or * as parameter", loption);
+	}
+	auto &children = ListValue::GetChildren(value);
+	// accept '*' as single argument
+	if (children.size() == 1 && children[0].type().id() == LogicalTypeId::VARCHAR &&
+	    children[0].GetValue<string>() == "*") {
+		result.resize(names.size(), 0);
+		std::iota(std::begin(result), std::end(result), 0);
+		return result;
+	}
+	return ParseColumnsOrdered(children, names, loption);
+}
+
 } // namespace duckdb

package/src/duckdb/src/common/enums/logical_operator_type.cpp

@@ -100,6 +100,8 @@ string LogicalOperatorToString(LogicalOperatorType type) {
 		return "CREATE_SCHEMA";
 	case LogicalOperatorType::LOGICAL_ATTACH:
 		return "ATTACH";
+	case LogicalOperatorType::LOGICAL_DETACH:
+		return "ATTACH";
 	case LogicalOperatorType::LOGICAL_DROP:
 		return "DROP";
 	case LogicalOperatorType::LOGICAL_PRAGMA:

package/src/duckdb/src/common/enums/physical_operator_type.cpp

@@ -133,6 +133,8 @@ string PhysicalOperatorToString(PhysicalOperatorType type) {
 		return "CREATE_TYPE";
 	case PhysicalOperatorType::ATTACH:
 		return "ATTACH";
+	case PhysicalOperatorType::DETACH:
+		return "DETACH";
 	case PhysicalOperatorType::RESULT_COLLECTOR:
 		return "RESULT_COLLECTOR";
 	case PhysicalOperatorType::EXTENSION:

package/src/duckdb/src/common/enums/statement_type.cpp

@@ -57,6 +57,8 @@ string StatementTypeToString(StatementType type) {
 		return "LOGICAL_PLAN";
 	case StatementType::ATTACH_STATEMENT:
 		return "ATTACH";
+	case StatementType::DETACH_STATEMENT:
+		return "DETACH";
 	case StatementType::INVALID_STATEMENT:
 		break;
 	}

package/src/duckdb/src/common/file_system.cpp

@@ -10,6 +10,7 @@
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/client_data.hpp"
 #include "duckdb/main/database.hpp"
+#include "duckdb/main/extension_helper.hpp"
 
 #include <cstdint>
 #include <cstdio>
@@ -335,6 +336,33 @@ bool FileSystem::CanHandleFile(const string &fpath) {
 	throw NotImplementedException("%s: CanHandleFile is not implemented!", GetName());
 }
 
+vector<string> FileSystem::GlobFiles(const string &pattern, ClientContext &context) {
+	auto result = Glob(pattern, context);
+	if (result.empty()) {
+		string required_extension;
+		const string prefixes[] = {"http://", "https://", "s3://"};
+		for (auto &prefix : prefixes) {
+			if (StringUtil::StartsWith(pattern, prefix)) {
+				required_extension = "httpfs";
+				break;
+			}
+		}
+		if (!required_extension.empty() && !context.db->ExtensionIsLoaded(required_extension)) {
+			// an extension is required to read this file but it is not loaded - try to load it
+			ExtensionHelper::LoadExternalExtension(context, required_extension);
+			// success! glob again
+			// check the extension is loaded just in case to prevent an infinite loop here
+			if (!context.db->ExtensionIsLoaded(required_extension)) {
+				throw InternalException("Extension load \"%s\" did not throw but somehow the extension was not loaded",
+				                        required_extension);
+			}
+			return GlobFiles(pattern, context);
+		}
+		throw IOException("No files found that match the pattern \"%s\"", pattern);
+	}
+	return result;
+}
+
 void FileSystem::Seek(FileHandle &handle, idx_t location) {
 	throw NotImplementedException("%s: Seek is not implemented!", GetName());
 }

package/src/duckdb/src/common/hive_partitioning.cpp

@@ -150,6 +150,7 @@ HivePartitionedColumnData::HivePartitionedColumnData(const HivePartitionedColumn
 void HivePartitionedColumnData::ComputePartitionIndices(PartitionedColumnDataAppendState &state, DataChunk &input) {
 	Vector hashes(LogicalType::HASH, input.size());
 	input.Hash(group_by_columns, hashes);
+	hashes.Flatten(input.size());
 
 	for (idx_t i = 0; i < input.size(); i++) {
 		HivePartitionKey key;

package/src/duckdb/src/common/local_file_system.cpp

@@ -833,8 +833,8 @@ static bool HasGlob(const string &str) {
 	return false;
 }
 
-static void GlobFiles(FileSystem &fs, const string &path, const string &glob, bool match_directory,
-                      vector<string> &result, bool join_path) {
+static void GlobFilesInternal(FileSystem &fs, const string &path, const string &glob, bool match_directory,
+                              vector<string> &result, bool join_path) {
 	fs.ListFiles(path, [&](const string &fname, bool is_directory) {
 		if (is_directory != match_directory) {
 			return;
@@ -951,12 +951,12 @@ vector<string> LocalFileSystem::Glob(const string &path, FileOpener *opener) {
 		} else {
 			if (previous_directories.empty()) {
 				// no previous directories: list in the current path
-				GlobFiles(*this, ".", splits[i], !is_last_chunk, result, false);
+				GlobFilesInternal(*this, ".", splits[i], !is_last_chunk, result, false);
 			} else {
 				// previous directories
 				// we iterate over each of the previous directories, and apply the glob of the current directory
 				for (auto &prev_directory : previous_directories) {
-					GlobFiles(*this, prev_directory, splits[i], !is_last_chunk, result, true);
+					GlobFilesInternal(*this, prev_directory, splits[i], !is_last_chunk, result, true);
 				}
 			}
 		}

package/src/duckdb/src/common/operator/cast_operators.cpp

@@ -946,13 +946,13 @@ static bool IntegerCastLoop(const char *buf, idx_t len, T &result, bool strict)
 					ExponentData exponent {0, false};
 					int negative = buf[pos] == '-';
 					if (negative) {
-						if (!IntegerCastLoop<ExponentData, true, false, IntegerCastOperation>(buf + pos, len - pos,
-						                                                                      exponent, strict)) {
+						if (!IntegerCastLoop<ExponentData, true, false, IntegerCastOperation, decimal_separator>(
+						        buf + pos, len - pos, exponent, strict)) {
 							return false;
 						}
 					} else {
-						if (!IntegerCastLoop<ExponentData, false, false, IntegerCastOperation>(buf + pos, len - pos,
-						                                                                       exponent, strict)) {
+						if (!IntegerCastLoop<ExponentData, false, false, IntegerCastOperation, decimal_separator>(
+						        buf + pos, len - pos, exponent, strict)) {
 							return false;
 						}
 					}
@@ -1554,16 +1554,22 @@ dtime_t Cast::Operation(string_t input) {
 //===--------------------------------------------------------------------===//
 template <>
 bool TryCastErrorMessage::Operation(string_t input, timestamp_t &result, string *error_message, bool strict) {
-	if (!TryCast::Operation<string_t, timestamp_t>(input, result, strict)) {
+	auto cast_result = Timestamp::TryConvertTimestamp(input.GetDataUnsafe(), input.GetSize(), result);
+	if (cast_result == TimestampCastResult::SUCCESS) {
+		return true;
+	}
+	if (cast_result == TimestampCastResult::ERROR_INCORRECT_FORMAT) {
 		HandleCastError::AssignError(Timestamp::ConversionError(input), error_message);
-		return false;
+	} else {
+		HandleCastError::AssignError(Timestamp::UnsupportedTimezoneError(input), error_message);
 	}
-	return true;
+	return false;
 }
 
 template <>
 bool TryCast::Operation(string_t input, timestamp_t &result, bool strict) {
-	return Timestamp::TryConvertTimestamp(input.GetDataUnsafe(), input.GetSize(), result);
+	return Timestamp::TryConvertTimestamp(input.GetDataUnsafe(), input.GetSize(), result) ==
+	       TimestampCastResult::SUCCESS;
 }
 
 template <>

package/src/duckdb/src/common/printer.cpp

@@ -65,7 +65,7 @@ idx_t Printer::TerminalWidth() {
 	int columns, rows;
 
 	GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &csbi);
-	rows = csbi.srWindow.Bottom - csbi.srWindow.Top + 1;
+	rows = csbi.srWindow.Right - csbi.srWindow.Left + 1;
 	return rows;
 #else
 	struct winsize w;

package/src/duckdb/src/common/string_util.cpp

@@ -249,7 +249,7 @@ private:
 };
 
 // adapted from https://en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Levenshtein_distance#C++
-idx_t StringUtil::LevenshteinDistance(const string &s1_p, const string &s2_p) {
+idx_t StringUtil::LevenshteinDistance(const string &s1_p, const string &s2_p, idx_t not_equal_penalty) {
 	auto s1 = StringUtil::Lower(s1_p);
 	auto s2 = StringUtil::Lower(s2_p);
 	idx_t len1 = s1.size();
@@ -273,7 +273,7 @@ idx_t StringUtil::LevenshteinDistance(const string &s1_p, const string &s2_p) {
 			// d[i][j] = std::min({ d[i - 1][j] + 1,
 			//                      d[i][j - 1] + 1,
 			//                      d[i - 1][j - 1] + (s1[i - 1] == s2[j - 1] ? 0 : 1) });
-			int equal = s1[i - 1] == s2[j - 1] ? 0 : 1;
+			int equal = s1[i - 1] == s2[j - 1] ? 0 : not_equal_penalty;
 			idx_t adjacent_score1 = array.Score(i - 1, j) + 1;
 			idx_t adjacent_score2 = array.Score(i, j - 1) + 1;
 			idx_t adjacent_score3 = array.Score(i - 1, j - 1) + equal;
@@ -285,15 +285,19 @@ idx_t StringUtil::LevenshteinDistance(const string &s1_p, const string &s2_p) {
 	return array.Score(len1, len2);
 }
 
+idx_t StringUtil::SimilarityScore(const string &s1, const string &s2) {
+	return LevenshteinDistance(s1, s2, 3);
+}
+
 vector<string> StringUtil::TopNLevenshtein(const vector<string> &strings, const string &target, idx_t n,
                                            idx_t threshold) {
 	vector<pair<string, idx_t>> scores;
 	scores.reserve(strings.size());
 	for (auto &str : strings) {
 		if (target.size() < str.size()) {
-			scores.emplace_back(str, LevenshteinDistance(str.substr(0, target.size()), target));
+			scores.emplace_back(str, SimilarityScore(str.substr(0, target.size()), target));
 		} else {
-			scores.emplace_back(str, LevenshteinDistance(str, target));
+			scores.emplace_back(str, SimilarityScore(str, target));
 		}
 	}
 	return TopNStrings(scores, n, threshold);

package/src/duckdb/src/common/types/partitioned_column_data.cpp

@@ -137,6 +137,7 @@ void PartitionedColumnData::FlushAppendState(PartitionedColumnDataAppendState &s
 		auto &partition_buffer = *state.partition_buffers[i];
 		if (partition_buffer.size() > 0) {
 			partitions[i]->Append(partition_buffer);
+			partition_buffer.Reset();
 		}
 	}
 }

package/src/duckdb/src/common/types/time.cpp

@@ -115,7 +115,7 @@ bool Time::TryConvertTime(const char *buf, idx_t len, idx_t &pos, dtime_t &resul
 		if (!strict) {
 			// last chance, check if we can parse as timestamp
 			timestamp_t timestamp;
-			if (Timestamp::TryConvertTimestamp(buf, len, timestamp)) {
+			if (Timestamp::TryConvertTimestamp(buf, len, timestamp) == TimestampCastResult::SUCCESS) {
 				if (!Timestamp::IsFinite(timestamp)) {
 					return false;
 				}

package/src/duckdb/src/common/types/timestamp.cpp

@@ -88,11 +88,27 @@ bool Timestamp::TryConvertTimestampTZ(const char *str, idx_t len, timestamp_t &r
 	return true;
 }
 
-bool Timestamp::TryConvertTimestamp(const char *str, idx_t len, timestamp_t &result) {
+TimestampCastResult Timestamp::TryConvertTimestamp(const char *str, idx_t len, timestamp_t &result) {
 	string_t tz(nullptr, 0);
 	bool has_offset = false;
 	// We don't understand TZ without an extension, so fail if one was provided.
-	return TryConvertTimestampTZ(str, len, result, has_offset, tz) && !tz.GetSize();
+	auto success = TryConvertTimestampTZ(str, len, result, has_offset, tz);
+	if (!success) {
+		return TimestampCastResult::ERROR_INCORRECT_FORMAT;
+	}
+	if (tz.GetSize() == 0) {
+		// no timezone provided - success!
+		return TimestampCastResult::SUCCESS;
+	}
+	if (tz.GetSize() == 3) {
+		// we can ONLY handle UTC without ICU being loaded
+		auto tz_ptr = tz.GetDataUnsafe();
+		if ((tz_ptr[0] == 'u' || tz_ptr[0] == 'U') && (tz_ptr[1] == 't' || tz_ptr[1] == 'T') &&
+		    (tz_ptr[2] == 'c' || tz_ptr[2] == 'C')) {
+			return TimestampCastResult::SUCCESS;
+		}
+	}
+	return TimestampCastResult::ERROR_NON_UTC_TIMEZONE;
 }
 
 string Timestamp::ConversionError(const string &str) {
@@ -101,16 +117,31 @@ string Timestamp::ConversionError(const string &str) {
 	                          str);
 }
 
+string Timestamp::UnsupportedTimezoneError(const string &str) {
+	return StringUtil::Format("timestamp field value \"%s\" has a timestamp that is not UTC.\nUse the TIMESTAMPTZ type "
+	                          "with the ICU extension loaded to handle non-UTC timestamps.",
+	                          str);
+}
+
 string Timestamp::ConversionError(string_t str) {
 	return Timestamp::ConversionError(str.GetString());
 }
 
+string Timestamp::UnsupportedTimezoneError(string_t str) {
+	return Timestamp::UnsupportedTimezoneError(str.GetString());
+}
+
 timestamp_t Timestamp::FromCString(const char *str, idx_t len) {
 	timestamp_t result;
-	if (!Timestamp::TryConvertTimestamp(str, len, result)) {
+	auto cast_result = Timestamp::TryConvertTimestamp(str, len, result);
+	if (cast_result == TimestampCastResult::SUCCESS) {
+		return result;
+	}
+	if (cast_result == TimestampCastResult::ERROR_NON_UTC_TIMEZONE) {
+		throw ConversionException(Timestamp::UnsupportedTimezoneError(string(str, len)));
+	} else {
 		throw ConversionException(Timestamp::ConversionError(string(str, len)));
 	}
-	return result;
 }
 
 bool Timestamp::TryParseUTCOffset(const char *str, idx_t &pos, idx_t len, int &hour_offset, int &minute_offset) {

package/src/duckdb/src/common/types.cpp

@@ -424,7 +424,7 @@ string LogicalType::ToString() const {
 		auto &child_types = StructType::GetChildTypes(*this);
 		string ret = "STRUCT(";
 		for (size_t i = 0; i < child_types.size(); i++) {
-			ret += child_types[i].first + " " + child_types[i].second.ToString();
+			ret += KeywordHelper::WriteOptionallyQuoted(child_types[i].first) + " " + child_types[i].second.ToString();
 			if (i < child_types.size() - 1) {
 				ret += ", ";
 			}
@@ -504,11 +504,32 @@ LogicalType TransformStringToLogicalType(const string &str) {
 	return Parser::ParseColumnList("dummy " + str).GetColumn(LogicalIndex(0)).Type();
 }
 
+LogicalType GetUserTypeRecursive(const LogicalType &type, ClientContext &context) {
+	if (type.id() == LogicalTypeId::USER && type.HasAlias()) {
+		return Catalog::GetSystemCatalog(context).GetType(context, SYSTEM_CATALOG, DEFAULT_SCHEMA, type.GetAlias());
+	}
+	// Look for LogicalTypeId::USER in nested types
+	if (type.id() == LogicalTypeId::STRUCT) {
+		child_list_t<LogicalType> children;
+		children.reserve(StructType::GetChildCount(type));
+		for (auto &child : StructType::GetChildTypes(type)) {
+			children.emplace_back(child.first, GetUserTypeRecursive(child.second, context));
+		}
+		return LogicalType::STRUCT(std::move(children));
+	}
+	if (type.id() == LogicalTypeId::LIST) {
+		return LogicalType::LIST(GetUserTypeRecursive(ListType::GetChildType(type), context));
+	}
+	if (type.id() == LogicalTypeId::MAP) {
+		return LogicalType::MAP(GetUserTypeRecursive(MapType::KeyType(type), context),
+		                        GetUserTypeRecursive(MapType::ValueType(type), context));
+	}
+	// Not LogicalTypeId::USER or a nested type
+	return type;
+}
+
 LogicalType TransformStringToLogicalType(const string &str, ClientContext &context) {
-	auto type = TransformStringToLogicalType(str);
-	return type.id() == LogicalTypeId::USER
-	           ? Catalog::GetSystemCatalog(context).GetType(context, SYSTEM_CATALOG, DEFAULT_SCHEMA, str)
-	           : type;
+	return GetUserTypeRecursive(TransformStringToLogicalType(str), context);
 }
 
 bool LogicalType::IsIntegral() const {
@@ -888,18 +909,23 @@ void LogicalType::SetAlias(string alias) {
 }
 
 string LogicalType::GetAlias() const {
-	if (!type_info_) {
-		return string();
-	} else {
+	if (id() == LogicalTypeId::USER) {
+		return UserType::GetTypeName(*this);
+	}
+	if (type_info_) {
 		return type_info_->alias;
 	}
+	return string();
 }
 
 bool LogicalType::HasAlias() const {
-	if (!type_info_) {
-		return false;
+	if (id() == LogicalTypeId::USER) {
+		return !UserType::GetTypeName(*this).empty();
+	}
+	if (type_info_ && !type_info_->alias.empty()) {
+		return true;
 	}
-	return !type_info_->alias.empty();
+	return false;
 }
 
 void LogicalType::SetCatalog(LogicalType &type, TypeCatalogEntry *catalog_entry) {

package/src/duckdb/src/execution/column_binding_resolver.cpp

@@ -65,9 +65,12 @@ void ColumnBindingResolver::VisitOperator(LogicalOperator &op) {
 		// ON CONFLICT DO UPDATE clause
 		auto &insert_op = (LogicalInsert &)op;
 		if (insert_op.action_type != OnConflictAction::THROW) {
+			// Get the bindings from the children
 			VisitOperatorChildren(op);
-			auto dummy_bindings = LogicalOperator::GenerateColumnBindings(
-			    insert_op.excluded_table_index, insert_op.table->GetColumns().PhysicalColumnCount());
+			auto column_count = insert_op.table->GetColumns().PhysicalColumnCount();
+			auto dummy_bindings = LogicalOperator::GenerateColumnBindings(insert_op.excluded_table_index, column_count);
+			// Now insert our dummy bindings at the start of the bindings,
+			// so the first 'column_count' indices of the chunk are reserved for our 'excluded' columns
 			bindings.insert(bindings.begin(), dummy_bindings.begin(), dummy_bindings.end());
 			if (insert_op.on_conflict_condition) {
 				VisitExpression(&insert_op.on_conflict_condition);