duckdb 0.7.1-dev90.0 → 0.7.2-dev0.0

package/src/duckdb/src/common/bind_helpers.cpp

@@ -4,6 +4,7 @@
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/types/value.hpp"
 #include "duckdb/common/case_insensitive_map.hpp"
+#include <numeric>
 
 namespace duckdb {
 
@@ -64,4 +65,58 @@ vector<bool> ParseColumnList(const Value &value, vector<string> &names, const st
 	return ParseColumnList(children, names, loption);
 }
 
+vector<idx_t> ParseColumnsOrdered(const vector<Value> &set, vector<string> &names, const string &loption) {
+	vector<idx_t> result;
+
+	if (set.empty()) {
+		throw BinderException("\"%s\" expects a column list or * as parameter", loption);
+	}
+
+	// Maps option to bool indicating if its found and the index in the original set
+	case_insensitive_map_t<std::pair<bool, idx_t>> option_map;
+	for (idx_t i = 0; i < set.size(); i++) {
+		option_map[set[i].ToString()] = {false, i};
+	}
+	result.resize(option_map.size());
+
+	for (idx_t i = 0; i < names.size(); i++) {
+		auto entry = option_map.find(names[i]);
+		if (entry != option_map.end()) {
+			result[entry->second.second] = i;
+			entry->second.first = true;
+		}
+	}
+	for (auto &entry : option_map) {
+		if (!entry.second.first) {
+			throw BinderException("\"%s\" expected to find %s, but it was not found in the table", loption,
+			                      entry.first.c_str());
+		}
+	}
+	return result;
+}
+
+vector<idx_t> ParseColumnsOrdered(const Value &value, vector<string> &names, const string &loption) {
+	vector<idx_t> result;
+
+	// Only accept a list of arguments
+	if (value.type().id() != LogicalTypeId::LIST) {
+		// Support a single argument if it's '*'
+		if (value.type().id() == LogicalTypeId::VARCHAR && value.GetValue<string>() == "*") {
+			result.resize(names.size(), 0);
+			std::iota(std::begin(result), std::end(result), 0);
+			return result;
+		}
+		throw BinderException("\"%s\" expects a column list or * as parameter", loption);
+	}
+	auto &children = ListValue::GetChildren(value);
+	// accept '*' as single argument
+	if (children.size() == 1 && children[0].type().id() == LogicalTypeId::VARCHAR &&
+	    children[0].GetValue<string>() == "*") {
+		result.resize(names.size(), 0);
+		std::iota(std::begin(result), std::end(result), 0);
+		return result;
+	}
+	return ParseColumnsOrdered(children, names, loption);
+}
+
 } // namespace duckdb

package/src/duckdb/src/common/file_system.cpp

@@ -10,6 +10,7 @@
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/client_data.hpp"
 #include "duckdb/main/database.hpp"
+#include "duckdb/main/extension_helper.hpp"
 
 #include <cstdint>
 #include <cstdio>
@@ -335,18 +336,31 @@ bool FileSystem::CanHandleFile(const string &fpath) {
 	throw NotImplementedException("%s: CanHandleFile is not implemented!", GetName());
 }
 
-IOException FileSystem::MissingFileException(const string &file_path, ClientContext &context) {
-	const string prefixes[] = {"http://", "https://", "s3://"};
-	for (auto &prefix : prefixes) {
-		if (StringUtil::StartsWith(file_path, prefix)) {
-			if (!context.db->LoadedExtensions().count("httpfs")) {
-				return MissingExtensionException("No files found that match the pattern \"%s\", because the httpfs "
-				                                 "extension is not loaded. Try loading the extension: LOAD HTTPFS",
-				                                 file_path);
+vector<string> FileSystem::GlobFiles(const string &pattern, ClientContext &context) {
+	auto result = Glob(pattern, context);
+	if (result.empty()) {
+		string required_extension;
+		const string prefixes[] = {"http://", "https://", "s3://"};
+		for (auto &prefix : prefixes) {
+			if (StringUtil::StartsWith(pattern, prefix)) {
+				required_extension = "httpfs";
+				break;
 			}
 		}
+		if (!required_extension.empty() && !context.db->ExtensionIsLoaded(required_extension)) {
+			// an extension is required to read this file but it is not loaded - try to load it
+			ExtensionHelper::LoadExternalExtension(context, required_extension);
+			// success! glob again
+			// check the extension is loaded just in case to prevent an infinite loop here
+			if (!context.db->ExtensionIsLoaded(required_extension)) {
+				throw InternalException("Extension load \"%s\" did not throw but somehow the extension was not loaded",
+				                        required_extension);
+			}
+			return GlobFiles(pattern, context);
+		}
+		throw IOException("No files found that match the pattern \"%s\"", pattern);
 	}
-	return IOException("No files found that match the pattern \"%s\"", file_path);
+	return result;
 }
 
 void FileSystem::Seek(FileHandle &handle, idx_t location) {

package/src/duckdb/src/common/hive_partitioning.cpp

@@ -150,6 +150,7 @@ HivePartitionedColumnData::HivePartitionedColumnData(const HivePartitionedColumn
 void HivePartitionedColumnData::ComputePartitionIndices(PartitionedColumnDataAppendState &state, DataChunk &input) {
 	Vector hashes(LogicalType::HASH, input.size());
 	input.Hash(group_by_columns, hashes);
+	hashes.Flatten(input.size());
 
 	for (idx_t i = 0; i < input.size(); i++) {
 		HivePartitionKey key;

package/src/duckdb/src/common/local_file_system.cpp

@@ -833,8 +833,8 @@ static bool HasGlob(const string &str) {
 	return false;
 }
 
-static void GlobFiles(FileSystem &fs, const string &path, const string &glob, bool match_directory,
-                      vector<string> &result, bool join_path) {
+static void GlobFilesInternal(FileSystem &fs, const string &path, const string &glob, bool match_directory,
+                              vector<string> &result, bool join_path) {
 	fs.ListFiles(path, [&](const string &fname, bool is_directory) {
 		if (is_directory != match_directory) {
 			return;
@@ -951,12 +951,12 @@ vector<string> LocalFileSystem::Glob(const string &path, FileOpener *opener) {
 		} else {
 			if (previous_directories.empty()) {
 				// no previous directories: list in the current path
-				GlobFiles(*this, ".", splits[i], !is_last_chunk, result, false);
+				GlobFilesInternal(*this, ".", splits[i], !is_last_chunk, result, false);
 			} else {
 				// previous directories
 				// we iterate over each of the previous directories, and apply the glob of the current directory
 				for (auto &prev_directory : previous_directories) {
-					GlobFiles(*this, prev_directory, splits[i], !is_last_chunk, result, true);
+					GlobFilesInternal(*this, prev_directory, splits[i], !is_last_chunk, result, true);
 				}
 			}
 		}

package/src/duckdb/src/common/string_util.cpp

@@ -249,7 +249,7 @@ private:
 };
 
 // adapted from https://en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Levenshtein_distance#C++
-idx_t StringUtil::LevenshteinDistance(const string &s1_p, const string &s2_p) {
+idx_t StringUtil::LevenshteinDistance(const string &s1_p, const string &s2_p, idx_t not_equal_penalty) {
 	auto s1 = StringUtil::Lower(s1_p);
 	auto s2 = StringUtil::Lower(s2_p);
 	idx_t len1 = s1.size();
@@ -273,7 +273,7 @@ idx_t StringUtil::LevenshteinDistance(const string &s1_p, const string &s2_p) {
 			// d[i][j] = std::min({ d[i - 1][j] + 1,
 			//                      d[i][j - 1] + 1,
 			//                      d[i - 1][j - 1] + (s1[i - 1] == s2[j - 1] ? 0 : 1) });
-			int equal = s1[i - 1] == s2[j - 1] ? 0 : 1;
+			int equal = s1[i - 1] == s2[j - 1] ? 0 : not_equal_penalty;
 			idx_t adjacent_score1 = array.Score(i - 1, j) + 1;
 			idx_t adjacent_score2 = array.Score(i, j - 1) + 1;
 			idx_t adjacent_score3 = array.Score(i - 1, j - 1) + equal;
@@ -285,15 +285,19 @@ idx_t StringUtil::LevenshteinDistance(const string &s1_p, const string &s2_p) {
 	return array.Score(len1, len2);
 }
 
+idx_t StringUtil::SimilarityScore(const string &s1, const string &s2) {
+	return LevenshteinDistance(s1, s2, 3);
+}
+
 vector<string> StringUtil::TopNLevenshtein(const vector<string> &strings, const string &target, idx_t n,
                                            idx_t threshold) {
 	vector<pair<string, idx_t>> scores;
 	scores.reserve(strings.size());
 	for (auto &str : strings) {
 		if (target.size() < str.size()) {
-			scores.emplace_back(str, LevenshteinDistance(str.substr(0, target.size()), target));
+			scores.emplace_back(str, SimilarityScore(str.substr(0, target.size()), target));
 		} else {
-			scores.emplace_back(str, LevenshteinDistance(str, target));
+			scores.emplace_back(str, SimilarityScore(str, target));
 		}
 	}
 	return TopNStrings(scores, n, threshold);

package/src/duckdb/src/common/types/partitioned_column_data.cpp

@@ -137,6 +137,7 @@ void PartitionedColumnData::FlushAppendState(PartitionedColumnDataAppendState &s
 		auto &partition_buffer = *state.partition_buffers[i];
 		if (partition_buffer.size() > 0) {
 			partitions[i]->Append(partition_buffer);
+			partition_buffer.Reset();
 		}
 	}
 }

package/src/duckdb/src/common/types.cpp

@@ -424,7 +424,7 @@ string LogicalType::ToString() const {
 		auto &child_types = StructType::GetChildTypes(*this);
 		string ret = "STRUCT(";
 		for (size_t i = 0; i < child_types.size(); i++) {
-			ret += child_types[i].first + " " + child_types[i].second.ToString();
+			ret += KeywordHelper::WriteOptionallyQuoted(child_types[i].first) + " " + child_types[i].second.ToString();
 			if (i < child_types.size() - 1) {
 				ret += ", ";
 			}
@@ -504,11 +504,32 @@ LogicalType TransformStringToLogicalType(const string &str) {
 	return Parser::ParseColumnList("dummy " + str).GetColumn(LogicalIndex(0)).Type();
 }
 
+LogicalType GetUserTypeRecursive(const LogicalType &type, ClientContext &context) {
+	if (type.id() == LogicalTypeId::USER && type.HasAlias()) {
+		return Catalog::GetSystemCatalog(context).GetType(context, SYSTEM_CATALOG, DEFAULT_SCHEMA, type.GetAlias());
+	}
+	// Look for LogicalTypeId::USER in nested types
+	if (type.id() == LogicalTypeId::STRUCT) {
+		child_list_t<LogicalType> children;
+		children.reserve(StructType::GetChildCount(type));
+		for (auto &child : StructType::GetChildTypes(type)) {
+			children.emplace_back(child.first, GetUserTypeRecursive(child.second, context));
+		}
+		return LogicalType::STRUCT(std::move(children));
+	}
+	if (type.id() == LogicalTypeId::LIST) {
+		return LogicalType::LIST(GetUserTypeRecursive(ListType::GetChildType(type), context));
+	}
+	if (type.id() == LogicalTypeId::MAP) {
+		return LogicalType::MAP(GetUserTypeRecursive(MapType::KeyType(type), context),
+		                        GetUserTypeRecursive(MapType::ValueType(type), context));
+	}
+	// Not LogicalTypeId::USER or a nested type
+	return type;
+}
+
 LogicalType TransformStringToLogicalType(const string &str, ClientContext &context) {
-	auto type = TransformStringToLogicalType(str);
-	return type.id() == LogicalTypeId::USER
-	           ? Catalog::GetSystemCatalog(context).GetType(context, SYSTEM_CATALOG, DEFAULT_SCHEMA, str)
-	           : type;
+	return GetUserTypeRecursive(TransformStringToLogicalType(str), context);
 }
 
 bool LogicalType::IsIntegral() const {
@@ -888,18 +909,23 @@ void LogicalType::SetAlias(string alias) {
 }
 
 string LogicalType::GetAlias() const {
-	if (!type_info_) {
-		return string();
-	} else {
+	if (id() == LogicalTypeId::USER) {
+		return UserType::GetTypeName(*this);
+	}
+	if (type_info_) {
 		return type_info_->alias;
 	}
+	return string();
 }
 
 bool LogicalType::HasAlias() const {
-	if (!type_info_) {
-		return false;
+	if (id() == LogicalTypeId::USER) {
+		return !UserType::GetTypeName(*this).empty();
+	}
+	if (type_info_ && !type_info_->alias.empty()) {
+		return true;
 	}
-	return !type_info_->alias.empty();
+	return false;
 }
 
 void LogicalType::SetCatalog(LogicalType &type, TypeCatalogEntry *catalog_entry) {

package/src/duckdb/src/execution/column_binding_resolver.cpp

@@ -65,9 +65,12 @@ void ColumnBindingResolver::VisitOperator(LogicalOperator &op) {
 		// ON CONFLICT DO UPDATE clause
 		auto &insert_op = (LogicalInsert &)op;
 		if (insert_op.action_type != OnConflictAction::THROW) {
+			// Get the bindings from the children
 			VisitOperatorChildren(op);
-			auto dummy_bindings = LogicalOperator::GenerateColumnBindings(
-			    insert_op.excluded_table_index, insert_op.table->GetColumns().PhysicalColumnCount());
+			auto column_count = insert_op.table->GetColumns().PhysicalColumnCount();
+			auto dummy_bindings = LogicalOperator::GenerateColumnBindings(insert_op.excluded_table_index, column_count);
+			// Now insert our dummy bindings at the start of the bindings,
+			// so the first 'column_count' indices of the chunk are reserved for our 'excluded' columns
 			bindings.insert(bindings.begin(), dummy_bindings.begin(), dummy_bindings.end());
 			if (insert_op.on_conflict_condition) {
 				VisitExpression(&insert_op.on_conflict_condition);