duckdb 0.7.1-dev284.0 → 0.7.1-dev341.0

package/README.md

@@ -1,6 +1,6 @@
 # DuckDB Node Bindings
 
-This package provides a node.js API for [DuckDB](https://github.com/cwida/duckdb), the "SQLite for Analytics". The API for this client is somewhat compliant to the SQLite node.js client for easier transition (and transition you must eventually).
+This package provides a node.js API for [DuckDB](https://github.com/duckdb/duckdb), the "SQLite for Analytics". The API for this client is somewhat compliant to the SQLite node.js client for easier transition (and transition you must eventually).
 
 Load the package and create a database object:
 

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.7.1-dev284.0",
+  "version": "0.7.1-dev341.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {
@@ -41,7 +41,7 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/cwida/duckdb.git"
+    "url": "git+https://github.com/duckdb/duckdb.git"
   },
   "ts-node": {
     "require": [
@@ -56,7 +56,7 @@
   "author": "Hannes Mühleisen",
   "license": "MPL-2.0",
   "bugs": {
-    "url": "https://github.com/cwida/duckdb/issues"
+    "url": "https://github.com/duckdb/duckdb/issues"
   },
   "homepage": "https://www.duckdb.org"
 }

package/src/duckdb/extension/json/json_scan.cpp

@@ -75,10 +75,7 @@ void JSONScanData::InitializeFilePaths(ClientContext &context, const vector<stri
                                        vector<string> &file_paths) {
 	auto &fs = FileSystem::GetFileSystem(context);
 	for (auto &file_pattern : patterns) {
-		auto found_files = fs.Glob(file_pattern, context);
-		if (found_files.empty()) {
-			throw FileSystem::MissingFileException(file_pattern, context);
-		}
+		auto found_files = fs.GlobFiles(file_pattern, context);
 		file_paths.insert(file_paths.end(), found_files.begin(), found_files.end());
 	}
 }

package/src/duckdb/extension/parquet/column_reader.cpp

@@ -589,6 +589,7 @@ void StringColumnReader::PrepareDeltaLengthByteArray(ResizeableBuffer &buffer) {
 	}
 	auto length_data = (uint32_t *)length_buffer->ptr;
 	byte_array_data = make_unique<Vector>(LogicalType::VARCHAR, value_count);
+	byte_array_count = value_count;
 	auto string_data = FlatVector::GetData<string_t>(*byte_array_data);
 	for (idx_t i = 0; i < value_count; i++) {
 		auto str_len = length_data[i];
@@ -615,6 +616,7 @@ void StringColumnReader::PrepareDeltaByteArray(ResizeableBuffer &buffer) {
 	auto prefix_data = (uint32_t *)prefix_buffer->ptr;
 	auto suffix_data = (uint32_t *)suffix_buffer->ptr;
 	byte_array_data = make_unique<Vector>(LogicalType::VARCHAR, prefix_count);
+	byte_array_count = prefix_count;
 	auto string_data = FlatVector::GetData<string_t>(*byte_array_data);
 	for (idx_t i = 0; i < prefix_count; i++) {
 		auto str_len = prefix_data[i] + suffix_data[i];
@@ -646,6 +648,11 @@ void StringColumnReader::DeltaByteArray(uint8_t *defines, idx_t num_values, parq
 			continue;
 		}
 		if (filter[row_idx + result_offset]) {
+			if (delta_offset >= byte_array_count) {
+				throw IOException("DELTA_BYTE_ARRAY - length mismatch between values and byte array lengths (attempted "
+				                  "read of %d from %d entries) - corrupt file?",
+				                  delta_offset + 1, byte_array_count);
+			}
 			result_ptr[row_idx + result_offset] = string_data[delta_offset++];
 		} else {
 			delta_offset++;

package/src/duckdb/extension/parquet/include/column_reader.hpp

@@ -131,6 +131,7 @@ protected:
 	ParquetReader &reader;
 	LogicalType type;
 	unique_ptr<Vector> byte_array_data;
+	idx_t byte_array_count = 0;
 
 	idx_t pending_skips = 0;
 

package/src/duckdb/extension/parquet/parquet-extension.cpp

@@ -221,10 +221,7 @@ public:
 		}
 
 		FileSystem &fs = FileSystem::GetFileSystem(context);
-		auto files = fs.Glob(info.file_path, context);
-		if (files.empty()) {
-			throw FileSystem::MissingFileException(info.file_path, context);
-		}
+		auto files = fs.GlobFiles(info.file_path, context);
 
 		// The most likely path (Parquet read without union by name option)
 		if (!parquet_options.union_by_name) {
@@ -362,12 +359,7 @@ public:
 	}
 
 	static vector<string> ParquetGlob(FileSystem &fs, const string &glob, ClientContext &context) {
-		auto files = fs.Glob(glob, FileSystem::GetFileOpener(context));
-
-		if (files.empty()) {
-			throw FileSystem::MissingFileException(glob, context);
-		}
-		return files;
+		return fs.GlobFiles(glob, context);
 	}
 
 	static unique_ptr<FunctionData> ParquetScanBind(ClientContext &context, TableFunctionBindInput &input,

package/src/duckdb/src/catalog/catalog.cpp

@@ -27,7 +27,7 @@
 #include "duckdb/planner/parsed_data/bound_create_table_info.hpp"
 #include "duckdb/planner/binder.hpp"
 #include "duckdb/catalog/default/default_types.hpp"
-#include "duckdb/main/extension_functions.hpp"
+#include "duckdb/main/extension_entries.hpp"
 #include "duckdb/main/connection.hpp"
 #include "duckdb/main/attached_database.hpp"
 #include "duckdb/main/database_manager.hpp"
@@ -332,17 +332,26 @@ SimilarCatalogEntry Catalog::SimilarEntryInSchemas(ClientContext &context, const
 	return result;
 }
 
-string FindExtension(const string &function_name) {
-	auto size = sizeof(EXTENSION_FUNCTIONS) / sizeof(ExtensionFunction);
-	auto it = std::lower_bound(
-	    EXTENSION_FUNCTIONS, EXTENSION_FUNCTIONS + size, function_name,
-	    [](const ExtensionFunction &element, const string &value) { return element.function < value; });
-	if (it != EXTENSION_FUNCTIONS + size && it->function == function_name) {
+string FindExtensionGeneric(const string &name, const ExtensionEntry entries[], idx_t size) {
+	auto lcase = StringUtil::Lower(name);
+	auto it = std::lower_bound(entries, entries + size, lcase,
+	                           [](const ExtensionEntry &element, const string &value) { return element.name < value; });
+	if (it != entries + size && it->name == lcase) {
 		return it->extension;
 	}
 	return "";
 }
 
+string FindExtensionForFunction(const string &name) {
+	idx_t size = sizeof(EXTENSION_FUNCTIONS) / sizeof(ExtensionEntry);
+	return FindExtensionGeneric(name, EXTENSION_FUNCTIONS, size);
+}
+
+string FindExtensionForSetting(const string &name) {
+	idx_t size = sizeof(EXTENSION_SETTINGS) / sizeof(ExtensionEntry);
+	return FindExtensionGeneric(name, EXTENSION_SETTINGS, size);
+}
+
 vector<CatalogSearchEntry> GetCatalogEntries(ClientContext &context, const string &catalog, const string &schema) {
 	vector<CatalogSearchEntry> entries;
 	auto &search_path = *context.client_data->catalog_search_path;
@@ -407,6 +416,26 @@ void FindMinimalQualification(ClientContext &context, const string &catalog_name
 	qualify_schema = true;
 }
 
+CatalogException Catalog::UnrecognizedConfigurationError(ClientContext &context, const string &name) {
+	// check if the setting exists in any extensions
+	auto extension_name = FindExtensionForSetting(name);
+	if (!extension_name.empty()) {
+		return CatalogException(
+		    "Setting with name \"%s\" is not in the catalog, but it exists in the %s extension.\n\nTo "
+		    "install and load the extension, run:\nINSTALL %s;\nLOAD %s;",
+		    name, extension_name, extension_name, extension_name);
+	}
+	// the setting is not in an extension
+	// get a list of all options
+	vector<string> potential_names = DBConfig::GetOptionNames();
+	for (auto &entry : DBConfig::GetConfig(context).extension_parameters) {
+		potential_names.push_back(entry.first);
+	}
+
+	throw CatalogException("unrecognized configuration parameter \"%s\"\n%s", name,
+	                       StringUtil::CandidatesErrorMessage(potential_names, name, "Did you mean"));
+}
+
 CatalogException Catalog::CreateMissingEntryException(ClientContext &context, const string &entry_name,
                                                       CatalogType type,
                                                       const unordered_set<SchemaCatalogEntry *> &schemas,
@@ -423,13 +452,18 @@ CatalogException Catalog::CreateMissingEntryException(ClientContext &context, co
 			unseen_schemas.insert(current_schema);
 		}
 	}
-	auto unseen_entry = SimilarEntryInSchemas(context, entry_name, type, unseen_schemas);
-	auto extension_name = FindExtension(entry_name);
-	if (!extension_name.empty()) {
-		return CatalogException("Function with name %s is not on the catalog, but it exists in the %s extension. To "
-		                        "Install and Load the extension, run: INSTALL %s; LOAD %s;",
-		                        entry_name, extension_name, extension_name, extension_name);
+	// check if the entry exists in any extension
+	if (type == CatalogType::TABLE_FUNCTION_ENTRY || type == CatalogType::SCALAR_FUNCTION_ENTRY ||
+	    type == CatalogType::AGGREGATE_FUNCTION_ENTRY) {
+		auto extension_name = FindExtensionForFunction(entry_name);
+		if (!extension_name.empty()) {
+			return CatalogException(
+			    "Function with name \"%s\" is not in the catalog, but it exists in the %s extension.\n\nTo "
+			    "install and load the extension, run:\nINSTALL %s;\nLOAD %s;",
+			    entry_name, extension_name, extension_name, extension_name);
+		}
 	}
+	auto unseen_entry = SimilarEntryInSchemas(context, entry_name, type, unseen_schemas);
 	string did_you_mean;
 	if (unseen_entry.Found() && unseen_entry.distance < entry.distance) {
 		// the closest matching entry requires qualification as it is not in the default search path

package/src/duckdb/src/catalog/default/default_views.cpp

@@ -48,7 +48,7 @@ static DefaultView internal_views[] = {
     {"pg_catalog", "pg_views", "SELECT schema_name schemaname, view_name viewname, 'duckdb' viewowner, sql definition FROM duckdb_views()"},
     {"information_schema", "columns", "SELECT database_name table_catalog, schema_name table_schema, table_name, column_name, column_index ordinal_position, column_default, CASE WHEN is_nullable THEN 'YES' ELSE 'NO' END is_nullable, data_type, character_maximum_length, NULL character_octet_length, numeric_precision, numeric_precision_radix, numeric_scale, NULL datetime_precision, NULL interval_type, NULL interval_precision, NULL character_set_catalog, NULL character_set_schema, NULL character_set_name, NULL collation_catalog, NULL collation_schema, NULL collation_name, NULL domain_catalog, NULL domain_schema, NULL domain_name, NULL udt_catalog, NULL udt_schema, NULL udt_name, NULL scope_catalog, NULL scope_schema, NULL scope_name, NULL maximum_cardinality, NULL dtd_identifier, NULL is_self_referencing, NULL is_identity, NULL identity_generation, NULL identity_start, NULL identity_increment, NULL identity_maximum, NULL identity_minimum, NULL identity_cycle, NULL is_generated, NULL generation_expression, NULL is_updatable FROM duckdb_columns;"},
     {"information_schema", "schemata", "SELECT database_name catalog_name, schema_name, 'duckdb' schema_owner, NULL default_character_set_catalog, NULL default_character_set_schema, NULL default_character_set_name, sql sql_path FROM duckdb_schemas()"},
-    {"information_schema", "tables", "SELECT database_name table_catalog, schema_name table_schema, table_name, CASE WHEN temporary THEN 'LOCAL TEMPORARY' ELSE 'BASE TABLE' END table_type, NULL self_referencing_column_name, NULL reference_generation, NULL user_defined_type_catalog, NULL user_defined_type_schema, NULL user_defined_type_name, 'YES' is_insertable_into, 'NO' is_typed, CASE WHEN temporary THEN 'PRESERVE' ELSE NULL END commit_action FROM duckdb_tables() UNION ALL SELECT NULL table_catalog, schema_name table_schema, view_name table_name, 'VIEW' table_type, NULL self_referencing_column_name, NULL reference_generation, NULL user_defined_type_catalog, NULL user_defined_type_schema, NULL user_defined_type_name, 'NO' is_insertable_into, 'NO' is_typed, NULL commit_action FROM duckdb_views;"},
+    {"information_schema", "tables", "SELECT database_name table_catalog, schema_name table_schema, table_name, CASE WHEN temporary THEN 'LOCAL TEMPORARY' ELSE 'BASE TABLE' END table_type, NULL self_referencing_column_name, NULL reference_generation, NULL user_defined_type_catalog, NULL user_defined_type_schema, NULL user_defined_type_name, 'YES' is_insertable_into, 'NO' is_typed, CASE WHEN temporary THEN 'PRESERVE' ELSE NULL END commit_action FROM duckdb_tables() UNION ALL SELECT database_name table_catalog, schema_name table_schema, view_name table_name, 'VIEW' table_type, NULL self_referencing_column_name, NULL reference_generation, NULL user_defined_type_catalog, NULL user_defined_type_schema, NULL user_defined_type_name, 'NO' is_insertable_into, 'NO' is_typed, NULL commit_action FROM duckdb_views;"},
     {nullptr, nullptr, nullptr}};
 
 static unique_ptr<CreateViewInfo> GetDefaultView(ClientContext &context, const string &input_schema, const string &input_name) {

package/src/duckdb/src/common/file_system.cpp

@@ -10,6 +10,7 @@
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/client_data.hpp"
 #include "duckdb/main/database.hpp"
+#include "duckdb/main/extension_helper.hpp"
 
 #include <cstdint>
 #include <cstdio>
@@ -335,18 +336,31 @@ bool FileSystem::CanHandleFile(const string &fpath) {
 	throw NotImplementedException("%s: CanHandleFile is not implemented!", GetName());
 }
 
-IOException FileSystem::MissingFileException(const string &file_path, ClientContext &context) {
-	const string prefixes[] = {"http://", "https://", "s3://"};
-	for (auto &prefix : prefixes) {
-		if (StringUtil::StartsWith(file_path, prefix)) {
-			if (!context.db->LoadedExtensions().count("httpfs")) {
-				return MissingExtensionException("No files found that match the pattern \"%s\", because the httpfs "
-				                                 "extension is not loaded. Try loading the extension: LOAD HTTPFS",
-				                                 file_path);
+vector<string> FileSystem::GlobFiles(const string &pattern, ClientContext &context) {
+	auto result = Glob(pattern, context);
+	if (result.empty()) {
+		string required_extension;
+		const string prefixes[] = {"http://", "https://", "s3://"};
+		for (auto &prefix : prefixes) {
+			if (StringUtil::StartsWith(pattern, prefix)) {
+				required_extension = "httpfs";
+				break;
 			}
 		}
+		if (!required_extension.empty() && !context.db->ExtensionIsLoaded(required_extension)) {
+			// an extension is required to read this file but it is not loaded - try to load it
+			ExtensionHelper::LoadExternalExtension(context, required_extension);
+			// success! glob again
+			// check the extension is loaded just in case to prevent an infinite loop here
+			if (!context.db->ExtensionIsLoaded(required_extension)) {
+				throw InternalException("Extension load \"%s\" did not throw but somehow the extension was not loaded",
+				                        required_extension);
+			}
+			return GlobFiles(pattern, context);
+		}
+		throw IOException("No files found that match the pattern \"%s\"", pattern);
 	}
-	return IOException("No files found that match the pattern \"%s\"", file_path);
+	return result;
 }
 
 void FileSystem::Seek(FileHandle &handle, idx_t location) {

package/src/duckdb/src/common/local_file_system.cpp

@@ -833,8 +833,8 @@ static bool HasGlob(const string &str) {
 	return false;
 }
 
-static void GlobFiles(FileSystem &fs, const string &path, const string &glob, bool match_directory,
-                      vector<string> &result, bool join_path) {
+static void GlobFilesInternal(FileSystem &fs, const string &path, const string &glob, bool match_directory,
+                              vector<string> &result, bool join_path) {
 	fs.ListFiles(path, [&](const string &fname, bool is_directory) {
 		if (is_directory != match_directory) {
 			return;
@@ -951,12 +951,12 @@ vector<string> LocalFileSystem::Glob(const string &path, FileOpener *opener) {
 		} else {
 			if (previous_directories.empty()) {
 				// no previous directories: list in the current path
-				GlobFiles(*this, ".", splits[i], !is_last_chunk, result, false);
+				GlobFilesInternal(*this, ".", splits[i], !is_last_chunk, result, false);
 			} else {
 				// previous directories
 				// we iterate over each of the previous directories, and apply the glob of the current directory
 				for (auto &prev_directory : previous_directories) {
-					GlobFiles(*this, prev_directory, splits[i], !is_last_chunk, result, true);
+					GlobFilesInternal(*this, prev_directory, splits[i], !is_last_chunk, result, true);
 				}
 			}
 		}

package/src/duckdb/src/common/types/partitioned_column_data.cpp

@@ -137,6 +137,7 @@ void PartitionedColumnData::FlushAppendState(PartitionedColumnDataAppendState &s
 		auto &partition_buffer = *state.partition_buffers[i];
 		if (partition_buffer.size() > 0) {
 			partitions[i]->Append(partition_buffer);
+			partition_buffer.Reset();
 		}
 	}
 }

package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp

@@ -169,6 +169,10 @@ private:
 };
 
 void WindowGlobalSinkState::ResizeGroupingData(idx_t cardinality) {
+	//	Have we started to combine? Then just live with it.
+	if (grouping_data && !grouping_data->GetPartitions().empty()) {
+		return;
+	}
 	//	Is the average partition size too large?
 	const idx_t partition_size = STANDARD_ROW_GROUPS_SIZE;
 	const auto bits = grouping_data ? grouping_data->GetRadixBits() : 0;
@@ -180,31 +184,7 @@ void WindowGlobalSinkState::ResizeGroupingData(idx_t cardinality) {
 	// Repartition the grouping data
 	if (new_bits != bits) {
 		const auto hash_col_idx = payload_types.size();
-		auto new_grouping_data =
-		    make_unique<RadixPartitionedColumnData>(context, grouping_types, new_bits, hash_col_idx);
-
-		// We have to append to a shared copy for some reason
-		if (grouping_data) {
-			auto new_shared = new_grouping_data->CreateShared();
-			PartitionedColumnDataAppendState shared_append;
-			new_shared->InitializeAppendState(shared_append);
-
-			auto &partitions = grouping_data->GetPartitions();
-			for (auto &partition : partitions) {
-				ColumnDataScanState scanner;
-				partition->InitializeScan(scanner);
-
-				DataChunk scan_chunk;
-				partition->InitializeScanChunk(scan_chunk);
-				for (scan_chunk.Reset(); partition->Scan(scanner, scan_chunk); scan_chunk.Reset()) {
-					new_shared->Append(shared_append, scan_chunk);
-				}
-			}
-			new_shared->FlushAppendState(shared_append);
-			new_grouping_data->Combine(*new_shared);
-		}
-
-		grouping_data = std::move(new_grouping_data);
+		grouping_data = make_unique<RadixPartitionedColumnData>(context, grouping_types, new_bits, hash_col_idx);
 	}
 }
 
@@ -432,8 +412,6 @@ void WindowLocalSinkState::Sink(DataChunk &input_chunk, WindowGlobalSinkState &g
 	}
 
 	// OVER(...)
-	gstate.UpdateLocalPartition(local_partition, local_append);
-
 	payload_chunk.Reset();
 	auto &hash_vector = payload_chunk.data.back();
 	Hash(input_chunk, hash_vector);
@@ -442,6 +420,7 @@ void WindowLocalSinkState::Sink(DataChunk &input_chunk, WindowGlobalSinkState &g
 	}
 	payload_chunk.SetCardinality(input_chunk);
 
+	gstate.UpdateLocalPartition(local_partition, local_append);
 	local_partition->Append(*local_append, payload_chunk);
 }
 

package/src/duckdb/src/execution/operator/helper/physical_reset.cpp

@@ -27,15 +27,7 @@ void PhysicalReset::GetData(ExecutionContext &context, DataChunk &chunk, GlobalS
 		auto &config = DBConfig::GetConfig(context.client);
 		auto entry = config.extension_parameters.find(name);
 		if (entry == config.extension_parameters.end()) {
-			// it is not!
-			// get a list of all options
-			vector<string> potential_names = DBConfig::GetOptionNames();
-			for (auto &entry : config.extension_parameters) {
-				potential_names.push_back(entry.first);
-			}
-
-			throw CatalogException("unrecognized configuration parameter \"%s\"\n%s", name,
-			                       StringUtil::CandidatesErrorMessage(potential_names, name, "Did you mean"));
+			throw Catalog::UnrecognizedConfigurationError(context.client, name);
 		}
 		ResetExtensionVariable(context, config, entry->second);
 		return;

package/src/duckdb/src/execution/operator/helper/physical_set.cpp

@@ -30,15 +30,7 @@ void PhysicalSet::GetData(ExecutionContext &context, DataChunk &chunk, GlobalSou
 		auto &config = DBConfig::GetConfig(context.client);
 		auto entry = config.extension_parameters.find(name);
 		if (entry == config.extension_parameters.end()) {
-			// it is not!
-			// get a list of all options
-			vector<string> potential_names = DBConfig::GetOptionNames();
-			for (auto &entry : config.extension_parameters) {
-				potential_names.push_back(entry.first);
-			}
-
-			throw CatalogException("unrecognized configuration parameter \"%s\"\n%s", name,
-			                       StringUtil::CandidatesErrorMessage(potential_names, name, "Did you mean"));
+			throw Catalog::UnrecognizedConfigurationError(context.client, name);
 		}
 		SetExtensionVariable(context.client, entry->second, name, scope, value);
 		return;

package/src/duckdb/src/function/scalar/generic/current_setting.cpp

@@ -4,7 +4,7 @@
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/planner/expression/bound_function_expression.hpp"
 #include "duckdb/execution/expression_executor.hpp"
-
+#include "duckdb/catalog/catalog.hpp"
 namespace duckdb {
 
 struct CurrentSettingBindData : public FunctionData {
@@ -51,7 +51,7 @@ unique_ptr<FunctionData> CurrentSettingBind(ClientContext &context, ScalarFuncti
 	auto key = StringUtil::Lower(key_str);
 	Value val;
 	if (!context.TryGetCurrentSetting(key, val)) {
-		throw InvalidInputException("unrecognized configuration parameter \"%s\"", key_str);
+		throw Catalog::UnrecognizedConfigurationError(context, key);
 	}
 
 	bound_function.return_type = val.type();

package/src/duckdb/src/function/scalar/map/map.cpp

@@ -66,46 +66,94 @@ void MapConversionVerify(Vector &vector, idx_t count) {
 	}
 }
 
-static void MapFunction(DataChunk &args, ExpressionState &state, Vector &result) {
-	D_ASSERT(result.GetType().id() == LogicalTypeId::MAP);
-
-	//! Otherwise if its not a constant vector, this breaks the optimizer
-	result.SetVectorType(VectorType::CONSTANT_VECTOR);
-	for (idx_t i = 0; i < args.ColumnCount(); i++) {
-		if (args.data[i].GetVectorType() != VectorType::CONSTANT_VECTOR) {
-			result.SetVectorType(VectorType::FLAT_VECTOR);
+// Example:
+// source: [1,2,3], expansion_factor: 4
+// target (result): [1,2,3,1,2,3,1,2,3,1,2,3]
+static void CreateExpandedVector(const Vector &source, Vector &target, idx_t expansion_factor) {
+	idx_t count = ListVector::GetListSize(source);
+	auto &entry = ListVector::GetEntry(source);
+
+	idx_t target_idx = 0;
+	for (idx_t copy = 0; copy < expansion_factor; copy++) {
+		for (idx_t key_idx = 0; key_idx < count; key_idx++) {
+			target.SetValue(target_idx, entry.GetValue(key_idx));
+			target_idx++;
 		}
 	}
+	D_ASSERT(target_idx == count * expansion_factor);
+}
+
+static void AlignVectorToReference(const Vector &original, const Vector &reference, idx_t tuple_count, Vector &result) {
+	auto original_length = ListVector::GetListSize(original);
+	auto new_length = ListVector::GetListSize(reference);
+
+	Vector expanded_const(ListType::GetChildType(original.GetType()), new_length);
+
+	auto expansion_factor = new_length / original_length;
+	if (expansion_factor != tuple_count) {
+		throw InvalidInputException("Error in MAP creation: key list and value list do not align. i.e. different "
+		                            "size or incompatible structure");
+	}
+	CreateExpandedVector(original, expanded_const, expansion_factor);
+	result.Reference(expanded_const);
+}
+
+static void MapFunction(DataChunk &args, ExpressionState &state, Vector &result) {
+	D_ASSERT(result.GetType().id() == LogicalTypeId::MAP);
 
 	auto &key_vector = MapVector::GetKeys(result);
 	auto &value_vector = MapVector::GetValues(result);
-	auto list_data = ListVector::GetData(result);
+	auto result_data = ListVector::GetData(result);
 
+	result.SetVectorType(VectorType::CONSTANT_VECTOR);
 	if (args.data.empty()) {
 		ListVector::SetListSize(result, 0);
-		list_data->offset = 0;
-		list_data->length = 0;
+		result_data->offset = 0;
+		result_data->length = 0;
 		result.Verify(args.size());
 		return;
 	}
 
-	auto args_data = ListVector::GetData(args.data[0]);
+	bool keys_are_const = args.data[0].GetVectorType() == VectorType::CONSTANT_VECTOR;
+	bool values_are_const = args.data[1].GetVectorType() == VectorType::CONSTANT_VECTOR;
+	if (!keys_are_const || !values_are_const) {
+		result.SetVectorType(VectorType::FLAT_VECTOR);
+	}
+
 	auto key_count = ListVector::GetListSize(args.data[0]);
 	auto value_count = ListVector::GetListSize(args.data[1]);
-	if (key_count != value_count) {
-		throw InvalidInputException(
-		    "Error in MAP creation: key list has a different size from value list (%lld keys, %lld values)", key_count,
-		    value_count);
+	auto key_data = ListVector::GetData(args.data[0]);
+	auto value_data = ListVector::GetData(args.data[1]);
+	auto src_data = key_data;
+
+	if (keys_are_const && !values_are_const) {
+		AlignVectorToReference(args.data[0], args.data[1], args.size(), key_vector);
+		src_data = value_data;
+	} else if (values_are_const && !keys_are_const) {
+		AlignVectorToReference(args.data[1], args.data[0], args.size(), value_vector);
+	} else {
+		if (key_count != value_count || memcmp(key_data, value_data, args.size() * sizeof(list_entry_t)) != 0) {
+			throw InvalidInputException("Error in MAP creation: key list and value list do not align. i.e. different "
+			                            "size or incompatible structure");
+		}
 	}
-	ListVector::Reserve(result, key_count);
-	ListVector::SetListSize(result, key_count);
 
+	ListVector::SetListSize(result, MaxValue(key_count, value_count));
+
+	result_data = ListVector::GetData(result);
 	for (idx_t i = 0; i < args.size(); i++) {
-		list_data[i] = args_data[i];
+		result_data[i] = src_data[i];
+	}
+
+	// check whether one of the vectors has already been referenced to an expanded vector in the case of const/non-const
+	// combination. If not, then referencing is still necessary
+	if (!(keys_are_const && !values_are_const)) {
+		key_vector.Reference(ListVector::GetEntry(args.data[0]));
+	}
+	if (!(values_are_const && !keys_are_const)) {
+		value_vector.Reference(ListVector::GetEntry(args.data[1]));
 	}
 
-	key_vector.Reference(ListVector::GetEntry(args.data[0]));
-	value_vector.Reference(ListVector::GetEntry(args.data[1]));
 	MapConversionVerify(result, args.size());
 	result.Verify(args.size());
 }

package/src/duckdb/src/function/table/read_csv.cpp

@@ -28,10 +28,7 @@ unique_ptr<CSVFileHandle> ReadCSV::OpenCSV(const string &file_path, FileCompress
 void ReadCSVData::InitializeFiles(ClientContext &context, const vector<string> &patterns) {
 	auto &fs = FileSystem::GetFileSystem(context);
 	for (auto &file_pattern : patterns) {
-		auto found_files = fs.Glob(file_pattern, context);
-		if (found_files.empty()) {
-			throw FileSystem::MissingFileException(file_pattern, context);
-		}
+		auto found_files = fs.GlobFiles(file_pattern, context);
 		files.insert(files.end(), found_files.begin(), found_files.end());
 	}
 }

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.7.1-dev284"
+#define DUCKDB_VERSION "0.7.1-dev341"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "6b7362d2ca"
+#define DUCKDB_SOURCE_ID "d58ab188ff"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/catalog/catalog.hpp

@@ -255,6 +255,8 @@ public:
 
 	virtual void Verify();
 
+	static CatalogException UnrecognizedConfigurationError(ClientContext &context, const string &name);
+
 protected:
 	//! Reference to the database
 	AttachedDatabase &db;

package/src/duckdb/src/include/duckdb/common/file_system.hpp

@@ -188,6 +188,7 @@ public:
 	//! Runs a glob on the file system, returning a list of matching files
 	DUCKDB_API virtual vector<string> Glob(const string &path, FileOpener *opener = nullptr);
 	DUCKDB_API virtual vector<string> Glob(const string &path, ClientContext &context);
+	DUCKDB_API vector<string> GlobFiles(const string &path, ClientContext &context);
 
 	//! registers a sub-file system to handle certain file name prefixes, e.g. http:// etc.
 	DUCKDB_API virtual void RegisterSubSystem(unique_ptr<FileSystem> sub_fs);
@@ -201,7 +202,6 @@ public:
 
 	//! Whether or not a sub-system can handle a specific file path
 	DUCKDB_API virtual bool CanHandleFile(const string &fpath);
-	DUCKDB_API static IOException MissingFileException(const string &file_path, ClientContext &context);
 
 	//! Set the file pointer of a file handle to a specified location. Reads and writes will happen from this location
 	DUCKDB_API virtual void Seek(FileHandle &handle, idx_t location);

package/src/duckdb/src/include/duckdb/main/{extension_functions.hpp → extension_entries.hpp}

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/main/extension_functions.hpp
+// duckdb/main/extension_entries.hpp
 //
 //
 //===----------------------------------------------------------------------===//
@@ -12,12 +12,12 @@
 
 namespace duckdb {
 
-struct ExtensionFunction {
-	char function[48];
+struct ExtensionEntry {
+	char name[48];
 	char extension[48];
 };
 
-static constexpr ExtensionFunction EXTENSION_FUNCTIONS[] = {
+static constexpr ExtensionEntry EXTENSION_FUNCTIONS[] = {
     {"->>", "json"},
     {"array_to_json", "json"},
     {"create_fts_index", "fts"},
@@ -30,9 +30,9 @@ static constexpr ExtensionFunction EXTENSION_FUNCTIONS[] = {
     {"from_json", "json"},
     {"from_json_strict", "json"},
     {"from_substrait", "substrait"},
+    {"from_substrait_json", "substrait"},
     {"get_substrait", "substrait"},
     {"get_substrait_json", "substrait"},
-    {"from_substrait_json", "substrait"},
     {"icu_calendar_names", "icu"},
     {"icu_sort_key", "icu"},
     {"json", "json"},
@@ -88,4 +88,25 @@ static constexpr ExtensionFunction EXTENSION_FUNCTIONS[] = {
     {"visualize_json_profiling_output", "visualizer"},
     {"visualize_last_profiling_output", "visualizer"},
 };
+
+static constexpr ExtensionEntry EXTENSION_SETTINGS[] = {
+    {"binary_as_string", "parquet"},
+    {"calendar", "icu"},
+    {"http_retries", "httpfs"},
+    {"http_retry_backoff", "httpfs"},
+    {"http_retry_wait_ms", "httpfs"},
+    {"http_timeout", "httpfs"},
+    {"s3_access_key_id", "httpfs"},
+    {"s3_endpoint", "httpfs"},
+    {"s3_region", "httpfs"},
+    {"s3_secret_access_key", "httpfs"},
+    {"s3_session_token", "httpfs"},
+    {"s3_uploader_max_filesize", "httpfs"},
+    {"s3_uploader_max_parts_per_file", "httpfs"},
+    {"s3_uploader_thread_limit", "httpfs"},
+    {"s3_url_style", "httpfs"},
+    {"s3_use_ssl", "httpfs"},
+    {"sqlite_all_varchar", "sqlite_scanner"},
+    {"timezone", "icu"},
+};
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/extension_helper.hpp

@@ -62,6 +62,9 @@ public:
 	//! Apply any known extension aliases
 	static string ApplyExtensionAlias(string extension_name);
 
+	static string GetExtensionName(const string &extension);
+	static bool IsFullPath(const string &extension);
+
 private:
 	static const vector<string> PathComponents();
 	static ExtensionInitResult InitialLoad(DBConfig &context, FileOpener *opener, const string &extension);

package/src/duckdb/src/main/database.cpp

@@ -356,7 +356,8 @@ idx_t DuckDB::NumberOfThreads() {
 }
 
 bool DatabaseInstance::ExtensionIsLoaded(const std::string &name) {
-	return loaded_extensions.find(name) != loaded_extensions.end();
+	auto extension_name = ExtensionHelper::GetExtensionName(name);
+	return loaded_extensions.find(extension_name) != loaded_extensions.end();
 }
 
 bool DuckDB::ExtensionIsLoaded(const std::string &name) {
@@ -364,7 +365,8 @@ bool DuckDB::ExtensionIsLoaded(const std::string &name) {
 }
 
 void DatabaseInstance::SetExtensionLoaded(const std::string &name) {
-	loaded_extensions.insert(name);
+	auto extension_name = ExtensionHelper::GetExtensionName(name);
+	loaded_extensions.insert(extension_name);
 }
 
 bool DatabaseInstance::TryGetCurrentSetting(const std::string &key, Value &result) {

package/src/duckdb/src/main/extension/extension_load.cpp

@@ -31,7 +31,7 @@ ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileOpener *o
 	auto filename = fs.ConvertSeparators(extension);
 
 	// shorthand case
-	if (!StringUtil::Contains(extension, ".") && !StringUtil::Contains(extension, fs.PathSeparator())) {
+	if (!ExtensionHelper::IsFullPath(extension)) {
 		string local_path = fs.GetHomeDirectory(opener);
 		auto path_components = PathComponents();
 		for (auto &path_ele : path_components) {
@@ -121,9 +121,28 @@ ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileOpener *o
 	return res;
 }
 
+bool ExtensionHelper::IsFullPath(const string &extension) {
+	return StringUtil::Contains(extension, ".") || StringUtil::Contains(extension, "/") ||
+	       StringUtil::Contains(extension, "\\");
+}
+
+string ExtensionHelper::GetExtensionName(const string &extension) {
+	if (!IsFullPath(extension)) {
+		return extension;
+	}
+	auto splits = StringUtil::Split(StringUtil::Replace(extension, "\\", "/"), '/');
+	if (splits.empty()) {
+		return extension;
+	}
+	splits = StringUtil::Split(splits.back(), '.');
+	if (splits.empty()) {
+		return extension;
+	}
+	return StringUtil::Lower(splits.front());
+}
+
 void ExtensionHelper::LoadExternalExtension(DatabaseInstance &db, FileOpener *opener, const string &extension) {
-	auto &loaded_extensions = db.LoadedExtensions();
-	if (loaded_extensions.find(extension) != loaded_extensions.end()) {
+	if (db.ExtensionIsLoaded(extension)) {
 		return;
 	}
 

package/src/duckdb/src/planner/binder/statement/bind_drop.cpp

@@ -54,7 +54,7 @@ BoundStatement Binder::Bind(DropStatement &stmt) {
 		auto &config = DBConfig::GetConfig(context);
 		// for now assume only one storage extension provides the custom drop_database impl
 		for (auto &extension_entry : config.storage_extensions) {
-			if (extension_entry.second->drop_database != nullptr) {
+			if (extension_entry.second->drop_database == nullptr) {
 				continue;
 			}
 			auto &storage_extension = extension_entry.second;
@@ -64,7 +64,7 @@ BoundStatement Binder::Bind(DropStatement &stmt) {
 				auto bound_drop_database_func = Bind(*drop_database_function_ref);
 				result.plan = CreatePlan(*bound_drop_database_func);
 				result.names = {"Success"};
-				result.types = {LogicalType::BOOLEAN};
+				result.types = {LogicalType::BIGINT};
 				properties.allow_stream_result = false;
 				properties.return_type = StatementReturnType::NOTHING;
 				return result;

package/src/duckdb/src/storage/buffer_manager.cpp

@@ -230,6 +230,7 @@ public:
 private:
 	DatabaseInstance &db;
 	string temp_directory;
+	bool created_directory = false;
 	unique_ptr<TemporaryFileManager> temp_file;
 };
 
@@ -868,7 +869,10 @@ TemporaryDirectoryHandle::TemporaryDirectoryHandle(DatabaseInstance &db, string
     : db(db), temp_directory(std::move(path_p)), temp_file(make_unique<TemporaryFileManager>(db, temp_directory)) {
 	auto &fs = FileSystem::GetFileSystem(db);
 	if (!temp_directory.empty()) {
-		fs.CreateDirectory(temp_directory);
+		if (!fs.DirectoryExists(temp_directory)) {
+			fs.CreateDirectory(temp_directory);
+			created_directory = true;
+		}
 	}
 }
 TemporaryDirectoryHandle::~TemporaryDirectoryHandle() {
@@ -877,7 +881,30 @@ TemporaryDirectoryHandle::~TemporaryDirectoryHandle() {
 	// then delete the temporary file directory
 	auto &fs = FileSystem::GetFileSystem(db);
 	if (!temp_directory.empty()) {
-		fs.RemoveDirectory(temp_directory);
+		bool delete_directory = created_directory;
+		vector<string> files_to_delete;
+		if (!created_directory) {
+			bool deleted_everything = true;
+			fs.ListFiles(temp_directory, [&](const string &path, bool isdir) {
+				if (isdir) {
+					deleted_everything = false;
+					return;
+				}
+				if (!StringUtil::StartsWith(path, "duckdb_temp_")) {
+					deleted_everything = false;
+					return;
+				}
+				files_to_delete.push_back(path);
+			});
+		}
+		if (delete_directory) {
+			// we want to remove all files in the directory
+			fs.RemoveDirectory(temp_directory);
+		} else {
+			for (auto &file : files_to_delete) {
+				fs.RemoveFile(fs.JoinPath(temp_directory, file));
+			}
+		}
 	}
 }
 
@@ -887,7 +914,7 @@ TemporaryFileManager &TemporaryDirectoryHandle::GetTempFile() {
 
 string BufferManager::GetTemporaryPath(block_id_t id) {
 	auto &fs = FileSystem::GetFileSystem(db);
-	return fs.JoinPath(temp_directory, to_string(id) + ".block");
+	return fs.JoinPath(temp_directory, "duckdb_temp_block-" + to_string(id) + ".block");
 }
 
 void BufferManager::RequireTemporaryDirectory() {

package/src/duckdb/third_party/concurrentqueue/blockingconcurrentqueue.h

@@ -24,8 +24,8 @@ template<typename T, typename Traits = ConcurrentQueueDefaultTraits>
 class BlockingConcurrentQueue
 {
 private:
-	typedef ::duckdb_moodycamelmoodycamel::ConcurrentQueue<T, Traits> ConcurrentQueue;
-	typedef ::duckdb_moodycamelmoodycamel::LightweightSemaphore LightweightSemaphore;
+	typedef ::duckdb_moodycamel::ConcurrentQueue<T, Traits> ConcurrentQueue;
+	typedef ::duckdb_moodycamel::LightweightSemaphore LightweightSemaphore;
 
 public:
 	typedef typename ConcurrentQueue::producer_token_t producer_token_t;

package/test/arrow.test.ts

@@ -20,13 +20,13 @@ describe('arrow IPC API fails neatly when extension not loaded', function() {
         db.arrowIPCStream(query).then(
             () => Promise.reject(new Error('Expected method to reject.')),
             err => {
-                assert.ok(err.message.includes("Catalog Error: Function with name to_arrow_ipc is not on the catalog, but it exists in the arrow extension. To Install and Load the extension, run: INSTALL arrow; LOAD arrow;"))
+                assert.ok(err.message.includes("arrow"))
             }
         );
 
         db.arrowIPCAll(`SELECT * FROM ipc_table`, function (err: null | Error, result: ArrowArray) {
             if (err) {
-                assert.ok(err.message.includes("Catalog Error: Function with name to_arrow_ipc is not on the catalog, but it exists in the arrow extension. To Install and Load the extension, run: INSTALL arrow; LOAD arrow;"))
+                assert.ok(err.message.includes("arrow"))
             } else {
                 assert.fail("Expected error");
             }
@@ -39,7 +39,7 @@ describe('arrow IPC API fails neatly when extension not loaded', function() {
     it('register buffer should be disabled currently', function(done) {
         db.register_buffer("test", [new Uint8Array(new ArrayBuffer(10))], true, (err: null | Error) => {
             assert.ok(err)
-            assert.ok(err.toString().includes("Function with name scan_arrow_ipc is not on the catalog, but it exists in the arrow extension. To Install and Load the extension, run: INSTALL arrow; LOAD arrow;"));
+            assert.ok(err.toString().includes("arrow"));
             done()
         });
     });