duckdb 0.5.2-dev1275.0 → 0.5.2-dev1292.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.5.2-dev1275.0",
+  "version": "0.5.2-dev1292.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -623,7 +623,7 @@ public:
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// extension_functions.hpp
+// duckdb/main/extension_functions.hpp
 //
 //
 //===----------------------------------------------------------------------===//
@@ -20261,7 +20261,7 @@ public:
 	//! Read exactly nr_bytes from the specified location in the file. Fails if nr_bytes could not be read. This is
 	//! equivalent to calling SetFilePointer(location) followed by calling Read().
 	void Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override;
-	//! Write exactly nr_bytes to the specified location in the file. Fails if nr_bytes could not be read. This is
+	//! Write exactly nr_bytes to the specified location in the file. Fails if nr_bytes could not be written. This is
 	//! equivalent to calling SetFilePointer(location) followed by calling Write().
 	void Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override;
 	//! Read nr_bytes from the specified file into the buffer, moving the file pointer forward by nr_bytes. Returns the
@@ -47492,6 +47492,66 @@ private:
 
 } // namespace duckdb
 
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/main/error_manager.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+
+
+namespace duckdb {
+class ClientContext;
+class DatabaseInstance;
+
+enum class ErrorType : uint16_t {
+	// error message types
+	UNSIGNED_EXTENSION = 0,
+
+	// this should always be the last value
+	ERROR_COUNT,
+	INVALID = 65535,
+};
+
+//! The error manager class is responsible for formatting error messages
+//! It allows for error messages to be overridden by extensions and clients
+class ErrorManager {
+public:
+	template <typename... Args>
+	string FormatException(ErrorType error_type, Args... params) {
+		vector<ExceptionFormatValue> values;
+		return FormatExceptionRecursive(error_type, values, params...);
+	}
+
+	DUCKDB_API string FormatExceptionRecursive(ErrorType error_type, vector<ExceptionFormatValue> &values);
+
+	template <class T, typename... Args>
+	string FormatExceptionRecursive(ErrorType error_type, vector<ExceptionFormatValue> &values, T param,
+	                                Args... params) {
+		values.push_back(ExceptionFormatValue::CreateFormatValue<T>(param));
+		return FormatExceptionRecursive(error_type, values, params...);
+	}
+
+	DUCKDB_API static string InvalidUnicodeError(const string &input, const string &context);
+
+	//! Adds a custom error for a specific error type
+	void AddCustomError(ErrorType type, string new_error);
+
+	DUCKDB_API ErrorManager &Get(ClientContext &context);
+	DUCKDB_API ErrorManager &Get(DatabaseInstance &context);
+
+private:
+	map<ErrorType, string> custom_errors;
+};
+
+} // namespace duckdb
+
 
 #include <utility>
 #include <cmath>
@@ -47528,7 +47588,7 @@ Value::Value(string_t val) : Value(string(val.GetDataUnsafe(), val.GetSize())) {
 
 Value::Value(string val) : type_(LogicalType::VARCHAR), is_null(false), str_value(move(val)) {
 	if (!Value::StringIsValid(str_value.c_str(), str_value.size())) {
-		throw Exception("String value is not valid UTF8");
+		throw Exception(ErrorManager::InvalidUnicodeError(str_value, "value construction"));
 	}
 }
 
@@ -76473,6 +76533,7 @@ string PhysicalTopN::ParamsToString() const {
 
 
 
+
 #include <algorithm>
 #include <cctype>
 #include <cstring>
@@ -77506,8 +77567,10 @@ void BufferedCSVReader::DetectCandidateTypes(const vector<LogicalType> &type_can
 					// try cast from string to sql_type
 					Value dummy_val;
 					if (is_header_row) {
+						VerifyUTF8(col, 0, header_row, -int64_t(parse_chunk.size()));
 						dummy_val = header_row.GetValue(col, 0);
 					} else {
+						VerifyUTF8(col, row, parse_chunk);
 						dummy_val = parse_chunk.GetValue(col, row);
 					}
 					// try formatting for date types if the user did not specify one and it starts with numeric values.
@@ -78361,9 +78424,9 @@ void BufferedCSVReader::AddValue(char *str_val, idx_t length, idx_t &column, vec
 			error_column_overflow = true;
 			return;
 		} else {
-			throw InvalidInputException("Error on line %s: expected %lld values per row, but got more. (%s)",
-			                            GetLineNumberStr(linenr, linenr_estimated).c_str(), sql_types.size(),
-			                            options.ToString());
+			throw InvalidInputException(
+			    "Error in file \"%s\", on line %s: expected %lld values per row, but got more. (%s)", options.file_path,
+			    GetLineNumberStr(linenr, linenr_estimated).c_str(), sql_types.size(), options.ToString());
 		}
 	}
 
@@ -78433,9 +78496,9 @@ bool BufferedCSVReader::AddRow(DataChunk &insert_chunk, idx_t &column) {
 			column = 0;
 			return false;
 		} else {
-			throw InvalidInputException("Error on line %s: expected %lld values per row, but got %d. (%s)",
-			                            GetLineNumberStr(linenr, linenr_estimated).c_str(), sql_types.size(), column,
-			                            options.ToString());
+			throw InvalidInputException(
+			    "Error in file \"%s\" on line %s: expected %lld values per row, but got %d. (%s)", options.file_path,
+			    GetLineNumberStr(linenr, linenr_estimated).c_str(), sql_types.size(), column, options.ToString());
 		}
 	}
 
@@ -78473,6 +78536,38 @@ void BufferedCSVReader::SetNullUnionCols(DataChunk &insert_chunk) {
 	}
 }
 
+void BufferedCSVReader::VerifyUTF8(idx_t col_idx, idx_t row_idx, DataChunk &chunk, int64_t offset) {
+	D_ASSERT(col_idx < chunk.data.size());
+	D_ASSERT(row_idx < chunk.size());
+	auto &v = chunk.data[col_idx];
+	if (FlatVector::IsNull(v, row_idx)) {
+		return;
+	}
+
+	auto parse_data = FlatVector::GetData<string_t>(chunk.data[col_idx]);
+	auto s = parse_data[row_idx];
+	auto utf_type = Utf8Proc::Analyze(s.GetDataUnsafe(), s.GetSize());
+	if (utf_type == UnicodeType::INVALID) {
+		string col_name = to_string(col_idx);
+		if (col_idx < col_names.size()) {
+			col_name = "\"" + col_names[col_idx] + "\"";
+		}
+		int64_t error_line = linenr - (chunk.size() - row_idx) + 1 + offset;
+		D_ASSERT(error_line >= 0);
+		throw InvalidInputException("Error in file \"%s\" at line %llu in column \"%s\": "
+		                            "%s. Parser options: %s",
+		                            options.file_path, error_line, col_name,
+		                            ErrorManager::InvalidUnicodeError(s.GetString(), "CSV file"), options.ToString());
+	}
+}
+
+void BufferedCSVReader::VerifyUTF8(idx_t col_idx) {
+	D_ASSERT(col_idx < parse_chunk.data.size());
+	for (idx_t i = 0; i < parse_chunk.size(); i++) {
+		VerifyUTF8(col_idx, i, parse_chunk);
+	}
+}
+
 void BufferedCSVReader::Flush(DataChunk &insert_chunk) {
 	if (parse_chunk.size() == 0) {
 		return;
@@ -78486,23 +78581,7 @@ void BufferedCSVReader::Flush(DataChunk &insert_chunk) {
 		if (sql_types[col_idx].id() == LogicalTypeId::VARCHAR) {
 			// target type is varchar: no need to convert
 			// just test that all strings are valid utf-8 strings
-			auto parse_data = FlatVector::GetData<string_t>(parse_chunk.data[col_idx]);
-			for (idx_t i = 0; i < parse_chunk.size(); i++) {
-				if (!FlatVector::IsNull(parse_chunk.data[col_idx], i)) {
-					auto s = parse_data[i];
-					auto utf_type = Utf8Proc::Analyze(s.GetDataUnsafe(), s.GetSize());
-					if (utf_type == UnicodeType::INVALID) {
-						string col_name = to_string(col_idx);
-						if (col_idx < col_names.size()) {
-							col_name = "\"" + col_names[col_idx] + "\"";
-						}
-						throw InvalidInputException("Error in file \"%s\" between line %llu and %llu in column \"%s\": "
-						                            "file is not valid UTF8. Parser options: %s",
-						                            options.file_path, linenr - parse_chunk.size(), linenr, col_name,
-						                            options.ToString());
-					}
-				}
-			}
+			VerifyUTF8(col_idx);
 			insert_chunk.data[insert_cols_idx[col_idx]].Reference(parse_chunk.data[col_idx]);
 		} else {
 			string error_message;
@@ -78536,17 +78615,27 @@ void BufferedCSVReader::Flush(DataChunk &insert_chunk) {
 				col_name = "\"" + col_names[col_idx] + "\"";
 			}
 
+			// figure out the exact line number
+			idx_t row_idx;
+			for (row_idx = 0; row_idx < parse_chunk.size(); row_idx++) {
+				auto &inserted_column = insert_chunk.data[col_idx];
+				auto &parsed_column = parse_chunk.data[col_idx];
+
+				if (FlatVector::IsNull(inserted_column, row_idx) && !FlatVector::IsNull(parsed_column, row_idx)) {
+					break;
+				}
+			}
+			auto error_line = linenr - (parse_chunk.size() - row_idx) + 1;
+
 			if (options.auto_detect) {
-				throw InvalidInputException("%s in column %s, between line %llu and %llu. Parser "
+				throw InvalidInputException("%s in column %s, at line %llu. Parser "
 				                            "options: %s. Consider either increasing the sample size "
 				                            "(SAMPLE_SIZE=X [X rows] or SAMPLE_SIZE=-1 [all rows]), "
 				                            "or skipping column conversion (ALL_VARCHAR=1)",
-				                            error_message, col_name, linenr - parse_chunk.size() + 1, linenr,
-				                            options.ToString());
+				                            error_message, col_name, error_line, options.ToString());
 			} else {
-				throw InvalidInputException("%s between line %llu and %llu in column %s. Parser options: %s ",
-				                            error_message, linenr - parse_chunk.size(), linenr, col_name,
-				                            options.ToString());
+				throw InvalidInputException("%s at line %llu in column %s. Parser options: %s ", error_message,
+				                            error_line, col_name, options.ToString());
 			}
 		}
 	}
@@ -132105,14 +132194,6 @@ struct DebugForceNoCrossProduct {
 	static Value GetSetting(ClientContext &context);
 };
 
-struct DebugManyFreeListBlocks {
-	static constexpr const char *Name = "debug_many_free_list_blocks";
-	static constexpr const char *Description = "DEBUG SETTING: add additional blocks to the free list";
-	static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN;
-	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
-	static Value GetSetting(ClientContext &context);
-};
-
 struct DebugWindowMode {
 	static constexpr const char *Name = "debug_window_mode";
 	static constexpr const char *Description = "DEBUG SETTING: switch window mode to use";
@@ -132396,7 +132477,6 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_GLOBAL(DebugCheckpointAbort),
                                                  DUCKDB_LOCAL(DebugForceExternal),
                                                  DUCKDB_LOCAL(DebugForceNoCrossProduct),
-                                                 DUCKDB_GLOBAL(DebugManyFreeListBlocks),
                                                  DUCKDB_GLOBAL(DebugWindowMode),
                                                  DUCKDB_GLOBAL_LOCAL(DefaultCollationSetting),
                                                  DUCKDB_GLOBAL(DefaultOrderSetting),
@@ -133130,6 +133210,7 @@ public:
 
 
 
+
 #ifndef DUCKDB_NO_THREADS
 
 #endif
@@ -133140,6 +133221,7 @@ DBConfig::DBConfig() {
 	compression_functions = make_unique<CompressionFunctionSet>();
 	replacement_opens.push_back(ExtensionPrefixReplacementOpen());
 	cast_functions = make_unique<CastFunctionSet>();
+	error_manager = make_unique<ErrorManager>();
 }
 
 DBConfig::DBConfig(std::unordered_map<string, string> &config_dict, bool read_only) {
@@ -133357,17 +133439,15 @@ Allocator &Allocator::Get(DatabaseInstance &db) {
 }
 
 void DatabaseInstance::Configure(DBConfig &new_config) {
-	config.options.database_path = new_config.options.database_path;
-	config.options.access_mode = AccessMode::READ_WRITE;
-	if (new_config.options.access_mode != AccessMode::UNDEFINED) {
-		config.options.access_mode = new_config.options.access_mode;
+	config.options = new_config.options;
+	if (config.options.access_mode == AccessMode::UNDEFINED) {
+		config.options.access_mode = AccessMode::READ_WRITE;
 	}
 	if (new_config.file_system) {
 		config.file_system = move(new_config.file_system);
 	} else {
 		config.file_system = make_unique<VirtualFileSystem>();
 	}
-	config.options.maximum_memory = new_config.options.maximum_memory;
 	if (config.options.maximum_memory == (idx_t)-1) {
 		auto memory = FileSystem::GetAvailableMemory();
 		if (memory != DConstants::INVALID_INDEX) {
@@ -133380,29 +133460,15 @@ void DatabaseInstance::Configure(DBConfig &new_config) {
 #else
 		config.options.maximum_threads = 1;
 #endif
-	} else {
-		config.options.maximum_threads = new_config.options.maximum_threads;
 	}
-	config.options.external_threads = new_config.options.external_threads;
-	config.options.load_extensions = new_config.options.load_extensions;
-	config.options.force_compression = new_config.options.force_compression;
 	config.allocator = move(new_config.allocator);
 	if (!config.allocator) {
 		config.allocator = make_unique<Allocator>();
 	}
-	config.options.checkpoint_wal_size = new_config.options.checkpoint_wal_size;
-	config.options.use_direct_io = new_config.options.use_direct_io;
-	config.options.temporary_directory = new_config.options.temporary_directory;
-	config.options.collation = new_config.options.collation;
-	config.options.default_order_type = new_config.options.default_order_type;
-	config.options.default_null_order = new_config.options.default_null_order;
-	config.options.enable_external_access = new_config.options.enable_external_access;
-	config.options.allow_unsigned_extensions = new_config.options.allow_unsigned_extensions;
 	config.replacement_scans = move(new_config.replacement_scans);
-	config.replacement_opens = move(new_config.replacement_opens); // TODO is this okay?
-	config.options.initialize_default_database = new_config.options.initialize_default_database;
-	config.options.disabled_optimizers = move(new_config.options.disabled_optimizers);
+	config.replacement_opens = move(new_config.replacement_opens);
 	config.parser_extensions = move(new_config.parser_extensions);
+	config.error_manager = move(new_config.error_manager);
 }
 
 DBConfig &DBConfig::GetConfig(ClientContext &context) {
@@ -133558,6 +133624,76 @@ shared_ptr<DuckDB> DBInstanceCache::CreateInstance(const string &database, DBCon
 
 
 
+namespace duckdb {
+
+struct DefaultError {
+	ErrorType type;
+	const char *error;
+};
+
+static DefaultError internal_errors[] = {
+    {ErrorType::UNSIGNED_EXTENSION,
+     "Extension \"%s\" could not be loaded because its signature is either missing or invalid and unsigned extensions "
+     "are disabled by configuration (allow_unsigned_extensions)"},
+    {ErrorType::INVALID, nullptr}};
+
+string ErrorManager::FormatExceptionRecursive(ErrorType error_type, vector<ExceptionFormatValue> &values) {
+	if (error_type >= ErrorType::ERROR_COUNT) {
+		throw InternalException("Invalid error type passed to ErrorManager::FormatError");
+	}
+	auto entry = custom_errors.find(error_type);
+	string error;
+	if (entry == custom_errors.end()) {
+		// error was not overwritten
+		error = internal_errors[int(error_type)].error;
+	} else {
+		// error was overwritten
+		error = entry->second;
+	}
+	return ExceptionFormatValue::Format(error, values);
+}
+
+string ErrorManager::InvalidUnicodeError(const string &input, const string &context) {
+	UnicodeInvalidReason reason;
+	size_t pos;
+	auto unicode = Utf8Proc::Analyze((const char *)input.c_str(), input.size(), &reason, &pos);
+	if (unicode != UnicodeType::INVALID) {
+		return "Invalid unicode error thrown but no invalid unicode detected in " + context;
+	}
+	string base_message;
+	switch (reason) {
+	case UnicodeInvalidReason::NULL_BYTE:
+		base_message = "Null-byte (\\0)";
+		break;
+	case UnicodeInvalidReason::BYTE_MISMATCH:
+		base_message = "Invalid unicode (byte sequence mismatch)";
+		break;
+	case UnicodeInvalidReason::INVALID_UNICODE:
+		base_message = "Invalid unicode";
+		break;
+	default:
+		break;
+	}
+	return base_message + " detected in " + context;
+}
+
+void ErrorManager::AddCustomError(ErrorType type, string new_error) {
+	custom_errors.insert(make_pair(type, move(new_error)));
+}
+
+ErrorManager &ErrorManager::Get(ClientContext &context) {
+	return *DBConfig::GetConfig(context).error_manager;
+}
+
+ErrorManager &ErrorManager::Get(DatabaseInstance &context) {
+	return *DBConfig::GetConfig(context).error_manager;
+}
+
+} // namespace duckdb
+
+
+
+
 
 
 
@@ -142416,6 +142552,7 @@ inline std::string GetDLError(void) {
 
 
 
+
 // LICENSE_CHANGE_BEGIN
 // The following code up to LICENSE_CHANGE_END is subject to THIRD PARTY LICENSE #12
 // See the end of this file for a list
@@ -142513,10 +142650,7 @@ ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileOpener *o
 			}
 		}
 		if (!any_valid && !config.options.allow_unsigned_extensions) {
-			throw IOException(
-			    "Extension \"%s\" could not be loaded because its signature is either missing or "
-			    "invalid and unsigned extensions are disabled by configuration (allow_unsigned_extensions)",
-			    filename);
+			throw IOException(config.error_manager->FormatException(ErrorType::UNSIGNED_EXTENSION, filename));
 		}
 	}
 	auto lib_hdl = dlopen(filename.c_str(), RTLD_NOW | RTLD_LOCAL);
@@ -146358,18 +146492,6 @@ Value DebugForceNoCrossProduct::GetSetting(ClientContext &context) {
 	return Value::BOOLEAN(ClientConfig::GetConfig(context).force_no_cross_product);
 }
 
-//===--------------------------------------------------------------------===//
-// Debug Many Free List blocks
-//===--------------------------------------------------------------------===//
-void DebugManyFreeListBlocks::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
-	config.options.debug_many_free_list_blocks = input.GetValue<bool>();
-}
-
-Value DebugManyFreeListBlocks::GetSetting(ClientContext &context) {
-	auto &config = DBConfig::GetConfig(context);
-	return Value::BOOLEAN(config.options.debug_many_free_list_blocks);
-}
-
 //===--------------------------------------------------------------------===//
 // Debug Window Mode
 //===--------------------------------------------------------------------===//
@@ -207987,10 +208109,6 @@ vector<block_id_t> SingleFileBlockManager::GetFreeListBlocks() {
 		// a bit from the max block size
 		auto space_in_block = Storage::BLOCK_SIZE - 4 * sizeof(block_id_t);
 		auto total_blocks = (total_size + space_in_block - 1) / space_in_block;
-		auto &config = DBConfig::GetConfig(db);
-		if (config.options.debug_many_free_list_blocks) {
-			total_blocks++;
-		}
 		D_ASSERT(total_size > 0);
 		D_ASSERT(total_blocks > 0);
 
@@ -208749,6 +208867,7 @@ void SegmentStatistics::Reset() {
 
 
 
+
 namespace duckdb {
 
 StringStatistics::StringStatistics(LogicalType type_p, StatisticsType stats_type)
@@ -208837,7 +208956,8 @@ void StringStatistics::Update(const string_t &value) {
 		if (unicode == UnicodeType::UNICODE) {
 			has_unicode = true;
 		} else if (unicode == UnicodeType::INVALID) {
-			throw InternalException("Invalid unicode detected in segment statistics update!");
+			throw InternalException(
+			    ErrorManager::InvalidUnicodeError(string((char *)data, size), "segment statistics update"));
 		}
 	}
 }

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "469752987"
-#define DUCKDB_VERSION "v0.5.2-dev1275"
+#define DUCKDB_SOURCE_ID "d4790e5b65"
+#define DUCKDB_VERSION "v0.5.2-dev1292"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -2149,7 +2149,7 @@ public:
 	//! Read exactly nr_bytes from the specified location in the file. Fails if nr_bytes could not be read. This is
 	//! equivalent to calling SetFilePointer(location) followed by calling Read().
 	DUCKDB_API virtual void Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location);
-	//! Write exactly nr_bytes to the specified location in the file. Fails if nr_bytes could not be read. This is
+	//! Write exactly nr_bytes to the specified location in the file. Fails if nr_bytes could not be written. This is
 	//! equivalent to calling SetFilePointer(location) followed by calling Write().
 	DUCKDB_API virtual void Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location);
 	//! Read nr_bytes from the specified file into the buffer, moving the file pointer forward by nr_bytes. Returns the
@@ -18644,6 +18644,7 @@ public:
 namespace duckdb {
 class CastFunctionSet;
 class ClientContext;
+class ErrorManager;
 class CompressionFunction;
 class TableFunctionRef;
 
@@ -18728,8 +18729,6 @@ struct DBConfigOptions {
 	set<OptimizerType> disabled_optimizers;
 	//! Force a specific compression method to be used when checkpointing (if available)
 	CompressionType force_compression = CompressionType::COMPRESSION_AUTO;
-	//! Debug flag that adds additional (unnecessary) free_list blocks to the storage
-	bool debug_many_free_list_blocks = false;
 	//! Debug setting for window aggregation mode: (window, combine, separate)
 	WindowAggregationMode window_mode = WindowAggregationMode::WINDOW;
 	//! Whether or not preserving insertion order should be preserved
@@ -18772,6 +18771,8 @@ public:
 	vector<ParserExtension> parser_extensions;
 	//! Extensions made to the optimizer
 	vector<OptimizerExtension> optimizer_extensions;
+	//! Error manager
+	unique_ptr<ErrorManager> error_manager;
 
 	DUCKDB_API void AddExtensionOption(string name, string description, LogicalType parameter,
 	                                   set_option_callback_t function = nullptr);
@@ -27846,6 +27847,9 @@ private:
 	                                        vector<vector<LogicalType>> &best_sql_types_candidates,
 	                                        map<LogicalTypeId, vector<string>> &best_format_candidates);
 
+	void VerifyUTF8(idx_t col_idx);
+	void VerifyUTF8(idx_t col_idx, idx_t row_idx, DataChunk &chunk, int64_t offset = 0);
+
 private:
 	//! Whether or not the current row's columns have overflown sql_types.size()
 	bool error_column_overflow = false;