duckdb 0.3.2-dev277.0 → 0.3.2-dev292.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.2-dev277.0",
+  "version": "0.3.2-dev292.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -11086,42 +11086,53 @@ string LogicalOperatorToString(LogicalOperatorType type) {
 
 
 
+
 namespace duckdb {
 
+struct DefaultOptimizerType {
+	const char *name;
+	OptimizerType type;
+};
+
+static DefaultOptimizerType internal_optimizer_types[] = {
+    {"expression_rewriter", OptimizerType::EXPRESSION_REWRITER},
+    {"filter_pullup", OptimizerType::FILTER_PULLUP},
+    {"filter_pushdown", OptimizerType::FILTER_PUSHDOWN},
+    {"regex_range", OptimizerType::REGEX_RANGE},
+    {"in_clause", OptimizerType::IN_CLAUSE},
+    {"join_order", OptimizerType::JOIN_ORDER},
+    {"deliminator", OptimizerType::DELIMINATOR},
+    {"unused_columns", OptimizerType::UNUSED_COLUMNS},
+    {"statistics_propagation", OptimizerType::STATISTICS_PROPAGATION},
+    {"common_subexpressions", OptimizerType::COMMON_SUBEXPRESSIONS},
+    {"common_aggregate", OptimizerType::COMMON_AGGREGATE},
+    {"column_lifetime", OptimizerType::COLUMN_LIFETIME},
+    {"top_n", OptimizerType::TOP_N},
+    {"reorder_filter", OptimizerType::REORDER_FILTER},
+    {nullptr, OptimizerType::INVALID}};
+
 string OptimizerTypeToString(OptimizerType type) {
-	switch (type) {
-	case OptimizerType::EXPRESSION_REWRITER:
-		return "expression_rewriter";
-	case OptimizerType::FILTER_PULLUP:
-		return "filter_pullup";
-	case OptimizerType::FILTER_PUSHDOWN:
-		return "filter_pushdown";
-	case OptimizerType::REGEX_RANGE:
-		return "regex_range";
-	case OptimizerType::IN_CLAUSE:
-		return "in_clause";
-	case OptimizerType::JOIN_ORDER:
-		return "join_order";
-	case OptimizerType::DELIMINATOR:
-		return "deliminator";
-	case OptimizerType::UNUSED_COLUMNS:
-		return "unused_columns";
-	case OptimizerType::STATISTICS_PROPAGATION:
-		return "statistics_propagation";
-	case OptimizerType::COMMON_SUBEXPRESSIONS:
-		return "common_subexpressions";
-	case OptimizerType::COMMON_AGGREGATE:
-		return "common_aggregate";
-	case OptimizerType::COLUMN_LIFETIME:
-		return "column_lifetime";
-	case OptimizerType::TOP_N:
-		return "top_n";
-	case OptimizerType::REORDER_FILTER:
-		return "reorder_filter";
-	case OptimizerType::INVALID: // LCOV_EXCL_START
-		break;
+	for (idx_t i = 0; internal_optimizer_types[i].name; i++) {
+		if (internal_optimizer_types[i].type == type) {
+			return internal_optimizer_types[i].name;
+		}
 	}
-	return "INVALID"; // LCOV_EXCL_STOP
+	throw InternalException("Invalid optimizer type");
+}
+
+OptimizerType OptimizerTypeFromString(const string &str) {
+	for (idx_t i = 0; internal_optimizer_types[i].name; i++) {
+		if (internal_optimizer_types[i].name == str) {
+			return internal_optimizer_types[i].type;
+		}
+	}
+	// optimizer not found, construct candidate list
+	vector<string> optimizer_names;
+	for (idx_t i = 0; internal_optimizer_types[i].name; i++) {
+		optimizer_names.emplace_back(internal_optimizer_types[i].name);
+	}
+	throw ParserException("Optimizer type \"%s\" not recognized\n%s", str,
+	                      StringUtil::CandidatesErrorMessage(optimizer_names, str, "Candidate optimizers"));
 }
 
 } // namespace duckdb
@@ -21229,21 +21240,37 @@ idx_t LocalFileSystem::GetFilePointer(FileHandle &handle) {
 	return ((WindowsFileHandle &)handle).position;
 }
 
-void LocalFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) {
-	HANDLE hFile = ((WindowsFileHandle &)handle).fd;
-	DWORD bytes_read;
+static DWORD FSInternalRead(FileHandle &handle, HANDLE hFile, void *buffer, int64_t nr_bytes, idx_t location) {
+	DWORD bytes_read = 0;
+	bool overlapped_read = false;
 	OVERLAPPED ov = {};
 	ov.Internal = 0;
 	ov.InternalHigh = 0;
 	ov.Offset = location & 0xFFFFFFFF;
 	ov.OffsetHigh = location >> 32;
 	ov.hEvent = 0;
-	ReadFile(hFile, buffer, (DWORD)nr_bytes, NULL, &ov);
-	auto rc = GetOverlappedResult(hFile, &ov, &bytes_read, true);
+	auto rc = ReadFile(hFile, buffer, (DWORD)nr_bytes, &bytes_read, &ov);
 	if (rc == 0) {
-		auto error = GetLastErrorAsString();
-		throw IOException("Could not read file \"%s\": %s", handle.path, error);
+		if (GetLastError() != ERROR_IO_PENDING) {
+			auto error = GetLastErrorAsString();
+			throw IOException("Could not read file \"%s\" (error in ReadFile): %s", handle.path, error);
+		} else {
+			overlapped_read = true;
+		}
+	}
+	if (overlapped_read) {
+		rc = GetOverlappedResult(hFile, &ov, &bytes_read, true);
+		if (rc == 0) {
+			auto error = GetLastErrorAsString();
+			throw IOException("Could not read file \"%s\" (error in GetOverlappedResult): %s", handle.path, error);
+		}
 	}
+	return bytes_read;
+}
+
+void LocalFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) {
+	HANDLE hFile = ((WindowsFileHandle &)handle).fd;
+	auto bytes_read = FSInternalRead(handle, hFile, buffer, nr_bytes, location);
 	if (bytes_read != nr_bytes) {
 		throw IOException("Could not read all bytes from file \"%s\": wanted=%lld read=%lld", handle.path, nr_bytes,
 		                  bytes_read);
@@ -21252,40 +21279,44 @@ void LocalFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, i
 
 int64_t LocalFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes) {
 	HANDLE hFile = ((WindowsFileHandle &)handle).fd;
-	DWORD bytes_read;
 	auto &pos = ((WindowsFileHandle &)handle).position;
-	OVERLAPPED ov = {};
-	ov.Internal = 0;
-	ov.InternalHigh = 0;
-	ov.Offset = pos & 0xFFFFFFFF;
-	ov.OffsetHigh = pos >> 32;
-	ov.hEvent = 0;
 	auto n = std::min<idx_t>(std::max<idx_t>(GetFileSize(handle), pos) - pos, nr_bytes);
-	ReadFile(hFile, buffer, (DWORD)n, NULL, &ov);
-	auto rc = GetOverlappedResult(hFile, &ov, &bytes_read, true);
-	if (rc == 0) {
-		auto error = GetLastErrorAsString();
-		throw IOException("Could not read file \"%s\": %s", handle.path, error);
-	}
+	auto bytes_read = FSInternalRead(handle, hFile, buffer, n, pos);
 	pos += bytes_read;
 	return bytes_read;
 }
 
-void LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) {
-	HANDLE hFile = ((WindowsFileHandle &)handle).fd;
-	DWORD bytes_written;
+static DWORD FSInternalWrite(FileHandle &handle, HANDLE hFile, void *buffer, int64_t nr_bytes, idx_t location) {
+	DWORD bytes_written = 0;
+	bool overlapped_write = false;
 	OVERLAPPED ov = {};
 	ov.Internal = 0;
 	ov.InternalHigh = 0;
 	ov.Offset = location & 0xFFFFFFFF;
 	ov.OffsetHigh = location >> 32;
 	ov.hEvent = 0;
-	WriteFile(hFile, buffer, (DWORD)nr_bytes, NULL, &ov);
-	auto rc = GetOverlappedResult(hFile, &ov, &bytes_written, true);
+	auto rc = WriteFile(hFile, buffer, (DWORD)nr_bytes, &bytes_written, &ov);
 	if (rc == 0) {
-		auto error = GetLastErrorAsString();
-		throw IOException("Could not write file \"%s\": %s", handle.path, error);
+		if (GetLastError() != ERROR_IO_PENDING) {
+			auto error = GetLastErrorAsString();
+			throw IOException("Could not write file \"%s\" (error in WriteFile): %s", handle.path, error);
+		} else {
+			overlapped_write = true;
+		}
+	}
+	if (overlapped_write) {
+		rc = GetOverlappedResult(hFile, &ov, &bytes_written, true);
+		if (rc == 0) {
+			auto error = GetLastErrorAsString();
+			throw IOException("Could not write file \"%s\" (error in GetOverlappedResult): %s", handle.path, error);
+		}
 	}
+	return bytes_written;
+}
+
+void LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) {
+	HANDLE hFile = ((WindowsFileHandle &)handle).fd;
+	auto bytes_written = FSInternalWrite(handle, hFile, buffer, nr_bytes, location);
 	if (bytes_written != nr_bytes) {
 		throw IOException("Could not write all bytes from file \"%s\": wanted=%lld wrote=%lld", handle.path, nr_bytes,
 		                  bytes_written);
@@ -21294,20 +21325,8 @@ void LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes,
 
 int64_t LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes) {
 	HANDLE hFile = ((WindowsFileHandle &)handle).fd;
-	DWORD bytes_written;
 	auto &pos = ((WindowsFileHandle &)handle).position;
-	OVERLAPPED ov = {};
-	ov.Internal = 0;
-	ov.InternalHigh = 0;
-	ov.Offset = pos & 0xFFFFFFFF;
-	ov.OffsetHigh = pos >> 32;
-	ov.hEvent = 0;
-	WriteFile(hFile, buffer, (DWORD)nr_bytes, NULL, &ov);
-	auto rc = GetOverlappedResult(hFile, &ov, &bytes_written, true);
-	if (rc == 0) {
-		auto error = GetLastErrorAsString();
-		throw IOException("Could not write file \"%s\": %s", handle.path, error);
-	}
+	auto bytes_written = FSInternalWrite(handle, hFile, buffer, nr_bytes, pos);
 	pos += bytes_written;
 	return bytes_written;
 }
@@ -32735,6 +32754,12 @@ string StringUtil::CandidatesMessage(const vector<string> &candidates, const str
 	return result_str;
 }
 
+string StringUtil::CandidatesErrorMessage(const vector<string> &strings, const string &target,
+                                          const string &message_prefix, idx_t n) {
+	auto closest_strings = StringUtil::TopNLevenshtein(strings, target, n);
+	return StringUtil::CandidatesMessage(closest_strings, message_prefix);
+}
+
 } // namespace duckdb
 //===----------------------------------------------------------------------===//
 //                         DuckDB
@@ -57789,9 +57814,9 @@ void PhysicalSet::GetData(ExecutionContext &context, DataChunk &chunk, GlobalSou
 			for (auto &entry : config.extension_parameters) {
 				potential_names.push_back(entry.first);
 			}
-			auto closest_settings = StringUtil::TopNLevenshtein(potential_names, name);
+
 			throw CatalogException("unrecognized configuration parameter \"%s\"\n%s", name,
-			                       StringUtil::CandidatesMessage(closest_settings, "Did you mean"));
+			                       StringUtil::CandidatesErrorMessage(potential_names, name, "Did you mean"));
 		}
 		//! it is!
 		auto &extension_option = entry->second;
@@ -103314,6 +103339,14 @@ struct DefaultNullOrderSetting {
 	static Value GetSetting(ClientContext &context);
 };
 
+struct DisabledOptimizersSetting {
+	static constexpr const char *Name = "disabled_optimizers";
+	static constexpr const char *Description = "DEBUG SETTING: disable a specific set of optimizers (comma separated)";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::VARCHAR;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static Value GetSetting(ClientContext &context);
+};
+
 struct EnableExternalAccessSetting {
 	static constexpr const char *Name = "enable_external_access";
 	static constexpr const char *Description =
@@ -103490,6 +103523,7 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_GLOBAL_LOCAL(DefaultCollationSetting),
                                                  DUCKDB_GLOBAL(DefaultOrderSetting),
                                                  DUCKDB_GLOBAL(DefaultNullOrderSetting),
+                                                 DUCKDB_GLOBAL(DisabledOptimizersSetting),
                                                  DUCKDB_GLOBAL(EnableExternalAccessSetting),
                                                  DUCKDB_GLOBAL(EnableObjectCacheSetting),
                                                  DUCKDB_LOCAL(EnableProfilingSetting),
@@ -115947,6 +115981,35 @@ Value DefaultNullOrderSetting::GetSetting(ClientContext &context) {
 	}
 }
 
+//===--------------------------------------------------------------------===//
+// Disabled Optimizer
+//===--------------------------------------------------------------------===//
+void DisabledOptimizersSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
+	auto list = StringUtil::Split(input.ToString(), ",");
+	set<OptimizerType> disabled_optimizers;
+	for (auto &entry : list) {
+		auto param = StringUtil::Lower(entry);
+		StringUtil::Trim(param);
+		if (param.empty()) {
+			continue;
+		}
+		disabled_optimizers.insert(OptimizerTypeFromString(param));
+	}
+	config.disabled_optimizers = move(disabled_optimizers);
+}
+
+Value DisabledOptimizersSetting::GetSetting(ClientContext &context) {
+	auto &config = DBConfig::GetConfig(context);
+	string result;
+	for (auto &optimizer : config.disabled_optimizers) {
+		if (!result.empty()) {
+			result += ",";
+		}
+		result += OptimizerTypeToString(optimizer);
+	}
+	return Value(result);
+}
+
 //===--------------------------------------------------------------------===//
 // Enable External Access
 //===--------------------------------------------------------------------===//
@@ -116715,6 +116778,9 @@ void CommonSubExpressionOptimizer::CountExpressions(Expression &expr, CSEReplace
 	case ExpressionClass::BOUND_COLUMN_REF:
 	case ExpressionClass::BOUND_CONSTANT:
 	case ExpressionClass::BOUND_PARAMETER:
+	// skip conjunctions and case, since short-circuiting might be incorrectly disabled otherwise
+	case ExpressionClass::BOUND_CONJUNCTION:
+	case ExpressionClass::BOUND_CASE:
 		return;
 	default:
 		break;
@@ -116754,7 +116820,9 @@ void CommonSubExpressionOptimizer::PerformCSEReplacement(unique_ptr<Expression>
 		return;
 	}
 	// check if this child is eligible for CSE elimination
-	if (state.expression_count.find(&expr) != state.expression_count.end()) {
+	bool can_cse = expr.expression_class != ExpressionClass::BOUND_CONJUNCTION &&
+	               expr.expression_class != ExpressionClass::BOUND_CASE;
+	if (can_cse && state.expression_count.find(&expr) != state.expression_count.end()) {
 		auto &node = state.expression_count[&expr];
 		if (node.count > 1) {
 			// this expression occurs more than once! push it into the projection
@@ -123045,7 +123113,7 @@ unique_ptr<Expression> InClauseSimplificationRule::Apply(LogicalOperator &op, ve
 	}
 	//! Here we check if we can apply the expression on the constant side
 	auto target_type = cast_expression->source_type();
-	if (!BoundCastExpression::CastIsInvertible(target_type, cast_expression->return_type)) {
+	if (!BoundCastExpression::CastIsInvertible(cast_expression->return_type, target_type)) {
 		return nullptr;
 	}
 	vector<unique_ptr<BoundConstantExpression>> cast_list;

package/src/duckdb.hpp

@@ -10,8 +10,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 #pragma once
 #define DUCKDB_AMALGAMATION 1
-#define DUCKDB_SOURCE_ID "b10ef8b11"
-#define DUCKDB_VERSION "0.3.2-dev277"
+#define DUCKDB_SOURCE_ID "43a2bb05d"
+#define DUCKDB_VERSION "0.3.2-dev292"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -9863,7 +9863,13 @@ public:
 	                                                 idx_t threshold = 5);
 	DUCKDB_API static string CandidatesMessage(const vector<string> &candidates,
 	                                           const string &candidate = "Candidate bindings");
+
+	//! Generate an error message in the form of "{message_prefix}: nearest_string, nearest_string2, ...
+	//! Equivalent to calling TopNLevenshtein followed by CandidatesMessage
+	DUCKDB_API static string CandidatesErrorMessage(const vector<string> &strings, const string &target,
+	                                                const string &message_prefix, idx_t n = 5);
 };
+
 } // namespace duckdb
 
 
@@ -9976,6 +9982,7 @@ enum class OptimizerType : uint32_t {
 };
 
 string OptimizerTypeToString(OptimizerType type);
+OptimizerType OptimizerTypeFromString(const string &str);
 
 } // namespace duckdb