duckdb 0.7.2-dev12.0 → 0.7.2-dev1244.0

package/src/duckdb/src/common/enums/expression_type.cpp

@@ -1,150 +1,17 @@
 #include "duckdb/common/enums/expression_type.hpp"
 
 #include "duckdb/common/exception.hpp"
+#include "duckdb/common/serializer/enum_serializer.hpp"
 
 namespace duckdb {
 
-// LCOV_EXCL_START
 string ExpressionTypeToString(ExpressionType type) {
-	switch (type) {
-	case ExpressionType::OPERATOR_CAST:
-		return "CAST";
-	case ExpressionType::OPERATOR_NOT:
-		return "NOT";
-	case ExpressionType::OPERATOR_IS_NULL:
-		return "IS_NULL";
-	case ExpressionType::OPERATOR_IS_NOT_NULL:
-		return "IS_NOT_NULL";
-	case ExpressionType::COMPARE_EQUAL:
-		return "EQUAL";
-	case ExpressionType::COMPARE_NOTEQUAL:
-		return "NOTEQUAL";
-	case ExpressionType::COMPARE_LESSTHAN:
-		return "LESSTHAN";
-	case ExpressionType::COMPARE_GREATERTHAN:
-		return "GREATERTHAN";
-	case ExpressionType::COMPARE_LESSTHANOREQUALTO:
-		return "LESSTHANOREQUALTO";
-	case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
-		return "GREATERTHANOREQUALTO";
-	case ExpressionType::COMPARE_IN:
-		return "IN";
-	case ExpressionType::COMPARE_DISTINCT_FROM:
-		return "DISTINCT_FROM";
-	case ExpressionType::COMPARE_NOT_DISTINCT_FROM:
-		return "NOT_DISTINCT_FROM";
-	case ExpressionType::CONJUNCTION_AND:
-		return "AND";
-	case ExpressionType::CONJUNCTION_OR:
-		return "OR";
-	case ExpressionType::VALUE_CONSTANT:
-		return "CONSTANT";
-	case ExpressionType::VALUE_PARAMETER:
-		return "PARAMETER";
-	case ExpressionType::VALUE_TUPLE:
-		return "TUPLE";
-	case ExpressionType::VALUE_TUPLE_ADDRESS:
-		return "TUPLE_ADDRESS";
-	case ExpressionType::VALUE_NULL:
-		return "NULL";
-	case ExpressionType::VALUE_VECTOR:
-		return "VECTOR";
-	case ExpressionType::VALUE_SCALAR:
-		return "SCALAR";
-	case ExpressionType::AGGREGATE:
-		return "AGGREGATE";
-	case ExpressionType::WINDOW_AGGREGATE:
-		return "WINDOW_AGGREGATE";
-	case ExpressionType::WINDOW_RANK:
-		return "RANK";
-	case ExpressionType::WINDOW_RANK_DENSE:
-		return "RANK_DENSE";
-	case ExpressionType::WINDOW_PERCENT_RANK:
-		return "PERCENT_RANK";
-	case ExpressionType::WINDOW_ROW_NUMBER:
-		return "ROW_NUMBER";
-	case ExpressionType::WINDOW_FIRST_VALUE:
-		return "FIRST_VALUE";
-	case ExpressionType::WINDOW_LAST_VALUE:
-		return "LAST_VALUE";
-	case ExpressionType::WINDOW_NTH_VALUE:
-		return "NTH_VALUE";
-	case ExpressionType::WINDOW_CUME_DIST:
-		return "CUME_DIST";
-	case ExpressionType::WINDOW_LEAD:
-		return "LEAD";
-	case ExpressionType::WINDOW_LAG:
-		return "LAG";
-	case ExpressionType::WINDOW_NTILE:
-		return "NTILE";
-	case ExpressionType::FUNCTION:
-		return "FUNCTION";
-	case ExpressionType::CASE_EXPR:
-		return "CASE";
-	case ExpressionType::OPERATOR_NULLIF:
-		return "NULLIF";
-	case ExpressionType::OPERATOR_COALESCE:
-		return "COALESCE";
-	case ExpressionType::ARRAY_EXTRACT:
-		return "ARRAY_EXTRACT";
-	case ExpressionType::ARRAY_SLICE:
-		return "ARRAY_SLICE";
-	case ExpressionType::STRUCT_EXTRACT:
-		return "STRUCT_EXTRACT";
-	case ExpressionType::SUBQUERY:
-		return "SUBQUERY";
-	case ExpressionType::STAR:
-		return "STAR";
-	case ExpressionType::PLACEHOLDER:
-		return "PLACEHOLDER";
-	case ExpressionType::COLUMN_REF:
-		return "COLUMN_REF";
-	case ExpressionType::FUNCTION_REF:
-		return "FUNCTION_REF";
-	case ExpressionType::TABLE_REF:
-		return "TABLE_REF";
-	case ExpressionType::CAST:
-		return "CAST";
-	case ExpressionType::COMPARE_NOT_IN:
-		return "COMPARE_NOT_IN";
-	case ExpressionType::COMPARE_BETWEEN:
-		return "COMPARE_BETWEEN";
-	case ExpressionType::COMPARE_NOT_BETWEEN:
-		return "COMPARE_NOT_BETWEEN";
-	case ExpressionType::VALUE_DEFAULT:
-		return "VALUE_DEFAULT";
-	case ExpressionType::BOUND_REF:
-		return "BOUND_REF";
-	case ExpressionType::BOUND_COLUMN_REF:
-		return "BOUND_COLUMN_REF";
-	case ExpressionType::BOUND_FUNCTION:
-		return "BOUND_FUNCTION";
-	case ExpressionType::BOUND_AGGREGATE:
-		return "BOUND_AGGREGATE";
-	case ExpressionType::GROUPING_FUNCTION:
-		return "GROUPING";
-	case ExpressionType::ARRAY_CONSTRUCTOR:
-		return "ARRAY_CONSTRUCTOR";
-	case ExpressionType::TABLE_STAR:
-		return "TABLE_STAR";
-	case ExpressionType::BOUND_UNNEST:
-		return "BOUND_UNNEST";
-	case ExpressionType::COLLATE:
-		return "COLLATE";
-	case ExpressionType::POSITIONAL_REFERENCE:
-		return "POSITIONAL_REFERENCE";
-	case ExpressionType::BOUND_LAMBDA_REF:
-		return "BOUND_LAMBDA_REF";
-	case ExpressionType::LAMBDA:
-		return "LAMBDA";
-	case ExpressionType::ARROW:
-		return "ARROW";
-	case ExpressionType::INVALID:
-		break;
-	}
-	return "INVALID";
+	return EnumSerializer::EnumToString(type);
+}
+
+string ExpressionClassToString(ExpressionClass type) {
+	return EnumSerializer::EnumToString(type);
 }
-// LCOV_EXCL_STOP
 
 string ExpressionTypeToOperator(ExpressionType type) {
 	switch (type) {
@@ -173,7 +40,7 @@ string ExpressionTypeToOperator(ExpressionType type) {
 	}
 }
 
-ExpressionType NegateComparisionExpression(ExpressionType type) {
+ExpressionType NegateComparisonExpression(ExpressionType type) {
 	ExpressionType negated_type = ExpressionType::INVALID;
 	switch (type) {
 	case ExpressionType::COMPARE_EQUAL:
@@ -200,7 +67,7 @@ ExpressionType NegateComparisionExpression(ExpressionType type) {
 	return negated_type;
 }
 
-ExpressionType FlipComparisionExpression(ExpressionType type) {
+ExpressionType FlipComparisonExpression(ExpressionType type) {
 	ExpressionType flipped_type = ExpressionType::INVALID;
 	switch (type) {
 	case ExpressionType::COMPARE_NOT_DISTINCT_FROM:
@@ -244,85 +111,4 @@ ExpressionType OperatorToExpressionType(const string &op) {
 	return ExpressionType::INVALID;
 }
 
-string ExpressionClassToString(ExpressionClass type) {
-	switch (type) {
-	case ExpressionClass::INVALID:
-		return "INVALID";
-	case ExpressionClass::AGGREGATE:
-		return "AGGREGATE";
-	case ExpressionClass::CASE:
-		return "CASE";
-	case ExpressionClass::CAST:
-		return "CAST";
-	case ExpressionClass::COLUMN_REF:
-		return "COLUMN_REF";
-	case ExpressionClass::COMPARISON:
-		return "COMPARISON";
-	case ExpressionClass::CONJUNCTION:
-		return "CONJUNCTION";
-	case ExpressionClass::CONSTANT:
-		return "CONSTANT";
-	case ExpressionClass::DEFAULT:
-		return "DEFAULT";
-	case ExpressionClass::FUNCTION:
-		return "FUNCTION";
-	case ExpressionClass::OPERATOR:
-		return "OPERATOR";
-	case ExpressionClass::STAR:
-		return "STAR";
-	case ExpressionClass::SUBQUERY:
-		return "SUBQUERY";
-	case ExpressionClass::WINDOW:
-		return "WINDOW";
-	case ExpressionClass::PARAMETER:
-		return "PARAMETER";
-	case ExpressionClass::COLLATE:
-		return "COLLATE";
-	case ExpressionClass::LAMBDA:
-		return "LAMBDA";
-	case ExpressionClass::POSITIONAL_REFERENCE:
-		return "POSITIONAL_REFERENCE";
-	case ExpressionClass::BETWEEN:
-		return "BETWEEN";
-	case ExpressionClass::BOUND_AGGREGATE:
-		return "BOUND_AGGREGATE";
-	case ExpressionClass::BOUND_CASE:
-		return "BOUND_CASE";
-	case ExpressionClass::BOUND_CAST:
-		return "BOUND_CAST";
-	case ExpressionClass::BOUND_COLUMN_REF:
-		return "BOUND_COLUMN_REF";
-	case ExpressionClass::BOUND_COMPARISON:
-		return "BOUND_COMPARISON";
-	case ExpressionClass::BOUND_CONJUNCTION:
-		return "BOUND_CONJUNCTION";
-	case ExpressionClass::BOUND_CONSTANT:
-		return "BOUND_CONSTANT";
-	case ExpressionClass::BOUND_DEFAULT:
-		return "BOUND_DEFAULT";
-	case ExpressionClass::BOUND_FUNCTION:
-		return "BOUND_FUNCTION";
-	case ExpressionClass::BOUND_OPERATOR:
-		return "BOUND_OPERATOR";
-	case ExpressionClass::BOUND_PARAMETER:
-		return "BOUND_PARAMETER";
-	case ExpressionClass::BOUND_REF:
-		return "BOUND_REF";
-	case ExpressionClass::BOUND_SUBQUERY:
-		return "BOUND_SUBQUERY";
-	case ExpressionClass::BOUND_WINDOW:
-		return "BOUND_WINDOW";
-	case ExpressionClass::BOUND_BETWEEN:
-		return "BOUND_BETWEEN";
-	case ExpressionClass::BOUND_UNNEST:
-		return "BOUND_UNNEST";
-	case ExpressionClass::BOUND_LAMBDA:
-		return "BOUND_LAMBDA";
-	case ExpressionClass::BOUND_EXPRESSION:
-		return "BOUND_EXPRESSION";
-	default:
-		return "ExpressionClass::!!UNIMPLEMENTED_CASE!!";
-	}
-}
-
 } // namespace duckdb

package/src/duckdb/src/common/enums/join_type.cpp

@@ -1,30 +1,11 @@
 #include "duckdb/common/enums/join_type.hpp"
+#include "duckdb/common/serializer/enum_serializer.hpp"
 
 namespace duckdb {
 
 string JoinTypeToString(JoinType type) {
-	switch (type) {
-	case JoinType::LEFT:
-		return "LEFT";
-	case JoinType::RIGHT:
-		return "RIGHT";
-	case JoinType::INNER:
-		return "INNER";
-	case JoinType::OUTER:
-		return "FULL";
-	case JoinType::SEMI:
-		return "SEMI";
-	case JoinType::ANTI:
-		return "ANTI";
-	case JoinType::SINGLE:
-		return "SINGLE";
-	case JoinType::MARK:
-		return "MARK";
-	case JoinType::INVALID: // LCOV_EXCL_START
-		break;
-	}
-	return "INVALID";
-} // LCOV_EXCL_STOP
+	return EnumSerializer::EnumToString(type);
+}
 
 bool IsLeftOuterJoin(JoinType type) {
 	return type == JoinType::LEFT || type == JoinType::OUTER;

package/src/duckdb/src/common/enums/logical_operator_type.cpp

@@ -20,6 +20,8 @@ string LogicalOperatorToString(LogicalOperatorType type) {
 		return "EXPRESSION_GET";
 	case LogicalOperatorType::LOGICAL_ANY_JOIN:
 		return "ANY_JOIN";
+	case LogicalOperatorType::LOGICAL_ASOF_JOIN:
+		return "ASOF_JOIN";
 	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
 		return "COMPARISON_JOIN";
 	case LogicalOperatorType::LOGICAL_DELIM_JOIN:

package/src/duckdb/src/common/enums/statement_type.cpp

@@ -59,6 +59,8 @@ string StatementTypeToString(StatementType type) {
 		return "ATTACH";
 	case StatementType::DETACH_STATEMENT:
 		return "DETACH";
+	case StatementType::MULTI_STATEMENT:
+		return "MULTI";
 	case StatementType::INVALID_STATEMENT:
 		break;
 	}

package/src/duckdb/src/common/exception.cpp

@@ -138,12 +138,23 @@ string Exception::ExceptionTypeToString(ExceptionType type) {
 		return "Parameter Not Allowed";
 	case ExceptionType::DEPENDENCY:
 		return "Dependency";
+	case ExceptionType::HTTP:
+		return "HTTP";
 	default:
 		return "Unknown";
 	}
 }
 
-void Exception::ThrowAsTypeWithMessage(ExceptionType type, const string &message) {
+const HTTPException &Exception::AsHTTPException() const {
+	D_ASSERT(type == ExceptionType::HTTP);
+	const auto &e = static_cast<const HTTPException *>(this);
+	D_ASSERT(e->GetStatusCode() != 0);
+	D_ASSERT(e->GetHeaders().size() > 0);
+	return *e;
+}
+
+void Exception::ThrowAsTypeWithMessage(ExceptionType type, const string &message,
+                                       const std::shared_ptr<Exception> &original) {
 	switch (type) {
 	case ExceptionType::OUT_OF_RANGE:
 		throw OutOfRangeException(message);
@@ -191,6 +202,9 @@ void Exception::ThrowAsTypeWithMessage(ExceptionType type, const string &message
 		throw FatalException(message);
 	case ExceptionType::DEPENDENCY:
 		throw DependencyException(message);
+	case ExceptionType::HTTP: {
+		original->AsHTTPException().Throw();
+	}
 	default:
 		throw Exception(type, message);
 	}

package/src/duckdb/src/common/field_writer.cpp

@@ -8,6 +8,7 @@ namespace duckdb {
 FieldWriter::FieldWriter(Serializer &serializer_p)
     : serializer(serializer_p), buffer(make_unique<BufferedSerializer>()), field_count(0), finalized(false) {
 	buffer->SetVersion(serializer.GetVersion());
+	buffer->is_query_plan = serializer.is_query_plan;
 }
 
 FieldWriter::~FieldWriter() {

package/src/duckdb/src/common/hive_partitioning.cpp

@@ -1,9 +1,11 @@
 #include "duckdb/common/hive_partitioning.hpp"
-#include "duckdb/optimizer/statistics_propagator.hpp"
 #include "duckdb/planner/table_filter.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/optimizer/filter_combiner.hpp"
 #include "duckdb/planner/expression_iterator.hpp"
+#include "duckdb/planner/expression/bound_constant_expression.hpp"
+#include "duckdb/planner/expression/bound_columnref_expression.hpp"
+#include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "re2/re2.h"
 
 namespace duckdb {

package/src/duckdb/src/common/local_file_system.cpp

@@ -832,6 +832,46 @@ static bool HasGlob(const string &str) {
 	}
 	return false;
 }
+static bool IsCrawl(const string &glob) {
+	// glob must match exactly
+	return glob == "**";
+}
+static bool HasMultipleCrawl(const vector<string> &splits) {
+	return std::count(splits.begin(), splits.end(), "**") > 1;
+}
+static bool IsSymbolicLink(const string &path) {
+#ifndef _WIN32
+	struct stat status;
+	return (lstat(path.c_str(), &status) != -1 && S_ISLNK(status.st_mode));
+#else
+	auto attributes = WindowsGetFileAttributes(path);
+	if (attributes == INVALID_FILE_ATTRIBUTES)
+		return false;
+	return attributes & FILE_ATTRIBUTE_REPARSE_POINT;
+#endif
+}
+
+static void RecursiveGlobDirectories(FileSystem &fs, const string &path, vector<string> &result, bool match_directory,
+                                     bool join_path) {
+
+	fs.ListFiles(path, [&](const string &fname, bool is_directory) {
+		string concat;
+		if (join_path) {
+			concat = fs.JoinPath(path, fname);
+		} else {
+			concat = fname;
+		}
+		if (IsSymbolicLink(concat)) {
+			return;
+		}
+		if (is_directory == match_directory) {
+			result.push_back(concat);
+		}
+		if (is_directory) {
+			RecursiveGlobDirectories(fs, concat, result, match_directory, true);
+		}
+	});
+}
 
 static void GlobFilesInternal(FileSystem &fs, const string &path, const string &glob, bool match_directory,
                               vector<string> &result, bool join_path) {
@@ -933,6 +973,10 @@ vector<string> LocalFileSystem::Glob(const string &path, FileOpener *opener) {
 		}
 	}
 
+	if (HasMultipleCrawl(splits)) {
+		throw IOException("Cannot use multiple \'**\' in one path");
+	}
+
 	for (idx_t i = absolute_path ? 1 : 0; i < splits.size(); i++) {
 		bool is_last_chunk = i + 1 == splits.size();
 		bool has_glob = HasGlob(splits[i]);
@@ -949,14 +993,27 @@ vector<string> LocalFileSystem::Glob(const string &path, FileOpener *opener) {
 				}
 			}
 		} else {
-			if (previous_directories.empty()) {
-				// no previous directories: list in the current path
-				GlobFilesInternal(*this, ".", splits[i], !is_last_chunk, result, false);
+			if (IsCrawl(splits[i])) {
+				if (!is_last_chunk) {
+					result = previous_directories;
+				}
+				if (previous_directories.empty()) {
+					RecursiveGlobDirectories(*this, ".", result, !is_last_chunk, false);
+				} else {
+					for (auto &prev_dir : previous_directories) {
+						RecursiveGlobDirectories(*this, prev_dir, result, !is_last_chunk, true);
+					}
+				}
 			} else {
-				// previous directories
-				// we iterate over each of the previous directories, and apply the glob of the current directory
-				for (auto &prev_directory : previous_directories) {
-					GlobFilesInternal(*this, prev_directory, splits[i], !is_last_chunk, result, true);
+				if (previous_directories.empty()) {
+					// no previous directories: list in the current path
+					GlobFilesInternal(*this, ".", splits[i], !is_last_chunk, result, false);
+				} else {
+					// previous directories
+					// we iterate over each of the previous directories, and apply the glob of the current directory
+					for (auto &prev_directory : previous_directories) {
+						GlobFilesInternal(*this, prev_directory, splits[i], !is_last_chunk, result, true);
+					}
 				}
 			}
 		}

package/src/duckdb/src/common/operator/cast_operators.cpp

@@ -1477,7 +1477,7 @@ bool TryCastToBit::Operation(string_t input, string_t &result, Vector &result_ve
 	}
 
 	result = StringVector::EmptyString(result_vector, result_size);
-	Bit::ToBit(input, (data_ptr_t)result.GetDataWriteable());
+	Bit::ToBit(input, result);
 	result.Finalize();
 	return true;
 }

package/src/duckdb/src/common/preserved_error.cpp

@@ -7,15 +7,17 @@
 
 namespace duckdb {
 
-PreservedError::PreservedError() : initialized(false) {
+PreservedError::PreservedError() : initialized(false), exception_instance(nullptr) {
 }
 
 PreservedError::PreservedError(const Exception &exception)
-    : initialized(true), type(exception.type), raw_message(SanitizeErrorMessage(exception.RawMessage())) {
+    : initialized(true), type(exception.type), raw_message(SanitizeErrorMessage(exception.RawMessage())),
+      exception_instance(exception.Copy()) {
 }
 
 PreservedError::PreservedError(const string &message)
-    : initialized(true), type(ExceptionType::INVALID), raw_message(SanitizeErrorMessage(message)) {
+    : initialized(true), type(ExceptionType::INVALID), raw_message(SanitizeErrorMessage(message)),
+      exception_instance(nullptr) {
 }
 
 const string &PreservedError::Message() {
@@ -33,9 +35,9 @@ void PreservedError::Throw(const string &prepended_message) const {
 	D_ASSERT(initialized);
 	if (!prepended_message.empty()) {
 		string new_message = prepended_message + raw_message;
-		Exception::ThrowAsTypeWithMessage(type, new_message);
+		Exception::ThrowAsTypeWithMessage(type, new_message, exception_instance);
 	}
-	Exception::ThrowAsTypeWithMessage(type, raw_message);
+	Exception::ThrowAsTypeWithMessage(type, raw_message, exception_instance);
 }
 
 const ExceptionType &PreservedError::Type() const {

package/src/duckdb/src/common/progress_bar/progress_bar.cpp

@@ -4,6 +4,13 @@
 
 namespace duckdb {
 
+void ProgressBar::SystemOverrideCheck(ClientConfig &config) {
+	if (config.system_progress_bar_disable_reason != nullptr) {
+		throw InvalidInputException("Could not change the progress bar setting because: '%s'",
+		                            config.system_progress_bar_disable_reason);
+	}
+}
+
 unique_ptr<ProgressBarDisplay> ProgressBar::DefaultProgressBarDisplay() {
 	return make_unique<TerminalProgressBarDisplay>();
 }

package/src/duckdb/src/common/serializer/buffered_deserializer.cpp

@@ -20,4 +20,8 @@ void BufferedDeserializer::ReadData(data_ptr_t buffer, idx_t read_size) {
 	ptr += read_size;
 }
 
+ClientContext &BufferedContextDeserializer::GetContext() {
+	return context;
+}
+
 } // namespace duckdb

package/src/duckdb/src/common/serializer/buffered_file_reader.cpp

@@ -7,8 +7,10 @@
 
 namespace duckdb {
 
-BufferedFileReader::BufferedFileReader(FileSystem &fs, const char *path, FileLockType lock_type, FileOpener *opener)
-    : fs(fs), data(unique_ptr<data_t[]>(new data_t[FILE_BUFFER_SIZE])), offset(0), read_data(0), total_read(0) {
+BufferedFileReader::BufferedFileReader(FileSystem &fs, const char *path, ClientContext *context, FileLockType lock_type,
+                                       FileOpener *opener)
+    : fs(fs), data(unique_ptr<data_t[]>(new data_t[FILE_BUFFER_SIZE])), offset(0), read_data(0), context(context),
+      total_read(0) {
 	handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ, lock_type, FileSystem::DEFAULT_COMPRESSION, opener);
 	file_size = fs.GetFileSize(*handle);
 }
@@ -54,4 +56,15 @@ uint64_t BufferedFileReader::CurrentOffset() {
 	return total_read + offset;
 }
 
+ClientContext &BufferedFileReader::GetContext() {
+	if (!context) {
+		throw InternalException("Trying to acquire a client context that does not exist");
+	}
+	return *context;
+}
+
+Catalog *BufferedFileReader::GetCatalog() {
+	return catalog;
+}
+
 } // namespace duckdb