duckdb 0.9.3-dev2.0 → 0.10.0

package/src/duckdb/src/common/sort/comparators.cpp

@@ -2,6 +2,7 @@
 
 #include "duckdb/common/fast_mem.hpp"
 #include "duckdb/common/sort/sort.hpp"
+#include "duckdb/common/uhugeint.hpp"
 
 namespace duckdb {
 
@@ -55,6 +56,7 @@ int Comparators::CompareVal(const data_ptr_t l_ptr, const data_ptr_t r_ptr, cons
 	case PhysicalType::VARCHAR:
 		return TemplatedCompareVal<string_t>(l_ptr, r_ptr);
 	case PhysicalType::LIST:
+	case PhysicalType::ARRAY:
 	case PhysicalType::STRUCT: {
 		auto l_nested_ptr = Load<data_ptr_t>(l_ptr);
 		auto r_nested_ptr = Load<data_ptr_t>(r_ptr);
@@ -134,6 +136,8 @@ int Comparators::CompareValAndAdvance(data_ptr_t &l_ptr, data_ptr_t &r_ptr, cons
 		return TemplatedCompareAndAdvance<uint64_t>(l_ptr, r_ptr);
 	case PhysicalType::INT128:
 		return TemplatedCompareAndAdvance<hugeint_t>(l_ptr, r_ptr);
+	case PhysicalType::UINT128:
+		return TemplatedCompareAndAdvance<uhugeint_t>(l_ptr, r_ptr);
 	case PhysicalType::FLOAT:
 		return TemplatedCompareAndAdvance<float>(l_ptr, r_ptr);
 	case PhysicalType::DOUBLE:
@@ -146,6 +150,8 @@ int Comparators::CompareValAndAdvance(data_ptr_t &l_ptr, data_ptr_t &r_ptr, cons
 		return CompareListAndAdvance(l_ptr, r_ptr, ListType::GetChildType(type), valid);
 	case PhysicalType::STRUCT:
 		return CompareStructAndAdvance(l_ptr, r_ptr, StructType::GetChildTypes(type), valid);
+	case PhysicalType::ARRAY:
+		return CompareArrayAndAdvance(l_ptr, r_ptr, ArrayType::GetChildType(type), valid, ArrayType::GetSize(type));
 	default:
 		throw NotImplementedException("Unimplemented CompareValAndAdvance for type %s", type.ToString());
 	}
@@ -222,6 +228,119 @@ int Comparators::CompareStructAndAdvance(data_ptr_t &left_ptr, data_ptr_t &right
 	return comp_res;
 }
 
+int Comparators::CompareArrayAndAdvance(data_ptr_t &left_ptr, data_ptr_t &right_ptr, const LogicalType &type,
+                                        bool valid, idx_t array_size) {
+	if (!valid) {
+		return 0;
+	}
+
+	// Load array validity masks
+	ValidityBytes left_validity(left_ptr);
+	ValidityBytes right_validity(right_ptr);
+	left_ptr += (array_size + 7) / 8;
+	right_ptr += (array_size + 7) / 8;
+
+	int comp_res = 0;
+	if (TypeIsConstantSize(type.InternalType())) {
+		// Templated code for fixed-size types
+		switch (type.InternalType()) {
+		case PhysicalType::BOOL:
+		case PhysicalType::INT8:
+			comp_res = TemplatedCompareListLoop<int8_t>(left_ptr, right_ptr, left_validity, right_validity, array_size);
+			break;
+		case PhysicalType::INT16:
+			comp_res =
+			    TemplatedCompareListLoop<int16_t>(left_ptr, right_ptr, left_validity, right_validity, array_size);
+			break;
+		case PhysicalType::INT32:
+			comp_res =
+			    TemplatedCompareListLoop<int32_t>(left_ptr, right_ptr, left_validity, right_validity, array_size);
+			break;
+		case PhysicalType::INT64:
+			comp_res =
+			    TemplatedCompareListLoop<int64_t>(left_ptr, right_ptr, left_validity, right_validity, array_size);
+			break;
+		case PhysicalType::UINT8:
+			comp_res =
+			    TemplatedCompareListLoop<uint8_t>(left_ptr, right_ptr, left_validity, right_validity, array_size);
+			break;
+		case PhysicalType::UINT16:
+			comp_res =
+			    TemplatedCompareListLoop<uint16_t>(left_ptr, right_ptr, left_validity, right_validity, array_size);
+			break;
+		case PhysicalType::UINT32:
+			comp_res =
+			    TemplatedCompareListLoop<uint32_t>(left_ptr, right_ptr, left_validity, right_validity, array_size);
+			break;
+		case PhysicalType::UINT64:
+			comp_res =
+			    TemplatedCompareListLoop<uint64_t>(left_ptr, right_ptr, left_validity, right_validity, array_size);
+			break;
+		case PhysicalType::INT128:
+			comp_res =
+			    TemplatedCompareListLoop<hugeint_t>(left_ptr, right_ptr, left_validity, right_validity, array_size);
+			break;
+		case PhysicalType::FLOAT:
+			comp_res = TemplatedCompareListLoop<float>(left_ptr, right_ptr, left_validity, right_validity, array_size);
+			break;
+		case PhysicalType::DOUBLE:
+			comp_res = TemplatedCompareListLoop<double>(left_ptr, right_ptr, left_validity, right_validity, array_size);
+			break;
+		case PhysicalType::INTERVAL:
+			comp_res =
+			    TemplatedCompareListLoop<interval_t>(left_ptr, right_ptr, left_validity, right_validity, array_size);
+			break;
+		default:
+			throw NotImplementedException("CompareListAndAdvance for fixed-size type %s", type.ToString());
+		}
+	} else {
+		// Variable-sized array entries
+		bool left_valid;
+		bool right_valid;
+		idx_t entry_idx;
+		idx_t idx_in_entry;
+		// Size (in bytes) of all variable-sizes entries is stored before the entries begin,
+		// to make deserialization easier. We need to skip over them
+		left_ptr += array_size * sizeof(idx_t);
+		right_ptr += array_size * sizeof(idx_t);
+		for (idx_t i = 0; i < array_size; i++) {
+			ValidityBytes::GetEntryIndex(i, entry_idx, idx_in_entry);
+			left_valid = left_validity.RowIsValid(left_validity.GetValidityEntry(entry_idx), idx_in_entry);
+			right_valid = right_validity.RowIsValid(right_validity.GetValidityEntry(entry_idx), idx_in_entry);
+			if (left_valid && right_valid) {
+				switch (type.InternalType()) {
+				case PhysicalType::LIST:
+					comp_res = CompareListAndAdvance(left_ptr, right_ptr, ListType::GetChildType(type), left_valid);
+					break;
+				case PhysicalType::ARRAY:
+					comp_res = CompareArrayAndAdvance(left_ptr, right_ptr, ArrayType::GetChildType(type), left_valid,
+					                                  ArrayType::GetSize(type));
+					break;
+				case PhysicalType::VARCHAR:
+					comp_res = CompareStringAndAdvance(left_ptr, right_ptr, left_valid);
+					break;
+				case PhysicalType::STRUCT:
+					comp_res =
+					    CompareStructAndAdvance(left_ptr, right_ptr, StructType::GetChildTypes(type), left_valid);
+					break;
+				default:
+					throw NotImplementedException("CompareArrayAndAdvance for variable-size type %s", type.ToString());
+				}
+			} else if (!left_valid && !right_valid) {
+				comp_res = 0;
+			} else if (left_valid) {
+				comp_res = -1;
+			} else {
+				comp_res = 1;
+			}
+			if (comp_res != 0) {
+				break;
+			}
+		}
+	}
+	return comp_res;
+}
+
 int Comparators::CompareListAndAdvance(data_ptr_t &left_ptr, data_ptr_t &right_ptr, const LogicalType &type,
                                        bool valid) {
 	if (!valid) {
@@ -271,6 +390,9 @@ int Comparators::CompareListAndAdvance(data_ptr_t &left_ptr, data_ptr_t &right_p
 		case PhysicalType::INT128:
 			comp_res = TemplatedCompareListLoop<hugeint_t>(left_ptr, right_ptr, left_validity, right_validity, count);
 			break;
+		case PhysicalType::UINT128:
+			comp_res = TemplatedCompareListLoop<uhugeint_t>(left_ptr, right_ptr, left_validity, right_validity, count);
+			break;
 		case PhysicalType::FLOAT:
 			comp_res = TemplatedCompareListLoop<float>(left_ptr, right_ptr, left_validity, right_validity, count);
 			break;
@@ -302,6 +424,10 @@ int Comparators::CompareListAndAdvance(data_ptr_t &left_ptr, data_ptr_t &right_p
 				case PhysicalType::LIST:
 					comp_res = CompareListAndAdvance(left_ptr, right_ptr, ListType::GetChildType(type), left_valid);
 					break;
+				case PhysicalType::ARRAY:
+					comp_res = CompareArrayAndAdvance(left_ptr, right_ptr, ArrayType::GetChildType(type), left_valid,
+					                                  ArrayType::GetSize(type));
+					break;
 				case PhysicalType::VARCHAR:
 					comp_res = CompareStringAndAdvance(left_ptr, right_ptr, left_valid);
 					break;

package/src/duckdb/src/common/sort/partition_state.cpp

@@ -22,35 +22,35 @@ PartitionGlobalHashGroup::PartitionGlobalHashGroup(BufferManager &buffer_manager
 	partition_layout = global_sort->sort_layout.GetPrefixComparisonLayout(partitions.size());
 }
 
-int PartitionGlobalHashGroup::ComparePartitions(const SBIterator &left, const SBIterator &right) const {
-	int part_cmp = 0;
-	if (partition_layout.all_constant) {
-		part_cmp = FastMemcmp(left.entry_ptr, right.entry_ptr, partition_layout.comparison_size);
-	} else {
-		part_cmp = Comparators::CompareTuple(left.scan, right.scan, left.entry_ptr, right.entry_ptr, partition_layout,
-		                                     left.external);
-	}
-	return part_cmp;
-}
-
-void PartitionGlobalHashGroup::ComputeMasks(ValidityMask &partition_mask, ValidityMask &order_mask) {
+void PartitionGlobalHashGroup::ComputeMasks(ValidityMask &partition_mask, OrderMasks &order_masks) {
 	D_ASSERT(count > 0);
 
 	SBIterator prev(*global_sort, ExpressionType::COMPARE_LESSTHAN);
 	SBIterator curr(*global_sort, ExpressionType::COMPARE_LESSTHAN);
 
 	partition_mask.SetValidUnsafe(0);
-	order_mask.SetValidUnsafe(0);
+	unordered_map<idx_t, SortLayout> prefixes;
+	for (auto &order_mask : order_masks) {
+		order_mask.second.SetValidUnsafe(0);
+		D_ASSERT(order_mask.first >= partition_layout.column_count);
+		prefixes[order_mask.first] = global_sort->sort_layout.GetPrefixComparisonLayout(order_mask.first);
+	}
+
 	for (++curr; curr.GetIndex() < count; ++curr) {
 		//	Compare the partition subset first because if that differs, then so does the full ordering
 		const auto part_cmp = ComparePartitions(prev, curr);
-		;
 
 		if (part_cmp) {
 			partition_mask.SetValidUnsafe(curr.GetIndex());
-			order_mask.SetValidUnsafe(curr.GetIndex());
-		} else if (prev.Compare(curr)) {
-			order_mask.SetValidUnsafe(curr.GetIndex());
+			for (auto &order_mask : order_masks) {
+				order_mask.second.SetValidUnsafe(curr.GetIndex());
+			}
+		} else {
+			for (auto &order_mask : order_masks) {
+				if (prev.Compare(curr, prefixes[order_mask.first])) {
+					order_mask.second.SetValidUnsafe(curr.GetIndex());
+				}
+			}
 		}
 		++prev;
 	}

package/src/duckdb/src/common/sort/radix_sort.cpp

@@ -248,7 +248,8 @@ void RadixSort(BufferManager &buffer_manager, const data_ptr_t &dataptr, const i
 	} else if (sorting_size <= SortConstants::MSD_RADIX_SORT_SIZE_THRESHOLD) {
 		RadixSortLSD(buffer_manager, dataptr, count, col_offset, sort_layout.entry_size, sorting_size);
 	} else {
-		auto temp_block = buffer_manager.Allocate(MaxValue(count * sort_layout.entry_size, (idx_t)Storage::BLOCK_SIZE));
+		auto temp_block = buffer_manager.Allocate(MemoryTag::ORDER_BY,
+		                                          MaxValue(count * sort_layout.entry_size, (idx_t)Storage::BLOCK_SIZE));
 		auto preallocated_array = make_unsafe_uniq_array<idx_t>(sorting_size * SortConstants::MSD_RADIX_LOCATIONS);
 		RadixSortMSD(dataptr, temp_block.Ptr(), count, col_offset, sort_layout.entry_size, sorting_size, 0,
 		             preallocated_array.get(), false);

package/src/duckdb/src/common/sort/sort_state.cpp

@@ -3,6 +3,7 @@
 #include "duckdb/common/row_operations/row_operations.hpp"
 #include "duckdb/common/sort/sort.hpp"
 #include "duckdb/common/sort/sorted_block.hpp"
+#include "duckdb/storage/buffer/buffer_pool.hpp"
 
 #include <algorithm>
 #include <numeric>
@@ -31,6 +32,10 @@ idx_t GetNestedSortingColSize(idx_t &col_size, const LogicalType &type) {
 			// Structs get 1 bytes (null)
 			col_size++;
 			return GetNestedSortingColSize(col_size, StructType::GetChildType(type, 0));
+		case PhysicalType::ARRAY:
+			// Arrays get 1 bytes (null)
+			col_size++;
+			return GetNestedSortingColSize(col_size, ArrayType::GetChildType(type));
 		default:
 			throw NotImplementedException("Unable to order column with type %s", type.ToString());
 		}
@@ -264,7 +269,7 @@ unique_ptr<RowDataBlock> LocalSortState::ConcatenateBlocks(RowDataCollection &ro
 	auto buffer_manager = &row_data.buffer_manager;
 	const idx_t &entry_size = row_data.entry_size;
 	idx_t capacity = MaxValue(((idx_t)Storage::BLOCK_SIZE + entry_size - 1) / entry_size, row_data.count);
-	auto new_block = make_uniq<RowDataBlock>(*buffer_manager, capacity, entry_size);
+	auto new_block = make_uniq<RowDataBlock>(MemoryTag::ORDER_BY, *buffer_manager, capacity, entry_size);
 	new_block->count = row_data.count;
 	auto new_block_handle = buffer_manager->Pin(new_block->block);
 	data_ptr_t new_block_ptr = new_block_handle.Ptr();
@@ -289,8 +294,8 @@ void LocalSortState::ReOrder(SortedData &sd, data_ptr_t sorting_ptr, RowDataColl
 	auto unordered_data_handle = buffer_manager->Pin(unordered_data_block->block);
 	const data_ptr_t unordered_data_ptr = unordered_data_handle.Ptr();
 	// Create new block that will hold re-ordered row data
-	auto ordered_data_block =
-	    make_uniq<RowDataBlock>(*buffer_manager, unordered_data_block->capacity, unordered_data_block->entry_size);
+	auto ordered_data_block = make_uniq<RowDataBlock>(MemoryTag::ORDER_BY, *buffer_manager,
+	                                                  unordered_data_block->capacity, unordered_data_block->entry_size);
 	ordered_data_block->count = count;
 	auto ordered_data_handle = buffer_manager->Pin(ordered_data_block->block);
 	data_ptr_t ordered_data_ptr = ordered_data_handle.Ptr();
@@ -318,7 +323,7 @@ void LocalSortState::ReOrder(SortedData &sd, data_ptr_t sorting_ptr, RowDataColl
 		    std::accumulate(heap.blocks.begin(), heap.blocks.end(), (idx_t)0,
 		                    [](idx_t a, const unique_ptr<RowDataBlock> &b) { return a + b->byte_offset; });
 		idx_t heap_block_size = MaxValue(total_byte_offset, (idx_t)Storage::BLOCK_SIZE);
-		auto ordered_heap_block = make_uniq<RowDataBlock>(*buffer_manager, heap_block_size, 1);
+		auto ordered_heap_block = make_uniq<RowDataBlock>(MemoryTag::ORDER_BY, *buffer_manager, heap_block_size, 1);
 		ordered_heap_block->count = count;
 		ordered_heap_block->byte_offset = total_byte_offset;
 		auto ordered_heap_handle = buffer_manager->Pin(ordered_heap_block->block);
@@ -396,7 +401,7 @@ void GlobalSortState::PrepareMergePhase() {
 	idx_t total_heap_size =
 	    std::accumulate(sorted_blocks.begin(), sorted_blocks.end(), (idx_t)0,
 	                    [](idx_t a, const unique_ptr<SortedBlock> &b) { return a + b->HeapSize(); });
-	if (external || (pinned_blocks.empty() && total_heap_size > 0.25 * buffer_manager.GetMaxMemory())) {
+	if (external || (pinned_blocks.empty() && total_heap_size > 0.25 * buffer_manager.GetQueryMaxMemory())) {
 		external = true;
 	}
 	// Use the data that we have to determine which partition size to use during the merge

package/src/duckdb/src/common/sort/sorted_block.cpp

@@ -27,9 +27,10 @@ idx_t SortedData::Count() {
 void SortedData::CreateBlock() {
 	auto capacity =
 	    MaxValue(((idx_t)Storage::BLOCK_SIZE + layout.GetRowWidth() - 1) / layout.GetRowWidth(), state.block_capacity);
-	data_blocks.push_back(make_uniq<RowDataBlock>(buffer_manager, capacity, layout.GetRowWidth()));
+	data_blocks.push_back(make_uniq<RowDataBlock>(MemoryTag::ORDER_BY, buffer_manager, capacity, layout.GetRowWidth()));
 	if (!layout.AllConstant() && state.external) {
-		heap_blocks.push_back(make_uniq<RowDataBlock>(buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1));
+		heap_blocks.push_back(
+		    make_uniq<RowDataBlock>(MemoryTag::ORDER_BY, buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1));
 		D_ASSERT(data_blocks.size() == heap_blocks.size());
 	}
 }
@@ -105,7 +106,8 @@ void SortedBlock::InitializeWrite() {
 void SortedBlock::CreateBlock() {
 	auto capacity = MaxValue(((idx_t)Storage::BLOCK_SIZE + sort_layout.entry_size - 1) / sort_layout.entry_size,
 	                         state.block_capacity);
-	radix_sorting_data.push_back(make_uniq<RowDataBlock>(buffer_manager, capacity, sort_layout.entry_size));
+	radix_sorting_data.push_back(
+	    make_uniq<RowDataBlock>(MemoryTag::ORDER_BY, buffer_manager, capacity, sort_layout.entry_size));
 }
 
 void SortedBlock::AppendSortedBlocks(vector<unique_ptr<SortedBlock>> &sorted_blocks) {
@@ -373,9 +375,8 @@ static idx_t GetBlockCountWithEmptyCheck(const GlobalSortState &gss) {
 
 SBIterator::SBIterator(GlobalSortState &gss, ExpressionType comparison, idx_t entry_idx_p)
     : sort_layout(gss.sort_layout), block_count(GetBlockCountWithEmptyCheck(gss)), block_capacity(gss.block_capacity),
-      cmp_size(sort_layout.comparison_size), entry_size(sort_layout.entry_size), all_constant(sort_layout.all_constant),
-      external(gss.external), cmp(ComparisonValue(comparison)), scan(gss.buffer_manager, gss), block_ptr(nullptr),
-      entry_ptr(nullptr) {
+      entry_size(sort_layout.entry_size), all_constant(sort_layout.all_constant), external(gss.external),
+      cmp(ComparisonValue(comparison)), scan(gss.buffer_manager, gss), block_ptr(nullptr), entry_ptr(nullptr) {
 
 	scan.sb = gss.sorted_blocks[0].get();
 	scan.block_idx = block_count;

package/src/duckdb/src/common/string_util.cpp

@@ -4,6 +4,7 @@
 #include "duckdb/common/pair.hpp"
 #include "duckdb/common/to_string.hpp"
 #include "duckdb/common/helper.hpp"
+#include "duckdb/function/scalar/string_functions.hpp"
 
 #include <algorithm>
 #include <cctype>
@@ -154,31 +155,45 @@ string StringUtil::Join(const vector<string> &input, const string &separator) {
 	return StringUtil::Join(input, input.size(), separator, [](const string &s) { return s; });
 }
 
-string StringUtil::BytesToHumanReadableString(idx_t bytes) {
+string StringUtil::Join(const set<string> &input, const string &separator) {
+	// The result
+	std::string result;
+
+	auto it = input.begin();
+	while (it != input.end()) {
+		result += *it;
+		it++;
+		if (it == input.end()) {
+			break;
+		}
+		result += separator;
+	}
+	return result;
+}
+
+string StringUtil::BytesToHumanReadableString(idx_t bytes, idx_t multiplier) {
+	D_ASSERT(multiplier == 1000 || multiplier == 1024);
 	string db_size;
-	auto kilobytes = bytes / 1000;
-	auto megabytes = kilobytes / 1000;
-	kilobytes -= megabytes * 1000;
-	auto gigabytes = megabytes / 1000;
-	megabytes -= gigabytes * 1000;
-	auto terabytes = gigabytes / 1000;
-	gigabytes -= terabytes * 1000;
-	auto petabytes = terabytes / 1000;
-	terabytes -= petabytes * 1000;
-	if (petabytes > 0) {
-		return to_string(petabytes) + "." + to_string(terabytes / 100) + "PB";
-	}
-	if (terabytes > 0) {
-		return to_string(terabytes) + "." + to_string(gigabytes / 100) + "TB";
-	} else if (gigabytes > 0) {
-		return to_string(gigabytes) + "." + to_string(megabytes / 100) + "GB";
-	} else if (megabytes > 0) {
-		return to_string(megabytes) + "." + to_string(kilobytes / 100) + "MB";
-	} else if (kilobytes > 0) {
-		return to_string(kilobytes) + "KB";
-	} else {
-		return to_string(bytes) + (bytes == 1 ? " byte" : " bytes");
+	idx_t array[6] = {};
+	const char *unit[2][6] = {{"bytes", "KiB", "MiB", "GiB", "TiB", "PiB"}, {"bytes", "kB", "MB", "GB", "TB", "PB"}};
+
+	const int sel = (multiplier == 1000);
+
+	array[0] = bytes;
+	for (idx_t i = 1; i < 6; i++) {
+		array[i] = array[i - 1] / multiplier;
+		array[i - 1] %= multiplier;
 	}
+
+	for (idx_t i = 5; i >= 1; i--) {
+		if (array[i]) {
+			// Map 0 -> 0 and (multiplier-1) -> 9
+			idx_t fractional_part = (array[i - 1] * 10) / multiplier;
+			return to_string(array[i]) + "." + to_string(fractional_part) + " " + unit[sel][i];
+		}
+	}
+
+	return to_string(array[0]) + (bytes == 1 ? " byte" : " bytes");
 }
 
 string StringUtil::Upper(const string &str) {
@@ -215,14 +230,32 @@ bool StringUtil::CIEquals(const string &l1, const string &l2) {
 	if (l1.size() != l2.size()) {
 		return false;
 	}
+	const auto charmap = LowerFun::ascii_to_lower_map;
 	for (idx_t c = 0; c < l1.size(); c++) {
-		if (StringUtil::CharacterToLower(l1[c]) != StringUtil::CharacterToLower(l2[c])) {
+		if (charmap[(uint8_t)l1[c]] != charmap[(uint8_t)l2[c]]) {
 			return false;
 		}
 	}
 	return true;
 }
 
+bool StringUtil::CILessThan(const string &s1, const string &s2) {
+	const auto charmap = UpperFun::ascii_to_upper_map;
+
+	unsigned char u1, u2;
+
+	idx_t length = MinValue<idx_t>(s1.length(), s2.length());
+	length += s1.length() != s2.length();
+	for (idx_t i = 0; i < length; i++) {
+		u1 = (unsigned char)s1[i];
+		u2 = (unsigned char)s2[i];
+		if (charmap[u1] != charmap[u2]) {
+			break;
+		}
+	}
+	return (charmap[u1] - charmap[u2]) < 0;
+}
+
 vector<string> StringUtil::Split(const string &input, const string &split) {
 	vector<string> splits;
 
@@ -372,4 +405,249 @@ string StringUtil::CandidatesErrorMessage(const vector<string> &strings, const s
 	return StringUtil::CandidatesMessage(closest_strings, message_prefix);
 }
 
+static void SkipSpaces(const string &message, idx_t &pos) {
+	for (; pos < message.size() && StringUtil::CharacterIsSpace(message[pos]); pos++) {
+	}
+}
+
+static bool MatchCharacter(const string &message, idx_t &pos, char c) {
+	if (pos >= message.size()) {
+		return false;
+	}
+	return message[pos] == c;
+}
+
+static string ParseJSONValue(const string &message, idx_t &pos) {
+	string result;
+	if (!MatchCharacter(message, pos, '"')) {
+		// values need to start with a quote
+		D_ASSERT(0);
+		return result;
+	}
+	pos++;
+	for (; pos < message.size(); pos++) {
+		if (message[pos] == '\\') {
+			// escape
+			pos++;
+			if (pos >= message.size()) {
+				// escape at end of string!?
+				D_ASSERT(0);
+				return result;
+			}
+			switch (message[pos]) {
+			case 'r':
+				result += '\r';
+				break;
+			case 'n':
+				result += '\n';
+				break;
+			case 't':
+				result += '\t';
+				break;
+			case 'b':
+				result += '\b';
+				break;
+			case 'f':
+				result += '\f';
+				break;
+			case '0':
+				result += '\0';
+				break;
+			case '\\':
+			case '"':
+			case '/':
+				result += message[pos];
+				break;
+			default:
+				// unsupported escape character
+				// NOTE: we do not support unicode escape sequences here
+				D_ASSERT(0);
+				result += message[pos];
+				break;
+			}
+		} else if (message[pos] == '"') {
+			// end of message
+			pos++;
+			return result;
+		} else {
+			result += message[pos];
+		}
+	}
+	// no end-of-value found
+	D_ASSERT(0);
+	return result;
+}
+
+unordered_map<string, string> StringUtil::ParseJSONMap(const string &json) {
+	unordered_map<string, string> result;
+	if (json.empty()) {
+		return result;
+	}
+	idx_t pos = 0;
+	SkipSpaces(json, pos);
+	if (!MatchCharacter(json, pos, '{')) {
+		D_ASSERT(0);
+		return result;
+	}
+	pos++;
+	while (true) {
+		SkipSpaces(json, pos);
+		if (MatchCharacter(json, pos, '}')) {
+			// end of object
+			break;
+		}
+		if (!result.empty()) {
+			// objects are comma separated
+			if (!MatchCharacter(json, pos, ',')) {
+				D_ASSERT(0);
+				return result;
+			}
+			pos++;
+		}
+		string key = ParseJSONValue(json, pos);
+		SkipSpaces(json, pos);
+		if (!MatchCharacter(json, pos, ':')) {
+			D_ASSERT(0);
+			return result;
+		}
+		pos++;
+		string value = ParseJSONValue(json, pos);
+		auto entry = result.find(key);
+		if (entry != result.end()) {
+			// entry already exists
+			D_ASSERT(0);
+			continue;
+		}
+		result.insert(make_pair(std::move(key), std::move(value)));
+	}
+	return result;
+}
+
+static void WriteJSONValue(const string &value, string &result) {
+	result += '"';
+	for (auto c : value) {
+		// check for characters we need to escape
+		switch (c) {
+		case '\0':
+			result += "\\0";
+			break;
+		case '\\':
+			result += "\\\\";
+			break;
+		case '\b':
+			result += "\\b";
+			break;
+		case '\f':
+			result += "\\f";
+			break;
+		case '\t':
+			result += "\\t";
+			break;
+		case '\r':
+			result += "\\r";
+			break;
+		case '\n':
+			result += "\\n";
+			break;
+		case '"':
+			result += "\\\"";
+			break;
+		default:
+			result += c;
+			break;
+		}
+	}
+	result += '"';
+}
+
+static void WriteJSONPair(const string &key, const string &value, string &result) {
+	WriteJSONValue(key, result);
+	result += ":";
+	WriteJSONValue(value, result);
+}
+
+string StringUtil::ToJSONMap(ExceptionType type, const string &message, const unordered_map<string, string> &map) {
+	D_ASSERT(map.find("exception_type") == map.end());
+	D_ASSERT(map.find("exception_message") == map.end());
+	string result;
+	result += "{";
+	// we always write exception type/message
+	WriteJSONPair("exception_type", Exception::ExceptionTypeToString(type), result);
+	result += ",";
+	WriteJSONPair("exception_message", message, result);
+	for (auto &entry : map) {
+		result += ",";
+		WriteJSONPair(entry.first, entry.second, result);
+	}
+	result += "}";
+	return result;
+}
+
+string StringUtil::GetFileName(const string &file_path) {
+
+	idx_t pos = file_path.find_last_of("/\\");
+	if (pos == string::npos) {
+		return file_path;
+	}
+	auto end = file_path.size() - 1;
+
+	// If the rest of the string is just slashes or dots, trim them
+	if (file_path.find_first_not_of("/\\.", pos) == string::npos) {
+		// Trim the trailing slashes and dots
+		while (end > 0 && (file_path[end] == '/' || file_path[end] == '.' || file_path[end] == '\\')) {
+			end--;
+		}
+
+		// Now find the next slash
+		pos = file_path.find_last_of("/\\", end);
+		if (pos == string::npos) {
+			return file_path.substr(0, end + 1);
+		}
+	}
+
+	return file_path.substr(pos + 1, end - pos);
+}
+
+string StringUtil::GetFileExtension(const string &file_name) {
+	auto name = GetFileName(file_name);
+	idx_t pos = name.find_last_of('.');
+	// We dont consider e.g. `.gitignore` to have an extension
+	if (pos == string::npos || pos == 0) {
+		return "";
+	}
+	return name.substr(pos + 1);
+}
+
+string StringUtil::GetFileStem(const string &file_name) {
+	auto name = GetFileName(file_name);
+	if (name.size() > 1 && name[0] == '.') {
+		return name;
+	}
+	idx_t pos = name.find_last_of('.');
+	if (pos == string::npos) {
+		return name;
+	}
+	return name.substr(0, pos);
+}
+
+string StringUtil::GetFilePath(const string &file_path) {
+
+	// Trim the trailing slashes
+	auto end = file_path.size() - 1;
+	while (end > 0 && (file_path[end] == '/' || file_path[end] == '\\')) {
+		end--;
+	}
+
+	auto pos = file_path.find_last_of("/\\", end);
+	if (pos == string::npos) {
+		return "";
+	}
+
+	while (pos > 0 && (file_path[pos] == '/' || file_path[pos] == '\\')) {
+		pos--;
+	}
+
+	return file_path.substr(0, pos + 1);
+}
+
 } // namespace duckdb