duckdb 0.7.2-dev1734.0 → 0.7.2-dev1803.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.7.2-dev1734.0",
+  "version": "0.7.2-dev1803.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/common/types.cpp

@@ -1187,8 +1187,8 @@ idx_t StructType::GetChildCount(const LogicalType &type) {
 	return StructType::GetChildTypes(type).size();
 }
 
-LogicalType LogicalType::STRUCT(const child_list_t<LogicalType> &children) {
-	auto info = make_shared<StructTypeInfo>(children);
+LogicalType LogicalType::STRUCT(child_list_t<LogicalType> children) {
+	auto info = make_shared<StructTypeInfo>(std::move(children));
 	return LogicalType(LogicalTypeId::STRUCT, std::move(info));
 }
 

package/src/duckdb/src/function/scalar/math/numeric.cpp

@@ -443,6 +443,63 @@ void FloorFun::RegisterFunction(BuiltinFunctions &set) {
 	set.AddFunction(floor);
 }
 
+//===--------------------------------------------------------------------===//
+// trunc
+//===--------------------------------------------------------------------===//
+struct TruncOperator {
+	// Integer truncation is a NOP
+	template <class TA, class TR>
+	static inline TR Operation(TA left) {
+		return std::trunc(left);
+	}
+};
+
+struct TruncDecimalOperator {
+	template <class T, class POWERS_OF_TEN_CLASS>
+	static void Operation(DataChunk &input, uint8_t scale, Vector &result) {
+		T power_of_ten = POWERS_OF_TEN_CLASS::POWERS_OF_TEN[scale];
+		UnaryExecutor::Execute<T, T>(input.data[0], result, input.size(), [&](T input) {
+			//	Always floor
+			return (input / power_of_ten);
+		});
+	}
+};
+
+void TruncFun::RegisterFunction(BuiltinFunctions &set) {
+	ScalarFunctionSet floor("trunc");
+	for (auto &type : LogicalType::Numeric()) {
+		scalar_function_t func = nullptr;
+		bind_scalar_function_t bind_func = nullptr;
+		//	Truncation of integers gets generated by some tools (e.g., Tableau/JDBC:Postgres)
+		switch (type.id()) {
+		case LogicalTypeId::FLOAT:
+			func = ScalarFunction::UnaryFunction<float, float, TruncOperator>;
+			break;
+		case LogicalTypeId::DOUBLE:
+			func = ScalarFunction::UnaryFunction<double, double, TruncOperator>;
+			break;
+		case LogicalTypeId::DECIMAL:
+			bind_func = BindGenericRoundFunctionDecimal<TruncDecimalOperator>;
+			break;
+		case LogicalTypeId::TINYINT:
+		case LogicalTypeId::SMALLINT:
+		case LogicalTypeId::INTEGER:
+		case LogicalTypeId::BIGINT:
+		case LogicalTypeId::HUGEINT:
+		case LogicalTypeId::UTINYINT:
+		case LogicalTypeId::USMALLINT:
+		case LogicalTypeId::UINTEGER:
+		case LogicalTypeId::UBIGINT:
+			func = ScalarFunction::NopFunction;
+			break;
+		default:
+			throw InternalException("Unimplemented numeric type for function \"trunc\"");
+		}
+		floor.AddFunction(ScalarFunction({type}, type, func, bind_func));
+	}
+	set.AddFunction(floor);
+}
+
 //===--------------------------------------------------------------------===//
 // round
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/function/scalar/math_functions.cpp

@@ -10,6 +10,7 @@ void BuiltinFunctions::RegisterMathFunctions() {
 	Register<CeilFun>();
 	Register<FloorFun>();
 	Register<RoundFun>();
+	Register<TruncFun>();
 
 	Register<DegreesFun>();
 	Register<RadiansFun>();

package/src/duckdb/src/function/scalar/string/hex.cpp

@@ -1,3 +1,4 @@
+#include "duckdb/common/bit_utils.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/common/types/blob.hpp"
@@ -7,6 +8,58 @@
 
 namespace duckdb {
 
+static void WriteHexBytes(uint64_t x, char *&output, idx_t buffer_size) {
+	idx_t offset = buffer_size * 4;
+
+	for (; offset >= 4; offset -= 4) {
+		uint8_t byte = (x >> (offset - 4)) & 0x0F;
+		*output = Blob::HEX_TABLE[byte];
+		output++;
+	}
+}
+
+static void WriteHugeIntHexBytes(hugeint_t x, char *&output, idx_t buffer_size) {
+	idx_t offset = buffer_size * 4;
+	auto upper = x.upper;
+	auto lower = x.lower;
+
+	for (; offset >= 68; offset -= 4) {
+		uint8_t byte = (upper >> (offset - 68)) & 0x0F;
+		*output = Blob::HEX_TABLE[byte];
+		output++;
+	}
+
+	for (; offset >= 4; offset -= 4) {
+		uint8_t byte = (lower >> (offset - 4)) & 0x0F;
+		*output = Blob::HEX_TABLE[byte];
+		output++;
+	}
+}
+
+static void WriteBinBytes(uint64_t x, char *&output, idx_t buffer_size) {
+	idx_t offset = buffer_size;
+	for (; offset >= 1; offset -= 1) {
+		*output = ((x >> (offset - 1)) & 0x01) + '0';
+		output++;
+	}
+}
+
+static void WriteHugeIntBinBytes(hugeint_t x, char *&output, idx_t buffer_size) {
+	auto upper = x.upper;
+	auto lower = x.lower;
+	idx_t offset = buffer_size;
+
+	for (; offset >= 65; offset -= 1) {
+		*output = ((upper >> (offset - 65)) & 0x01) + '0';
+		output++;
+	}
+
+	for (; offset >= 1; offset -= 1) {
+		*output = ((lower >> (offset - 1)) & 0x01) + '0';
+		output++;
+	}
+}
+
 struct HexStrOperator {
 	template <class INPUT_TYPE, class RESULT_TYPE>
 	static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) {
@@ -29,6 +82,149 @@ struct HexStrOperator {
 	}
 };
 
+struct HexIntegralOperator {
+	template <class INPUT_TYPE, class RESULT_TYPE>
+	static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) {
+
+		idx_t num_leading_zero = CountZeros<uint64_t>::Leading(input);
+		idx_t num_bits_to_check = 64 - num_leading_zero;
+		D_ASSERT(num_bits_to_check <= sizeof(INPUT_TYPE) * 8);
+
+		idx_t buffer_size = (num_bits_to_check + 3) / 4;
+
+		// Special case: All bits are zero
+		if (buffer_size == 0) {
+			auto target = StringVector::EmptyString(result, 1);
+			auto output = target.GetDataWriteable();
+			*output = '0';
+			target.Finalize();
+			return target;
+		}
+
+		D_ASSERT(buffer_size > 0);
+		auto target = StringVector::EmptyString(result, buffer_size);
+		auto output = target.GetDataWriteable();
+
+		WriteHexBytes(input, output, buffer_size);
+
+		target.Finalize();
+		return target;
+	}
+};
+
+struct HexHugeIntOperator {
+	template <class INPUT_TYPE, class RESULT_TYPE>
+	static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) {
+
+		idx_t num_leading_zero = CountZeros<hugeint_t>::Leading(input);
+		idx_t buffer_size = sizeof(INPUT_TYPE) * 2 - (num_leading_zero / 4);
+
+		// Special case: All bits are zero
+		if (buffer_size == 0) {
+			auto target = StringVector::EmptyString(result, 1);
+			auto output = target.GetDataWriteable();
+			*output = '0';
+			target.Finalize();
+			return target;
+		}
+
+		D_ASSERT(buffer_size > 0);
+		auto target = StringVector::EmptyString(result, buffer_size);
+		auto output = target.GetDataWriteable();
+
+		WriteHugeIntHexBytes(input, output, buffer_size);
+
+		target.Finalize();
+		return target;
+	}
+};
+
+template <class INPUT, class OP>
+static void ToHexFunction(DataChunk &args, ExpressionState &state, Vector &result) {
+	D_ASSERT(args.ColumnCount() == 1);
+	auto &input = args.data[0];
+	idx_t count = args.size();
+	UnaryExecutor::ExecuteString<INPUT, string_t, OP>(input, result, count);
+}
+
+struct BinaryStrOperator {
+	template <class INPUT_TYPE, class RESULT_TYPE>
+	static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) {
+		auto data = input.GetDataUnsafe();
+		auto size = input.GetSize();
+
+		// Allocate empty space
+		auto target = StringVector::EmptyString(result, size * 8);
+		auto output = target.GetDataWriteable();
+
+		for (idx_t i = 0; i < size; ++i) {
+			uint8_t byte = data[i];
+			for (idx_t i = 8; i >= 1; --i) {
+				*output = ((byte >> (i - 1)) & 0x01) + '0';
+				output++;
+			}
+		}
+
+		target.Finalize();
+		return target;
+	}
+};
+
+struct BinaryIntegralOperator {
+	template <class INPUT_TYPE, class RESULT_TYPE>
+	static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) {
+
+		idx_t num_leading_zero = CountZeros<uint64_t>::Leading(input);
+		idx_t num_bits_to_check = 64 - num_leading_zero;
+		D_ASSERT(num_bits_to_check <= sizeof(INPUT_TYPE) * 8);
+
+		idx_t buffer_size = num_bits_to_check;
+
+		// Special case: All bits are zero
+		if (buffer_size == 0) {
+			auto target = StringVector::EmptyString(result, 1);
+			auto output = target.GetDataWriteable();
+			*output = '0';
+			target.Finalize();
+			return target;
+		}
+
+		D_ASSERT(buffer_size > 0);
+		auto target = StringVector::EmptyString(result, buffer_size);
+		auto output = target.GetDataWriteable();
+
+		WriteBinBytes(input, output, buffer_size);
+
+		target.Finalize();
+		return target;
+	}
+};
+
+struct BinaryHugeIntOperator {
+	template <class INPUT_TYPE, class RESULT_TYPE>
+	static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) {
+		idx_t num_leading_zero = CountZeros<hugeint_t>::Leading(input);
+		idx_t buffer_size = sizeof(INPUT_TYPE) * 8 - num_leading_zero;
+
+		// Special case: All bits are zero
+		if (buffer_size == 0) {
+			auto target = StringVector::EmptyString(result, 1);
+			auto output = target.GetDataWriteable();
+			*output = '0';
+			target.Finalize();
+			return target;
+		}
+
+		auto target = StringVector::EmptyString(result, buffer_size);
+		auto output = target.GetDataWriteable();
+
+		WriteHugeIntBinBytes(input, output, buffer_size);
+
+		target.Finalize();
+		return target;
+	}
+};
+
 struct FromHexOperator {
 	template <class INPUT_TYPE, class RESULT_TYPE>
 	static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) {
@@ -66,103 +262,65 @@ struct FromHexOperator {
 	}
 };
 
-struct HexIntegralOperator {
+struct FromBinaryOperator {
 	template <class INPUT_TYPE, class RESULT_TYPE>
 	static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) {
-		// Sufficient space for maximum length
-		char buffer[sizeof(INPUT_TYPE) * 2];
-		char *ptr = buffer;
-		idx_t buffer_size = 0;
-
-		bool seen_non_zero = false;
-		for (idx_t offset = sizeof(INPUT_TYPE) * 8; offset >= 4; offset -= 4) {
-			uint8_t byte = (input >> (offset - 4)) & 0x0F;
-			if (byte == 0 && !seen_non_zero && offset > 4) {
-				continue;
-			}
-			seen_non_zero = true;
-			*ptr = Blob::HEX_TABLE[byte];
-			ptr++;
-			buffer_size++;
+		auto data = input.GetDataUnsafe();
+		auto size = input.GetSize();
+
+		if (size > NumericLimits<uint32_t>::Maximum()) {
+			throw InvalidInputException("Binary input length larger than 2^32 are not supported");
 		}
 
+		D_ASSERT(size <= NumericLimits<uint32_t>::Maximum());
+		auto buffer_size = (size + 7) / 8;
+
 		// Allocate empty space
 		auto target = StringVector::EmptyString(result, buffer_size);
 		auto output = target.GetDataWriteable();
-		memcpy(output, buffer, buffer_size);
-
-		target.Finalize();
-		return target;
-	}
-};
 
-struct HexHugeIntOperator {
-	template <class INPUT_TYPE, class RESULT_TYPE>
-	static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) {
-		char buffer[sizeof(INPUT_TYPE) * 2];
-		char *ptr = buffer;
-		idx_t buffer_size = 0;
-
-		uint64_t lower = input.lower;
-		int64_t upper = input.upper;
-
-		bool seen_non_zero = false;
-		for (idx_t offset = 64; offset >= 4; offset -= 4) {
-			uint8_t byte = (upper >> (offset - 4)) & 0x0F;
-
-			if (byte == 0 && !seen_non_zero) {
-				continue;
+		// Treated as a single byte
+		idx_t i = 0;
+		if (size % 8 != 0) {
+			uint8_t byte = 0;
+			for (idx_t j = size % 8; j > 0; --j) {
+				byte |= StringUtil::GetBinaryValue(data[i]) << (j - 1);
+				i++;
 			}
-			seen_non_zero = true;
-			*ptr = Blob::HEX_TABLE[byte];
-			ptr++;
-			buffer_size++;
+			*output = byte;
+			output++;
 		}
 
-		for (idx_t offset = 64; offset >= 4; offset -= 4) {
-			uint8_t byte = (lower >> (offset - 4)) & 0x0F;
-
-			// at least one byte space
-			if (byte == 0 && !seen_non_zero && offset > 4) {
-				continue;
+		while (i < size) {
+			uint8_t byte = 0;
+			for (idx_t j = 8; j > 0; --j) {
+				byte |= StringUtil::GetBinaryValue(data[i]) << (j - 1);
+				i++;
 			}
-			seen_non_zero = true;
-			*ptr = Blob::HEX_TABLE[byte];
-			ptr++;
-			buffer_size++;
+			*output = byte;
+			output++;
 		}
 
-		// Allocate empty space
-		auto target = StringVector::EmptyString(result, buffer_size);
-		auto output = target.GetDataWriteable();
-		memcpy(output, buffer, buffer_size);
-
 		target.Finalize();
 		return target;
 	}
 };
 
-static void ToHexFunction(DataChunk &args, ExpressionState &state, Vector &result) {
+template <class INPUT, class OP>
+static void ToBinaryFunction(DataChunk &args, ExpressionState &state, Vector &result) {
 	D_ASSERT(args.ColumnCount() == 1);
 	auto &input = args.data[0];
 	idx_t count = args.size();
+	UnaryExecutor::ExecuteString<INPUT, string_t, OP>(input, result, count);
+}
 
-	switch (input.GetType().InternalType()) {
-	case PhysicalType::VARCHAR:
-		UnaryExecutor::ExecuteString<string_t, string_t, HexStrOperator>(input, result, count);
-		break;
-	case PhysicalType::INT64:
-		UnaryExecutor::ExecuteString<int64_t, string_t, HexIntegralOperator>(input, result, count);
-		break;
-	case PhysicalType::INT128:
-		UnaryExecutor::ExecuteString<hugeint_t, string_t, HexHugeIntOperator>(input, result, count);
-		break;
-	case PhysicalType::UINT64:
-		UnaryExecutor::ExecuteString<uint64_t, string_t, HexIntegralOperator>(input, result, count);
-		break;
-	default:
-		throw NotImplementedException("Specifier type not implemented");
-	}
+static void FromBinaryFunction(DataChunk &args, ExpressionState &state, Vector &result) {
+	D_ASSERT(args.ColumnCount() == 1);
+	D_ASSERT(args.data[0].GetType().InternalType() == PhysicalType::VARCHAR);
+	auto &input = args.data[0];
+	idx_t count = args.size();
+
+	UnaryExecutor::ExecuteString<string_t, string_t, FromBinaryOperator>(input, result, count);
 }
 
 static void FromHexFunction(DataChunk &args, ExpressionState &state, Vector &result) {
@@ -178,13 +336,17 @@ void HexFun::RegisterFunction(BuiltinFunctions &set) {
 	ScalarFunctionSet to_hex("to_hex");
 	ScalarFunctionSet from_hex("from_hex");
 
-	to_hex.AddFunction(ScalarFunction({LogicalType::VARCHAR}, LogicalType::VARCHAR, ToHexFunction));
+	to_hex.AddFunction(
+	    ScalarFunction({LogicalType::VARCHAR}, LogicalType::VARCHAR, ToHexFunction<string_t, HexStrOperator>));
 
-	to_hex.AddFunction(ScalarFunction({LogicalType::BIGINT}, LogicalType::VARCHAR, ToHexFunction));
+	to_hex.AddFunction(
+	    ScalarFunction({LogicalType::BIGINT}, LogicalType::VARCHAR, ToHexFunction<int64_t, HexIntegralOperator>));
 
-	to_hex.AddFunction(ScalarFunction({LogicalType::UBIGINT}, LogicalType::VARCHAR, ToHexFunction));
+	to_hex.AddFunction(
+	    ScalarFunction({LogicalType::UBIGINT}, LogicalType::VARCHAR, ToHexFunction<uint64_t, HexIntegralOperator>));
 
-	to_hex.AddFunction(ScalarFunction({LogicalType::HUGEINT}, LogicalType::VARCHAR, ToHexFunction));
+	to_hex.AddFunction(
+	    ScalarFunction({LogicalType::HUGEINT}, LogicalType::VARCHAR, ToHexFunction<hugeint_t, HexHugeIntOperator>));
 
 	from_hex.AddFunction(ScalarFunction({LogicalType::VARCHAR}, LogicalType::BLOB, FromHexFunction));
 
@@ -196,6 +358,27 @@ void HexFun::RegisterFunction(BuiltinFunctions &set) {
 	from_hex.name = "unhex";
 	set.AddFunction(to_hex);
 	set.AddFunction(from_hex);
+
+	ScalarFunctionSet to_binary("to_binary");
+	ScalarFunctionSet from_binary("from_binary");
+	to_binary.AddFunction(
+	    ScalarFunction({LogicalType::VARCHAR}, LogicalType::VARCHAR, ToBinaryFunction<string_t, BinaryStrOperator>));
+	to_binary.AddFunction(ScalarFunction({LogicalType::UBIGINT}, LogicalType::VARCHAR,
+	                                     ToBinaryFunction<uint64_t, BinaryIntegralOperator>));
+	to_binary.AddFunction(
+	    ScalarFunction({LogicalType::BIGINT}, LogicalType::VARCHAR, ToBinaryFunction<int64_t, BinaryIntegralOperator>));
+	to_binary.AddFunction(ScalarFunction({LogicalType::HUGEINT}, LogicalType::VARCHAR,
+	                                     ToBinaryFunction<hugeint_t, BinaryHugeIntOperator>));
+
+	from_binary.AddFunction(ScalarFunction({LogicalType::VARCHAR}, LogicalType::BLOB, FromBinaryFunction));
+
+	set.AddFunction(to_binary);
+	set.AddFunction(from_binary);
+
+	to_binary.name = "bin";
+	from_binary.name = "unbin";
+	set.AddFunction(to_binary);
+	set.AddFunction(from_binary);
 }
 
 } // namespace duckdb

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.7.2-dev1734"
+#define DUCKDB_VERSION "0.7.2-dev1803"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "e8610c85fb"
+#define DUCKDB_SOURCE_ID "cc6160599b"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/common/bit_utils.hpp

@@ -0,0 +1,147 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/bit_utils.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/hugeint.hpp"
+
+#ifdef _MSC_VER
+#define __restrict__
+#define __BYTE_ORDER__          __ORDER_LITTLE_ENDIAN__
+#define __ORDER_LITTLE_ENDIAN__ 2
+#include <intrin.h>
+static inline int __builtin_ctzll(unsigned long long x) {
+#ifdef _WIN64
+	unsigned long ret;
+	_BitScanForward64(&ret, x);
+	return (int)ret;
+#else
+	unsigned long low, high;
+	bool low_set = _BitScanForward(&low, (unsigned __int32)(x)) != 0;
+	_BitScanForward(&high, (unsigned __int32)(x >> 32));
+	high += 32;
+	return low_set ? low : high;
+#endif
+}
+static inline int __builtin_clzll(unsigned long long mask) {
+	unsigned long where;
+// BitScanReverse scans from MSB to LSB for first set bit.
+// Returns 0 if no set bit is found.
+#if defined(_WIN64)
+	if (_BitScanReverse64(&where, mask))
+		return static_cast<int>(63 - where);
+#elif defined(_WIN32)
+	// Scan the high 32 bits.
+	if (_BitScanReverse(&where, static_cast<unsigned long>(mask >> 32)))
+		return static_cast<int>(63 - (where + 32)); // Create a bit offset from the MSB.
+	// Scan the low 32 bits.
+	if (_BitScanReverse(&where, static_cast<unsigned long>(mask)))
+		return static_cast<int>(63 - where);
+#else
+#error "Implementation of __builtin_clzll required"
+#endif
+	return 64; // Undefined Behavior.
+}
+
+static inline int __builtin_ctz(unsigned int value) {
+	unsigned long trailing_zero = 0;
+
+	if (_BitScanForward(&trailing_zero, value)) {
+		return trailing_zero;
+	} else {
+		// This is undefined, I better choose 32 than 0
+		return 32;
+	}
+}
+
+static inline int __builtin_clz(unsigned int value) {
+	unsigned long leading_zero = 0;
+
+	if (_BitScanReverse(&leading_zero, value)) {
+		return 31 - leading_zero;
+	} else {
+		// Same remarks as above
+		return 32;
+	}
+}
+
+#endif
+
+namespace duckdb {
+
+template <class T>
+struct CountZeros {};
+
+template <>
+struct CountZeros<uint32_t> {
+	inline static int Leading(uint32_t value) {
+		if (!value) {
+			return 32;
+		}
+		return __builtin_clz(value);
+	}
+	inline static int Trailing(uint32_t value) {
+		if (!value) {
+			return 32;
+		}
+		return __builtin_ctz(value);
+	}
+};
+
+template <>
+struct CountZeros<uint64_t> {
+	inline static int Leading(uint64_t value) {
+		if (!value) {
+			return 64;
+		}
+		return __builtin_clzll(value);
+	}
+	inline static int Trailing(uint64_t value) {
+		if (!value) {
+			return 64;
+		}
+		return __builtin_ctzll(value);
+	}
+};
+
+template <>
+struct CountZeros<hugeint_t> {
+	inline static int Leading(hugeint_t value) {
+		if (value == 0) {
+			return 128;
+		}
+
+		uint64_t upper = (uint64_t)value.upper;
+		uint64_t lower = value.lower;
+
+		int res = __builtin_clzll(upper);
+		if (res == 64) {
+			res += __builtin_clzll(lower);
+		}
+
+		return res;
+	}
+
+	inline static int Trailing(hugeint_t value) {
+		if (value == 0) {
+			return 128;
+		}
+
+		uint64_t upper = (uint64_t)value.upper;
+		uint64_t lower = value.lower;
+
+		int res = __builtin_ctzll(lower);
+		if (res == 64) {
+			res += __builtin_ctzll(upper);
+		}
+
+		return res;
+	}
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/hugeint.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "duckdb/common/winapi.hpp"
+#include "duckdb/common/string.hpp"
 #include <stdint.h>
 
 namespace duckdb {

package/src/duckdb/src/include/duckdb/common/string_util.hpp

@@ -38,6 +38,13 @@ public:
 		throw InvalidInputException("Invalid input for hex digit: %s", string(c, 1));
 	}
 
+	static uint8_t GetBinaryValue(char c) {
+		if (c >= '0' && c <= '1') {
+			return c - '0';
+		}
+		throw InvalidInputException("Invalid input for binary digit: %s", string(c, 1));
+	}
+
 	DUCKDB_API static bool CharacterIsSpace(char c) {
 		return c == ' ' || c == '\t' || c == '\n' || c == '\v' || c == '\f' || c == '\r';
 	}

package/src/duckdb/src/include/duckdb/common/types.hpp

@@ -387,7 +387,7 @@ public:
 	DUCKDB_API static LogicalType DECIMAL(int width, int scale);                 // NOLINT
 	DUCKDB_API static LogicalType VARCHAR_COLLATION(string collation);           // NOLINT
 	DUCKDB_API static LogicalType LIST(const LogicalType &child);                       // NOLINT
-	DUCKDB_API static LogicalType STRUCT(const child_list_t<LogicalType> &children);    // NOLINT
+	DUCKDB_API static LogicalType STRUCT(child_list_t<LogicalType> children);    // NOLINT
 	DUCKDB_API static LogicalType AGGREGATE_STATE(aggregate_state_t state_type);    // NOLINT
 	DUCKDB_API static LogicalType MAP(const LogicalType &child);				// NOLINT
 	DUCKDB_API static LogicalType MAP( child_list_t<LogicalType> children);       // NOLINT

package/src/duckdb/src/include/duckdb/function/scalar/math_functions.hpp

@@ -30,6 +30,10 @@ struct FloorFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
 
+struct TruncFun {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
 struct RoundFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };

package/src/duckdb/src/include/duckdb/storage/buffer/block_handle.hpp

@@ -24,8 +24,9 @@ enum class BlockState : uint8_t { BLOCK_UNLOADED = 0, BLOCK_LOADED = 1 };
 
 struct BufferPoolReservation {
 	idx_t size {0};
+	BufferPool &pool;
 
-	BufferPoolReservation() {
+	BufferPoolReservation(BufferPool &pool) : pool(pool) {
 	}
 	BufferPoolReservation(const BufferPoolReservation &) = delete;
 	BufferPoolReservation &operator=(const BufferPoolReservation &) = delete;
@@ -35,18 +36,17 @@ struct BufferPoolReservation {
 
 	~BufferPoolReservation();
 
-	void Resize(atomic<idx_t> &counter, idx_t new_size);
+	void Resize(idx_t new_size);
 	void Merge(BufferPoolReservation &&src);
 };
 
 struct TempBufferPoolReservation : BufferPoolReservation {
-	atomic<idx_t> &counter;
-	TempBufferPoolReservation(atomic<idx_t> &counter, idx_t size) : counter(counter) {
-		Resize(counter, size);
+	TempBufferPoolReservation(BufferPool &pool, idx_t size) : BufferPoolReservation(pool) {
+		Resize(size);
 	}
 	TempBufferPoolReservation(TempBufferPoolReservation &&) = default;
 	~TempBufferPoolReservation() {
-		Resize(counter, 0);
+		Resize(0);
 	}
 };
 

package/src/duckdb/src/include/duckdb/storage/buffer/buffer_pool.hpp

@@ -39,6 +39,8 @@ public:
 	//! blocks can be evicted
 	void SetLimit(idx_t limit, const char *exception_postscript);
 
+	void IncreaseUsedMemory(idx_t size);
+
 	idx_t GetUsedMemory();
 
 	idx_t GetMaxMemory();

package/src/duckdb/src/include/duckdb/storage/compression/chimp/algorithm/chimp128.hpp

@@ -17,6 +17,7 @@
 #include "duckdb/common/likely.hpp"
 #include "duckdb/storage/compression/chimp/algorithm/packed_data.hpp"
 #include "duckdb/common/limits.hpp"
+#include "duckdb/common/bit_utils.hpp"
 
 #include "duckdb/storage/compression/chimp/algorithm/bit_reader.hpp"
 #include "duckdb/storage/compression/chimp/algorithm/output_bit_stream.hpp"

package/src/duckdb/src/include/duckdb/storage/compression/chimp/algorithm/chimp_utils.hpp

@@ -10,68 +10,6 @@
 
 #include "duckdb.h"
 
-#ifdef _MSC_VER
-#define __restrict__
-#define __BYTE_ORDER__          __ORDER_LITTLE_ENDIAN__
-#define __ORDER_LITTLE_ENDIAN__ 2
-#include <intrin.h>
-static inline int __builtin_ctzll(unsigned long long x) {
-#ifdef _WIN64
-	unsigned long ret;
-	_BitScanForward64(&ret, x);
-	return (int)ret;
-#else
-	unsigned long low, high;
-	bool low_set = _BitScanForward(&low, (unsigned __int32)(x)) != 0;
-	_BitScanForward(&high, (unsigned __int32)(x >> 32));
-	high += 32;
-	return low_set ? low : high;
-#endif
-}
-static inline int __builtin_clzll(unsigned long long mask) {
-	unsigned long where;
-// BitScanReverse scans from MSB to LSB for first set bit.
-// Returns 0 if no set bit is found.
-#if defined(_WIN64)
-	if (_BitScanReverse64(&where, mask))
-		return static_cast<int>(63 - where);
-#elif defined(_WIN32)
-	// Scan the high 32 bits.
-	if (_BitScanReverse(&where, static_cast<unsigned long>(mask >> 32)))
-		return static_cast<int>(63 - (where + 32)); // Create a bit offset from the MSB.
-	// Scan the low 32 bits.
-	if (_BitScanReverse(&where, static_cast<unsigned long>(mask)))
-		return static_cast<int>(63 - where);
-#else
-#error "Implementation of __builtin_clzll required"
-#endif
-	return 64; // Undefined Behavior.
-}
-
-static inline int __builtin_ctz(unsigned int value) {
-	unsigned long trailing_zero = 0;
-
-	if (_BitScanForward(&trailing_zero, value)) {
-		return trailing_zero;
-	} else {
-		// This is undefined, I better choose 32 than 0
-		return 32;
-	}
-}
-
-static inline int __builtin_clz(unsigned int value) {
-	unsigned long leading_zero = 0;
-
-	if (_BitScanReverse(&leading_zero, value)) {
-		return 31 - leading_zero;
-	} else {
-		// Same remarks as above
-		return 32;
-	}
-}
-
-#endif
-
 namespace duckdb {
 
 template <class T>
@@ -89,41 +27,6 @@ struct SignificantBits<uint32_t> {
 	static constexpr uint8_t mask = ((uint8_t)1 << size) - 1;
 };
 
-template <class T>
-struct CountZeros {};
-
-template <>
-struct CountZeros<uint32_t> {
-	inline static int Leading(uint32_t value) {
-		if (!value) {
-			return 32;
-		}
-		return __builtin_clz(value);
-	}
-	inline static int Trailing(uint32_t value) {
-		if (!value) {
-			return 32;
-		}
-		return __builtin_ctz(value);
-	}
-};
-
-template <>
-struct CountZeros<uint64_t> {
-	inline static int Leading(uint64_t value) {
-		if (!value) {
-			return 64;
-		}
-		return __builtin_clzll(value);
-	}
-	inline static int Trailing(uint64_t value) {
-		if (!value) {
-			return 64;
-		}
-		return __builtin_ctzll(value);
-	}
-};
-
 struct ChimpConstants {
 	struct Compression {
 		static constexpr uint8_t LEADING_ROUND[] = {0,  0,  0,  0,  0,  0,  0,  0,  8,  8,  8,  8,  12, 12, 12, 12,

package/src/duckdb/src/include/duckdb/storage/compression/patas/algorithm/patas.hpp

@@ -14,6 +14,7 @@
 #include "duckdb/storage/compression/chimp/algorithm/chimp_utils.hpp"
 #include "duckdb/storage/compression/chimp/algorithm/packed_data.hpp"
 #include "duckdb/storage/compression/patas/shared.hpp"
+#include "duckdb/common/bit_utils.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp

@@ -44,17 +44,18 @@ unique_ptr<Expression> RegexOptimizationRule::Apply(LogicalOperator &op, vector<
 
 	if (pattern.Regexp()->op() == duckdb_re2::kRegexpLiteralString ||
 	    pattern.Regexp()->op() == duckdb_re2::kRegexpLiteral) {
-		auto contains = make_uniq<BoundFunctionExpression>(root.return_type, ContainsFun::GetFunction(),
-		                                                   std::move(root.children), nullptr);
 
 		string min;
 		string max;
-		pattern.PossibleMatchRange(&min, &max, patt_str.size());
-		if (min == max) {
-			contains->children[1] = make_uniq<BoundConstantExpression>(Value(std::move(min)));
-		} else {
-			contains->children[1] = make_uniq<BoundConstantExpression>(Value(std::move(patt_str)));
+		pattern.PossibleMatchRange(&min, &max, patt_str.size() + 1);
+		if (min != max) {
+			return nullptr;
 		}
+		auto parameter = make_uniq<BoundConstantExpression>(Value(std::move(min)));
+		auto contains = make_uniq<BoundFunctionExpression>(root.return_type, ContainsFun::GetFunction(),
+		                                                   std::move(root.children), nullptr);
+		contains->children[1] = std::move(parameter);
+
 		return std::move(contains);
 	}
 	return nullptr;

package/src/duckdb/src/storage/buffer/block_handle.cpp

@@ -9,7 +9,7 @@ namespace duckdb {
 
 BlockHandle::BlockHandle(BlockManager &block_manager, block_id_t block_id_p)
     : block_manager(block_manager), readers(0), block_id(block_id_p), buffer(nullptr), eviction_timestamp(0),
-      can_destroy(false), unswizzled(nullptr) {
+      can_destroy(false), memory_charge(block_manager.buffer_manager.GetBufferPool()), unswizzled(nullptr) {
 	eviction_timestamp = 0;
 	state = BlockState::BLOCK_UNLOADED;
 	memory_usage = Storage::BLOCK_ALLOC_SIZE;
@@ -18,7 +18,7 @@ BlockHandle::BlockHandle(BlockManager &block_manager, block_id_t block_id_p)
 BlockHandle::BlockHandle(BlockManager &block_manager, block_id_t block_id_p, unique_ptr<FileBuffer> buffer_p,
                          bool can_destroy_p, idx_t block_size, BufferPoolReservation &&reservation)
     : block_manager(block_manager), readers(0), block_id(block_id_p), eviction_timestamp(0), can_destroy(can_destroy_p),
-      unswizzled(nullptr) {
+      memory_charge(block_manager.buffer_manager.GetBufferPool()), unswizzled(nullptr) {
 	buffer = std::move(buffer_p);
 	state = BlockState::BLOCK_LOADED;
 	memory_usage = block_size;
@@ -34,7 +34,7 @@ BlockHandle::~BlockHandle() { // NOLINT: allow internal exceptions
 		D_ASSERT(memory_charge.size > 0);
 		// the block is still loaded in memory: erase it
 		buffer.reset();
-		memory_charge.Resize(buffer_manager.buffer_pool.current_memory, 0);
+		memory_charge.Resize(0);
 	} else {
 		D_ASSERT(memory_charge.size == 0);
 	}
@@ -97,7 +97,7 @@ unique_ptr<FileBuffer> BlockHandle::UnloadAndTakeBlock() {
 		// temporary block that cannot be destroyed: write to temporary file
 		block_manager.buffer_manager.WriteTemporaryBuffer(block_id, *buffer);
 	}
-	memory_charge.Resize(block_manager.buffer_manager.buffer_pool.current_memory, 0);
+	memory_charge.Resize(0);
 	state = BlockState::BLOCK_UNLOADED;
 	return std::move(buffer);
 }

package/src/duckdb/src/storage/buffer/buffer_pool.cpp

@@ -50,6 +50,10 @@ void BufferPool::AddToEvictionQueue(shared_ptr<BlockHandle> &handle) {
 	queue->q.enqueue(BufferEvictionNode(weak_ptr<BlockHandle>(handle), handle->eviction_timestamp));
 }
 
+void BufferPool::IncreaseUsedMemory(idx_t size) {
+	current_memory += size;
+}
+
 idx_t BufferPool::GetUsedMemory() {
 	return current_memory;
 }
@@ -60,12 +64,12 @@ idx_t BufferPool::GetMaxMemory() {
 BufferPool::EvictionResult BufferPool::EvictBlocks(idx_t extra_memory, idx_t memory_limit,
                                                    unique_ptr<FileBuffer> *buffer) {
 	BufferEvictionNode node;
-	TempBufferPoolReservation r(current_memory, extra_memory);
+	TempBufferPoolReservation r(*this, extra_memory);
 	while (current_memory > memory_limit) {
 		// get a block to unpin from the queue
 		if (!queue->q.try_dequeue(node)) {
 			// Failed to reserve. Adjust size of temp reservation to 0.
-			r.Resize(current_memory, 0);
+			r.Resize(0);
 			return {false, std::move(r)};
 		}
 		// get a reference to the underlying block pointer

package/src/duckdb/src/storage/buffer/buffer_pool_reservation.cpp

@@ -1,8 +1,9 @@
 #include "duckdb/storage/buffer/block_handle.hpp"
+#include "duckdb/storage/buffer/buffer_pool.hpp"
 
 namespace duckdb {
 
-BufferPoolReservation::BufferPoolReservation(BufferPoolReservation &&src) noexcept {
+BufferPoolReservation::BufferPoolReservation(BufferPoolReservation &&src) noexcept : pool(src.pool) {
 	size = src.size;
 	src.size = 0;
 }
@@ -17,10 +18,9 @@ BufferPoolReservation::~BufferPoolReservation() {
 	D_ASSERT(size == 0);
 }
 
-void BufferPoolReservation::Resize(atomic<idx_t> &counter, idx_t new_size) {
+void BufferPoolReservation::Resize(idx_t new_size) {
 	int64_t delta = (int64_t)new_size - size;
-	D_ASSERT(delta > 0 || (int64_t)counter >= -delta);
-	counter += delta;
+	pool.IncreaseUsedMemory(delta);
 	size = new_size;
 }
 

package/src/duckdb/src/storage/buffer_manager.cpp

@@ -131,7 +131,7 @@ void BufferManager::ReAllocate(shared_ptr<BlockHandle> &handle, idx_t block_size
 		handle->memory_charge.Merge(std::move(reservation));
 	} else {
 		// no need to evict blocks, but we do need to decrement 'current_memory'.
-		handle->memory_charge.Resize(buffer_pool.current_memory, req.alloc_size);
+		handle->memory_charge.Resize(req.alloc_size);
 	}
 
 	// resize and adjust current memory
@@ -163,7 +163,7 @@ BufferHandle BufferManager::Pin(shared_ptr<BlockHandle> &handle) {
 	if (handle->state == BlockState::BLOCK_LOADED) {
 		// the block is loaded, increment the reader count and return a pointer to the handle
 		handle->readers++;
-		reservation.Resize(buffer_pool.current_memory, 0);
+		reservation.Resize(0);
 		return handle->Load(handle);
 	}
 	// now we can actually load the current block
@@ -176,7 +176,7 @@ BufferHandle BufferManager::Pin(shared_ptr<BlockHandle> &handle) {
 	if (delta) {
 		D_ASSERT(delta < 0);
 		handle->memory_usage += delta;
-		handle->memory_charge.Resize(buffer_pool.current_memory, handle->memory_usage);
+		handle->memory_charge.Resize(handle->memory_usage);
 	}
 	D_ASSERT(handle->memory_usage == handle->buffer->AllocSize());
 	return buf;
@@ -731,9 +731,9 @@ data_ptr_t BufferManager::BufferAllocatorAllocate(PrivateAllocatorData *private_
 
 void BufferManager::BufferAllocatorFree(PrivateAllocatorData *private_data, data_ptr_t pointer, idx_t size) {
 	auto &data = (BufferAllocatorData &)*private_data;
-	BufferPoolReservation r;
+	BufferPoolReservation r(data.manager.GetBufferPool());
 	r.size = size;
-	r.Resize(data.manager.buffer_pool.current_memory, 0);
+	r.Resize(0);
 	return Allocator::Get(data.manager.db).FreeData(pointer, size);
 }
 
@@ -743,9 +743,9 @@ data_ptr_t BufferManager::BufferAllocatorRealloc(PrivateAllocatorData *private_d
 		return pointer;
 	}
 	auto &data = (BufferAllocatorData &)*private_data;
-	BufferPoolReservation r;
+	BufferPoolReservation r(data.manager.GetBufferPool());
 	r.size = old_size;
-	r.Resize(data.manager.buffer_pool.current_memory, size);
+	r.Resize(size);
 	r.size = 0;
 	return Allocator::Get(data.manager.db).ReallocateData(pointer, old_size, size);
 }