duckdb 0.8.2-dev5080.0 → 0.8.2-dev5120.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.8.2-dev5080.0",
+  "version": "0.8.2-dev5120.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/core_functions/aggregate/holistic/quantile.cpp

@@ -288,12 +288,40 @@ struct QuantileCompare {
 	}
 };
 
+//	Avoid using naked Values in inner loops...
+struct QuantileValue {
+	explicit QuantileValue(const Value &v) : val(v), dbl(v.GetValue<double>()) {
+		const auto &type = val.type();
+		switch (type.id()) {
+		case LogicalTypeId::DECIMAL: {
+			integral = IntegralValue::Get(v);
+			scaling = Hugeint::POWERS_OF_TEN[DecimalType::GetScale(type)];
+			break;
+		}
+		default:
+			break;
+		}
+	}
+
+	Value val;
+
+	//	DOUBLE
+	double dbl;
+
+	//	DECIMAL
+	hugeint_t integral;
+	hugeint_t scaling;
+};
+
+bool operator==(const QuantileValue &x, const QuantileValue &y) {
+	return x.val == y.val;
+}
+
 // Continuous interpolation
 template <bool DISCRETE>
 struct Interpolator {
-	Interpolator(const Value &q, const idx_t n_p, const bool desc_p)
-	    : desc(desc_p), RN((double)(n_p - 1) * q.GetValue<double>()), FRN(floor(RN)), CRN(ceil(RN)), begin(0),
-	      end(n_p) {
+	Interpolator(const QuantileValue &q, const idx_t n_p, const bool desc_p)
+	    : desc(desc_p), RN((double)(n_p - 1) * q.dbl), FRN(floor(RN)), CRN(ceil(RN)), begin(0), end(n_p) {
 	}
 
 	template <class INPUT_TYPE, class TARGET_TYPE, typename ACCESSOR = QuantileDirect<INPUT_TYPE>>
@@ -336,21 +364,20 @@ struct Interpolator {
 // Discrete "interpolation"
 template <>
 struct Interpolator<true> {
-	static inline idx_t Index(const Value &q, const idx_t n) {
+	static inline idx_t Index(const QuantileValue &q, const idx_t n) {
 		idx_t floored;
-		const auto &type = q.type();
-		switch (type.id()) {
+		switch (q.val.type().id()) {
 		case LogicalTypeId::DECIMAL: {
 			//	Integer arithmetic for accuracy
-			const auto integral = IntegralValue::Get(q);
-			const auto scaling = Hugeint::POWERS_OF_TEN[DecimalType::GetScale(type)];
+			const auto integral = q.integral;
+			const auto scaling = q.scaling;
 			const auto scaled_q = DecimalMultiplyOverflowCheck::Operation<hugeint_t, hugeint_t, hugeint_t>(n, integral);
 			const auto scaled_n = DecimalMultiplyOverflowCheck::Operation<hugeint_t, hugeint_t, hugeint_t>(n, scaling);
 			floored = Cast::Operation<hugeint_t, idx_t>((scaled_n - scaled_q) / scaling);
 			break;
 		}
 		default:
-			const auto scaled_q = (double)(n * q.GetValue<double>());
+			const auto scaled_q = (double)(n * q.dbl);
 			floored = floor(n - scaled_q);
 			break;
 		}
@@ -358,7 +385,7 @@ struct Interpolator<true> {
 		return MaxValue<idx_t>(1, n - floored) - 1;
 	}
 
-	Interpolator(const Value &q, const idx_t n_p, bool desc_p)
+	Interpolator(const QuantileValue &q, const idx_t n_p, bool desc_p)
 	    : desc(desc_p), FRN(Index(q, n_p)), CRN(FRN), begin(0), end(n_p) {
 	}
 
@@ -420,17 +447,18 @@ struct QuantileBindData : public FunctionData {
 	}
 
 	explicit QuantileBindData(const Value &quantile_p)
-	    : quantiles(1, QuantileAbs(quantile_p)), order(1, 0), desc(quantile_p < 0) {
+	    : quantiles(1, QuantileValue(QuantileAbs(quantile_p))), order(1, 0), desc(quantile_p < 0) {
 	}
 
 	explicit QuantileBindData(const vector<Value> &quantiles_p) {
+		vector<Value> normalised;
 		size_t pos = 0;
 		size_t neg = 0;
 		for (idx_t i = 0; i < quantiles_p.size(); ++i) {
 			const auto &q = quantiles_p[i];
 			pos += (q > 0);
 			neg += (q < 0);
-			quantiles.emplace_back(QuantileAbs(q));
+			normalised.emplace_back(QuantileAbs(q));
 			order.push_back(i);
 		}
 		if (pos && neg) {
@@ -438,8 +466,12 @@ struct QuantileBindData : public FunctionData {
 		}
 		desc = (neg > 0);
 
-		IndirectLess<Value> lt(quantiles.data());
+		IndirectLess<Value> lt(normalised.data());
 		std::sort(order.begin(), order.end(), lt);
+
+		for (const auto &q : normalised) {
+			quantiles.emplace_back(QuantileValue(q));
+		}
 	}
 
 	QuantileBindData(const QuantileBindData &other) : order(other.order), desc(other.desc) {
@@ -460,16 +492,24 @@ struct QuantileBindData : public FunctionData {
 	static void Serialize(Serializer &serializer, const optional_ptr<FunctionData> bind_data_p,
 	                      const AggregateFunction &function) {
 		auto &bind_data = bind_data_p->Cast<QuantileBindData>();
-		serializer.WriteProperty(100, "quantiles", bind_data.quantiles);
+		vector<Value> raw;
+		for (const auto &q : bind_data.quantiles) {
+			raw.emplace_back(q.val);
+		}
+		serializer.WriteProperty(100, "quantiles", raw);
 		serializer.WriteProperty(101, "order", bind_data.order);
 		serializer.WriteProperty(102, "desc", bind_data.desc);
 	}
 
 	static unique_ptr<FunctionData> Deserialize(Deserializer &deserializer, AggregateFunction &function) {
 		auto result = make_uniq<QuantileBindData>();
-		deserializer.ReadProperty(100, "quantiles", result->quantiles);
+		vector<Value> raw;
+		deserializer.ReadProperty(100, "quantiles", raw);
 		deserializer.ReadProperty(101, "order", result->order);
 		deserializer.ReadProperty(102, "desc", result->desc);
+		for (const auto &r : raw) {
+			result->quantiles.emplace_back(QuantileValue(r));
+		}
 		return std::move(result);
 	}
 
@@ -478,7 +518,7 @@ struct QuantileBindData : public FunctionData {
 		throw NotImplementedException("FIXME: serializing quantiles with decimals is not supported right now");
 	}
 
-	vector<Value> quantiles;
+	vector<QuantileValue> quantiles;
 	vector<idx_t> order;
 	bool desc;
 };
@@ -566,7 +606,7 @@ struct QuantileScalarOperation : public QuantileOperation {
 		auto &bind_data = aggr_input_data.bind_data->Cast<QuantileBindData>();
 
 		// Find the two positions needed
-		const auto q = bind_data.quantiles[0];
+		const auto &q = bind_data.quantiles[0];
 
 		bool replace = false;
 		if (frame.start == prev.start + 1 && frame.end == prev.end + 1) {
@@ -1041,7 +1081,11 @@ struct MedianAbsoluteDeviationOperation : public QuantileOperation {
 			return;
 		}
 		using SAVE_TYPE = typename STATE::SaveType;
-		Interpolator<false> interp(0.5, state.v.size(), false);
+		D_ASSERT(finalize_data.input.bind_data);
+		auto &bind_data = finalize_data.input.bind_data->Cast<QuantileBindData>();
+		D_ASSERT(bind_data.quantiles.size() == 1);
+		const auto &q = bind_data.quantiles[0];
+		Interpolator<false> interp(q, state.v.size(), false);
 		const auto med = interp.template Operation<SAVE_TYPE, MEDIAN_TYPE>(state.v.data(), finalize_data.result);
 
 		MadAccessor<SAVE_TYPE, T, MEDIAN_TYPE> accessor(med);
@@ -1050,8 +1094,8 @@ struct MedianAbsoluteDeviationOperation : public QuantileOperation {
 
 	template <class STATE, class INPUT_TYPE, class RESULT_TYPE>
 	static void Window(const INPUT_TYPE *data, const ValidityMask &fmask, const ValidityMask &dmask,
-	                   AggregateInputData &, STATE &state, const FrameBounds &frame, const FrameBounds &prev,
-	                   Vector &result, idx_t ridx, idx_t bias) {
+	                   AggregateInputData &aggr_input_data, STATE &state, const FrameBounds &frame,
+	                   const FrameBounds &prev, Vector &result, idx_t ridx, idx_t bias) {
 		auto rdata = FlatVector::GetData<RESULT_TYPE>(result);
 		auto &rmask = FlatVector::Validity(result);
 
@@ -1079,7 +1123,10 @@ struct MedianAbsoluteDeviationOperation : public QuantileOperation {
 		std::partition(index2, index2 + state.pos, included);
 
 		// Find the two positions needed for the median
-		const float q = 0.5;
+		D_ASSERT(aggr_input_data.bind_data);
+		auto &bind_data = aggr_input_data.bind_data->Cast<QuantileBindData>();
+		D_ASSERT(bind_data.quantiles.size() == 1);
+		const auto &q = bind_data.quantiles[0];
 
 		bool replace = false;
 		if (frame.start == prev.start + 1 && frame.end == prev.end + 1) {
@@ -1124,12 +1171,18 @@ struct MedianAbsoluteDeviationOperation : public QuantileOperation {
 	}
 };
 
+unique_ptr<FunctionData> BindMedian(ClientContext &context, AggregateFunction &function,
+                                    vector<unique_ptr<Expression>> &arguments) {
+	return make_uniq<QuantileBindData>(Value::DECIMAL(int16_t(5), 2, 1));
+}
+
 template <typename INPUT_TYPE, typename MEDIAN_TYPE, typename TARGET_TYPE>
 AggregateFunction GetTypedMedianAbsoluteDeviationAggregateFunction(const LogicalType &input_type,
                                                                    const LogicalType &target_type) {
 	using STATE = QuantileState<INPUT_TYPE>;
 	using OP = MedianAbsoluteDeviationOperation<MEDIAN_TYPE>;
 	auto fun = AggregateFunction::UnaryAggregateDestructor<STATE, INPUT_TYPE, TARGET_TYPE, OP>(input_type, target_type);
+	fun.bind = BindMedian;
 	fun.order_dependent = AggregateOrderDependent::NOT_ORDER_DEPENDENT;
 	fun.window = AggregateFunction::UnaryWindow<STATE, INPUT_TYPE, TARGET_TYPE, OP>;
 	return fun;
@@ -1173,11 +1226,6 @@ AggregateFunction GetMedianAbsoluteDeviationAggregateFunction(const LogicalType
 	}
 }
 
-unique_ptr<FunctionData> BindMedian(ClientContext &context, AggregateFunction &function,
-                                    vector<unique_ptr<Expression>> &arguments) {
-	return make_uniq<QuantileBindData>(Value::DECIMAL(int16_t(5), 2, 1));
-}
-
 unique_ptr<FunctionData> BindMedianDecimal(ClientContext &context, AggregateFunction &function,
                                            vector<unique_ptr<Expression>> &arguments) {
 	auto bind_data = BindMedian(context, function, arguments);
@@ -1195,7 +1243,7 @@ unique_ptr<FunctionData> BindMedianAbsoluteDeviationDecimal(ClientContext &conte
 	function = GetMedianAbsoluteDeviationAggregateFunction(arguments[0]->return_type);
 	function.name = "mad";
 	function.order_dependent = AggregateOrderDependent::NOT_ORDER_DEPENDENT;
-	return nullptr;
+	return BindMedian(context, function, arguments);
 }
 
 static const Value &CheckQuantile(const Value &quantile_val) {

package/src/duckdb/src/core_functions/function_list.cpp

@@ -281,7 +281,7 @@ static StaticFunctionDefinition internal_functions[] = {
 	DUCKDB_SCALAR_FUNCTION(RightFun),
 	DUCKDB_SCALAR_FUNCTION(RightGraphemeFun),
 	DUCKDB_SCALAR_FUNCTION_SET(RoundFun),
-	DUCKDB_SCALAR_FUNCTION_ALIAS(RowFun),
+	DUCKDB_SCALAR_FUNCTION(RowFun),
 	DUCKDB_SCALAR_FUNCTION(RpadFun),
 	DUCKDB_SCALAR_FUNCTION_SET(RtrimFun),
 	DUCKDB_SCALAR_FUNCTION_SET(SecondsFun),

package/src/duckdb/src/core_functions/scalar/struct/struct_pack.cpp

@@ -30,6 +30,7 @@ static void StructPackFunction(DataChunk &args, ExpressionState &state, Vector &
 	result.Verify(args.size());
 }
 
+template <bool IS_STRUCT_PACK>
 static unique_ptr<FunctionData> StructPackBind(ClientContext &context, ScalarFunction &bound_function,
                                                vector<unique_ptr<Expression>> &arguments) {
 	case_insensitive_set_t name_collision_set;
@@ -39,25 +40,20 @@ static unique_ptr<FunctionData> StructPackBind(ClientContext &context, ScalarFun
 		throw Exception("Can't pack nothing into a struct");
 	}
 	child_list_t<LogicalType> struct_children;
-	bool unnamed = false;
 	for (idx_t i = 0; i < arguments.size(); i++) {
 		auto &child = arguments[i];
-		if (child->alias.empty()) {
-			if (bound_function.name == "struct_pack") {
+		string alias;
+		if (IS_STRUCT_PACK) {
+			if (child->alias.empty()) {
 				throw BinderException("Need named argument for struct pack, e.g. STRUCT_PACK(a := b)");
-			} else {
-				D_ASSERT(bound_function.name == "row");
-				if (i > 1) {
-					D_ASSERT(unnamed);
-				}
-				unnamed = true;
 			}
+			alias = child->alias;
+			if (name_collision_set.find(alias) != name_collision_set.end()) {
+				throw BinderException("Duplicate struct entry name \"%s\"", alias);
+			}
+			name_collision_set.insert(alias);
 		}
-		if (!child->alias.empty() && name_collision_set.find(child->alias) != name_collision_set.end()) {
-			throw BinderException("Duplicate struct entry name \"%s\"", child->alias);
-		}
-		name_collision_set.insert(child->alias);
-		struct_children.push_back(make_pair(child->alias, arguments[i]->return_type));
+		struct_children.push_back(make_pair(alias, arguments[i]->return_type));
 	}
 
 	// this is more for completeness reasons
@@ -75,10 +71,10 @@ unique_ptr<BaseStatistics> StructPackStats(ClientContext &context, FunctionStati
 	return struct_stats.ToUnique();
 }
 
-ScalarFunction StructPackFun::GetFunction() {
-	// the arguments and return types are actually set in the binder function
-	ScalarFunction fun("struct_pack", {}, LogicalTypeId::STRUCT, StructPackFunction, StructPackBind, nullptr,
-	                   StructPackStats);
+template <bool IS_STRUCT_PACK>
+ScalarFunction GetStructPackFunction() {
+	ScalarFunction fun(IS_STRUCT_PACK ? "struct_pack" : "row", {}, LogicalTypeId::STRUCT, StructPackFunction,
+	                   StructPackBind<IS_STRUCT_PACK>, nullptr, StructPackStats);
 	fun.varargs = LogicalType::ANY;
 	fun.null_handling = FunctionNullHandling::SPECIAL_HANDLING;
 	fun.serialize = VariableReturnBindData::Serialize;
@@ -86,4 +82,12 @@ ScalarFunction StructPackFun::GetFunction() {
 	return fun;
 }
 
+ScalarFunction StructPackFun::GetFunction() {
+	return GetStructPackFunction<true>();
+}
+
+ScalarFunction RowFun::GetFunction() {
+	return GetStructPackFunction<false>();
+}
+
 } // namespace duckdb

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.8.2-dev5080"
+#define DUCKDB_VERSION "0.8.2-dev5120"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "e07b94a748"
+#define DUCKDB_SOURCE_ID "fc2e4b26a6"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp

@@ -34,9 +34,12 @@ struct StructPackFun {
 };
 
 struct RowFun {
-	using ALIAS = StructPackFun;
-
 	static constexpr const char *Name = "row";
+	static constexpr const char *Parameters = "any";
+	static constexpr const char *Description = "Creates an unnamed STRUCT containing the argument values.";
+	static constexpr const char *Example = "row(4, 'hello')";
+
+	static ScalarFunction GetFunction();
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/config.hpp

@@ -39,6 +39,7 @@ class CompressionFunction;
 class TableFunctionRef;
 class OperatorExtension;
 class StorageExtension;
+class ExtensionCallback;
 
 struct CompressionFunctionSet;
 struct DBConfig;
@@ -212,6 +213,8 @@ public:
 	case_insensitive_map_t<duckdb::unique_ptr<StorageExtension>> storage_extensions;
 	//! A buffer pool can be shared across multiple databases (if desired).
 	shared_ptr<BufferPool> buffer_pool;
+	//! Set of callbacks that can be installed by extensions
+	vector<unique_ptr<ExtensionCallback>> extension_callbacks;
 
 public:
 	DUCKDB_API static DBConfig &GetConfig(ClientContext &context);

package/src/duckdb/src/include/duckdb/planner/extension_callback.hpp

@@ -0,0 +1,26 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/planner/extension_callback.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+
+namespace duckdb {
+class DatabaseInstance;
+
+class ExtensionCallback {
+public:
+	virtual ~ExtensionCallback() {
+	}
+
+	//! Called after an extension is finished loading
+	virtual void OnExtensionLoaded(DatabaseInstance &db, const string &name) {
+	}
+};
+
+} // namespace duckdb

package/src/duckdb/src/main/database.cpp

@@ -19,6 +19,7 @@
 #include "duckdb/storage/storage_extension.hpp"
 #include "duckdb/storage/storage_manager.hpp"
 #include "duckdb/transaction/transaction_manager.hpp"
+#include "duckdb/planner/extension_callback.hpp"
 
 #ifndef DUCKDB_NO_THREADS
 #include "duckdb/common/thread.hpp"
@@ -381,6 +382,11 @@ bool DuckDB::ExtensionIsLoaded(const std::string &name) {
 void DatabaseInstance::SetExtensionLoaded(const std::string &name) {
 	auto extension_name = ExtensionHelper::GetExtensionName(name);
 	loaded_extensions.insert(extension_name);
+
+	auto &callbacks = DBConfig::GetConfig(*this).extension_callbacks;
+	for (auto &callback : callbacks) {
+		callback->OnExtensionLoaded(*this, name);
+	}
 }
 
 bool DatabaseInstance::TryGetCurrentSetting(const std::string &key, Value &result) {

package/src/duckdb/src/main/settings/settings.cpp

@@ -21,6 +21,10 @@ namespace duckdb {
 // Access Mode
 //===--------------------------------------------------------------------===//
 void AccessModeSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
+	if (db) {
+		throw InvalidInputException("Cannot change access_mode setting while database is running - it must be set when "
+		                            "opening or attaching the database");
+	}
 	auto parameter = StringUtil::Lower(input.ToString());
 	if (parameter == "automatic") {
 		config.options.access_mode = AccessMode::AUTOMATIC;

package/src/duckdb/src/storage/single_file_block_manager.cpp

@@ -8,6 +8,7 @@
 #include "duckdb/storage/metadata/metadata_writer.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/main/config.hpp"
+#include "duckdb/main/database.hpp"
 
 #include <algorithm>
 #include <cstring>
@@ -16,12 +17,22 @@ namespace duckdb {
 
 const char MainHeader::MAGIC_BYTES[] = "DUCK";
 
+void SerializeVersionNumber(WriteStream &ser, const string &version_str) {
+	constexpr const idx_t MAX_VERSION_SIZE = 32;
+	data_t version[MAX_VERSION_SIZE];
+	memset(version, 0, MAX_VERSION_SIZE);
+	memcpy(version, version_str.c_str(), MinValue<idx_t>(version_str.size(), MAX_VERSION_SIZE));
+	ser.WriteData(version, MAX_VERSION_SIZE);
+}
+
 void MainHeader::Write(WriteStream &ser) {
 	ser.WriteData(const_data_ptr_cast(MAGIC_BYTES), MAGIC_BYTE_SIZE);
 	ser.Write<uint64_t>(version_number);
 	for (idx_t i = 0; i < FLAG_COUNT; i++) {
 		ser.Write<uint64_t>(flags[i]);
 	}
+	SerializeVersionNumber(ser, DuckDB::LibraryVersion());
+	SerializeVersionNumber(ser, DuckDB::SourceID());
 }
 
 void MainHeader::CheckMagicBytes(FileHandle &handle) {