duckdb 0.8.2-dev2509.0 → 0.8.2-dev2669.0

package/binding.gyp

@@ -87,6 +87,7 @@
                 "src/duckdb/ub_src_function_scalar_string_regexp.cpp", 
                 "src/duckdb/ub_src_function_scalar_struct.cpp", 
                 "src/duckdb/ub_src_function_scalar_system.cpp", 
+                "src/duckdb/ub_src_function_table_arrow.cpp", 
                 "src/duckdb/ub_src_function_table.cpp", 
                 "src/duckdb/ub_src_function_table_system.cpp", 
                 "src/duckdb/ub_src_function_table_version.cpp", 

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.8.2-dev2509.0",
+  "version": "0.8.2-dev2669.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/extension/icu/icu-datepart.cpp

@@ -54,13 +54,13 @@ struct ICUDatePart : public ICUDateFunc {
 	}
 
 	static int64_t ExtractDayOfWeek(icu::Calendar *calendar, const uint64_t micros) {
-		calendar->setFirstDayOfWeek(UCAL_SUNDAY);
+		// [Sun(0), Sat(6)]
 		return ExtractField(calendar, UCAL_DAY_OF_WEEK) - UCAL_SUNDAY;
 	}
 
 	static int64_t ExtractISODayOfWeek(icu::Calendar *calendar, const uint64_t micros) {
-		calendar->setFirstDayOfWeek(UCAL_MONDAY);
-		return ExtractField(calendar, UCAL_DAY_OF_WEEK);
+		// [Mon(1), Sun(7)]
+		return 1 + (ExtractField(calendar, UCAL_DAY_OF_WEEK) + 7 - UCAL_MONDAY) % 7;
 	}
 
 	static int64_t ExtractWeek(icu::Calendar *calendar, const uint64_t micros) {

package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp

@@ -43,7 +43,7 @@ void AddDataTableIndex(DataTable &storage, const ColumnList &columns, const vect
 	// create an adaptive radix tree around the expressions
 	if (index_block) {
 		art = make_uniq<ART>(column_ids, TableIOManager::Get(storage), std::move(unbound_expressions), constraint_type,
-		                     storage.db, index_block->block_id, index_block->offset);
+		                     storage.db, nullptr, index_block->block_id, index_block->offset);
 	} else {
 		art = make_uniq<ART>(column_ids, TableIOManager::Get(storage), std::move(unbound_expressions), constraint_type,
 		                     storage.db);

package/src/duckdb/src/catalog/default/default_functions.cpp

@@ -98,6 +98,11 @@ static DefaultMacro internal_macros[] = {
 	{DEFAULT_SCHEMA, "fmod", {"x", "y", nullptr}, "(x-y*floor(x/y))"},
 	{DEFAULT_SCHEMA, "count_if", {"l", nullptr}, "sum(if(l, 1, 0))"},
 	{DEFAULT_SCHEMA, "split_part", {"string", "delimiter", "position", nullptr}, "coalesce(string_split(string, delimiter)[position],'')"},
+	{DEFAULT_SCHEMA, "geomean", {"x", nullptr}, "exp(avg(ln(x)))"},
+	{DEFAULT_SCHEMA, "geometric_mean", {"x", nullptr}, "geomean(x)"},
+
+    {DEFAULT_SCHEMA, "list_reverse", {"l", nullptr}, "l[:-:-1]"},
+    {DEFAULT_SCHEMA, "array_reverse", {"l", nullptr}, "list_reverse(l)"},
 
     // FIXME implement as actual function if we encounter a lot of performance issues. Complexity now: n * m, with hashing possibly n + m
     {DEFAULT_SCHEMA, "list_intersect", {"l1", "l2", nullptr}, "list_filter(l1, (x) -> list_contains(l2, x))"},

package/src/duckdb/src/common/enum_util.cpp

@@ -71,7 +71,7 @@
 #include "duckdb/function/macro_function.hpp"
 #include "duckdb/function/scalar/compressed_materialization_functions.hpp"
 #include "duckdb/function/scalar/strftime_format.hpp"
-#include "duckdb/function/table/arrow.hpp"
+#include "duckdb/function/table/arrow/arrow_duck_schema.hpp"
 #include "duckdb/main/appender.hpp"
 #include "duckdb/main/capi/capi_internal.hpp"
 #include "duckdb/main/config.hpp"
@@ -81,6 +81,7 @@
 #include "duckdb/parallel/interrupt.hpp"
 #include "duckdb/parallel/task.hpp"
 #include "duckdb/parser/constraint.hpp"
+#include "duckdb/parser/expression/parameter_expression.hpp"
 #include "duckdb/parser/expression/window_expression.hpp"
 #include "duckdb/parser/parsed_data/alter_info.hpp"
 #include "duckdb/parser/parsed_data/alter_scalar_function_info.hpp"
@@ -4404,6 +4405,39 @@ PragmaType EnumUtil::FromString<PragmaType>(const char *value) {
 	throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value));
 }
 
+template<>
+const char* EnumUtil::ToChars<PreparedParamType>(PreparedParamType value) {
+	switch(value) {
+	case PreparedParamType::AUTO_INCREMENT:
+		return "AUTO_INCREMENT";
+	case PreparedParamType::POSITIONAL:
+		return "POSITIONAL";
+	case PreparedParamType::NAMED:
+		return "NAMED";
+	case PreparedParamType::INVALID:
+		return "INVALID";
+	default:
+		throw NotImplementedException(StringUtil::Format("Enum value: '%d' not implemented", value));
+	}
+}
+
+template<>
+PreparedParamType EnumUtil::FromString<PreparedParamType>(const char *value) {
+	if (StringUtil::Equals(value, "AUTO_INCREMENT")) {
+		return PreparedParamType::AUTO_INCREMENT;
+	}
+	if (StringUtil::Equals(value, "POSITIONAL")) {
+		return PreparedParamType::POSITIONAL;
+	}
+	if (StringUtil::Equals(value, "NAMED")) {
+		return PreparedParamType::NAMED;
+	}
+	if (StringUtil::Equals(value, "INVALID")) {
+		return PreparedParamType::INVALID;
+	}
+	throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value));
+}
+
 template<>
 const char* EnumUtil::ToChars<ProfilerPrintFormat>(ProfilerPrintFormat value) {
 	switch(value) {

package/src/duckdb/src/core_functions/scalar/list/array_slice.cpp

@@ -308,6 +308,12 @@ static void ArraySliceFunction(DataChunk &args, ExpressionState &state, Vector &
 	auto count = args.size();
 
 	Vector &list_or_str_vector = args.data[0];
+	if (list_or_str_vector.GetType().id() == LogicalTypeId::SQLNULL) {
+		auto &result_validity = FlatVector::Validity(result);
+		result_validity.SetInvalid(0);
+		return;
+	}
+
 	Vector &begin_vector = args.data[1];
 	Vector &end_vector = args.data[2];
 

package/src/duckdb/src/execution/expression_executor/execute_parameter.cpp

@@ -15,8 +15,8 @@ void ExpressionExecutor::Execute(const BoundParameterExpression &expr, Expressio
                                  const SelectionVector *sel, idx_t count, Vector &result) {
 	D_ASSERT(expr.parameter_data);
 	D_ASSERT(expr.parameter_data->return_type == expr.return_type);
-	D_ASSERT(expr.parameter_data->value.type() == expr.return_type);
-	result.Reference(expr.parameter_data->value);
+	D_ASSERT(expr.parameter_data->GetValue().type() == expr.return_type);
+	result.Reference(expr.parameter_data->GetValue());
 }
 
 } // namespace duckdb

package/src/duckdb/src/execution/index/art/art.cpp

@@ -33,21 +33,27 @@ struct ARTIndexScanState : public IndexScanState {
 
 ART::ART(const vector<column_t> &column_ids, TableIOManager &table_io_manager,
          const vector<unique_ptr<Expression>> &unbound_expressions, const IndexConstraintType constraint_type,
-         AttachedDatabase &db, const idx_t block_id, const idx_t block_offset)
+         AttachedDatabase &db, const shared_ptr<vector<FixedSizeAllocator>> &allocators_ptr, const idx_t block_id,
+         const idx_t block_offset)
 
-    : Index(db, IndexType::ART, table_io_manager, column_ids, unbound_expressions, constraint_type) {
+    : Index(db, IndexType::ART, table_io_manager, column_ids, unbound_expressions, constraint_type),
+      allocators(allocators_ptr), owns_data(false) {
 
 	if (!Radix::IsLittleEndian()) {
 		throw NotImplementedException("ART indexes are not supported on big endian architectures");
 	}
 
 	// initialize all allocators
-	allocators.emplace_back(make_uniq<FixedSizeAllocator>(sizeof(Prefix), buffer_manager.GetBufferAllocator()));
-	allocators.emplace_back(make_uniq<FixedSizeAllocator>(sizeof(Leaf), buffer_manager.GetBufferAllocator()));
-	allocators.emplace_back(make_uniq<FixedSizeAllocator>(sizeof(Node4), buffer_manager.GetBufferAllocator()));
-	allocators.emplace_back(make_uniq<FixedSizeAllocator>(sizeof(Node16), buffer_manager.GetBufferAllocator()));
-	allocators.emplace_back(make_uniq<FixedSizeAllocator>(sizeof(Node48), buffer_manager.GetBufferAllocator()));
-	allocators.emplace_back(make_uniq<FixedSizeAllocator>(sizeof(Node256), buffer_manager.GetBufferAllocator()));
+	if (!allocators) {
+		owns_data = true;
+		allocators = make_shared<vector<FixedSizeAllocator>>();
+		allocators->emplace_back(FixedSizeAllocator(sizeof(Prefix), buffer_manager.GetBufferAllocator()));
+		allocators->emplace_back(FixedSizeAllocator(sizeof(Leaf), buffer_manager.GetBufferAllocator()));
+		allocators->emplace_back(FixedSizeAllocator(sizeof(Node4), buffer_manager.GetBufferAllocator()));
+		allocators->emplace_back(FixedSizeAllocator(sizeof(Node16), buffer_manager.GetBufferAllocator()));
+		allocators->emplace_back(FixedSizeAllocator(sizeof(Node48), buffer_manager.GetBufferAllocator()));
+		allocators->emplace_back(FixedSizeAllocator(sizeof(Node256), buffer_manager.GetBufferAllocator()));
+	}
 
 	// set the root node of the tree
 	tree = make_uniq<Node>();
@@ -986,26 +992,28 @@ BlockPointer ART::Serialize(MetaBlockWriter &writer) {
 
 void ART::InitializeVacuum(ARTFlags &flags) {
 
-	flags.vacuum_flags.reserve(allocators.size());
-	for (auto &allocator : allocators) {
-		flags.vacuum_flags.push_back(allocator->InitializeVacuum());
+	flags.vacuum_flags.reserve(allocators->size());
+	for (auto &allocator : *allocators) {
+		flags.vacuum_flags.push_back(allocator.InitializeVacuum());
 	}
 }
 
 void ART::FinalizeVacuum(const ARTFlags &flags) {
 
-	for (idx_t i = 0; i < allocators.size(); i++) {
+	for (idx_t i = 0; i < allocators->size(); i++) {
 		if (flags.vacuum_flags[i]) {
-			allocators[i]->FinalizeVacuum();
+			(*allocators)[i].FinalizeVacuum();
 		}
 	}
 }
 
 void ART::Vacuum(IndexLock &state) {
 
+	D_ASSERT(owns_data);
+
 	if (!tree->IsSet()) {
-		for (auto &allocator : allocators) {
-			allocator->Reset();
+		for (auto &allocator : *allocators) {
+			allocator.Reset();
 		}
 		return;
 	}
@@ -1032,8 +1040,8 @@ void ART::Vacuum(IndexLock &state) {
 	// finalize the vacuum operation
 	FinalizeVacuum(flags);
 
-	for (auto &allocator : allocators) {
-		allocator->Verify();
+	for (auto &allocator : *allocators) {
+		allocator.Verify();
 	}
 }
 
@@ -1043,9 +1051,11 @@ void ART::Vacuum(IndexLock &state) {
 
 void ART::InitializeMerge(ARTFlags &flags) {
 
-	flags.merge_buffer_counts.reserve(allocators.size());
-	for (auto &allocator : allocators) {
-		flags.merge_buffer_counts.emplace_back(allocator->buffers.size());
+	D_ASSERT(owns_data);
+
+	flags.merge_buffer_counts.reserve(allocators->size());
+	for (auto &allocator : *allocators) {
+		flags.merge_buffer_counts.emplace_back(allocator.buffers.size());
 	}
 }
 
@@ -1056,16 +1066,18 @@ bool ART::MergeIndexes(IndexLock &state, Index &other_index) {
 		return true;
 	}
 
-	if (tree->IsSet()) {
-		//  fully deserialize other_index, and traverse it to increment its buffer IDs
-		ARTFlags flags;
-		InitializeMerge(flags);
-		other_art.tree->InitializeMerge(other_art, flags);
-	}
+	if (other_art.owns_data) {
+		if (tree->IsSet()) {
+			//  fully deserialize other_index, and traverse it to increment its buffer IDs
+			ARTFlags flags;
+			InitializeMerge(flags);
+			other_art.tree->InitializeMerge(other_art, flags);
+		}
 
-	// merge the node storage
-	for (idx_t i = 0; i < allocators.size(); i++) {
-		allocators[i]->Merge(*other_art.allocators[i]);
+		// merge the node storage
+		for (idx_t i = 0; i < allocators->size(); i++) {
+			(*allocators)[i].Merge((*other_art.allocators)[i]);
+		}
 	}
 
 	// merge the ARTs
@@ -1073,8 +1085,8 @@ bool ART::MergeIndexes(IndexLock &state, Index &other_index) {
 		return false;
 	}
 
-	for (auto &allocator : allocators) {
-		allocator->Verify();
+	for (auto &allocator : *allocators) {
+		allocator.Verify();
 	}
 	return true;
 }

package/src/duckdb/src/execution/index/art/leaf.cpp

@@ -42,19 +42,30 @@ void Leaf::New(ART &art, reference<Node> &node, const row_t *row_ids, idx_t coun
 
 void Leaf::Free(ART &art, Node &node) {
 
-	D_ASSERT(node.IsSet() && !node.IsSerialized());
-	auto &child = Leaf::Get(art, node).ptr;
-	Node::Free(art, child);
+	Node current_node = node;
+	Node next_node;
+	while (current_node.IsSet() && !current_node.IsSerialized()) {
+		next_node = Leaf::Get(art, current_node).ptr;
+		Node::GetAllocator(art, NType::LEAF).Free(current_node);
+		current_node = next_node;
+	}
+
+	node.Reset();
 }
 
 void Leaf::InitializeMerge(ART &art, Node &node, const ARTFlags &flags) {
 
-	D_ASSERT(node.IsSet() && !node.IsSerialized());
-	D_ASSERT(node.GetType() == NType::LEAF);
+	auto merge_buffer_count = flags.merge_buffer_counts[(uint8_t)NType::LEAF - 1];
 
-	auto &leaf = Leaf::Get(art, node);
-	if (leaf.ptr.IsSet()) {
-		leaf.ptr.InitializeMerge(art, flags);
+	Node next_node = node;
+	node.AddToBufferID(merge_buffer_count);
+
+	while (next_node.IsSet()) {
+		auto &leaf = Leaf::Get(art, next_node);
+		next_node = leaf.ptr;
+		if (leaf.ptr.IsSet()) {
+			leaf.ptr.AddToBufferID(merge_buffer_count);
+		}
 	}
 }
 
@@ -290,7 +301,6 @@ string Leaf::VerifyAndToString(ART &art, Node &node) {
 		return "Leaf [count: 1, row ID: " + to_string(node.GetRowId()) + "]";
 	}
 
-	// NOTE: we could do this recursively, but the function-call overhead can become kinda crazy
 	string str = "";
 
 	reference<Node> node_ref(node);
@@ -322,46 +332,51 @@ BlockPointer Leaf::Serialize(ART &art, Node &node, MetaBlockWriter &writer) {
 		return block_pointer;
 	}
 
-	// recurse into the child and retrieve its block pointer
-	auto &leaf = Leaf::Get(art, node);
-	auto child_block_pointer = leaf.ptr.Serialize(art, writer);
-
-	// get pointer and write fields
 	auto block_pointer = writer.GetBlockPointer();
 	writer.Write(NType::LEAF);
-	writer.Write<uint8_t>(leaf.count);
+	idx_t total_count = Leaf::TotalCount(art, node);
+	writer.Write<idx_t>(total_count);
 
-	// write row IDs
-	for (idx_t i = 0; i < leaf.count; i++) {
-		writer.Write(leaf.row_ids[i]);
-	}
+	// iterate all leaves and write their row IDs
+	reference<Node> ref_node(node);
+	while (ref_node.get().IsSet()) {
+		D_ASSERT(!ref_node.get().IsSerialized());
+		auto &leaf = Leaf::Get(art, ref_node);
 
-	// write child block pointer
-	writer.Write(child_block_pointer.block_id);
-	writer.Write(child_block_pointer.offset);
+		// write row IDs
+		for (idx_t i = 0; i < leaf.count; i++) {
+			writer.Write(leaf.row_ids[i]);
+		}
+		ref_node = leaf.ptr;
+	}
 
 	return block_pointer;
 }
 
 void Leaf::Deserialize(ART &art, Node &node, MetaBlockReader &reader) {
 
-	D_ASSERT(node.GetType() == NType::LEAF);
+	auto total_count = reader.Read<idx_t>();
+	reference<Node> ref_node(node);
 
-	auto &leaf = Leaf::Get(art, node);
-	leaf.count = reader.Read<uint8_t>();
+	while (total_count) {
+		ref_node.get() = Node::GetAllocator(art, NType::LEAF).New();
+		ref_node.get().SetType((uint8_t)NType::LEAF);
 
-	// read row IDs
-	for (idx_t i = 0; i < leaf.count; i++) {
-		leaf.row_ids[i] = reader.Read<row_t>();
-	}
+		auto &leaf = Leaf::Get(art, ref_node);
+
+		leaf.count = MinValue((idx_t)Node::LEAF_SIZE, total_count);
+		for (idx_t i = 0; i < leaf.count; i++) {
+			leaf.row_ids[i] = reader.Read<row_t>();
+		}
 
-	// read child block pointer
-	leaf.ptr = Node(reader);
+		total_count -= leaf.count;
+		ref_node = leaf.ptr;
+		leaf.ptr.Reset();
+	}
 }
 
 void Leaf::Vacuum(ART &art, Node &node) {
 
-	// NOTE: we could do this recursively, but the function-call overhead can become kinda crazy
 	auto &allocator = Node::GetAllocator(art, NType::LEAF);
 
 	reference<Node> node_ref(node);
@@ -373,7 +388,6 @@ void Leaf::Vacuum(ART &art, Node &node) {
 		auto &leaf = Leaf::Get(art, node_ref);
 		node_ref = leaf.ptr;
 	}
-	return;
 }
 
 void Leaf::MoveInlinedToLeaf(ART &art, Node &node) {

package/src/duckdb/src/execution/index/art/node.cpp

@@ -61,7 +61,6 @@ void Node::New(ART &art, Node &node, const NType type) {
 
 void Node::Free(ART &art, Node &node) {
 
-	// recursively free all nodes that are in-memory, and skip serialized and empty nodes
 	if (!node.IsSet()) {
 		return;
 	}
@@ -72,11 +71,11 @@ void Node::Free(ART &art, Node &node) {
 		auto type = node.GetType();
 		switch (type) {
 		case NType::PREFIX:
-			Prefix::Free(art, node);
-			break;
+			// iterative
+			return Prefix::Free(art, node);
 		case NType::LEAF:
-			Leaf::Free(art, node);
-			break;
+			// iterative
+			return Leaf::Free(art, node);
 		case NType::NODE_4:
 			Node4::Free(art, node);
 			break;
@@ -90,8 +89,7 @@ void Node::Free(ART &art, Node &node) {
 			Node256::Free(art, node);
 			break;
 		case NType::LEAF_INLINED:
-			node.Reset();
-			return;
+			return node.Reset();
 		}
 
 		Node::GetAllocator(art, type).Free(node);
@@ -236,8 +234,10 @@ BlockPointer Node::Serialize(ART &art, MetaBlockWriter &writer) {
 
 	switch (GetType()) {
 	case NType::PREFIX:
-		return Prefix::Get(art, *this).Serialize(art, writer);
+		// iterative
+		return Prefix::Serialize(art, *this, writer);
 	case NType::LEAF:
+		// iterative
 		return Leaf::Serialize(art, *this, writer);
 	case NType::NODE_4:
 		return Node4::Get(art, *this).Serialize(art, writer);
@@ -263,19 +263,23 @@ void Node::Deserialize(ART &art) {
 	SetType(reader.Read<uint8_t>());
 
 	auto decoded_type = GetType();
+
+	// iterative functions
+	if (decoded_type == NType::PREFIX) {
+		return Prefix::Deserialize(art, *this, reader);
+	}
 	if (decoded_type == NType::LEAF_INLINED) {
-		SetRowId(reader.Read<row_t>());
-		return;
+		return SetRowId(reader.Read<row_t>());
+	}
+	if (decoded_type == NType::LEAF) {
+		return Leaf::Deserialize(art, *this, reader);
 	}
 
 	*this = Node::GetAllocator(art, decoded_type).New();
 	SetType((uint8_t)decoded_type);
 
+	// recursive functions
 	switch (decoded_type) {
-	case NType::PREFIX:
-		return Prefix::Get(art, *this).Deserialize(reader);
-	case NType::LEAF:
-		return Leaf::Deserialize(art, *this, reader);
 	case NType::NODE_4:
 		return Node4::Get(art, *this).Deserialize(reader);
 	case NType::NODE_16:
@@ -363,7 +367,7 @@ NType Node::GetARTNodeTypeByCount(const idx_t count) {
 }
 
 FixedSizeAllocator &Node::GetAllocator(const ART &art, NType type) {
-	return *art.allocators[(uint8_t)type - 1];
+	return (*art.allocators)[(uint8_t)type - 1];
 }
 
 //===--------------------------------------------------------------------===//
@@ -377,11 +381,11 @@ void Node::InitializeMerge(ART &art, const ARTFlags &flags) {
 
 	switch (GetType()) {
 	case NType::PREFIX:
-		Prefix::Get(art, *this).InitializeMerge(art, flags);
-		break;
+		// iterative
+		return Prefix::InitializeMerge(art, *this, flags);
 	case NType::LEAF:
-		Leaf::InitializeMerge(art, *this, flags);
-		break;
+		// iterative
+		return Leaf::InitializeMerge(art, *this, flags);
 	case NType::NODE_4:
 		Node4::Get(art, *this).InitializeMerge(art, flags);
 		break;
@@ -398,8 +402,7 @@ void Node::InitializeMerge(ART &art, const ARTFlags &flags) {
 		return;
 	}
 
-	// NOTE: this works because the rightmost 32 bits contain the buffer ID
-	data += flags.merge_buffer_counts[(uint8_t)GetType() - 1];
+	AddToBufferID(flags.merge_buffer_counts[(uint8_t)GetType() - 1]);
 }
 
 bool Node::Merge(ART &art, Node &other) {
@@ -572,11 +575,16 @@ void Node::Vacuum(ART &art, const ARTFlags &flags) {
 	}
 
 	auto node_type = GetType();
+
+	// iterative functions
+	if (node_type == NType::PREFIX) {
+		return Prefix::Vacuum(art, *this, flags);
+	}
 	if (node_type == NType::LEAF_INLINED) {
 		return;
 	}
 	if (node_type == NType::LEAF) {
-		if (flags.vacuum_flags[(uint8_t)GetType() - 1]) {
+		if (flags.vacuum_flags[(uint8_t)node_type - 1]) {
 			Leaf::Vacuum(art, *this);
 		}
 		return;
@@ -589,9 +597,8 @@ void Node::Vacuum(ART &art, const ARTFlags &flags) {
 		SetType((uint8_t)node_type);
 	}
 
+	// recursive functions
 	switch (node_type) {
-	case NType::PREFIX:
-		return Prefix::Get(art, *this).Vacuum(art, flags);
 	case NType::NODE_4:
 		return Node4::Get(art, *this).Vacuum(art, flags);
 	case NType::NODE_16: