duckdb 0.8.2-dev2068.0 → 0.8.2-dev2090.0

package/binding.gyp

@@ -14,6 +14,7 @@
                 "src/duckdb/ub_src_catalog_default.cpp", 
                 "src/duckdb/ub_src_common_adbc.cpp", 
                 "src/duckdb/ub_src_common.cpp", 
+                "src/duckdb/ub_src_common_arrow_appender.cpp", 
                 "src/duckdb/ub_src_common_arrow.cpp", 
                 "src/duckdb/ub_src_common_crypto.cpp", 
                 "src/duckdb/ub_src_common_enums.cpp", 

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.8.2-dev2068.0",
+  "version": "0.8.2-dev2090.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/common/arrow/appender/bool_data.cpp

@@ -0,0 +1,44 @@
+#include "duckdb/common/arrow/arrow_appender.hpp"
+#include "duckdb/common/arrow/appender/bool_data.hpp"
+
+namespace duckdb {
+
+void ArrowBoolData::Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity) {
+	auto byte_count = (capacity + 7) / 8;
+	result.main_buffer.reserve(byte_count);
+}
+
+void ArrowBoolData::Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
+	idx_t size = to - from;
+	UnifiedVectorFormat format;
+	input.ToUnifiedFormat(input_size, format);
+
+	// we initialize both the validity and the bit set to 1's
+	ResizeValidity(append_data.validity, append_data.row_count + size);
+	ResizeValidity(append_data.main_buffer, append_data.row_count + size);
+	auto data = UnifiedVectorFormat::GetData<bool>(format);
+
+	auto result_data = append_data.main_buffer.GetData<uint8_t>();
+	auto validity_data = append_data.validity.GetData<uint8_t>();
+	uint8_t current_bit;
+	idx_t current_byte;
+	GetBitPosition(append_data.row_count, current_byte, current_bit);
+	for (idx_t i = from; i < to; i++) {
+		auto source_idx = format.sel->get_index(i);
+		// append the validity mask
+		if (!format.validity.RowIsValid(source_idx)) {
+			SetNull(append_data, validity_data, current_byte, current_bit);
+		} else if (!data[source_idx]) {
+			UnsetBit(result_data, current_byte, current_bit);
+		}
+		NextBit(current_byte, current_bit);
+	}
+	append_data.row_count += size;
+}
+
+void ArrowBoolData::Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result) {
+	result->n_buffers = 2;
+	result->buffers[1] = append_data.main_buffer.data();
+}
+
+} // namespace duckdb

package/src/duckdb/src/common/arrow/appender/list_data.cpp

@@ -0,0 +1,78 @@
+#include "duckdb/common/arrow/arrow_appender.hpp"
+#include "duckdb/common/arrow/appender/list_data.hpp"
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// Lists
+//===--------------------------------------------------------------------===//
+void ArrowListData::AppendOffsets(ArrowAppendData &append_data, UnifiedVectorFormat &format, idx_t from, idx_t to,
+                                  vector<sel_t> &child_sel) {
+	// resize the offset buffer - the offset buffer holds the offsets into the child array
+	idx_t size = to - from;
+	append_data.main_buffer.resize(append_data.main_buffer.size() + sizeof(uint32_t) * (size + 1));
+	auto data = UnifiedVectorFormat::GetData<list_entry_t>(format);
+	auto offset_data = append_data.main_buffer.GetData<uint32_t>();
+	if (append_data.row_count == 0) {
+		// first entry
+		offset_data[0] = 0;
+	}
+	// set up the offsets using the list entries
+	auto last_offset = offset_data[append_data.row_count];
+	for (idx_t i = from; i < to; i++) {
+		auto source_idx = format.sel->get_index(i);
+		auto offset_idx = append_data.row_count + i + 1 - from;
+
+		if (!format.validity.RowIsValid(source_idx)) {
+			offset_data[offset_idx] = last_offset;
+			continue;
+		}
+
+		// append the offset data
+		auto list_length = data[source_idx].length;
+		last_offset += list_length;
+		offset_data[offset_idx] = last_offset;
+
+		for (idx_t k = 0; k < list_length; k++) {
+			child_sel.push_back(data[source_idx].offset + k);
+		}
+	}
+}
+
+void ArrowListData::Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity) {
+	auto &child_type = ListType::GetChildType(type);
+	result.main_buffer.reserve((capacity + 1) * sizeof(uint32_t));
+	auto child_buffer = ArrowAppender::InitializeChild(child_type, capacity, result.options);
+	result.child_data.push_back(std::move(child_buffer));
+}
+
+void ArrowListData::Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
+	UnifiedVectorFormat format;
+	input.ToUnifiedFormat(input_size, format);
+	idx_t size = to - from;
+	vector<sel_t> child_indices;
+	AppendValidity(append_data, format, from, to);
+	ArrowListData::AppendOffsets(append_data, format, from, to, child_indices);
+
+	// append the child vector of the list
+	SelectionVector child_sel(child_indices.data());
+	auto &child = ListVector::GetEntry(input);
+	auto child_size = child_indices.size();
+	Vector child_copy(child.GetType());
+	child_copy.Slice(child, child_sel, child_size);
+	append_data.child_data[0]->append_vector(*append_data.child_data[0], child_copy, 0, child_size, child_size);
+	append_data.row_count += size;
+}
+
+void ArrowListData::Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result) {
+	result->n_buffers = 2;
+	result->buffers[1] = append_data.main_buffer.data();
+
+	auto &child_type = ListType::GetChildType(type);
+	append_data.child_pointers.resize(1);
+	result->children = append_data.child_pointers.data();
+	result->n_children = 1;
+	append_data.child_pointers[0] = ArrowAppender::FinalizeChild(child_type, *append_data.child_data[0]);
+}
+
+} // namespace duckdb

package/src/duckdb/src/common/arrow/appender/map_data.cpp

@@ -0,0 +1,86 @@
+#include "duckdb/common/arrow/arrow_appender.hpp"
+#include "duckdb/common/arrow/appender/map_data.hpp"
+#include "duckdb/common/arrow/appender/list_data.hpp"
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// Maps
+//===--------------------------------------------------------------------===//
+void ArrowMapData::Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity) {
+	// map types are stored in a (too) clever way
+	// the main buffer holds the null values and the offsets
+	// then we have a single child, which is a struct of the map_type, and the key_type
+	result.main_buffer.reserve((capacity + 1) * sizeof(uint32_t));
+
+	auto &key_type = MapType::KeyType(type);
+	auto &value_type = MapType::ValueType(type);
+	auto internal_struct = make_uniq<ArrowAppendData>(result.options);
+	internal_struct->child_data.push_back(ArrowAppender::InitializeChild(key_type, capacity, result.options));
+	internal_struct->child_data.push_back(ArrowAppender::InitializeChild(value_type, capacity, result.options));
+
+	result.child_data.push_back(std::move(internal_struct));
+}
+
+void ArrowMapData::Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
+	UnifiedVectorFormat format;
+	input.ToUnifiedFormat(input_size, format);
+	idx_t size = to - from;
+	AppendValidity(append_data, format, from, to);
+	vector<sel_t> child_indices;
+	ArrowListData::AppendOffsets(append_data, format, from, to, child_indices);
+
+	SelectionVector child_sel(child_indices.data());
+	auto &key_vector = MapVector::GetKeys(input);
+	auto &value_vector = MapVector::GetValues(input);
+	auto list_size = child_indices.size();
+
+	auto &struct_data = *append_data.child_data[0];
+	auto &key_data = *struct_data.child_data[0];
+	auto &value_data = *struct_data.child_data[1];
+
+	Vector key_vector_copy(key_vector.GetType());
+	key_vector_copy.Slice(key_vector, child_sel, list_size);
+	Vector value_vector_copy(value_vector.GetType());
+	value_vector_copy.Slice(value_vector, child_sel, list_size);
+	key_data.append_vector(key_data, key_vector_copy, 0, list_size, list_size);
+	value_data.append_vector(value_data, value_vector_copy, 0, list_size, list_size);
+
+	append_data.row_count += size;
+	struct_data.row_count += size;
+}
+
+void ArrowMapData::Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result) {
+	// set up the main map buffer
+	result->n_buffers = 2;
+	result->buffers[1] = append_data.main_buffer.data();
+
+	// the main map buffer has a single child: a struct
+	append_data.child_pointers.resize(1);
+	result->children = append_data.child_pointers.data();
+	result->n_children = 1;
+	append_data.child_pointers[0] = ArrowAppender::FinalizeChild(type, *append_data.child_data[0]);
+
+	// now that struct has two children: the key and the value type
+	auto &struct_data = *append_data.child_data[0];
+	auto &struct_result = append_data.child_pointers[0];
+	struct_data.child_pointers.resize(2);
+	struct_result->n_buffers = 1;
+	struct_result->n_children = 2;
+	struct_result->length = struct_data.child_data[0]->row_count;
+	struct_result->children = struct_data.child_pointers.data();
+
+	D_ASSERT(struct_data.child_data[0]->row_count == struct_data.child_data[1]->row_count);
+
+	auto &key_type = MapType::KeyType(type);
+	auto &value_type = MapType::ValueType(type);
+	struct_data.child_pointers[0] = ArrowAppender::FinalizeChild(key_type, *struct_data.child_data[0]);
+	struct_data.child_pointers[1] = ArrowAppender::FinalizeChild(value_type, *struct_data.child_data[1]);
+
+	// keys cannot have null values
+	if (struct_data.child_pointers[0]->null_count > 0) {
+		throw std::runtime_error("Arrow doesn't accept NULL keys on Maps");
+	}
+}
+
+} // namespace duckdb

package/src/duckdb/src/common/arrow/appender/struct_data.cpp

@@ -0,0 +1,45 @@
+#include "duckdb/common/arrow/arrow_appender.hpp"
+#include "duckdb/common/arrow/appender/struct_data.hpp"
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// Structs
+//===--------------------------------------------------------------------===//
+void ArrowStructData::Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity) {
+	auto &children = StructType::GetChildTypes(type);
+	for (auto &child : children) {
+		auto child_buffer = ArrowAppender::InitializeChild(child.second, capacity, result.options);
+		result.child_data.push_back(std::move(child_buffer));
+	}
+}
+
+void ArrowStructData::Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
+	UnifiedVectorFormat format;
+	input.ToUnifiedFormat(input_size, format);
+	idx_t size = to - from;
+	AppendValidity(append_data, format, from, to);
+	// append the children of the struct
+	auto &children = StructVector::GetEntries(input);
+	for (idx_t child_idx = 0; child_idx < children.size(); child_idx++) {
+		auto &child = children[child_idx];
+		auto &child_data = *append_data.child_data[child_idx];
+		child_data.append_vector(child_data, *child, from, to, size);
+	}
+	append_data.row_count += size;
+}
+
+void ArrowStructData::Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result) {
+	result->n_buffers = 1;
+
+	auto &child_types = StructType::GetChildTypes(type);
+	append_data.child_pointers.resize(child_types.size());
+	result->children = append_data.child_pointers.data();
+	result->n_children = child_types.size();
+	for (idx_t i = 0; i < child_types.size(); i++) {
+		auto &child_type = child_types[i].second;
+		append_data.child_pointers[i] = ArrowAppender::FinalizeChild(child_type, *append_data.child_data[i]);
+	}
+}
+
+} // namespace duckdb

package/src/duckdb/src/common/arrow/appender/union_data.cpp

@@ -0,0 +1,70 @@
+#include "duckdb/common/arrow/arrow_appender.hpp"
+#include "duckdb/common/arrow/appender/union_data.hpp"
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// Unions
+//===--------------------------------------------------------------------===//
+void ArrowUnionData::Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity) {
+	result.main_buffer.reserve(capacity * sizeof(int8_t));
+
+	for (auto &child : UnionType::CopyMemberTypes(type)) {
+		auto child_buffer = ArrowAppender::InitializeChild(child.second, capacity, result.options);
+		result.child_data.push_back(std::move(child_buffer));
+	}
+}
+
+void ArrowUnionData::Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
+	UnifiedVectorFormat format;
+	input.ToUnifiedFormat(input_size, format);
+	idx_t size = to - from;
+
+	auto &types_buffer = append_data.main_buffer;
+
+	duckdb::vector<Vector> child_vectors;
+	for (const auto &child : UnionType::CopyMemberTypes(input.GetType())) {
+		child_vectors.emplace_back(child.second);
+	}
+
+	for (idx_t input_idx = from; input_idx < to; input_idx++) {
+		const auto &val = input.GetValue(input_idx);
+
+		idx_t tag = 0;
+		Value resolved_value(nullptr);
+		if (!val.IsNull()) {
+			tag = UnionValue::GetTag(val);
+
+			resolved_value = UnionValue::GetValue(val);
+		}
+
+		for (idx_t child_idx = 0; child_idx < child_vectors.size(); child_idx++) {
+			child_vectors[child_idx].SetValue(input_idx, child_idx == tag ? resolved_value : Value(nullptr));
+		}
+
+		types_buffer.data()[input_idx] = tag;
+	}
+
+	for (idx_t child_idx = 0; child_idx < child_vectors.size(); child_idx++) {
+		auto &child_buffer = append_data.child_data[child_idx];
+		auto &child = child_vectors[child_idx];
+		child_buffer->append_vector(*child_buffer, child, from, to, size);
+	}
+	append_data.row_count += size;
+}
+
+void ArrowUnionData::Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result) {
+	result->n_buffers = 2;
+	result->buffers[1] = append_data.main_buffer.data();
+
+	auto &child_types = UnionType::CopyMemberTypes(type);
+	append_data.child_pointers.resize(child_types.size());
+	result->children = append_data.child_pointers.data();
+	result->n_children = child_types.size();
+	for (idx_t i = 0; i < child_types.size(); i++) {
+		auto &child_type = child_types[i].second;
+		append_data.child_pointers[i] = ArrowAppender::FinalizeChild(child_type, *append_data.child_data[i]);
+	}
+}
+
+} // namespace duckdb