duckdb 0.8.2-dev2044.0 → 0.8.2-dev2090.0

package/binding.gyp

@@ -14,6 +14,7 @@
                 "src/duckdb/ub_src_catalog_default.cpp", 
                 "src/duckdb/ub_src_common_adbc.cpp", 
                 "src/duckdb/ub_src_common.cpp", 
+                "src/duckdb/ub_src_common_arrow_appender.cpp", 
                 "src/duckdb/ub_src_common_arrow.cpp", 
                 "src/duckdb/ub_src_common_crypto.cpp", 
                 "src/duckdb/ub_src_common_enums.cpp", 

package/configure.py

@@ -31,6 +31,7 @@ if os.environ.get('DUCKDB_NODE_BUILD_CACHE') == '1' and os.path.isfile(cache_fil
     windows_options = cache['windows_options']
     cflags = cache['cflags']
 elif 'DUCKDB_NODE_BINDIR' in os.environ:
+
     def find_library_path(libdir, libname):
         flist = os.listdir(libdir)
         for fname in flist:
@@ -38,6 +39,7 @@ elif 'DUCKDB_NODE_BINDIR' in os.environ:
             if os.path.isfile(fpath) and package_build.file_is_lib(fname, libname):
                 return fpath
         raise Exception(f"Failed to find library {libname} in {libdir}")
+
     # existing build
     existing_duckdb_dir = os.environ['DUCKDB_NODE_BINDIR']
     cflags = os.environ['DUCKDB_NODE_CFLAGS']
@@ -48,7 +50,7 @@ elif 'DUCKDB_NODE_BINDIR' in os.environ:
 
     result_libraries = package_build.get_libraries(existing_duckdb_dir, libraries, extensions)
     libraries = []
-    for (libdir, libname) in result_libraries:
+    for libdir, libname in result_libraries:
         if libdir is None:
             continue
         libraries.append(find_library_path(libdir, libname))
@@ -72,7 +74,7 @@ elif 'DUCKDB_NODE_BINDIR' in os.environ:
             'include_list': include_list,
             'libraries': libraries,
             'cflags': cflags,
-            'windows_options': windows_options
+            'windows_options': windows_options,
         }
         with open(cache_file, 'wb+') as f:
             pickle.dump(cache, f)
@@ -90,9 +92,11 @@ else:
     windows_options = ['/GR']
     cflags = ['-frtti']
 
+
 def sanitize_path(x):
     return x.replace('\\', '/')
 
+
 source_list = [sanitize_path(x) for x in source_list]
 include_list = [sanitize_path(x) for x in include_list]
 libraries = [sanitize_path(x) for x in libraries]
@@ -100,6 +104,7 @@ libraries = [sanitize_path(x) for x in libraries]
 with open(gyp_in, 'r') as f:
     input_json = json.load(f)
 
+
 def replace_entries(node, replacement_map):
     if type(node) == type([]):
         for key in replacement_map.keys():

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.8.2-dev2044.0",
+  "version": "0.8.2-dev2090.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/common/adbc/adbc.cpp

@@ -52,12 +52,14 @@ duckdb_adbc::AdbcStatusCode duckdb_adbc_init(size_t count, struct duckdb_adbc::A
 
 namespace duckdb_adbc {
 
+enum class IngestionMode { CREATE = 0, APPEND = 1 };
 struct DuckDBAdbcStatementWrapper {
 	::duckdb_connection connection;
 	::duckdb_arrow result;
 	::duckdb_prepared_statement statement;
 	char *ingestion_table_name;
 	ArrowArrayStream *ingestion_stream;
+	IngestionMode ingestion_mode = IngestionMode::CREATE;
 };
 static AdbcStatusCode QueryInternal(struct AdbcConnection *connection, struct ArrowArrayStream *out, const char *query,
                                     struct AdbcError *error);
@@ -428,7 +430,7 @@ void stream_schema(uintptr_t factory_ptr, duckdb::ArrowSchemaWrapper &schema) {
 }
 
 AdbcStatusCode Ingest(duckdb_connection connection, const char *table_name, struct ArrowArrayStream *input,
-                      struct AdbcError *error) {
+                      struct AdbcError *error, IngestionMode ingestion_mode) {
 
 	auto status = SetErrorMaybe(connection, error, "Invalid connection");
 	if (status != ADBC_STATUS_OK) {
@@ -446,12 +448,11 @@ AdbcStatusCode Ingest(duckdb_connection connection, const char *table_name, stru
 	}
 	auto cconn = (duckdb::Connection *)connection;
 
-	auto has_table = cconn->TableInfo(table_name);
 	auto arrow_scan = cconn->TableFunction("arrow_scan", {duckdb::Value::POINTER((uintptr_t)input),
 	                                                      duckdb::Value::POINTER((uintptr_t)stream_produce),
 	                                                      duckdb::Value::POINTER((uintptr_t)get_schema)});
 	try {
-		if (!has_table) {
+		if (ingestion_mode == IngestionMode::CREATE) {
 			// We create the table based on an Arrow Scanner
 			arrow_scan->Create(table_name);
 		} else {
@@ -505,6 +506,7 @@ AdbcStatusCode StatementNew(struct AdbcConnection *connection, struct AdbcStatem
 	statement_wrapper->result = nullptr;
 	statement_wrapper->ingestion_stream = nullptr;
 	statement_wrapper->ingestion_table_name = nullptr;
+	statement_wrapper->ingestion_mode = IngestionMode::CREATE;
 	return ADBC_STATUS_OK;
 }
 
@@ -557,7 +559,7 @@ AdbcStatusCode StatementExecuteQuery(struct AdbcStatement *statement, struct Arr
 	if (wrapper->ingestion_stream && wrapper->ingestion_table_name) {
 		auto stream = wrapper->ingestion_stream;
 		wrapper->ingestion_stream = nullptr;
-		return Ingest(wrapper->connection, wrapper->ingestion_table_name, stream, error);
+		return Ingest(wrapper->connection, wrapper->ingestion_table_name, stream, error, wrapper->ingestion_mode);
 	}
 
 	auto res = duckdb_execute_prepared_arrow(wrapper->statement, &wrapper->result);
@@ -643,6 +645,18 @@ AdbcStatusCode StatementSetOption(struct AdbcStatement *statement, const char *k
 		wrapper->ingestion_table_name = strdup(value);
 		return ADBC_STATUS_OK;
 	}
+	if (strcmp(key, ADBC_INGEST_OPTION_MODE) == 0) {
+		if (strcmp(value, ADBC_INGEST_OPTION_MODE_CREATE) == 0) {
+			wrapper->ingestion_mode = IngestionMode::CREATE;
+			return ADBC_STATUS_OK;
+		} else if (strcmp(value, ADBC_INGEST_OPTION_MODE_APPEND) == 0) {
+			wrapper->ingestion_mode = IngestionMode::APPEND;
+			return ADBC_STATUS_OK;
+		} else {
+			SetError(error, "Invalid ingestion mode");
+			return ADBC_STATUS_INVALID_ARGUMENT;
+		}
+	}
 	return ADBC_STATUS_INVALID_ARGUMENT;
 }
 

package/src/duckdb/src/common/arrow/appender/bool_data.cpp

@@ -0,0 +1,44 @@
+#include "duckdb/common/arrow/arrow_appender.hpp"
+#include "duckdb/common/arrow/appender/bool_data.hpp"
+
+namespace duckdb {
+
+void ArrowBoolData::Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity) {
+	auto byte_count = (capacity + 7) / 8;
+	result.main_buffer.reserve(byte_count);
+}
+
+void ArrowBoolData::Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
+	idx_t size = to - from;
+	UnifiedVectorFormat format;
+	input.ToUnifiedFormat(input_size, format);
+
+	// we initialize both the validity and the bit set to 1's
+	ResizeValidity(append_data.validity, append_data.row_count + size);
+	ResizeValidity(append_data.main_buffer, append_data.row_count + size);
+	auto data = UnifiedVectorFormat::GetData<bool>(format);
+
+	auto result_data = append_data.main_buffer.GetData<uint8_t>();
+	auto validity_data = append_data.validity.GetData<uint8_t>();
+	uint8_t current_bit;
+	idx_t current_byte;
+	GetBitPosition(append_data.row_count, current_byte, current_bit);
+	for (idx_t i = from; i < to; i++) {
+		auto source_idx = format.sel->get_index(i);
+		// append the validity mask
+		if (!format.validity.RowIsValid(source_idx)) {
+			SetNull(append_data, validity_data, current_byte, current_bit);
+		} else if (!data[source_idx]) {
+			UnsetBit(result_data, current_byte, current_bit);
+		}
+		NextBit(current_byte, current_bit);
+	}
+	append_data.row_count += size;
+}
+
+void ArrowBoolData::Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result) {
+	result->n_buffers = 2;
+	result->buffers[1] = append_data.main_buffer.data();
+}
+
+} // namespace duckdb

package/src/duckdb/src/common/arrow/appender/list_data.cpp

@@ -0,0 +1,78 @@
+#include "duckdb/common/arrow/arrow_appender.hpp"
+#include "duckdb/common/arrow/appender/list_data.hpp"
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// Lists
+//===--------------------------------------------------------------------===//
+void ArrowListData::AppendOffsets(ArrowAppendData &append_data, UnifiedVectorFormat &format, idx_t from, idx_t to,
+                                  vector<sel_t> &child_sel) {
+	// resize the offset buffer - the offset buffer holds the offsets into the child array
+	idx_t size = to - from;
+	append_data.main_buffer.resize(append_data.main_buffer.size() + sizeof(uint32_t) * (size + 1));
+	auto data = UnifiedVectorFormat::GetData<list_entry_t>(format);
+	auto offset_data = append_data.main_buffer.GetData<uint32_t>();
+	if (append_data.row_count == 0) {
+		// first entry
+		offset_data[0] = 0;
+	}
+	// set up the offsets using the list entries
+	auto last_offset = offset_data[append_data.row_count];
+	for (idx_t i = from; i < to; i++) {
+		auto source_idx = format.sel->get_index(i);
+		auto offset_idx = append_data.row_count + i + 1 - from;
+
+		if (!format.validity.RowIsValid(source_idx)) {
+			offset_data[offset_idx] = last_offset;
+			continue;
+		}
+
+		// append the offset data
+		auto list_length = data[source_idx].length;
+		last_offset += list_length;
+		offset_data[offset_idx] = last_offset;
+
+		for (idx_t k = 0; k < list_length; k++) {
+			child_sel.push_back(data[source_idx].offset + k);
+		}
+	}
+}
+
+void ArrowListData::Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity) {
+	auto &child_type = ListType::GetChildType(type);
+	result.main_buffer.reserve((capacity + 1) * sizeof(uint32_t));
+	auto child_buffer = ArrowAppender::InitializeChild(child_type, capacity, result.options);
+	result.child_data.push_back(std::move(child_buffer));
+}
+
+void ArrowListData::Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
+	UnifiedVectorFormat format;
+	input.ToUnifiedFormat(input_size, format);
+	idx_t size = to - from;
+	vector<sel_t> child_indices;
+	AppendValidity(append_data, format, from, to);
+	ArrowListData::AppendOffsets(append_data, format, from, to, child_indices);
+
+	// append the child vector of the list
+	SelectionVector child_sel(child_indices.data());
+	auto &child = ListVector::GetEntry(input);
+	auto child_size = child_indices.size();
+	Vector child_copy(child.GetType());
+	child_copy.Slice(child, child_sel, child_size);
+	append_data.child_data[0]->append_vector(*append_data.child_data[0], child_copy, 0, child_size, child_size);
+	append_data.row_count += size;
+}
+
+void ArrowListData::Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result) {
+	result->n_buffers = 2;
+	result->buffers[1] = append_data.main_buffer.data();
+
+	auto &child_type = ListType::GetChildType(type);
+	append_data.child_pointers.resize(1);
+	result->children = append_data.child_pointers.data();
+	result->n_children = 1;
+	append_data.child_pointers[0] = ArrowAppender::FinalizeChild(child_type, *append_data.child_data[0]);
+}
+
+} // namespace duckdb

package/src/duckdb/src/common/arrow/appender/map_data.cpp

@@ -0,0 +1,86 @@
+#include "duckdb/common/arrow/arrow_appender.hpp"
+#include "duckdb/common/arrow/appender/map_data.hpp"
+#include "duckdb/common/arrow/appender/list_data.hpp"
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// Maps
+//===--------------------------------------------------------------------===//
+void ArrowMapData::Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity) {
+	// map types are stored in a (too) clever way
+	// the main buffer holds the null values and the offsets
+	// then we have a single child, which is a struct of the map_type, and the key_type
+	result.main_buffer.reserve((capacity + 1) * sizeof(uint32_t));
+
+	auto &key_type = MapType::KeyType(type);
+	auto &value_type = MapType::ValueType(type);
+	auto internal_struct = make_uniq<ArrowAppendData>(result.options);
+	internal_struct->child_data.push_back(ArrowAppender::InitializeChild(key_type, capacity, result.options));
+	internal_struct->child_data.push_back(ArrowAppender::InitializeChild(value_type, capacity, result.options));
+
+	result.child_data.push_back(std::move(internal_struct));
+}
+
+void ArrowMapData::Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
+	UnifiedVectorFormat format;
+	input.ToUnifiedFormat(input_size, format);
+	idx_t size = to - from;
+	AppendValidity(append_data, format, from, to);
+	vector<sel_t> child_indices;
+	ArrowListData::AppendOffsets(append_data, format, from, to, child_indices);
+
+	SelectionVector child_sel(child_indices.data());
+	auto &key_vector = MapVector::GetKeys(input);
+	auto &value_vector = MapVector::GetValues(input);
+	auto list_size = child_indices.size();
+
+	auto &struct_data = *append_data.child_data[0];
+	auto &key_data = *struct_data.child_data[0];
+	auto &value_data = *struct_data.child_data[1];
+
+	Vector key_vector_copy(key_vector.GetType());
+	key_vector_copy.Slice(key_vector, child_sel, list_size);
+	Vector value_vector_copy(value_vector.GetType());
+	value_vector_copy.Slice(value_vector, child_sel, list_size);
+	key_data.append_vector(key_data, key_vector_copy, 0, list_size, list_size);
+	value_data.append_vector(value_data, value_vector_copy, 0, list_size, list_size);
+
+	append_data.row_count += size;
+	struct_data.row_count += size;
+}
+
+void ArrowMapData::Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result) {
+	// set up the main map buffer
+	result->n_buffers = 2;
+	result->buffers[1] = append_data.main_buffer.data();
+
+	// the main map buffer has a single child: a struct
+	append_data.child_pointers.resize(1);
+	result->children = append_data.child_pointers.data();
+	result->n_children = 1;
+	append_data.child_pointers[0] = ArrowAppender::FinalizeChild(type, *append_data.child_data[0]);
+
+	// now that struct has two children: the key and the value type
+	auto &struct_data = *append_data.child_data[0];
+	auto &struct_result = append_data.child_pointers[0];
+	struct_data.child_pointers.resize(2);
+	struct_result->n_buffers = 1;
+	struct_result->n_children = 2;
+	struct_result->length = struct_data.child_data[0]->row_count;
+	struct_result->children = struct_data.child_pointers.data();
+
+	D_ASSERT(struct_data.child_data[0]->row_count == struct_data.child_data[1]->row_count);
+
+	auto &key_type = MapType::KeyType(type);
+	auto &value_type = MapType::ValueType(type);
+	struct_data.child_pointers[0] = ArrowAppender::FinalizeChild(key_type, *struct_data.child_data[0]);
+	struct_data.child_pointers[1] = ArrowAppender::FinalizeChild(value_type, *struct_data.child_data[1]);
+
+	// keys cannot have null values
+	if (struct_data.child_pointers[0]->null_count > 0) {
+		throw std::runtime_error("Arrow doesn't accept NULL keys on Maps");
+	}
+}
+
+} // namespace duckdb

package/src/duckdb/src/common/arrow/appender/struct_data.cpp

@@ -0,0 +1,45 @@
+#include "duckdb/common/arrow/arrow_appender.hpp"
+#include "duckdb/common/arrow/appender/struct_data.hpp"
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// Structs
+//===--------------------------------------------------------------------===//
+void ArrowStructData::Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity) {
+	auto &children = StructType::GetChildTypes(type);
+	for (auto &child : children) {
+		auto child_buffer = ArrowAppender::InitializeChild(child.second, capacity, result.options);
+		result.child_data.push_back(std::move(child_buffer));
+	}
+}
+
+void ArrowStructData::Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
+	UnifiedVectorFormat format;
+	input.ToUnifiedFormat(input_size, format);
+	idx_t size = to - from;
+	AppendValidity(append_data, format, from, to);
+	// append the children of the struct
+	auto &children = StructVector::GetEntries(input);
+	for (idx_t child_idx = 0; child_idx < children.size(); child_idx++) {
+		auto &child = children[child_idx];
+		auto &child_data = *append_data.child_data[child_idx];
+		child_data.append_vector(child_data, *child, from, to, size);
+	}
+	append_data.row_count += size;
+}
+
+void ArrowStructData::Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result) {
+	result->n_buffers = 1;
+
+	auto &child_types = StructType::GetChildTypes(type);
+	append_data.child_pointers.resize(child_types.size());
+	result->children = append_data.child_pointers.data();
+	result->n_children = child_types.size();
+	for (idx_t i = 0; i < child_types.size(); i++) {
+		auto &child_type = child_types[i].second;
+		append_data.child_pointers[i] = ArrowAppender::FinalizeChild(child_type, *append_data.child_data[i]);
+	}
+}
+
+} // namespace duckdb

package/src/duckdb/src/common/arrow/appender/union_data.cpp

@@ -0,0 +1,70 @@
+#include "duckdb/common/arrow/arrow_appender.hpp"
+#include "duckdb/common/arrow/appender/union_data.hpp"
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// Unions
+//===--------------------------------------------------------------------===//
+void ArrowUnionData::Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity) {
+	result.main_buffer.reserve(capacity * sizeof(int8_t));
+
+	for (auto &child : UnionType::CopyMemberTypes(type)) {
+		auto child_buffer = ArrowAppender::InitializeChild(child.second, capacity, result.options);
+		result.child_data.push_back(std::move(child_buffer));
+	}
+}
+
+void ArrowUnionData::Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
+	UnifiedVectorFormat format;
+	input.ToUnifiedFormat(input_size, format);
+	idx_t size = to - from;
+
+	auto &types_buffer = append_data.main_buffer;
+
+	duckdb::vector<Vector> child_vectors;
+	for (const auto &child : UnionType::CopyMemberTypes(input.GetType())) {
+		child_vectors.emplace_back(child.second);
+	}
+
+	for (idx_t input_idx = from; input_idx < to; input_idx++) {
+		const auto &val = input.GetValue(input_idx);
+
+		idx_t tag = 0;
+		Value resolved_value(nullptr);
+		if (!val.IsNull()) {
+			tag = UnionValue::GetTag(val);
+
+			resolved_value = UnionValue::GetValue(val);
+		}
+
+		for (idx_t child_idx = 0; child_idx < child_vectors.size(); child_idx++) {
+			child_vectors[child_idx].SetValue(input_idx, child_idx == tag ? resolved_value : Value(nullptr));
+		}
+
+		types_buffer.data()[input_idx] = tag;
+	}
+
+	for (idx_t child_idx = 0; child_idx < child_vectors.size(); child_idx++) {
+		auto &child_buffer = append_data.child_data[child_idx];
+		auto &child = child_vectors[child_idx];
+		child_buffer->append_vector(*child_buffer, child, from, to, size);
+	}
+	append_data.row_count += size;
+}
+
+void ArrowUnionData::Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result) {
+	result->n_buffers = 2;
+	result->buffers[1] = append_data.main_buffer.data();
+
+	auto &child_types = UnionType::CopyMemberTypes(type);
+	append_data.child_pointers.resize(child_types.size());
+	result->children = append_data.child_pointers.data();
+	result->n_children = child_types.size();
+	for (idx_t i = 0; i < child_types.size(); i++) {
+		auto &child_type = child_types[i].second;
+		append_data.child_pointers[i] = ArrowAppender::FinalizeChild(child_type, *append_data.child_data[i]);
+	}
+}
+
+} // namespace duckdb