duckdb 0.3.3-dev933.0 → 0.3.3-dev957.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.3-dev933.0",
+  "version": "0.3.3-dev957.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -4710,11 +4710,13 @@ data_ptr_t Allocator::ReallocateData(data_ptr_t pointer, idx_t size) {
 namespace duckdb {
 class ResultArrowArrayStreamWrapper {
 public:
-	explicit ResultArrowArrayStreamWrapper(unique_ptr<QueryResult> result, idx_t approx_batch_size);
+	explicit ResultArrowArrayStreamWrapper(unique_ptr<QueryResult> result, idx_t batch_size);
 	ArrowArrayStream stream;
 	unique_ptr<QueryResult> result;
 	std::string last_error;
-	idx_t vectors_per_chunk;
+	idx_t batch_size;
+	vector<LogicalType> column_types;
+	vector<string> column_names;
 
 private:
 	static int MyStreamGetSchema(struct ArrowArrayStream *stream, struct ArrowSchema *out);
@@ -4724,6 +4726,7 @@ private:
 };
 } // namespace duckdb
 
+
 namespace duckdb {
 
 ArrowSchemaWrapper::~ArrowSchemaWrapper() {
@@ -4792,6 +4795,11 @@ int ResultArrowArrayStreamWrapper::MyStreamGetSchema(struct ArrowArrayStream *st
 		return -1;
 	}
 	auto my_stream = (ResultArrowArrayStreamWrapper *)stream->private_data;
+	if (!my_stream->column_types.empty()) {
+		QueryResult::ToArrowSchema(out, my_stream->column_types, my_stream->column_names);
+		return 0;
+	}
+
 	auto &result = *my_stream->result;
 	if (!result.success) {
 		my_stream->last_error = "Query Failed";
@@ -4804,7 +4812,11 @@ int ResultArrowArrayStreamWrapper::MyStreamGetSchema(struct ArrowArrayStream *st
 			return -1;
 		}
 	}
-	result.ToArrowSchema(out);
+	if (my_stream->column_types.empty()) {
+		my_stream->column_types = result.types;
+		my_stream->column_names = result.names;
+	}
+	QueryResult::ToArrowSchema(out, my_stream->column_types, my_stream->column_names);
 	return 0;
 }
 
@@ -4826,13 +4838,17 @@ int ResultArrowArrayStreamWrapper::MyStreamGetNext(struct ArrowArrayStream *stre
 			return 0;
 		}
 	}
+	if (my_stream->column_types.empty()) {
+		my_stream->column_types = result.types;
+		my_stream->column_names = result.names;
+	}
 	unique_ptr<DataChunk> chunk_result = result.Fetch();
 	if (!chunk_result) {
 		// Nothing to output
 		out->release = nullptr;
 		return 0;
 	}
-	for (idx_t i = 1; i < my_stream->vectors_per_chunk; i++) {
+	while (chunk_result->size() < my_stream->batch_size) {
 		auto new_chunk = result.Fetch();
 		if (!new_chunk) {
 			break;
@@ -4860,15 +4876,15 @@ const char *ResultArrowArrayStreamWrapper::MyStreamGetLastError(struct ArrowArra
 	auto my_stream = (ResultArrowArrayStreamWrapper *)stream->private_data;
 	return my_stream->last_error.c_str();
 }
-ResultArrowArrayStreamWrapper::ResultArrowArrayStreamWrapper(unique_ptr<QueryResult> result_p, idx_t approx_batch_size)
+ResultArrowArrayStreamWrapper::ResultArrowArrayStreamWrapper(unique_ptr<QueryResult> result_p, idx_t batch_size_p)
     : result(move(result_p)) {
 	//! We first initialize the private data of the stream
 	stream.private_data = this;
 	//! Ceil Approx_Batch_Size/STANDARD_VECTOR_SIZE
-	if (approx_batch_size == 0) {
+	if (batch_size_p == 0) {
 		throw std::runtime_error("Approximate Batch Size of Record Batch MUST be higher than 0");
 	}
-	vectors_per_chunk = (approx_batch_size + STANDARD_VECTOR_SIZE - 1) / STANDARD_VECTOR_SIZE;
+	batch_size = batch_size_p;
 	//! We initialize the stream functions
 	stream.get_schema = ResultArrowArrayStreamWrapper::MyStreamGetSchema;
 	stream.get_next = ResultArrowArrayStreamWrapper::MyStreamGetNext;
@@ -4876,6 +4892,30 @@ ResultArrowArrayStreamWrapper::ResultArrowArrayStreamWrapper(unique_ptr<QueryRes
 	stream.get_last_error = ResultArrowArrayStreamWrapper::MyStreamGetLastError;
 }
 
+unique_ptr<DataChunk> ArrowUtil::FetchNext(QueryResult &result) {
+	auto chunk = result.Fetch();
+	if (!result.success) {
+		throw std::runtime_error(result.error);
+	}
+	return chunk;
+}
+
+unique_ptr<DataChunk> ArrowUtil::FetchChunk(QueryResult *result, idx_t chunk_size) {
+
+	auto data_chunk = FetchNext(*result);
+	if (!data_chunk) {
+		return data_chunk;
+	}
+	while (data_chunk->size() < chunk_size) {
+		auto next_chunk = FetchNext(*result);
+		if (!next_chunk || next_chunk->size() == 0) {
+			break;
+		}
+		data_chunk->Append(*next_chunk, true);
+	}
+	return data_chunk;
+}
+
 } // namespace duckdb
 
 
@@ -101378,7 +101418,7 @@ duckdb_state duckdb_query_arrow_schema(duckdb_arrow result, duckdb_arrow_schema
 		return DuckDBSuccess;
 	}
 	auto wrapper = (ArrowResultWrapper *)result;
-	wrapper->result->ToArrowSchema((ArrowSchema *)*out_schema);
+	QueryResult::ToArrowSchema((ArrowSchema *)*out_schema, wrapper->result->types, wrapper->result->names);
 	return DuckDBSuccess;
 }
 
@@ -115359,20 +115399,21 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 	}
 }
 
-void QueryResult::ToArrowSchema(ArrowSchema *out_schema) {
+void QueryResult::ToArrowSchema(ArrowSchema *out_schema, vector<LogicalType> &types, vector<string> &names) {
 	D_ASSERT(out_schema);
-
+	D_ASSERT(types.size() == names.size());
+	idx_t column_count = types.size();
 	// Allocate as unique_ptr first to cleanup properly on error
 	auto root_holder = make_unique<DuckDBArrowSchemaHolder>();
 
 	// Allocate the children
-	root_holder->children.resize(ColumnCount());
-	root_holder->children_ptrs.resize(ColumnCount(), nullptr);
-	for (size_t i = 0; i < ColumnCount(); ++i) {
+	root_holder->children.resize(column_count);
+	root_holder->children_ptrs.resize(column_count, nullptr);
+	for (size_t i = 0; i < column_count; ++i) {
 		root_holder->children_ptrs[i] = &root_holder->children[i];
 	}
 	out_schema->children = root_holder->children_ptrs.data();
-	out_schema->n_children = ColumnCount();
+	out_schema->n_children = column_count;
 
 	// Store the schema
 	out_schema->format = "+s"; // struct apparently
@@ -115382,7 +115423,7 @@ void QueryResult::ToArrowSchema(ArrowSchema *out_schema) {
 	out_schema->dictionary = nullptr;
 
 	// Configure all child schemas
-	for (idx_t col_idx = 0; col_idx < ColumnCount(); col_idx++) {
+	for (idx_t col_idx = 0; col_idx < column_count; col_idx++) {
 
 		auto &child = root_holder->children[col_idx];
 		InitializeChild(child, names[col_idx]);

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "068bdbbe2"
-#define DUCKDB_VERSION "v0.3.3-dev933"
+#define DUCKDB_SOURCE_ID "d13fbab42"
+#define DUCKDB_VERSION "v0.3.3-dev957"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -4014,6 +4014,9 @@ struct ArrowArrayStream {
 
 //! Here we have the internal duckdb classes that interact with Arrow's Internal Header (i.e., duckdb/commons/arrow.hpp)
 namespace duckdb {
+class QueryResult;
+class DataChunk;
+
 class ArrowSchemaWrapper {
 public:
 	ArrowSchema arrow_schema;
@@ -4049,6 +4052,14 @@ public:
 		arrow_array_stream.release = nullptr;
 	}
 };
+
+class ArrowUtil {
+public:
+	static unique_ptr<DataChunk> FetchChunk(QueryResult *result, idx_t chunk_size);
+
+private:
+	static unique_ptr<DataChunk> FetchNext(QueryResult &result);
+};
 } // namespace duckdb
 
 
@@ -9760,7 +9771,7 @@ public:
 		}
 	}
 
-	DUCKDB_API void ToArrowSchema(ArrowSchema *out_array);
+	DUCKDB_API static void ToArrowSchema(ArrowSchema *out_schema, vector<LogicalType> &types, vector<string> &names);
 
 private:
 	//! The current chunk used by the iterator
@@ -16507,6 +16518,8 @@ class LogicalOperator;
 class QueryNode;
 class TableRef;
 
+class ExtraDependencies {};
+
 class Relation : public std::enable_shared_from_this<Relation> {
 public:
 	DUCKDB_API Relation(const std::shared_ptr<ClientContext> &context, RelationType type)
@@ -16521,6 +16534,8 @@ public:
 
 	RelationType type;
 
+	unique_ptr<ExtraDependencies> extra_dependencies;
+
 public:
 	DUCKDB_API virtual const vector<ColumnDefinition> &Columns() = 0;
 	DUCKDB_API virtual unique_ptr<QueryNode> GetQueryNode();