duckdb 0.8.2-dev33.0 → 0.8.2-dev3334.0

package/src/duckdb/src/include/duckdb/common/arrow/appender/varchar_data.hpp

@@ -0,0 +1,105 @@
+#pragma once
+
+#include "duckdb/common/arrow/appender/append_data.hpp"
+#include "duckdb/common/arrow/appender/scalar_data.hpp"
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// Varchar
+//===--------------------------------------------------------------------===//
+struct ArrowVarcharConverter {
+	template <class SRC>
+	static idx_t GetLength(SRC input) {
+		return input.GetSize();
+	}
+
+	template <class SRC>
+	static void WriteData(data_ptr_t target, SRC input) {
+		memcpy(target, input.GetData(), input.GetSize());
+	}
+};
+
+struct ArrowUUIDConverter {
+	template <class SRC>
+	static idx_t GetLength(SRC input) {
+		return UUID::STRING_SIZE;
+	}
+
+	template <class SRC>
+	static void WriteData(data_ptr_t target, SRC input) {
+		UUID::ToString(input, char_ptr_cast(target));
+	}
+};
+
+template <class SRC = string_t, class OP = ArrowVarcharConverter, class BUFTYPE = uint64_t>
+struct ArrowVarcharData {
+	static void Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity) {
+		result.main_buffer.reserve((capacity + 1) * sizeof(BUFTYPE));
+
+		result.aux_buffer.reserve(capacity);
+	}
+
+	static void Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
+		idx_t size = to - from;
+		UnifiedVectorFormat format;
+		input.ToUnifiedFormat(input_size, format);
+
+		// resize the validity mask and set up the validity buffer for iteration
+		ResizeValidity(append_data.validity, append_data.row_count + size);
+		auto validity_data = (uint8_t *)append_data.validity.data();
+
+		// resize the offset buffer - the offset buffer holds the offsets into the child array
+		append_data.main_buffer.resize(append_data.main_buffer.size() + sizeof(BUFTYPE) * (size + 1));
+		auto data = UnifiedVectorFormat::GetData<SRC>(format);
+		auto offset_data = append_data.main_buffer.GetData<BUFTYPE>();
+		if (append_data.row_count == 0) {
+			// first entry
+			offset_data[0] = 0;
+		}
+		// now append the string data to the auxiliary buffer
+		// the auxiliary buffer's length depends on the string lengths, so we resize as required
+		auto last_offset = offset_data[append_data.row_count];
+		idx_t max_offset = append_data.row_count + to - from;
+		if (max_offset > NumericLimits<uint32_t>::Maximum() &&
+		    append_data.options.arrow_offset_size == ArrowOffsetSize::REGULAR) {
+			throw InvalidInputException("Arrow Appender: The maximum total string size for regular string buffers is "
+			                            "%u but the offset of %lu exceeds this.",
+			                            NumericLimits<uint32_t>::Maximum(), max_offset);
+		}
+		for (idx_t i = from; i < to; i++) {
+			auto source_idx = format.sel->get_index(i);
+			auto offset_idx = append_data.row_count + i + 1 - from;
+
+			if (!format.validity.RowIsValid(source_idx)) {
+				uint8_t current_bit;
+				idx_t current_byte;
+				GetBitPosition(append_data.row_count + i - from, current_byte, current_bit);
+				SetNull(append_data, validity_data, current_byte, current_bit);
+				offset_data[offset_idx] = last_offset;
+				continue;
+			}
+
+			auto string_length = OP::GetLength(data[source_idx]);
+
+			// append the offset data
+			auto current_offset = last_offset + string_length;
+			offset_data[offset_idx] = current_offset;
+
+			// resize the string buffer if required, and write the string data
+			append_data.aux_buffer.resize(current_offset);
+			OP::WriteData(append_data.aux_buffer.data() + last_offset, data[source_idx]);
+
+			last_offset = current_offset;
+		}
+		append_data.row_count += size;
+	}
+
+	static void Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result) {
+		result->n_buffers = 3;
+		result->buffers[1] = append_data.main_buffer.data();
+		result->buffers[2] = append_data.aux_buffer.data();
+	}
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/arrow/arrow_appender.hpp

@@ -9,8 +9,7 @@
 #pragma once
 
 #include "duckdb/common/arrow/arrow_converter.hpp"
-
-struct ArrowSchema;
+#include "duckdb/common/arrow/arrow.hpp"
 
 namespace duckdb {
 
@@ -19,7 +18,7 @@ struct ArrowAppendData;
 //! The ArrowAppender class can be used to incrementally construct an arrow array by appending data chunks into it
 class ArrowAppender {
 public:
-	DUCKDB_API ArrowAppender(vector<LogicalType> types, idx_t initial_capacity, ArrowOptions options);
+	DUCKDB_API ArrowAppender(vector<LogicalType> types, idx_t initial_capacity, ClientProperties options);
 	DUCKDB_API ~ArrowAppender();
 
 	//! Append a data chunk to the underlying arrow array
@@ -27,6 +26,12 @@ public:
 	//! Returns the underlying arrow array
 	DUCKDB_API ArrowArray Finalize();
 
+public:
+	static void ReleaseArray(ArrowArray *array);
+	static ArrowArray *FinalizeChild(const LogicalType &type, ArrowAppendData &append_data);
+	static unique_ptr<ArrowAppendData> InitializeChild(const LogicalType &type, idx_t capacity,
+	                                                   ClientProperties &options);
+
 private:
 	//! The types of the chunks that will be appended in
 	vector<LogicalType> types;
@@ -35,7 +40,7 @@ private:
 	//! The total row count that has been appended
 	idx_t row_count = 0;
 
-	ArrowOptions options;
+	ClientProperties options;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/arrow/arrow_converter.hpp

@@ -10,16 +10,14 @@
 
 #include "duckdb/common/types/data_chunk.hpp"
 #include "duckdb/common/arrow/arrow.hpp"
-#include "duckdb/common/arrow/arrow_options.hpp"
-
-struct ArrowSchema;
+#include "duckdb/main/client_properties.hpp"
 
 namespace duckdb {
 
 struct ArrowConverter {
 	DUCKDB_API static void ToArrowSchema(ArrowSchema *out_schema, const vector<LogicalType> &types,
-	                                     const vector<string> &names, const ArrowOptions &options);
-	DUCKDB_API static void ToArrowArray(DataChunk &input, ArrowArray *out_array, ArrowOptions options);
+	                                     const vector<string> &names, const ClientProperties &options);
+	DUCKDB_API static void ToArrowArray(DataChunk &input, ArrowArray *out_array, ClientProperties options);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/arrow/arrow_wrapper.hpp

@@ -10,6 +10,8 @@
 #include "duckdb/common/arrow/arrow.hpp"
 #include "duckdb/common/helper.hpp"
 #include "duckdb/common/preserved_error.hpp"
+#include "duckdb/main/chunk_scan_state.hpp"
+#include "duckdb/main/client_properties.hpp"
 
 //! Here we have the internal duckdb classes that interact with Arrow's Internal Header (i.e., duckdb/commons/arrow.hpp)
 namespace duckdb {
@@ -56,9 +58,9 @@ public:
 
 class ArrowUtil {
 public:
-	static bool TryFetchChunk(QueryResult *result, idx_t chunk_size, ArrowArray *out, idx_t &result_count,
-	                          PreservedError &error);
-	static idx_t FetchChunk(QueryResult *result, idx_t chunk_size, ArrowArray *out);
+	static bool TryFetchChunk(ChunkScanState &scan_state, ClientProperties options, idx_t chunk_size, ArrowArray *out,
+	                          idx_t &result_count, PreservedError &error);
+	static idx_t FetchChunk(ChunkScanState &scan_state, ClientProperties options, idx_t chunk_size, ArrowArray *out);
 
 private:
 	static bool TryFetchNext(QueryResult &result, unique_ptr<DataChunk> &out, PreservedError &error);

package/src/duckdb/src/include/duckdb/common/arrow/nanoarrow/nanoarrow.h

@@ -0,0 +1,462 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef NANOARROW_H_INCLUDED
+#define NANOARROW_H_INCLUDED
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#include "duckdb/common/arrow/arrow.hpp"
+
+namespace duckdb_nanoarrow {
+
+/// \file Arrow C Implementation
+///
+/// EXPERIMENTAL. Interface subject to change.
+
+/// \page object-model Object Model
+///
+/// Except where noted, objects are not thread-safe and clients should
+/// take care to serialize accesses to methods.
+///
+/// Because this library is intended to be vendored, it provides full type
+/// definitions and encourages clients to stack or statically allocate
+/// where convenient.
+
+/// \defgroup nanoarrow-malloc Memory management
+///
+/// Non-buffer members of a struct ArrowSchema and struct ArrowArray
+/// must be allocated using ArrowMalloc() or ArrowRealloc() and freed
+/// using ArrowFree for schemas and arrays allocated here. Buffer members
+/// are allocated using an ArrowBufferAllocator.
+
+/// \brief Allocate like malloc()
+void *ArrowMalloc(int64_t size);
+
+/// \brief Reallocate like realloc()
+void *ArrowRealloc(void *ptr, int64_t size);
+
+/// \brief Free a pointer allocated using ArrowMalloc() or ArrowRealloc().
+void ArrowFree(void *ptr);
+
+/// \brief Array buffer allocation and deallocation
+///
+/// Container for allocate, reallocate, and free methods that can be used
+/// to customize allocation and deallocation of buffers when constructing
+/// an ArrowArray.
+struct ArrowBufferAllocator {
+	/// \brief Allocate a buffer or return NULL if it cannot be allocated
+	uint8_t *(*allocate)(struct ArrowBufferAllocator *allocator, int64_t size);
+
+	/// \brief Reallocate a buffer or return NULL if it cannot be reallocated
+	uint8_t *(*reallocate)(struct ArrowBufferAllocator *allocator, uint8_t *ptr, int64_t old_size, int64_t new_size);
+
+	/// \brief Deallocate a buffer allocated by this allocator
+	void (*free)(struct ArrowBufferAllocator *allocator, uint8_t *ptr, int64_t size);
+
+	/// \brief Opaque data specific to the allocator
+	void *private_data;
+};
+
+/// \brief Return the default allocator
+///
+/// The default allocator uses ArrowMalloc(), ArrowRealloc(), and
+/// ArrowFree().
+struct ArrowBufferAllocator *ArrowBufferAllocatorDefault();
+
+/// }@
+
+/// \defgroup nanoarrow-errors Error handling primitives
+/// Functions generally return an errno-compatible error code; functions that
+/// need to communicate more verbose error information accept a pointer
+/// to an ArrowError. This can be stack or statically allocated. The
+/// content of the message is undefined unless an error code has been
+/// returned.
+
+/// \brief Error type containing a UTF-8 encoded message.
+struct ArrowError {
+	char message[1024];
+};
+
+/// \brief Return code for success.
+#define NANOARROW_OK 0
+
+/// \brief Represents an errno-compatible error code
+typedef int ArrowErrorCode;
+
+/// \brief Set the contents of an error using printf syntax
+ArrowErrorCode ArrowErrorSet(struct ArrowError *error, const char *fmt, ...);
+
+/// \brief Get the contents of an error
+const char *ArrowErrorMessage(struct ArrowError *error);
+
+/// }@
+
+/// \defgroup nanoarrow-utils Utility data structures
+
+/// \brief An non-owning view of a string
+struct ArrowStringView {
+	/// \brief A pointer to the start of the string
+	///
+	/// If n_bytes is 0, this value may be NULL.
+	const char *data;
+
+	/// \brief The size of the string in bytes,
+	///
+	/// (Not including the null terminator.)
+	int64_t n_bytes;
+};
+
+/// \brief Arrow type enumerator
+///
+/// These names are intended to map to the corresponding arrow::Type::type
+/// enumerator; however, the numeric values are specifically not equal
+/// (i.e., do not rely on numeric comparison).
+enum ArrowType {
+	NANOARROW_TYPE_UNINITIALIZED = 0,
+	NANOARROW_TYPE_NA = 1,
+	NANOARROW_TYPE_BOOL,
+	NANOARROW_TYPE_UINT8,
+	NANOARROW_TYPE_INT8,
+	NANOARROW_TYPE_UINT16,
+	NANOARROW_TYPE_INT16,
+	NANOARROW_TYPE_UINT32,
+	NANOARROW_TYPE_INT32,
+	NANOARROW_TYPE_UINT64,
+	NANOARROW_TYPE_INT64,
+	NANOARROW_TYPE_HALF_FLOAT,
+	NANOARROW_TYPE_FLOAT,
+	NANOARROW_TYPE_DOUBLE,
+	NANOARROW_TYPE_STRING,
+	NANOARROW_TYPE_BINARY,
+	NANOARROW_TYPE_FIXED_SIZE_BINARY,
+	NANOARROW_TYPE_DATE32,
+	NANOARROW_TYPE_DATE64,
+	NANOARROW_TYPE_TIMESTAMP,
+	NANOARROW_TYPE_TIME32,
+	NANOARROW_TYPE_TIME64,
+	NANOARROW_TYPE_INTERVAL_MONTHS,
+	NANOARROW_TYPE_INTERVAL_DAY_TIME,
+	NANOARROW_TYPE_DECIMAL128,
+	NANOARROW_TYPE_DECIMAL256,
+	NANOARROW_TYPE_LIST,
+	NANOARROW_TYPE_STRUCT,
+	NANOARROW_TYPE_SPARSE_UNION,
+	NANOARROW_TYPE_DENSE_UNION,
+	NANOARROW_TYPE_DICTIONARY,
+	NANOARROW_TYPE_MAP,
+	NANOARROW_TYPE_EXTENSION,
+	NANOARROW_TYPE_FIXED_SIZE_LIST,
+	NANOARROW_TYPE_DURATION,
+	NANOARROW_TYPE_LARGE_STRING,
+	NANOARROW_TYPE_LARGE_BINARY,
+	NANOARROW_TYPE_LARGE_LIST,
+	NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO
+};
+
+/// \brief Arrow time unit enumerator
+///
+/// These names and values map to the corresponding arrow::TimeUnit::type
+/// enumerator.
+enum ArrowTimeUnit {
+	NANOARROW_TIME_UNIT_SECOND = 0,
+	NANOARROW_TIME_UNIT_MILLI = 1,
+	NANOARROW_TIME_UNIT_MICRO = 2,
+	NANOARROW_TIME_UNIT_NANO = 3
+};
+
+/// }@
+
+/// \defgroup nanoarrow-schema Schema producer helpers
+/// These functions allocate, copy, and destroy ArrowSchema structures
+
+/// \brief Initialize the fields of a schema
+///
+/// Initializes the fields and release callback of schema_out. Caller
+/// is responsible for calling the schema->release callback if
+/// NANOARROW_OK is returned.
+ArrowErrorCode ArrowSchemaInit(struct ArrowSchema *schema, enum ArrowType type);
+
+/// \brief Initialize the fields of a fixed-size schema
+///
+/// Returns EINVAL for fixed_size <= 0 or for data_type that is not
+/// NANOARROW_TYPE_FIXED_SIZE_BINARY or NANOARROW_TYPE_FIXED_SIZE_LIST.
+ArrowErrorCode ArrowSchemaInitFixedSize(struct ArrowSchema *schema, enum ArrowType data_type, int32_t fixed_size);
+
+/// \brief Initialize the fields of a decimal schema
+///
+/// Returns EINVAL for scale <= 0 or for data_type that is not
+/// NANOARROW_TYPE_DECIMAL128 or NANOARROW_TYPE_DECIMAL256.
+ArrowErrorCode ArrowSchemaInitDecimal(struct ArrowSchema *schema, enum ArrowType data_type, int32_t decimal_precision,
+                                      int32_t decimal_scale);
+
+/// \brief Initialize the fields of a time, timestamp, or duration schema
+///
+/// Returns EINVAL for data_type that is not
+/// NANOARROW_TYPE_TIME32, NANOARROW_TYPE_TIME64,
+/// NANOARROW_TYPE_TIMESTAMP, or NANOARROW_TYPE_DURATION. The
+/// timezone parameter must be NULL for a non-timestamp data_type.
+ArrowErrorCode ArrowSchemaInitDateTime(struct ArrowSchema *schema, enum ArrowType data_type,
+                                       enum ArrowTimeUnit time_unit, const char *timezone);
+
+/// \brief Make a (recursive) copy of a schema
+///
+/// Allocates and copies fields of schema into schema_out.
+ArrowErrorCode ArrowSchemaDeepCopy(struct ArrowSchema *schema, struct ArrowSchema *schema_out);
+
+/// \brief Copy format into schema->format
+///
+/// schema must have been allocated using ArrowSchemaInit or
+/// ArrowSchemaDeepCopy.
+ArrowErrorCode ArrowSchemaSetFormat(struct ArrowSchema *schema, const char *format);
+
+/// \brief Copy name into schema->name
+///
+/// schema must have been allocated using ArrowSchemaInit or
+/// ArrowSchemaDeepCopy.
+ArrowErrorCode ArrowSchemaSetName(struct ArrowSchema *schema, const char *name);
+
+/// \brief Copy metadata into schema->metadata
+///
+/// schema must have been allocated using ArrowSchemaInit or
+/// ArrowSchemaDeepCopy.
+ArrowErrorCode ArrowSchemaSetMetadata(struct ArrowSchema *schema, const char *metadata);
+
+/// \brief Allocate the schema->children array
+///
+/// Includes the memory for each child struct ArrowSchema.
+/// schema must have been allocated using ArrowSchemaInit or
+/// ArrowSchemaDeepCopy.
+ArrowErrorCode ArrowSchemaAllocateChildren(struct ArrowSchema *schema, int64_t n_children);
+
+/// \brief Allocate the schema->dictionary member
+///
+/// schema must have been allocated using ArrowSchemaInit or
+/// ArrowSchemaDeepCopy.
+ArrowErrorCode ArrowSchemaAllocateDictionary(struct ArrowSchema *schema);
+
+/// \brief Reader for key/value pairs in schema metadata
+struct ArrowMetadataReader {
+	const char *metadata;
+	int64_t offset;
+	int32_t remaining_keys;
+};
+
+/// \brief Initialize an ArrowMetadataReader
+ArrowErrorCode ArrowMetadataReaderInit(struct ArrowMetadataReader *reader, const char *metadata);
+
+/// \brief Read the next key/value pair from an ArrowMetadataReader
+ArrowErrorCode ArrowMetadataReaderRead(struct ArrowMetadataReader *reader, struct ArrowStringView *key_out,
+                                       struct ArrowStringView *value_out);
+
+/// \brief The number of bytes in in a key/value metadata string
+int64_t ArrowMetadataSizeOf(const char *metadata);
+
+/// \brief Check for a key in schema metadata
+char ArrowMetadataHasKey(const char *metadata, const char *key);
+
+/// \brief Extract a value from schema metadata
+ArrowErrorCode ArrowMetadataGetValue(const char *metadata, const char *key, const char *default_value,
+                                     struct ArrowStringView *value_out);
+
+/// }@
+
+/// \defgroup nanoarrow-schema-view Schema consumer helpers
+
+/// \brief A non-owning view of a parsed ArrowSchema
+///
+/// Contains more readily extractable values than a raw ArrowSchema.
+/// Clients can stack or statically allocate this structure but are
+/// encouraged to use the provided getters to ensure forward
+/// compatiblity.
+struct ArrowSchemaView {
+	/// \brief A pointer to the schema represented by this view
+	struct ArrowSchema *schema;
+
+	/// \brief The data type represented by the schema
+	///
+	/// This value may be NANOARROW_TYPE_DICTIONARY if the schema has a
+	/// non-null dictionary member; datetime types are valid values.
+	/// This value will never be NANOARROW_TYPE_EXTENSION (see
+	/// extension_name and/or extension_metadata to check for
+	/// an extension type).
+	enum ArrowType data_type;
+
+	/// \brief The storage data type represented by the schema
+	///
+	/// This value will never be NANOARROW_TYPE_DICTIONARY, NANOARROW_TYPE_EXTENSION
+	/// or any datetime type. This value represents only the type required to
+	/// interpret the buffers in the array.
+	enum ArrowType storage_data_type;
+
+	/// \brief The extension type name if it exists
+	///
+	/// If the ARROW:extension:name key is present in schema.metadata,
+	/// extension_name.data will be non-NULL.
+	struct ArrowStringView extension_name;
+
+	/// \brief The extension type metadata if it exists
+	///
+	/// If the ARROW:extension:metadata key is present in schema.metadata,
+	/// extension_metadata.data will be non-NULL.
+	struct ArrowStringView extension_metadata;
+
+	/// \brief The expected number of buffers in a paired ArrowArray
+	int32_t n_buffers;
+
+	/// \brief The index of the validity buffer or -1 if one does not exist
+	int32_t validity_buffer_id;
+
+	/// \brief The index of the offset buffer or -1 if one does not exist
+	int32_t offset_buffer_id;
+
+	/// \brief The index of the data buffer or -1 if one does not exist
+	int32_t data_buffer_id;
+
+	/// \brief The index of the type_ids buffer or -1 if one does not exist
+	int32_t type_id_buffer_id;
+
+	/// \brief Format fixed size parameter
+	///
+	/// This value is set when parsing a fixed-size binary or fixed-size
+	/// list schema; this value is undefined for other types. For a
+	/// fixed-size binary schema this value is in bytes; for a fixed-size
+	/// list schema this value refers to the number of child elements for
+	/// each element of the parent.
+	int32_t fixed_size;
+
+	/// \brief Decimal bitwidth
+	///
+	/// This value is set when parsing a decimal type schema;
+	/// this value is undefined for other types.
+	int32_t decimal_bitwidth;
+
+	/// \brief Decimal precision
+	///
+	/// This value is set when parsing a decimal type schema;
+	/// this value is undefined for other types.
+	int32_t decimal_precision;
+
+	/// \brief Decimal scale
+	///
+	/// This value is set when parsing a decimal type schema;
+	/// this value is undefined for other types.
+	int32_t decimal_scale;
+
+	/// \brief Format time unit parameter
+	///
+	/// This value is set when parsing a date/time type. The value is
+	/// undefined for other types.
+	enum ArrowTimeUnit time_unit;
+
+	/// \brief Format timezone parameter
+	///
+	/// This value is set when parsing a timestamp type and represents
+	/// the timezone format parameter. The ArrowStrintgView points to
+	/// data within the schema and the value is undefined for other types.
+	struct ArrowStringView timezone;
+
+	/// \brief Union type ids parameter
+	///
+	/// This value is set when parsing a union type and represents
+	/// type ids parameter. The ArrowStringView points to
+	/// data within the schema and the value is undefined for other types.
+	struct ArrowStringView union_type_ids;
+};
+
+/// \brief Initialize an ArrowSchemaView
+ArrowErrorCode ArrowSchemaViewInit(struct ArrowSchemaView *schema_view, struct ArrowSchema *schema,
+                                   struct ArrowError *error);
+
+/// }@
+
+/// \defgroup nanoarrow-buffer-builder Growable buffer builders
+
+/// \brief An owning mutable view of a buffer
+struct ArrowBuffer {
+	/// \brief A pointer to the start of the buffer
+	///
+	/// If capacity_bytes is 0, this value may be NULL.
+	uint8_t *data;
+
+	/// \brief The size of the buffer in bytes
+	int64_t size_bytes;
+
+	/// \brief The capacity of the buffer in bytes
+	int64_t capacity_bytes;
+
+	/// \brief The allocator that will be used to reallocate and/or free the buffer
+	struct ArrowBufferAllocator *allocator;
+};
+
+/// \brief Initialize an ArrowBuffer
+///
+/// Initialize a buffer with a NULL, zero-size buffer using the default
+/// buffer allocator.
+void ArrowBufferInit(struct ArrowBuffer *buffer);
+
+/// \brief Set a newly-initialized buffer's allocator
+///
+/// Returns EINVAL if the buffer has already been allocated.
+ArrowErrorCode ArrowBufferSetAllocator(struct ArrowBuffer *buffer, struct ArrowBufferAllocator *allocator);
+
+/// \brief Reset an ArrowBuffer
+///
+/// Releases the buffer using the allocator's free method if
+/// the buffer's data member is non-null, sets the data member
+/// to NULL, and sets the buffer's size and capacity to 0.
+void ArrowBufferReset(struct ArrowBuffer *buffer);
+
+/// \brief Move an ArrowBuffer
+///
+/// Transfers the buffer data and lifecycle management to another
+/// address and resets buffer.
+void ArrowBufferMove(struct ArrowBuffer *buffer, struct ArrowBuffer *buffer_out);
+
+/// \brief Grow or shrink a buffer to a given capacity
+///
+/// When shrinking the capacity of the buffer, the buffer is only reallocated
+/// if shrink_to_fit is non-zero. Calling ArrowBufferResize() does not
+/// adjust the buffer's size member except to ensure that the invariant
+/// capacity >= size remains true.
+ArrowErrorCode ArrowBufferResize(struct ArrowBuffer *buffer, int64_t new_capacity_bytes, char shrink_to_fit);
+
+/// \brief Ensure a buffer has at least a given additional capacity
+///
+/// Ensures that the buffer has space to append at least
+/// additional_size_bytes, overallocating when required.
+ArrowErrorCode ArrowBufferReserve(struct ArrowBuffer *buffer, int64_t additional_size_bytes);
+
+/// \brief Write data to buffer and increment the buffer size
+///
+/// This function does not check that buffer has the required capacity
+void ArrowBufferAppendUnsafe(struct ArrowBuffer *buffer, const void *data, int64_t size_bytes);
+
+/// \brief Write data to buffer and increment the buffer size
+///
+/// This function writes and ensures that the buffer has the required capacity,
+/// possibly by reallocating the buffer. Like ArrowBufferReserve, this will
+/// overallocate when reallocation is required.
+ArrowErrorCode ArrowBufferAppend(struct ArrowBuffer *buffer, const void *data, int64_t size_bytes);
+
+/// }@
+
+} // namespace duckdb_nanoarrow
+
+#endif // NANOARROW_H_INCLUDED

package/src/duckdb/src/include/duckdb/common/arrow/nanoarrow/nanoarrow.hpp

@@ -0,0 +1,14 @@
+#pragma once
+
+#include "duckdb/common/arrow/nanoarrow/nanoarrow.h"
+
+// Bring in the symbols from duckdb_nanoarrow into duckdb
+namespace duckdb {
+
+// using duckdb_nanoarrow::ArrowBuffer; //We have a variant of this that should be renamed
+using duckdb_nanoarrow::ArrowBufferAllocator;
+using duckdb_nanoarrow::ArrowError;
+using duckdb_nanoarrow::ArrowSchemaView;
+using duckdb_nanoarrow::ArrowStringView;
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/arrow/result_arrow_wrapper.hpp

@@ -10,17 +10,21 @@
 
 #include "duckdb/main/query_result.hpp"
 #include "duckdb/common/arrow/arrow_wrapper.hpp"
+#include "duckdb/main/chunk_scan_state.hpp"
 
 namespace duckdb {
 class ResultArrowArrayStreamWrapper {
 public:
 	explicit ResultArrowArrayStreamWrapper(unique_ptr<QueryResult> result, idx_t batch_size);
+
+public:
 	ArrowArrayStream stream;
 	unique_ptr<QueryResult> result;
 	PreservedError last_error;
 	idx_t batch_size;
 	vector<LogicalType> column_types;
 	vector<string> column_names;
+	unique_ptr<ChunkScanState> scan_state;
 
 private:
 	static int MyStreamGetSchema(struct ArrowArrayStream *stream, struct ArrowSchema *out);