duckdb 0.8.2-dev2283.0 → 0.8.2-dev2356.0

package/binding.gyp

@@ -93,6 +93,7 @@
                 "src/duckdb/ub_src_main.cpp", 
                 "src/duckdb/ub_src_main_capi.cpp", 
                 "src/duckdb/ub_src_main_capi_cast.cpp", 
+                "src/duckdb/ub_src_main_chunk_scan_state.cpp", 
                 "src/duckdb/ub_src_main_extension.cpp", 
                 "src/duckdb/ub_src_main_relation.cpp", 
                 "src/duckdb/ub_src_main_settings.cpp", 

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.8.2-dev2283.0",
+  "version": "0.8.2-dev2356.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/extension/parquet/parquet_writer.cpp

@@ -297,6 +297,7 @@ void ParquetWriter::PrepareRowGroup(ColumnDataCollection &buffer, PreparedRowGro
 	// set up a new row group for this chunk collection
 	auto &row_group = result.row_group;
 	row_group.num_rows = buffer.Count();
+	row_group.total_byte_size = buffer.SizeInBytes();
 	row_group.__isset.file_offset = true;
 
 	auto &states = result.states;

package/src/duckdb/src/common/adbc/adbc.cpp

@@ -202,14 +202,15 @@ AdbcStatusCode ConnectionGetTableSchema(struct AdbcConnection *connection, const
 		SetError(error, "Connection is not set");
 		return ADBC_STATUS_INVALID_ARGUMENT;
 	}
+	if (db_schema == nullptr) {
+		// if schema is not set, we use the default schema
+		db_schema = "main";
+	}
 	if (catalog != nullptr && strlen(catalog) > 0) {
 		// In DuckDB this is the name of the database, not sure what's the expected functionality here, so for now,
 		// scream.
 		SetError(error, "Catalog Name is not used in DuckDB. It must be set to nullptr or an empty string");
 		return ADBC_STATUS_NOT_IMPLEMENTED;
-	} else if (db_schema == nullptr) {
-		SetError(error, "AdbcConnectionGetTableSchema: must provide db_schema");
-		return ADBC_STATUS_INVALID_ARGUMENT;
 	} else if (table_name == nullptr) {
 		SetError(error, "AdbcConnectionGetTableSchema: must provide table_name");
 		return ADBC_STATUS_INVALID_ARGUMENT;
@@ -686,6 +687,10 @@ AdbcStatusCode QueryInternal(struct AdbcConnection *connection, struct ArrowArra
 AdbcStatusCode ConnectionGetObjects(struct AdbcConnection *connection, int depth, const char *catalog,
                                     const char *db_schema, const char *table_name, const char **table_type,
                                     const char *column_name, struct ArrowArrayStream *out, struct AdbcError *error) {
+	if (depth != 0) {
+		SetError(error, "Depth parameter not yet supported");
+		return ADBC_STATUS_NOT_IMPLEMENTED;
+	}
 	if (catalog != nullptr) {
 		if (strcmp(catalog, "duckdb") == 0) {
 			SetError(error, "catalog must be NULL or 'duckdb'");

package/src/duckdb/src/common/arrow/arrow_appender.cpp

@@ -14,7 +14,7 @@ namespace duckdb {
 // ArrowAppender
 //===--------------------------------------------------------------------===//
 
-ArrowAppender::ArrowAppender(vector<LogicalType> types_p, idx_t initial_capacity, ArrowOptions options)
+ArrowAppender::ArrowAppender(vector<LogicalType> types_p, idx_t initial_capacity, ClientProperties options)
     : types(std::move(types_p)) {
 	for (auto &type : types) {
 		auto entry = ArrowAppender::InitializeChild(type, initial_capacity, options);
@@ -179,14 +179,14 @@ static void InitializeFunctionPointers(ArrowAppendData &append_data, const Logic
 	case LogicalTypeId::VARCHAR:
 	case LogicalTypeId::BLOB:
 	case LogicalTypeId::BIT:
-		if (append_data.options.offset_size == ArrowOffsetSize::LARGE) {
+		if (append_data.options.arrow_offset_size == ArrowOffsetSize::LARGE) {
 			InitializeAppenderForType<ArrowVarcharData<string_t>>(append_data);
 		} else {
 			InitializeAppenderForType<ArrowVarcharData<string_t, ArrowVarcharConverter, uint32_t>>(append_data);
 		}
 		break;
 	case LogicalTypeId::UUID:
-		if (append_data.options.offset_size == ArrowOffsetSize::LARGE) {
+		if (append_data.options.arrow_offset_size == ArrowOffsetSize::LARGE) {
 			InitializeAppenderForType<ArrowVarcharData<hugeint_t, ArrowUUIDConverter>>(append_data);
 		} else {
 			InitializeAppenderForType<ArrowVarcharData<hugeint_t, ArrowUUIDConverter, uint32_t>>(append_data);
@@ -228,7 +228,7 @@ static void InitializeFunctionPointers(ArrowAppendData &append_data, const Logic
 }
 
 unique_ptr<ArrowAppendData> ArrowAppender::InitializeChild(const LogicalType &type, idx_t capacity,
-                                                           ArrowOptions &options) {
+                                                           ClientProperties &options) {
 	auto result = make_uniq<ArrowAppendData>(options);
 	InitializeFunctionPointers(*result, type);
 

package/src/duckdb/src/common/arrow/arrow_converter.cpp

@@ -15,12 +15,21 @@
 
 namespace duckdb {
 
-void ArrowConverter::ToArrowArray(DataChunk &input, ArrowArray *out_array, ArrowOptions options) {
+void ArrowConverter::ToArrowArray(DataChunk &input, ArrowArray *out_array, ClientProperties options) {
 	ArrowAppender appender(input.GetTypes(), input.size(), std::move(options));
 	appender.Append(input, 0, input.size(), input.size());
 	*out_array = appender.Finalize();
 }
 
+unsafe_unique_array<char> AddName(const string &name) {
+	auto name_ptr = make_unsafe_uniq_array<char>(name.size() + 1);
+	for (size_t i = 0; i < name.size(); i++) {
+		name_ptr[i] = name[i];
+	}
+	name_ptr[name.size()] = '\0';
+	return name_ptr;
+}
+
 //===--------------------------------------------------------------------===//
 // Arrow Schema
 //===--------------------------------------------------------------------===//
@@ -45,24 +54,26 @@ static void ReleaseDuckDBArrowSchema(ArrowSchema *schema) {
 	delete holder;
 }
 
-void InitializeChild(ArrowSchema &child, const string &name = "") {
+void InitializeChild(ArrowSchema &child, DuckDBArrowSchemaHolder &root_holder, const string &name = "") {
 	//! Child is cleaned up by parent
 	child.private_data = nullptr;
 	child.release = ReleaseDuckDBArrowSchema;
 
 	//! Store the child schema
 	child.flags = ARROW_FLAG_NULLABLE;
-	child.name = name.c_str();
+	root_holder.owned_type_names.push_back(AddName(name));
+
+	child.name = root_holder.owned_type_names.back().get();
 	child.n_children = 0;
 	child.children = nullptr;
 	child.metadata = nullptr;
 	child.dictionary = nullptr;
 }
 void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, const LogicalType &type,
-                    const ArrowOptions &options);
+                    const ClientProperties &options);
 
 void SetArrowMapFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, const LogicalType &type,
-                       const ArrowOptions &options) {
+                       const ClientProperties &options) {
 	child.format = "+m";
 	//! Map has one child which is a struct
 	child.n_children = 1;
@@ -70,23 +81,14 @@ void SetArrowMapFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child,
 	root_holder.nested_children.back().resize(1);
 	root_holder.nested_children_ptr.emplace_back();
 	root_holder.nested_children_ptr.back().push_back(&root_holder.nested_children.back()[0]);
-	InitializeChild(root_holder.nested_children.back()[0]);
+	InitializeChild(root_holder.nested_children.back()[0], root_holder);
 	child.children = &root_holder.nested_children_ptr.back()[0];
 	child.children[0]->name = "entries";
 	SetArrowFormat(root_holder, **child.children, ListType::GetChildType(type), options);
 }
 
-unsafe_unique_array<char> AddName(const string &name) {
-	auto name_ptr = make_unsafe_uniq_array<char>(name.size() + 1);
-	for (size_t i = 0; i < name.size(); i++) {
-		name_ptr[i] = name[i];
-	}
-	name_ptr[name.size()] = '\0';
-	return name_ptr;
-}
-
 void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, const LogicalType &type,
-                    const ArrowOptions &options) {
+                    const ClientProperties &options) {
 	switch (type.id()) {
 	case LogicalTypeId::BOOLEAN:
 		child.format = "b";
@@ -126,7 +128,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 		break;
 	case LogicalTypeId::UUID:
 	case LogicalTypeId::VARCHAR:
-		if (options.offset_size == ArrowOffsetSize::LARGE) {
+		if (options.arrow_offset_size == ArrowOffsetSize::LARGE) {
 			child.format = "U";
 		} else {
 			child.format = "u";
@@ -136,7 +138,6 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 		child.format = "tdD";
 		break;
 	case LogicalTypeId::TIME:
-	case LogicalTypeId::TIME_TZ:
 		child.format = "ttu";
 		break;
 	case LogicalTypeId::TIMESTAMP:
@@ -174,7 +175,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 	}
 	case LogicalTypeId::BLOB:
 	case LogicalTypeId::BIT: {
-		if (options.offset_size == ArrowOffsetSize::LARGE) {
+		if (options.arrow_offset_size == ArrowOffsetSize::LARGE) {
 			child.format = "Z";
 		} else {
 			child.format = "z";
@@ -188,7 +189,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 		root_holder.nested_children.back().resize(1);
 		root_holder.nested_children_ptr.emplace_back();
 		root_holder.nested_children_ptr.back().push_back(&root_holder.nested_children.back()[0]);
-		InitializeChild(root_holder.nested_children.back()[0]);
+		InitializeChild(root_holder.nested_children.back()[0], root_holder);
 		child.children = &root_holder.nested_children_ptr.back()[0];
 		child.children[0]->name = "l";
 		SetArrowFormat(root_holder, **child.children, ListType::GetChildType(type), options);
@@ -208,7 +209,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 		child.children = &root_holder.nested_children_ptr.back()[0];
 		for (size_t type_idx = 0; type_idx < child_types.size(); type_idx++) {
 
-			InitializeChild(*child.children[type_idx]);
+			InitializeChild(*child.children[type_idx], root_holder);
 
 			root_holder.owned_type_names.push_back(AddName(child_types[type_idx].first));
 
@@ -236,7 +237,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 		child.children = &root_holder.nested_children_ptr.back()[0];
 		for (size_t type_idx = 0; type_idx < child_types.size(); type_idx++) {
 
-			InitializeChild(*child.children[type_idx]);
+			InitializeChild(*child.children[type_idx], root_holder);
 
 			root_holder.owned_type_names.push_back(AddName(child_types[type_idx].first));
 
@@ -272,18 +273,18 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 		root_holder.nested_children.back().resize(1);
 		root_holder.nested_children_ptr.emplace_back();
 		root_holder.nested_children_ptr.back().push_back(&root_holder.nested_children.back()[0]);
-		InitializeChild(root_holder.nested_children.back()[0]);
+		InitializeChild(root_holder.nested_children.back()[0], root_holder);
 		child.dictionary = root_holder.nested_children_ptr.back()[0];
 		child.dictionary->format = "u";
 		break;
 	}
 	default:
-		throw InternalException("Unsupported Arrow type " + type.ToString());
+		throw NotImplementedException("Unsupported Arrow type " + type.ToString());
 	}
 }
 
 void ArrowConverter::ToArrowSchema(ArrowSchema *out_schema, const vector<LogicalType> &types,
-                                   const vector<string> &names, const ArrowOptions &options) {
+                                   const vector<string> &names, const ClientProperties &options) {
 	D_ASSERT(out_schema);
 	D_ASSERT(types.size() == names.size());
 	idx_t column_count = types.size();
@@ -310,7 +311,7 @@ void ArrowConverter::ToArrowSchema(ArrowSchema *out_schema, const vector<Logical
 	for (idx_t col_idx = 0; col_idx < column_count; col_idx++) {
 
 		auto &child = root_holder->children[col_idx];
-		InitializeChild(child, names[col_idx]);
+		InitializeChild(child, *root_holder, names[col_idx]);
 		SetArrowFormat(*root_holder, child, types[col_idx], options);
 	}
 

package/src/duckdb/src/common/arrow/arrow_wrapper.cpp

@@ -9,6 +9,7 @@
 #include "duckdb/common/arrow/result_arrow_wrapper.hpp"
 #include "duckdb/common/arrow/arrow_appender.hpp"
 #include "duckdb/main/query_result.hpp"
+#include "duckdb/main/chunk_scan_state/query_result.hpp"
 
 namespace duckdb {
 
@@ -68,7 +69,7 @@ int ResultArrowArrayStreamWrapper::MyStreamGetSchema(struct ArrowArrayStream *st
 	auto my_stream = reinterpret_cast<ResultArrowArrayStreamWrapper *>(stream->private_data);
 	if (!my_stream->column_types.empty()) {
 		ArrowConverter::ToArrowSchema(out, my_stream->column_types, my_stream->column_names,
-		                              QueryResult::GetArrowOptions(*my_stream->result));
+		                              my_stream->result->client_properties);
 		return 0;
 	}
 
@@ -89,7 +90,7 @@ int ResultArrowArrayStreamWrapper::MyStreamGetSchema(struct ArrowArrayStream *st
 		my_stream->column_names = result.names;
 	}
 	ArrowConverter::ToArrowSchema(out, my_stream->column_types, my_stream->column_names,
-	                              QueryResult::GetArrowOptions(*my_stream->result));
+	                              my_stream->result->client_properties);
 	return 0;
 }
 
@@ -99,6 +100,7 @@ int ResultArrowArrayStreamWrapper::MyStreamGetNext(struct ArrowArrayStream *stre
 	}
 	auto my_stream = reinterpret_cast<ResultArrowArrayStreamWrapper *>(stream->private_data);
 	auto &result = *my_stream->result;
+	auto &scan_state = *my_stream->scan_state;
 	if (result.HasError()) {
 		my_stream->last_error = result.GetErrorObject();
 		return -1;
@@ -117,7 +119,8 @@ int ResultArrowArrayStreamWrapper::MyStreamGetNext(struct ArrowArrayStream *stre
 	}
 	idx_t result_count;
 	PreservedError error;
-	if (!ArrowUtil::TryFetchChunk(&result, my_stream->batch_size, out, result_count, error)) {
+	if (!ArrowUtil::TryFetchChunk(scan_state, result.client_properties, my_stream->batch_size, out, result_count,
+	                              error)) {
 		D_ASSERT(error);
 		my_stream->last_error = error;
 		return -1;
@@ -147,7 +150,7 @@ const char *ResultArrowArrayStreamWrapper::MyStreamGetLastError(struct ArrowArra
 }
 
 ResultArrowArrayStreamWrapper::ResultArrowArrayStreamWrapper(unique_ptr<QueryResult> result_p, idx_t batch_size_p)
-    : result(std::move(result_p)) {
+    : result(std::move(result_p)), scan_state(make_uniq<QueryResultChunkScanState>(*result)) {
 	//! We first initialize the private data of the stream
 	stream.private_data = this;
 	//! Ceil Approx_Batch_Size/STANDARD_VECTOR_SIZE
@@ -162,52 +165,43 @@ ResultArrowArrayStreamWrapper::ResultArrowArrayStreamWrapper(unique_ptr<QueryRes
 	stream.get_last_error = ResultArrowArrayStreamWrapper::MyStreamGetLastError;
 }
 
-bool ArrowUtil::TryFetchNext(QueryResult &result, unique_ptr<DataChunk> &chunk, PreservedError &error) {
-	if (result.type == QueryResultType::STREAM_RESULT) {
-		auto &stream_result = result.Cast<StreamQueryResult>();
-		if (!stream_result.IsOpen()) {
-			return true;
-		}
-	}
-	return result.TryFetch(chunk, error);
-}
-
-bool ArrowUtil::TryFetchChunk(QueryResult *result, idx_t chunk_size, ArrowArray *out, idx_t &count,
-                              PreservedError &error) {
+bool ArrowUtil::TryFetchChunk(ChunkScanState &scan_state, ClientProperties options, idx_t batch_size, ArrowArray *out,
+                              idx_t &count, PreservedError &error) {
 	count = 0;
-	ArrowAppender appender(result->types, chunk_size, QueryResult::GetArrowOptions(*result));
-	auto &current_chunk = result->current_chunk;
-	if (current_chunk.Valid()) {
+	ArrowAppender appender(scan_state.Types(), batch_size, std::move(options));
+	auto remaining_tuples_in_chunk = scan_state.RemainingInChunk();
+	if (remaining_tuples_in_chunk) {
 		// We start by scanning the non-finished current chunk
-		// Limit the amount we're fetching to the chunk_size
-		idx_t cur_consumption = MinValue<idx_t>(current_chunk.RemainingSize(), chunk_size);
+		idx_t cur_consumption = MinValue(remaining_tuples_in_chunk, batch_size);
 		count += cur_consumption;
-		appender.Append(*current_chunk.data_chunk, current_chunk.position, current_chunk.position + cur_consumption,
-		                current_chunk.data_chunk->size());
-		current_chunk.position += cur_consumption;
-	}
-	while (count < chunk_size) {
-		unique_ptr<DataChunk> data_chunk;
-		if (!TryFetchNext(*result, data_chunk, error)) {
-			if (result->HasError()) {
-				error = result->GetErrorObject();
+		auto &current_chunk = scan_state.CurrentChunk();
+		appender.Append(current_chunk, scan_state.CurrentOffset(), scan_state.CurrentOffset() + cur_consumption,
+		                current_chunk.size());
+		scan_state.IncreaseOffset(cur_consumption);
+	}
+	while (count < batch_size) {
+		if (!scan_state.LoadNextChunk(error)) {
+			if (scan_state.HasError()) {
+				error = scan_state.GetError();
 			}
 			return false;
 		}
-		if (!data_chunk || data_chunk->size() == 0) {
+		if (scan_state.ChunkIsEmpty()) {
+			// The scan was successful, but an empty chunk was returned
 			break;
 		}
-		if (count + data_chunk->size() > chunk_size) {
-			// We have to split the chunk between this and the next batch
-			idx_t available_space = chunk_size - count;
-			appender.Append(*data_chunk, 0, available_space, data_chunk->size());
-			count += available_space;
-			current_chunk.data_chunk = std::move(data_chunk);
-			current_chunk.position = available_space;
-		} else {
-			count += data_chunk->size();
-			appender.Append(*data_chunk, 0, data_chunk->size(), data_chunk->size());
+		auto &current_chunk = scan_state.CurrentChunk();
+		if (scan_state.Finished() || current_chunk.size() == 0) {
+			break;
 		}
+		// The amount we still need to append into this chunk
+		auto remaining = batch_size - count;
+
+		// The amount remaining, capped by the amount left in the current chunk
+		auto to_append_to_batch = MinValue(remaining, scan_state.RemainingInChunk());
+		appender.Append(current_chunk, 0, to_append_to_batch, current_chunk.size());
+		count += to_append_to_batch;
+		scan_state.IncreaseOffset(to_append_to_batch);
 	}
 	if (count > 0) {
 		*out = appender.Finalize();
@@ -215,10 +209,10 @@ bool ArrowUtil::TryFetchChunk(QueryResult *result, idx_t chunk_size, ArrowArray
 	return true;
 }
 
-idx_t ArrowUtil::FetchChunk(QueryResult *result, idx_t chunk_size, ArrowArray *out) {
+idx_t ArrowUtil::FetchChunk(ChunkScanState &scan_state, ClientProperties options, idx_t chunk_size, ArrowArray *out) {
 	PreservedError error;
 	idx_t result_count;
-	if (!TryFetchChunk(result, chunk_size, out, result_count, error)) {
+	if (!TryFetchChunk(scan_state, std::move(options), chunk_size, out, result_count, error)) {
 		error.Throw();
 	}
 	return result_count;

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.8.2-dev2283"
+#define DUCKDB_VERSION "0.8.2-dev2356"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "e89665e802"
+#define DUCKDB_SOURCE_ID "ef2efd1b9d"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/common/arrow/appender/append_data.hpp

@@ -4,7 +4,7 @@
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/common/arrow/arrow.hpp"
 #include "duckdb/common/arrow/arrow_buffer.hpp"
-#include "duckdb/common/arrow/arrow_options.hpp"
+#include "duckdb/main/client_properties.hpp"
 #include "duckdb/common/array.hpp"
 
 namespace duckdb {
@@ -26,7 +26,7 @@ typedef void (*finalize_t)(ArrowAppendData &append_data, const LogicalType &type
 // FIXME: we should separate the append state variables from the variables required by the ArrowArray into
 // ArrowAppendState
 struct ArrowAppendData {
-	explicit ArrowAppendData(ArrowOptions &options_p) : options(options_p) {
+	explicit ArrowAppendData(ClientProperties &options_p) : options(options_p) {
 	}
 	// the buffers of the arrow vector
 	ArrowBuffer validity;
@@ -49,7 +49,7 @@ struct ArrowAppendData {
 	duckdb::array<const void *, 3> buffers = {{nullptr, nullptr, nullptr}};
 	vector<ArrowArray *> child_pointers;
 
-	ArrowOptions options;
+	ClientProperties options;
 };
 
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/include/duckdb/common/arrow/appender/varchar_data.hpp

@@ -62,7 +62,7 @@ struct ArrowVarcharData {
 		auto last_offset = offset_data[append_data.row_count];
 		idx_t max_offset = append_data.row_count + to - from;
 		if (max_offset > NumericLimits<uint32_t>::Maximum() &&
-		    append_data.options.offset_size == ArrowOffsetSize::REGULAR) {
+		    append_data.options.arrow_offset_size == ArrowOffsetSize::REGULAR) {
 			throw InvalidInputException("Arrow Appender: The maximum total string size for regular string buffers is "
 			                            "%u but the offset of %lu exceeds this.",
 			                            NumericLimits<uint32_t>::Maximum(), max_offset);

package/src/duckdb/src/include/duckdb/common/arrow/arrow_appender.hpp

@@ -19,7 +19,7 @@ struct ArrowAppendData;
 //! The ArrowAppender class can be used to incrementally construct an arrow array by appending data chunks into it
 class ArrowAppender {
 public:
-	DUCKDB_API ArrowAppender(vector<LogicalType> types, idx_t initial_capacity, ArrowOptions options);
+	DUCKDB_API ArrowAppender(vector<LogicalType> types, idx_t initial_capacity, ClientProperties options);
 	DUCKDB_API ~ArrowAppender();
 
 	//! Append a data chunk to the underlying arrow array
@@ -30,7 +30,8 @@ public:
 public:
 	static void ReleaseArray(ArrowArray *array);
 	static ArrowArray *FinalizeChild(const LogicalType &type, ArrowAppendData &append_data);
-	static unique_ptr<ArrowAppendData> InitializeChild(const LogicalType &type, idx_t capacity, ArrowOptions &options);
+	static unique_ptr<ArrowAppendData> InitializeChild(const LogicalType &type, idx_t capacity,
+	                                                   ClientProperties &options);
 
 private:
 	//! The types of the chunks that will be appended in
@@ -40,7 +41,7 @@ private:
 	//! The total row count that has been appended
 	idx_t row_count = 0;
 
-	ArrowOptions options;
+	ClientProperties options;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/arrow/arrow_converter.hpp

@@ -10,7 +10,7 @@
 
 #include "duckdb/common/types/data_chunk.hpp"
 #include "duckdb/common/arrow/arrow.hpp"
-#include "duckdb/common/arrow/arrow_options.hpp"
+#include "duckdb/main/client_properties.hpp"
 
 struct ArrowSchema;
 
@@ -18,8 +18,8 @@ namespace duckdb {
 
 struct ArrowConverter {
 	DUCKDB_API static void ToArrowSchema(ArrowSchema *out_schema, const vector<LogicalType> &types,
-	                                     const vector<string> &names, const ArrowOptions &options);
-	DUCKDB_API static void ToArrowArray(DataChunk &input, ArrowArray *out_array, ArrowOptions options);
+	                                     const vector<string> &names, const ClientProperties &options);
+	DUCKDB_API static void ToArrowArray(DataChunk &input, ArrowArray *out_array, ClientProperties options);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/arrow/arrow_wrapper.hpp

@@ -10,6 +10,8 @@
 #include "duckdb/common/arrow/arrow.hpp"
 #include "duckdb/common/helper.hpp"
 #include "duckdb/common/preserved_error.hpp"
+#include "duckdb/main/chunk_scan_state.hpp"
+#include "duckdb/main/client_properties.hpp"
 
 //! Here we have the internal duckdb classes that interact with Arrow's Internal Header (i.e., duckdb/commons/arrow.hpp)
 namespace duckdb {
@@ -56,9 +58,9 @@ public:
 
 class ArrowUtil {
 public:
-	static bool TryFetchChunk(QueryResult *result, idx_t chunk_size, ArrowArray *out, idx_t &result_count,
-	                          PreservedError &error);
-	static idx_t FetchChunk(QueryResult *result, idx_t chunk_size, ArrowArray *out);
+	static bool TryFetchChunk(ChunkScanState &scan_state, ClientProperties options, idx_t chunk_size, ArrowArray *out,
+	                          idx_t &result_count, PreservedError &error);
+	static idx_t FetchChunk(ChunkScanState &scan_state, ClientProperties options, idx_t chunk_size, ArrowArray *out);
 
 private:
 	static bool TryFetchNext(QueryResult &result, unique_ptr<DataChunk> &out, PreservedError &error);

package/src/duckdb/src/include/duckdb/common/arrow/result_arrow_wrapper.hpp

@@ -10,17 +10,21 @@
 
 #include "duckdb/main/query_result.hpp"
 #include "duckdb/common/arrow/arrow_wrapper.hpp"
+#include "duckdb/main/chunk_scan_state.hpp"
 
 namespace duckdb {
 class ResultArrowArrayStreamWrapper {
 public:
 	explicit ResultArrowArrayStreamWrapper(unique_ptr<QueryResult> result, idx_t batch_size);
+
+public:
 	ArrowArrayStream stream;
 	unique_ptr<QueryResult> result;
 	PreservedError last_error;
 	idx_t batch_size;
 	vector<LogicalType> column_types;
 	vector<string> column_names;
+	unique_ptr<ChunkScanState> scan_state;
 
 private:
 	static int MyStreamGetSchema(struct ArrowArrayStream *stream, struct ArrowSchema *out);

package/src/duckdb/src/include/duckdb/common/stack_checker.hpp

@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/stack_checker.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+namespace duckdb {
+
+template <class RECURSIVE_CLASS>
+class StackChecker {
+public:
+	StackChecker(RECURSIVE_CLASS &recursive_class_p, idx_t stack_usage_p)
+	    : recursive_class(recursive_class_p), stack_usage(stack_usage_p) {
+		recursive_class.stack_depth += stack_usage;
+	}
+	~StackChecker() {
+		recursive_class.stack_depth -= stack_usage;
+	}
+	StackChecker(StackChecker &&other) noexcept
+	    : recursive_class(other.recursive_class), stack_usage(other.stack_usage) {
+		other.stack_usage = 0;
+	}
+	StackChecker(const StackChecker &) = delete;
+
+private:
+	RECURSIVE_CLASS &recursive_class;
+	idx_t stack_usage;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/main/capi/capi_internal.hpp

@@ -13,7 +13,6 @@
 #include "duckdb/common/types.hpp"
 #include "duckdb/common/types/data_chunk.hpp"
 #include "duckdb/main/appender.hpp"
-#include "duckdb/common/arrow/arrow_options.hpp"
 
 #include <cstring>
 #include <cassert>
@@ -48,7 +47,7 @@ struct PendingStatementWrapper {
 struct ArrowResultWrapper {
 	unique_ptr<MaterializedQueryResult> result;
 	unique_ptr<DataChunk> current_chunk;
-	ArrowOptions options;
+	ClientProperties options;
 };
 
 struct AppenderWrapper {

package/src/duckdb/src/include/duckdb/main/chunk_scan_state/query_result.hpp

@@ -0,0 +1,29 @@
+#pragma once
+
+#include "duckdb/main/chunk_scan_state.hpp"
+#include "duckdb/common/preserved_error.hpp"
+
+namespace duckdb {
+
+class QueryResult;
+
+class QueryResultChunkScanState : public ChunkScanState {
+public:
+	QueryResultChunkScanState(QueryResult &result);
+	~QueryResultChunkScanState();
+
+public:
+	bool LoadNextChunk(PreservedError &error) override;
+	bool HasError() const override;
+	PreservedError &GetError() override;
+	const vector<LogicalType> &Types() const override;
+	const vector<string> &Names() const override;
+
+private:
+	bool InternalLoad(PreservedError &error);
+
+private:
+	QueryResult &result;
+};
+
+} // namespace duckdb