duckdb 0.8.2-dev161.0 → 0.8.2-dev1764.0

package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp

@@ -23,7 +23,7 @@ struct FlattenDependentJoins {
 
 	//! Detects which Logical Operators have correlated expressions that they are dependent upon, filling the
 	//! has_correlated_expressions map.
-	bool DetectCorrelatedExpressions(LogicalOperator *op, bool lateral = false);
+	bool DetectCorrelatedExpressions(LogicalOperator *op, bool lateral = false, idx_t lateral_depth = 0);
 
 	//! Push the dependent join down a LogicalOperator
 	unique_ptr<LogicalOperator> PushDownDependentJoin(unique_ptr<LogicalOperator> plan);
@@ -43,7 +43,7 @@ struct FlattenDependentJoins {
 
 private:
 	unique_ptr<LogicalOperator> PushDownDependentJoinInternal(unique_ptr<LogicalOperator> plan,
-	                                                          bool &parent_propagate_null_values);
+	                                                          bool &parent_propagate_null_values, idx_t lateral_depth);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp

@@ -16,7 +16,8 @@ namespace duckdb {
 //! Helper class to recursively detect correlated expressions inside a single LogicalOperator
 class HasCorrelatedExpressions : public LogicalOperatorVisitor {
 public:
-	explicit HasCorrelatedExpressions(const vector<CorrelatedColumnInfo> &correlated, bool lateral = false);
+	explicit HasCorrelatedExpressions(const vector<CorrelatedColumnInfo> &correlated, bool lateral = false,
+	                                  idx_t lateral_depth = 0);
 
 	void VisitOperator(LogicalOperator &op) override;
 
@@ -28,6 +29,8 @@ protected:
 	unique_ptr<Expression> VisitReplace(BoundSubqueryExpression &expr, unique_ptr<Expression> *expr_ptr) override;
 
 	const vector<CorrelatedColumnInfo> &correlated_columns;
+	// Tracks number of nested laterals
+	idx_t lateral_depth;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp

@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/planner/subquery/recursive_dependent_join_planner.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/planner/logical_operator_visitor.hpp"
+
+namespace duckdb {
+
+class Binder;
+
+/*
+ * Recursively plan subqueries and flatten dependent joins from outermost to innermost (like peeling an onion).
+ */
+class RecursiveDependentJoinPlanner : public LogicalOperatorVisitor {
+public:
+	explicit RecursiveDependentJoinPlanner(Binder &binder) : binder(binder) {
+	}
+	void VisitOperator(LogicalOperator &op) override;
+	unique_ptr<Expression> VisitReplace(BoundSubqueryExpression &expr, unique_ptr<Expression> *expr_ptr) override;
+
+private:
+	unique_ptr<LogicalOperator> root;
+	Binder &binder;
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp

@@ -17,7 +17,8 @@ namespace duckdb {
 //! Helper class to rewrite correlated expressions within a single LogicalOperator
 class RewriteCorrelatedExpressions : public LogicalOperatorVisitor {
 public:
-	RewriteCorrelatedExpressions(ColumnBinding base_binding, column_binding_map_t<idx_t> &correlated_map);
+	RewriteCorrelatedExpressions(ColumnBinding base_binding, column_binding_map_t<idx_t> &correlated_map,
+	                             idx_t lateral_depth, bool recursive_rewrite = false);
 
 	void VisitOperator(LogicalOperator &op) override;
 
@@ -31,7 +32,7 @@ private:
 	public:
 		RewriteCorrelatedRecursive(BoundSubqueryExpression &parent, ColumnBinding base_binding,
 		                           column_binding_map_t<idx_t> &correlated_map);
-
+		void RewriteJoinRefRecursive(BoundTableRef &ref);
 		void RewriteCorrelatedSubquery(BoundSubqueryExpression &expr);
 		void RewriteCorrelatedExpressions(Expression &child);
 
@@ -43,6 +44,11 @@ private:
 private:
 	ColumnBinding base_binding;
 	column_binding_map_t<idx_t> &correlated_map;
+	// To keep track of the number of dependent joins encountered
+	idx_t lateral_depth;
+	// This flag is used to determine if the rewrite should recursively update the bindings for all
+	// bound columns ref in the plan, and update the depths to match the new source
+	bool recursive_rewrite;
 };
 
 //! Helper class that rewrites COUNT aggregates into a CASE expression turning NULL into 0 after a LEFT OUTER JOIN

package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp

@@ -17,8 +17,9 @@ public:
 	static constexpr const TableReferenceType TYPE = TableReferenceType::CTE;
 
 public:
-	BoundCTERef(idx_t bind_index, idx_t cte_index)
-	    : BoundTableRef(TableReferenceType::CTE), bind_index(bind_index), cte_index(cte_index) {
+	BoundCTERef(idx_t bind_index, idx_t cte_index, CTEMaterialize materialized_cte)
+	    : BoundTableRef(TableReferenceType::CTE), bind_index(bind_index), cte_index(cte_index),
+	      materialized_cte(materialized_cte) {
 	}
 
 	//! The set of columns bound to this base table reference
@@ -29,5 +30,7 @@ public:
 	idx_t bind_index;
 	//! The index of the cte
 	idx_t cte_index;
+	//! Is this a reference to a materialized CTE?
+	CTEMaterialize materialized_cte;
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp

@@ -45,7 +45,7 @@ public:
 	DUCKDB_API ArenaChunk *GetHead();
 	DUCKDB_API ArenaChunk *GetTail();
 
-	DUCKDB_API bool IsEmpty();
+	DUCKDB_API bool IsEmpty() const;
 
 	//! Returns an "Allocator" wrapper for this arena allocator
 	Allocator &GetAllocator() {

package/src/duckdb/src/include/duckdb/storage/block_manager.hpp

@@ -65,6 +65,9 @@ public:
 	//! Returns the number of free blocks
 	virtual idx_t FreeBlocks() = 0;
 
+	//! Truncate the underlying database file after a checkpoint
+	virtual void Truncate();
+
 	//! Register a block with the given block id in the base file
 	shared_ptr<BlockHandle> RegisterBlock(block_id_t block_id, bool is_meta_block = false);
 	//! Clear cached handles for meta blocks
@@ -74,9 +77,6 @@ public:
 
 	void UnregisterBlock(block_id_t block_id, bool can_destroy);
 
-	static BlockManager &GetBlockManager(ClientContext &context);
-	static BlockManager &GetBlockManager(DatabaseInstance &db);
-
 private:
 	//! The lock for the set of blocks
 	mutex blocks_lock;

package/src/duckdb/src/include/duckdb/storage/data_table.hpp

@@ -51,7 +51,7 @@ public:
 	          const string &table, vector<ColumnDefinition> column_definitions_p,
 	          unique_ptr<PersistentTableData> data = nullptr);
 	//! Constructs a DataTable as a delta on an existing data table with a newly added column
-	DataTable(ClientContext &context, DataTable &parent, ColumnDefinition &new_column, Expression *default_value);
+	DataTable(ClientContext &context, DataTable &parent, ColumnDefinition &new_column, Expression &default_value);
 	//! Constructs a DataTable as a delta on an existing data table but with one column removed
 	DataTable(ClientContext &context, DataTable &parent, idx_t removed_column);
 	//! Constructs a DataTable as a delta on an existing data table but with one column changed type

package/src/duckdb/src/include/duckdb/storage/object_cache.hpp

@@ -47,11 +47,33 @@ public:
 		return std::static_pointer_cast<T, ObjectCacheEntry>(object);
 	}
 
+	template <class T, class... Args>
+	shared_ptr<T> GetOrCreate(const string &key, Args &&...args) {
+		lock_guard<mutex> glock(lock);
+
+		auto entry = cache.find(key);
+		if (entry == cache.end()) {
+			auto value = make_shared<T>(args...);
+			cache[key] = value;
+			return value;
+		}
+		auto object = entry->second;
+		if (!object || object->GetObjectType() != T::ObjectType()) {
+			return nullptr;
+		}
+		return std::static_pointer_cast<T, ObjectCacheEntry>(object);
+	}
+
 	void Put(string key, shared_ptr<ObjectCacheEntry> value) {
 		lock_guard<mutex> glock(lock);
 		cache[key] = std::move(value);
 	}
 
+	void Delete(const string &key) {
+		lock_guard<mutex> glock(lock);
+		cache.erase(key);
+	}
+
 	DUCKDB_API static ObjectCache &GetObjectCache(ClientContext &context);
 	DUCKDB_API static bool ObjectCacheEnabled(ClientContext &context);
 

package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp

@@ -60,6 +60,8 @@ public:
 	void Write(FileBuffer &block, block_id_t block_id) override;
 	//! Write the header to disk, this is the final step of the checkpointing process
 	void WriteHeader(DatabaseHeader header) override;
+	//! Truncate the underlying database file after a checkpoint
+	void Truncate() override;
 
 	//! Returns the number of total blocks
 	idx_t TotalBlocks() override;

package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp

@@ -48,6 +48,10 @@ struct StringStats {
 	DUCKDB_API static uint32_t MaxStringLength(const BaseStatistics &stats);
 	//! Whether or not the strings can contain unicode
 	DUCKDB_API static bool CanContainUnicode(const BaseStatistics &stats);
+	//! Returns the min value (up to a length of StringStatsData::MAX_STRING_MINMAX_SIZE)
+	DUCKDB_API static string Min(const BaseStatistics &stats);
+	//! Returns the max value (up to a length of StringStatsData::MAX_STRING_MINMAX_SIZE)
+	DUCKDB_API static string Max(const BaseStatistics &stats);
 
 	//! Resets the max string length so HasMaxStringLength() is false
 	DUCKDB_API static void ResetMaxStringLength(BaseStatistics &stats);

package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp

@@ -41,6 +41,7 @@ public:
 	//! Returns whether or not a single row in the ChunkInfo should be used or not for the given transaction
 	virtual bool Fetch(TransactionData transaction, row_t row) = 0;
 	virtual void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) = 0;
+	virtual idx_t GetCommittedDeletedCount(idx_t max_count) = 0;
 
 	virtual void Serialize(Serializer &serialize) = 0;
 	static unique_ptr<ChunkInfo> Deserialize(Deserializer &source);
@@ -79,6 +80,7 @@ public:
 	                            SelectionVector &sel_vector, idx_t max_count) override;
 	bool Fetch(TransactionData transaction, row_t row) override;
 	void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) override;
+	idx_t GetCommittedDeletedCount(idx_t max_count) override;
 
 	void Serialize(Serializer &serialize) override;
 	static unique_ptr<ChunkInfo> Deserialize(Deserializer &source);
@@ -113,6 +115,7 @@ public:
 	                            SelectionVector &sel_vector, idx_t max_count) override;
 	bool Fetch(TransactionData transaction, row_t row) override;
 	void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) override;
+	idx_t GetCommittedDeletedCount(idx_t max_count) override;
 
 	void Append(idx_t start, idx_t end, transaction_t commit_id);
 

package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -74,7 +74,6 @@ public:
 	RowGroupCollection &GetCollection() {
 		return collection.get();
 	}
-	DatabaseInstance &GetDatabase();
 	BlockManager &GetBlockManager();
 	DataTableInfo &GetTableInfo();
 
@@ -82,7 +81,7 @@ public:
 	                               ExpressionExecutor &executor, CollectionScanState &scan_state,
 	                               DataChunk &scan_chunk);
 	unique_ptr<RowGroup> AddColumn(RowGroupCollection &collection, ColumnDefinition &new_column,
-	                               ExpressionExecutor &executor, Expression *default_value, Vector &intermediate);
+	                               ExpressionExecutor &executor, Expression &default_value, Vector &intermediate);
 	unique_ptr<RowGroup> RemoveColumn(RowGroupCollection &collection, idx_t removed_column);
 
 	void CommitDrop();
@@ -123,6 +122,7 @@ public:
 	idx_t Delete(TransactionData transaction, DataTable &table, row_t *row_ids, idx_t count);
 
 	RowGroupWriteData WriteToDisk(PartialBlockManager &manager, const vector<CompressionType> &compression_types);
+	bool AllDeleted();
 	RowGroupPointer Checkpoint(RowGroupWriter &writer, TableStatistics &global_stats);
 	static void Serialize(RowGroupPointer &pointer, Serializer &serializer);
 	static RowGroupPointer Deserialize(Deserializer &source, const vector<LogicalType> &columns);
@@ -170,6 +170,7 @@ struct VersionNode {
 	unique_ptr<ChunkInfo> info[RowGroup::ROW_GROUP_VECTOR_COUNT];
 
 	void SetStart(idx_t start);
+	idx_t GetCommittedDeletedCount(idx_t count);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp

@@ -45,7 +45,6 @@ public:
 	void AppendRowGroup(SegmentLock &l, idx_t start_row);
 	//! Get the nth row-group, negative numbers start from the back (so -1 is the last row group, etc)
 	RowGroup *GetRowGroup(int64_t index);
-	idx_t RowGroupCount();
 	void Verify();
 
 	void InitializeScan(CollectionScanState &state, const vector<column_t> &column_ids, TableFilterSet *table_filters);
@@ -93,7 +92,7 @@ public:
 	const vector<LogicalType> &GetTypes() const;
 
 	shared_ptr<RowGroupCollection> AddColumn(ClientContext &context, ColumnDefinition &new_column,
-	                                         Expression *default_value);
+	                                         Expression &default_value);
 	shared_ptr<RowGroupCollection> RemoveColumn(idx_t col_idx);
 	shared_ptr<RowGroupCollection> AlterType(ClientContext &context, idx_t changed_idx, const LogicalType &target_type,
 	                                         vector<column_t> bound_columns, Expression &cast_expr);
@@ -104,7 +103,6 @@ public:
 	void SetDistinct(column_t column_id, unique_ptr<DistinctStatistics> distinct_stats);
 
 	AttachedDatabase &GetAttached();
-	DatabaseInstance &GetDatabase();
 	BlockManager &GetBlockManager() {
 		return block_manager;
 	}

package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp

@@ -32,7 +32,7 @@ public:
 	LocalTableStorage(DataTable &table, LocalTableStorage &parent, idx_t drop_idx);
 	// Create a LocalTableStorage from an ADD COLUMN
 	LocalTableStorage(ClientContext &context, DataTable &table, LocalTableStorage &parent, ColumnDefinition &new_column,
-	                  optional_ptr<Expression> default_value);
+	                  Expression &default_value);
 	~LocalTableStorage();
 
 	reference<DataTable> table_ref;
@@ -143,8 +143,7 @@ public:
 
 	idx_t AddedRows(DataTable &table);
 
-	void AddColumn(DataTable &old_dt, DataTable &new_dt, ColumnDefinition &new_column,
-	               optional_ptr<Expression> default_value);
+	void AddColumn(DataTable &old_dt, DataTable &new_dt, ColumnDefinition &new_column, Expression &default_value);
 	void DropColumn(DataTable &old_dt, DataTable &new_dt, idx_t removed_column);
 	void ChangeType(DataTable &old_dt, DataTable &new_dt, idx_t changed_idx, const LogicalType &target_type,
 	                const vector<column_t> &bound_columns, Expression &cast_expr);

package/src/duckdb/src/include/duckdb.h

@@ -279,6 +279,9 @@ typedef struct _duckdb_appender {
 typedef struct _duckdb_arrow {
 	void *__arrw;
 } * duckdb_arrow;
+typedef struct _duckdb_arrow_stream {
+	void *__arrwstr;
+} * duckdb_arrow_stream;
 typedef struct _duckdb_config {
 	void *__cnfg;
 } * duckdb_config;
@@ -1078,6 +1081,31 @@ Executes the prepared statement with the given bound parameters, and returns an
 DUCKDB_API duckdb_state duckdb_execute_prepared_arrow(duckdb_prepared_statement prepared_statement,
                                                       duckdb_arrow *out_result);
 
+/*!
+Scans the Arrow stream and creates a view with the given name.
+
+* connection: The connection on which to execute the scan.
+* table_name: Name of the temporary view to create.
+* arrow: Arrow stream wrapper.
+* returns: `DuckDBSuccess` on success or `DuckDBError` on failure.
+*/
+DUCKDB_API duckdb_state duckdb_arrow_scan(duckdb_connection connection, const char *table_name,
+                                          duckdb_arrow_stream arrow);
+
+/*!
+Scans the Arrow array and creates a view with the given name.
+
+* connection: The connection on which to execute the scan.
+* table_name: Name of the temporary view to create.
+* arrow_schema: Arrow schema wrapper.
+* arrow_array: Arrow array wrapper.
+* out_stream: Output array stream that wraps around the passed schema, for releasing/deleting once done.
+* returns: `DuckDBSuccess` on success or `DuckDBError` on failure.
+*/
+DUCKDB_API duckdb_state duckdb_arrow_array_scan(duckdb_connection connection, const char *table_name,
+                                                duckdb_arrow_schema arrow_schema, duckdb_arrow_array arrow_array,
+                                                duckdb_arrow_stream *out_stream);
+
 //===--------------------------------------------------------------------===//
 // Extract Statements
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/main/capi/arrow-c.cpp

@@ -1,5 +1,6 @@
-#include "duckdb/main/capi/capi_internal.hpp"
 #include "duckdb/common/arrow/arrow_converter.hpp"
+#include "duckdb/function/table/arrow.hpp"
+#include "duckdb/main/capi/capi_internal.hpp"
 
 using duckdb::ArrowConverter;
 using duckdb::ArrowResultWrapper;
@@ -108,3 +109,156 @@ duckdb_state duckdb_execute_prepared_arrow(duckdb_prepared_statement prepared_st
 	*out_result = reinterpret_cast<duckdb_arrow>(arrow_wrapper);
 	return !arrow_wrapper->result->HasError() ? DuckDBSuccess : DuckDBError;
 }
+
+namespace arrow_array_stream_wrapper {
+namespace {
+struct PrivateData {
+	ArrowSchema *schema;
+	ArrowArray *array;
+	bool done = false;
+};
+
+// LCOV_EXCL_START
+// This function is never called, but used to set ArrowSchema's release functions to a non-null NOOP.
+void EmptySchemaRelease(ArrowSchema *) {
+}
+// LCOV_EXCL_STOP
+
+void EmptyArrayRelease(ArrowArray *) {
+}
+
+void EmptyStreamRelease(ArrowArrayStream *) {
+}
+
+void FactoryGetSchema(uintptr_t stream_factory_ptr, duckdb::ArrowSchemaWrapper &schema) {
+	auto private_data =
+	    reinterpret_cast<PrivateData *>(reinterpret_cast<ArrowArrayStream *>(stream_factory_ptr)->private_data);
+	schema.arrow_schema = *private_data->schema;
+
+	// Need to nullify the root schema's release function here, because streams don't allow us to set the release
+	// function. For the schema's children, we nullify the release functions in `duckdb_arrow_scan`, so we don't need to
+	// handle them again here. We set this to nullptr and not EmptySchemaRelease to prevent ArrowSchemaWrapper's
+	// destructor from destroying the schema (it's the caller's responsibility).
+	schema.arrow_schema.release = nullptr;
+}
+
+int GetSchema(struct ArrowArrayStream *stream, struct ArrowSchema *out) {
+	auto private_data = static_cast<arrow_array_stream_wrapper::PrivateData *>((stream->private_data));
+	if (private_data->schema == nullptr) {
+		return DuckDBError;
+	}
+
+	*out = *private_data->schema;
+	out->release = EmptySchemaRelease;
+	return DuckDBSuccess;
+}
+
+int GetNext(struct ArrowArrayStream *stream, struct ArrowArray *out) {
+	auto private_data = static_cast<arrow_array_stream_wrapper::PrivateData *>((stream->private_data));
+	*out = *private_data->array;
+	if (private_data->done) {
+		out->release = nullptr;
+	} else {
+		out->release = EmptyArrayRelease;
+	}
+
+	private_data->done = true;
+	return DuckDBSuccess;
+}
+
+duckdb::unique_ptr<duckdb::ArrowArrayStreamWrapper> FactoryGetNext(uintptr_t stream_factory_ptr,
+                                                                   duckdb::ArrowStreamParameters &parameters) {
+	auto stream = reinterpret_cast<ArrowArrayStream *>(stream_factory_ptr);
+	auto ret = duckdb::make_uniq<duckdb::ArrowArrayStreamWrapper>();
+	ret->arrow_array_stream = *stream;
+	ret->arrow_array_stream.release = EmptyStreamRelease;
+	return ret;
+}
+
+// LCOV_EXCL_START
+// This function is never be called, because it's used to construct a stream wrapping around a caller-supplied
+// ArrowArray. Thus, the stream itself cannot produce an error.
+const char *GetLastError(struct ArrowArrayStream *stream) {
+	return nullptr;
+}
+// LCOV_EXCL_STOP
+
+void Release(struct ArrowArrayStream *stream) {
+	if (stream->private_data != nullptr) {
+		delete reinterpret_cast<PrivateData *>(stream->private_data);
+	}
+
+	stream->private_data = nullptr;
+	stream->release = nullptr;
+}
+
+duckdb_state Ingest(duckdb_connection connection, const char *table_name, struct ArrowArrayStream *input) {
+	try {
+		auto cconn = reinterpret_cast<duckdb::Connection *>(connection);
+		cconn
+		    ->TableFunction("arrow_scan", {duckdb::Value::POINTER((uintptr_t)input),
+		                                   duckdb::Value::POINTER((uintptr_t)FactoryGetNext),
+		                                   duckdb::Value::POINTER((uintptr_t)FactoryGetSchema)})
+		    ->CreateView(table_name, true, false);
+	} catch (...) { // LCOV_EXCL_START
+		// Tried covering this in tests, but it proved harder than expected. At the time of writing:
+		// - Passing any name to `CreateView` worked without throwing an exception
+		// - Passing a null Arrow array worked without throwing an exception
+		// - Passing an invalid schema (without any columns) led to an InternalException with SIGABRT, which is meant to
+		//   be un-catchable. This case likely needs to be handled gracefully within `arrow_scan`.
+		// Ref: https://discord.com/channels/909674491309850675/921100573732909107/1115230468699336785
+		return DuckDBError;
+	} // LCOV_EXCL_STOP
+
+	return DuckDBSuccess;
+}
+} // namespace
+} // namespace arrow_array_stream_wrapper
+
+duckdb_state duckdb_arrow_scan(duckdb_connection connection, const char *table_name, duckdb_arrow_stream arrow) {
+	auto stream = reinterpret_cast<ArrowArrayStream *>(arrow);
+
+	// Backup release functions - we nullify children schema release functions because we don't want to release on
+	// behalf of the caller, downstream in our code. Note that Arrow releases target immediate children, but aren't
+	// recursive. So we only back up immediate children here and restore their functions.
+	ArrowSchema schema;
+	if (stream->get_schema(stream, &schema) == DuckDBError) {
+		return DuckDBError;
+	}
+
+	typedef void (*release_fn_t)(ArrowSchema *);
+	std::vector<release_fn_t> release_fns(schema.n_children);
+	for (int64_t i = 0; i < schema.n_children; i++) {
+		auto child = schema.children[i];
+		release_fns[i] = child->release;
+		child->release = arrow_array_stream_wrapper::EmptySchemaRelease;
+	}
+
+	auto ret = arrow_array_stream_wrapper::Ingest(connection, table_name, stream);
+
+	// Restore release functions.
+	for (int64_t i = 0; i < schema.n_children; i++) {
+		schema.children[i]->release = release_fns[i];
+	}
+
+	return ret;
+}
+
+duckdb_state duckdb_arrow_array_scan(duckdb_connection connection, const char *table_name,
+                                     duckdb_arrow_schema arrow_schema, duckdb_arrow_array arrow_array,
+                                     duckdb_arrow_stream *out_stream) {
+	auto private_data = new arrow_array_stream_wrapper::PrivateData;
+	private_data->schema = reinterpret_cast<ArrowSchema *>(arrow_schema);
+	private_data->array = reinterpret_cast<ArrowArray *>(arrow_array);
+	private_data->done = false;
+
+	ArrowArrayStream *stream = new ArrowArrayStream;
+	*out_stream = reinterpret_cast<duckdb_arrow_stream>(stream);
+	stream->get_schema = arrow_array_stream_wrapper::GetSchema;
+	stream->get_next = arrow_array_stream_wrapper::GetNext;
+	stream->get_last_error = arrow_array_stream_wrapper::GetLastError;
+	stream->release = arrow_array_stream_wrapper::Release;
+	stream->private_data = private_data;
+
+	return duckdb_arrow_scan(connection, table_name, reinterpret_cast<duckdb_arrow_stream>(stream));
+}

package/src/duckdb/src/main/capi/duckdb_value-c.cpp

@@ -26,7 +26,7 @@ char *duckdb_get_varchar(duckdb_value value) {
 	auto str_val = val->DefaultCastAs(duckdb::LogicalType::VARCHAR);
 	auto &str = duckdb::StringValue::Get(str_val);
 
-	auto result = reinterpret_cast<char *>(malloc(sizeof(char *) * (str.size() + 1)));
+	auto result = reinterpret_cast<char *>(malloc(sizeof(char) * (str.size() + 1)));
 	memcpy(result, str.c_str(), str.size());
 	result[str.size()] = '\0';
 	return result;

package/src/duckdb/src/main/config.cpp

@@ -59,10 +59,12 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_LOCAL(DebugForceExternal),
                                                  DUCKDB_LOCAL(DebugForceNoCrossProduct),
                                                  DUCKDB_LOCAL(DebugAsOfIEJoin),
+                                                 DUCKDB_LOCAL(PreferRangeJoins),
                                                  DUCKDB_GLOBAL(DebugWindowMode),
                                                  DUCKDB_GLOBAL_LOCAL(DefaultCollationSetting),
                                                  DUCKDB_GLOBAL(DefaultOrderSetting),
                                                  DUCKDB_GLOBAL(DefaultNullOrderSetting),
+                                                 DUCKDB_GLOBAL(DisabledFileSystemsSetting),
                                                  DUCKDB_GLOBAL(DisabledOptimizersSetting),
                                                  DUCKDB_GLOBAL(EnableExternalAccessSetting),
                                                  DUCKDB_GLOBAL(EnableFSSTVectors),
@@ -92,6 +94,7 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_LOCAL(OrderedAggregateThreshold),
                                                  DUCKDB_GLOBAL(PasswordSetting),
                                                  DUCKDB_LOCAL(PerfectHashThresholdSetting),
+                                                 DUCKDB_LOCAL(PivotFilterThreshold),
                                                  DUCKDB_LOCAL(PivotLimitSetting),
                                                  DUCKDB_LOCAL(PreserveIdentifierCase),
                                                  DUCKDB_GLOBAL(PreserveInsertionOrder),
@@ -109,6 +112,7 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_GLOBAL_ALIAS("user", UsernameSetting),
                                                  DUCKDB_GLOBAL_ALIAS("wal_autocheckpoint", CheckpointThresholdSetting),
                                                  DUCKDB_GLOBAL_ALIAS("worker_threads", ThreadsSetting),
+                                                 DUCKDB_GLOBAL(FlushAllocatorSetting),
                                                  FINAL_SETTING};
 
 vector<ConfigurationOption> DBConfig::GetOptions() {

package/src/duckdb/src/main/database.cpp

@@ -178,7 +178,7 @@ void DatabaseInstance::CreateMainDatabase() {
 void ThrowExtensionSetUnrecognizedOptions(const unordered_map<string, Value> &unrecognized_options) {
 	auto unrecognized_options_iter = unrecognized_options.begin();
 	string unrecognized_option_keys = unrecognized_options_iter->first;
-	for (; unrecognized_options_iter == unrecognized_options.end(); ++unrecognized_options_iter) {
+	while (++unrecognized_options_iter != unrecognized_options.end()) {
 		unrecognized_option_keys = "," + unrecognized_options_iter->first;
 	}
 	throw InvalidInputException("Unrecognized configuration property \"%s\"", unrecognized_option_keys);