duckdb 0.8.2-dev150.0 → 0.8.2-dev1559.0

package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp

@@ -60,6 +60,8 @@ public:
 	void Write(FileBuffer &block, block_id_t block_id) override;
 	//! Write the header to disk, this is the final step of the checkpointing process
 	void WriteHeader(DatabaseHeader header) override;
+	//! Truncate the underlying database file after a checkpoint
+	void Truncate() override;
 
 	//! Returns the number of total blocks
 	idx_t TotalBlocks() override;

package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp

@@ -48,6 +48,10 @@ struct StringStats {
 	DUCKDB_API static uint32_t MaxStringLength(const BaseStatistics &stats);
 	//! Whether or not the strings can contain unicode
 	DUCKDB_API static bool CanContainUnicode(const BaseStatistics &stats);
+	//! Returns the min value (up to a length of StringStatsData::MAX_STRING_MINMAX_SIZE)
+	DUCKDB_API static string Min(const BaseStatistics &stats);
+	//! Returns the max value (up to a length of StringStatsData::MAX_STRING_MINMAX_SIZE)
+	DUCKDB_API static string Max(const BaseStatistics &stats);
 
 	//! Resets the max string length so HasMaxStringLength() is false
 	DUCKDB_API static void ResetMaxStringLength(BaseStatistics &stats);

package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp

@@ -41,6 +41,7 @@ public:
 	//! Returns whether or not a single row in the ChunkInfo should be used or not for the given transaction
 	virtual bool Fetch(TransactionData transaction, row_t row) = 0;
 	virtual void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) = 0;
+	virtual idx_t GetCommittedDeletedCount(idx_t max_count) = 0;
 
 	virtual void Serialize(Serializer &serialize) = 0;
 	static unique_ptr<ChunkInfo> Deserialize(Deserializer &source);
@@ -79,6 +80,7 @@ public:
 	                            SelectionVector &sel_vector, idx_t max_count) override;
 	bool Fetch(TransactionData transaction, row_t row) override;
 	void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) override;
+	idx_t GetCommittedDeletedCount(idx_t max_count) override;
 
 	void Serialize(Serializer &serialize) override;
 	static unique_ptr<ChunkInfo> Deserialize(Deserializer &source);
@@ -113,6 +115,7 @@ public:
 	                            SelectionVector &sel_vector, idx_t max_count) override;
 	bool Fetch(TransactionData transaction, row_t row) override;
 	void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) override;
+	idx_t GetCommittedDeletedCount(idx_t max_count) override;
 
 	void Append(idx_t start, idx_t end, transaction_t commit_id);
 

package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -74,7 +74,6 @@ public:
 	RowGroupCollection &GetCollection() {
 		return collection.get();
 	}
-	DatabaseInstance &GetDatabase();
 	BlockManager &GetBlockManager();
 	DataTableInfo &GetTableInfo();
 
@@ -82,7 +81,7 @@ public:
 	                               ExpressionExecutor &executor, CollectionScanState &scan_state,
 	                               DataChunk &scan_chunk);
 	unique_ptr<RowGroup> AddColumn(RowGroupCollection &collection, ColumnDefinition &new_column,
-	                               ExpressionExecutor &executor, Expression *default_value, Vector &intermediate);
+	                               ExpressionExecutor &executor, Expression &default_value, Vector &intermediate);
 	unique_ptr<RowGroup> RemoveColumn(RowGroupCollection &collection, idx_t removed_column);
 
 	void CommitDrop();
@@ -123,6 +122,7 @@ public:
 	idx_t Delete(TransactionData transaction, DataTable &table, row_t *row_ids, idx_t count);
 
 	RowGroupWriteData WriteToDisk(PartialBlockManager &manager, const vector<CompressionType> &compression_types);
+	bool AllDeleted();
 	RowGroupPointer Checkpoint(RowGroupWriter &writer, TableStatistics &global_stats);
 	static void Serialize(RowGroupPointer &pointer, Serializer &serializer);
 	static RowGroupPointer Deserialize(Deserializer &source, const vector<LogicalType> &columns);
@@ -170,6 +170,7 @@ struct VersionNode {
 	unique_ptr<ChunkInfo> info[RowGroup::ROW_GROUP_VECTOR_COUNT];
 
 	void SetStart(idx_t start);
+	idx_t GetCommittedDeletedCount(idx_t count);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp

@@ -45,7 +45,6 @@ public:
 	void AppendRowGroup(SegmentLock &l, idx_t start_row);
 	//! Get the nth row-group, negative numbers start from the back (so -1 is the last row group, etc)
 	RowGroup *GetRowGroup(int64_t index);
-	idx_t RowGroupCount();
 	void Verify();
 
 	void InitializeScan(CollectionScanState &state, const vector<column_t> &column_ids, TableFilterSet *table_filters);
@@ -93,7 +92,7 @@ public:
 	const vector<LogicalType> &GetTypes() const;
 
 	shared_ptr<RowGroupCollection> AddColumn(ClientContext &context, ColumnDefinition &new_column,
-	                                         Expression *default_value);
+	                                         Expression &default_value);
 	shared_ptr<RowGroupCollection> RemoveColumn(idx_t col_idx);
 	shared_ptr<RowGroupCollection> AlterType(ClientContext &context, idx_t changed_idx, const LogicalType &target_type,
 	                                         vector<column_t> bound_columns, Expression &cast_expr);
@@ -104,7 +103,6 @@ public:
 	void SetDistinct(column_t column_id, unique_ptr<DistinctStatistics> distinct_stats);
 
 	AttachedDatabase &GetAttached();
-	DatabaseInstance &GetDatabase();
 	BlockManager &GetBlockManager() {
 		return block_manager;
 	}

package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp

@@ -32,7 +32,7 @@ public:
 	LocalTableStorage(DataTable &table, LocalTableStorage &parent, idx_t drop_idx);
 	// Create a LocalTableStorage from an ADD COLUMN
 	LocalTableStorage(ClientContext &context, DataTable &table, LocalTableStorage &parent, ColumnDefinition &new_column,
-	                  optional_ptr<Expression> default_value);
+	                  Expression &default_value);
 	~LocalTableStorage();
 
 	reference<DataTable> table_ref;
@@ -143,8 +143,7 @@ public:
 
 	idx_t AddedRows(DataTable &table);
 
-	void AddColumn(DataTable &old_dt, DataTable &new_dt, ColumnDefinition &new_column,
-	               optional_ptr<Expression> default_value);
+	void AddColumn(DataTable &old_dt, DataTable &new_dt, ColumnDefinition &new_column, Expression &default_value);
 	void DropColumn(DataTable &old_dt, DataTable &new_dt, idx_t removed_column);
 	void ChangeType(DataTable &old_dt, DataTable &new_dt, idx_t changed_idx, const LogicalType &target_type,
 	                const vector<column_t> &bound_columns, Expression &cast_expr);

package/src/duckdb/src/include/duckdb.h

@@ -279,6 +279,9 @@ typedef struct _duckdb_appender {
 typedef struct _duckdb_arrow {
 	void *__arrw;
 } * duckdb_arrow;
+typedef struct _duckdb_arrow_stream {
+	void *__arrwstr;
+} * duckdb_arrow_stream;
 typedef struct _duckdb_config {
 	void *__cnfg;
 } * duckdb_config;
@@ -1078,6 +1081,31 @@ Executes the prepared statement with the given bound parameters, and returns an
 DUCKDB_API duckdb_state duckdb_execute_prepared_arrow(duckdb_prepared_statement prepared_statement,
                                                       duckdb_arrow *out_result);
 
+/*!
+Scans the Arrow stream and creates a view with the given name.
+
+* connection: The connection on which to execute the scan.
+* table_name: Name of the temporary view to create.
+* arrow: Arrow stream wrapper.
+* returns: `DuckDBSuccess` on success or `DuckDBError` on failure.
+*/
+DUCKDB_API duckdb_state duckdb_arrow_scan(duckdb_connection connection, const char *table_name,
+                                          duckdb_arrow_stream arrow);
+
+/*!
+Scans the Arrow array and creates a view with the given name.
+
+* connection: The connection on which to execute the scan.
+* table_name: Name of the temporary view to create.
+* arrow_schema: Arrow schema wrapper.
+* arrow_array: Arrow array wrapper.
+* out_stream: Output array stream that wraps around the passed schema, for releasing/deleting once done.
+* returns: `DuckDBSuccess` on success or `DuckDBError` on failure.
+*/
+DUCKDB_API duckdb_state duckdb_arrow_array_scan(duckdb_connection connection, const char *table_name,
+                                                duckdb_arrow_schema arrow_schema, duckdb_arrow_array arrow_array,
+                                                duckdb_arrow_stream *out_stream);
+
 //===--------------------------------------------------------------------===//
 // Extract Statements
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/main/capi/arrow-c.cpp

@@ -1,5 +1,6 @@
-#include "duckdb/main/capi/capi_internal.hpp"
 #include "duckdb/common/arrow/arrow_converter.hpp"
+#include "duckdb/function/table/arrow.hpp"
+#include "duckdb/main/capi/capi_internal.hpp"
 
 using duckdb::ArrowConverter;
 using duckdb::ArrowResultWrapper;
@@ -108,3 +109,156 @@ duckdb_state duckdb_execute_prepared_arrow(duckdb_prepared_statement prepared_st
 	*out_result = reinterpret_cast<duckdb_arrow>(arrow_wrapper);
 	return !arrow_wrapper->result->HasError() ? DuckDBSuccess : DuckDBError;
 }
+
+namespace arrow_array_stream_wrapper {
+namespace {
+struct PrivateData {
+	ArrowSchema *schema;
+	ArrowArray *array;
+	bool done = false;
+};
+
+// LCOV_EXCL_START
+// This function is never called, but used to set ArrowSchema's release functions to a non-null NOOP.
+void EmptySchemaRelease(ArrowSchema *) {
+}
+// LCOV_EXCL_STOP
+
+void EmptyArrayRelease(ArrowArray *) {
+}
+
+void EmptyStreamRelease(ArrowArrayStream *) {
+}
+
+void FactoryGetSchema(uintptr_t stream_factory_ptr, duckdb::ArrowSchemaWrapper &schema) {
+	auto private_data =
+	    reinterpret_cast<PrivateData *>(reinterpret_cast<ArrowArrayStream *>(stream_factory_ptr)->private_data);
+	schema.arrow_schema = *private_data->schema;
+
+	// Need to nullify the root schema's release function here, because streams don't allow us to set the release
+	// function. For the schema's children, we nullify the release functions in `duckdb_arrow_scan`, so we don't need to
+	// handle them again here. We set this to nullptr and not EmptySchemaRelease to prevent ArrowSchemaWrapper's
+	// destructor from destroying the schema (it's the caller's responsibility).
+	schema.arrow_schema.release = nullptr;
+}
+
+int GetSchema(struct ArrowArrayStream *stream, struct ArrowSchema *out) {
+	auto private_data = static_cast<arrow_array_stream_wrapper::PrivateData *>((stream->private_data));
+	if (private_data->schema == nullptr) {
+		return DuckDBError;
+	}
+
+	*out = *private_data->schema;
+	out->release = EmptySchemaRelease;
+	return DuckDBSuccess;
+}
+
+int GetNext(struct ArrowArrayStream *stream, struct ArrowArray *out) {
+	auto private_data = static_cast<arrow_array_stream_wrapper::PrivateData *>((stream->private_data));
+	*out = *private_data->array;
+	if (private_data->done) {
+		out->release = nullptr;
+	} else {
+		out->release = EmptyArrayRelease;
+	}
+
+	private_data->done = true;
+	return DuckDBSuccess;
+}
+
+duckdb::unique_ptr<duckdb::ArrowArrayStreamWrapper> FactoryGetNext(uintptr_t stream_factory_ptr,
+                                                                   duckdb::ArrowStreamParameters &parameters) {
+	auto stream = reinterpret_cast<ArrowArrayStream *>(stream_factory_ptr);
+	auto ret = duckdb::make_uniq<duckdb::ArrowArrayStreamWrapper>();
+	ret->arrow_array_stream = *stream;
+	ret->arrow_array_stream.release = EmptyStreamRelease;
+	return ret;
+}
+
+// LCOV_EXCL_START
+// This function is never be called, because it's used to construct a stream wrapping around a caller-supplied
+// ArrowArray. Thus, the stream itself cannot produce an error.
+const char *GetLastError(struct ArrowArrayStream *stream) {
+	return nullptr;
+}
+// LCOV_EXCL_STOP
+
+void Release(struct ArrowArrayStream *stream) {
+	if (stream->private_data != nullptr) {
+		delete reinterpret_cast<PrivateData *>(stream->private_data);
+	}
+
+	stream->private_data = nullptr;
+	stream->release = nullptr;
+}
+
+duckdb_state Ingest(duckdb_connection connection, const char *table_name, struct ArrowArrayStream *input) {
+	try {
+		auto cconn = reinterpret_cast<duckdb::Connection *>(connection);
+		cconn
+		    ->TableFunction("arrow_scan", {duckdb::Value::POINTER((uintptr_t)input),
+		                                   duckdb::Value::POINTER((uintptr_t)FactoryGetNext),
+		                                   duckdb::Value::POINTER((uintptr_t)FactoryGetSchema)})
+		    ->CreateView(table_name, true, false);
+	} catch (...) { // LCOV_EXCL_START
+		// Tried covering this in tests, but it proved harder than expected. At the time of writing:
+		// - Passing any name to `CreateView` worked without throwing an exception
+		// - Passing a null Arrow array worked without throwing an exception
+		// - Passing an invalid schema (without any columns) led to an InternalException with SIGABRT, which is meant to
+		//   be un-catchable. This case likely needs to be handled gracefully within `arrow_scan`.
+		// Ref: https://discord.com/channels/909674491309850675/921100573732909107/1115230468699336785
+		return DuckDBError;
+	} // LCOV_EXCL_STOP
+
+	return DuckDBSuccess;
+}
+} // namespace
+} // namespace arrow_array_stream_wrapper
+
+duckdb_state duckdb_arrow_scan(duckdb_connection connection, const char *table_name, duckdb_arrow_stream arrow) {
+	auto stream = reinterpret_cast<ArrowArrayStream *>(arrow);
+
+	// Backup release functions - we nullify children schema release functions because we don't want to release on
+	// behalf of the caller, downstream in our code. Note that Arrow releases target immediate children, but aren't
+	// recursive. So we only back up immediate children here and restore their functions.
+	ArrowSchema schema;
+	if (stream->get_schema(stream, &schema) == DuckDBError) {
+		return DuckDBError;
+	}
+
+	typedef void (*release_fn_t)(ArrowSchema *);
+	std::vector<release_fn_t> release_fns(schema.n_children);
+	for (int64_t i = 0; i < schema.n_children; i++) {
+		auto child = schema.children[i];
+		release_fns[i] = child->release;
+		child->release = arrow_array_stream_wrapper::EmptySchemaRelease;
+	}
+
+	auto ret = arrow_array_stream_wrapper::Ingest(connection, table_name, stream);
+
+	// Restore release functions.
+	for (int64_t i = 0; i < schema.n_children; i++) {
+		schema.children[i]->release = release_fns[i];
+	}
+
+	return ret;
+}
+
+duckdb_state duckdb_arrow_array_scan(duckdb_connection connection, const char *table_name,
+                                     duckdb_arrow_schema arrow_schema, duckdb_arrow_array arrow_array,
+                                     duckdb_arrow_stream *out_stream) {
+	auto private_data = new arrow_array_stream_wrapper::PrivateData;
+	private_data->schema = reinterpret_cast<ArrowSchema *>(arrow_schema);
+	private_data->array = reinterpret_cast<ArrowArray *>(arrow_array);
+	private_data->done = false;
+
+	ArrowArrayStream *stream = new ArrowArrayStream;
+	*out_stream = reinterpret_cast<duckdb_arrow_stream>(stream);
+	stream->get_schema = arrow_array_stream_wrapper::GetSchema;
+	stream->get_next = arrow_array_stream_wrapper::GetNext;
+	stream->get_last_error = arrow_array_stream_wrapper::GetLastError;
+	stream->release = arrow_array_stream_wrapper::Release;
+	stream->private_data = private_data;
+
+	return duckdb_arrow_scan(connection, table_name, reinterpret_cast<duckdb_arrow_stream>(stream));
+}

package/src/duckdb/src/main/capi/duckdb_value-c.cpp

@@ -26,7 +26,7 @@ char *duckdb_get_varchar(duckdb_value value) {
 	auto str_val = val->DefaultCastAs(duckdb::LogicalType::VARCHAR);
 	auto &str = duckdb::StringValue::Get(str_val);
 
-	auto result = reinterpret_cast<char *>(malloc(sizeof(char *) * (str.size() + 1)));
+	auto result = reinterpret_cast<char *>(malloc(sizeof(char) * (str.size() + 1)));
 	memcpy(result, str.c_str(), str.size());
 	result[str.size()] = '\0';
 	return result;

package/src/duckdb/src/main/config.cpp

@@ -59,10 +59,12 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_LOCAL(DebugForceExternal),
                                                  DUCKDB_LOCAL(DebugForceNoCrossProduct),
                                                  DUCKDB_LOCAL(DebugAsOfIEJoin),
+                                                 DUCKDB_LOCAL(PreferRangeJoins),
                                                  DUCKDB_GLOBAL(DebugWindowMode),
                                                  DUCKDB_GLOBAL_LOCAL(DefaultCollationSetting),
                                                  DUCKDB_GLOBAL(DefaultOrderSetting),
                                                  DUCKDB_GLOBAL(DefaultNullOrderSetting),
+                                                 DUCKDB_GLOBAL(DisabledFileSystemsSetting),
                                                  DUCKDB_GLOBAL(DisabledOptimizersSetting),
                                                  DUCKDB_GLOBAL(EnableExternalAccessSetting),
                                                  DUCKDB_GLOBAL(EnableFSSTVectors),
@@ -92,6 +94,7 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_LOCAL(OrderedAggregateThreshold),
                                                  DUCKDB_GLOBAL(PasswordSetting),
                                                  DUCKDB_LOCAL(PerfectHashThresholdSetting),
+                                                 DUCKDB_LOCAL(PivotFilterThreshold),
                                                  DUCKDB_LOCAL(PivotLimitSetting),
                                                  DUCKDB_LOCAL(PreserveIdentifierCase),
                                                  DUCKDB_GLOBAL(PreserveInsertionOrder),
@@ -109,6 +112,7 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_GLOBAL_ALIAS("user", UsernameSetting),
                                                  DUCKDB_GLOBAL_ALIAS("wal_autocheckpoint", CheckpointThresholdSetting),
                                                  DUCKDB_GLOBAL_ALIAS("worker_threads", ThreadsSetting),
+                                                 DUCKDB_GLOBAL(FlushAllocatorSetting),
                                                  FINAL_SETTING};
 
 vector<ConfigurationOption> DBConfig::GetOptions() {

package/src/duckdb/src/main/database.cpp

@@ -178,7 +178,7 @@ void DatabaseInstance::CreateMainDatabase() {
 void ThrowExtensionSetUnrecognizedOptions(const unordered_map<string, Value> &unrecognized_options) {
 	auto unrecognized_options_iter = unrecognized_options.begin();
 	string unrecognized_option_keys = unrecognized_options_iter->first;
-	for (; unrecognized_options_iter == unrecognized_options.end(); ++unrecognized_options_iter) {
+	while (++unrecognized_options_iter != unrecognized_options.end()) {
 		unrecognized_option_keys = "," + unrecognized_options_iter->first;
 	}
 	throw InvalidInputException("Unrecognized configuration property \"%s\"", unrecognized_option_keys);