duckdb 1.5.5.0 → 1.5.6.0

data/ext/duckdb/appender.c

@@ -50,10 +50,20 @@ static VALUE appender__clear_columns(VALUE self);
 static VALUE appender__close(VALUE self);
 static VALUE state_to_rbool(duckdb_state state);
 
+/*
+ * Keeps RUBY_TYPED_FREE_IMMEDIATELY although duckdb_appender_destroy flushes
+ * buffered rows (blocking I/O): when an unclosed appender is collected in the
+ * same GC as its connection, freeing it during sweep runs before the deferred
+ * Connection/Database free, so its rows still reach the table. Deferring it too
+ * would let the connection close first and lose rows. (The appender holds no
+ * reference to its connection, so if the connection is collected in an earlier
+ * GC the flush is lost anyway.) Freeing in sweep cannot shut the database down:
+ * DuckDB's BaseAppender holds only a weak_ptr<ClientContext>.
+ */
 static const rb_data_type_t appender_data_type = {
     "DuckDB/Appender",
     {NULL, deallocate, memsize,},
-    0, 0, RUBY_TYPED_FREE_IMMEDIATELY
+    0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED
 };
 
 static void deallocate(void * ctx) {
@@ -94,6 +104,8 @@ static VALUE appender_s_create_query(VALUE klass, VALUE con, VALUE query, VALUE
     idx_t column_count = 0;
     duckdb_logical_type *type_array = NULL;
     VALUE appender = Qnil;
+    VALUE column_name_strings = Qnil;
+    duckdb_state state;
 
     if (!rb_obj_is_kind_of(con, cDuckDBConnection)) {
         rb_raise(rb_eTypeError, "1st argument should be instance of DackDB::Connection");
@@ -116,17 +128,29 @@ static VALUE appender_s_create_query(VALUE klass, VALUE con, VALUE query, VALUE
         if (rb_obj_is_kind_of(columns, rb_cArray) == Qfalse) {
             rb_raise(rb_eTypeError, "4th argument should be an Array or nil");
         }
-        idx_t col_count = RARRAY_LEN(columns);
-        column_names = ALLOCA_N(const char *, (size_t)col_count);
-        for (idx_t i = 0; i < col_count; i++) {
+        /* DuckDB reads column_count entries from column_names, so a shorter list
+         * makes it read uninitialized stack slots as column names. */
+        if ((idx_t)RARRAY_LEN(columns) != column_count) {
+            rb_raise(rb_eArgError, "column names size (%ld) must be the same as types size (%ld)",
+                     RARRAY_LEN(columns), RARRAY_LEN(types));
+        }
+        /* Keep the converted Strings alive: StringValue on a to_str object
+         * returns a String that nothing else references. */
+        column_name_strings = rb_ary_new_capa((long)column_count);
+        column_names = ALLOCA_N(const char *, (size_t)column_count);
+        for (idx_t i = 0; i < column_count; i++) {
             VALUE col_name_val = rb_ary_entry(columns, i);
-            column_names[i] = StringValuePtr(col_name_val);
+            StringValue(col_name_val);
+            rb_ary_push(column_name_strings, col_name_val);
+            column_names[i] = RSTRING_PTR(col_name_val);
         }
     }
     ctxcon = rbduckdb_get_struct_connection(con);
     appender = allocate(klass);
     TypedData_Get_Struct(appender, rubyDuckDBAppender, &appender_data_type, ctx);
-    if (duckdb_appender_create_query(ctxcon->con, query_str, column_count, type_array, table_name, column_names, &ctx->appender) == DuckDBError) {
+    state = duckdb_appender_create_query(ctxcon->con, query_str, column_count, type_array, table_name, column_names, &ctx->appender);
+    RB_GC_GUARD(column_name_strings);
+    if (state == DuckDBError) {
         rb_raise(eDuckDBError, "failed to create appender from query");
     }
 
@@ -333,11 +357,13 @@ static VALUE appender__append_double(VALUE self, VALUE val) {
 /* :nodoc: */
 static VALUE appender__append_varchar(VALUE self, VALUE val) {
     rubyDuckDBAppender *ctx;
+    /* Length-aware, as duckdb_append_varchar would stop at an embedded NUL. */
     char *pval = StringValuePtr(val);
+    long len = RSTRING_LEN(val);
 
     TypedData_Get_Struct(self, rubyDuckDBAppender, &appender_data_type, ctx);
 
-    return state_to_rbool(duckdb_append_varchar(ctx->appender, pval));
+    return state_to_rbool(duckdb_append_varchar_length(ctx->appender, pval, (idx_t)len));
 }
 
 /* :nodoc: */

data/ext/duckdb/arrow_array_stream.c

@@ -13,7 +13,7 @@ typedef struct {
  * the Ruby DuckDB::ArrowArrayStream object is gone, so this context is
  * freed only by the stream release callback, and it holds a reference on
  * the result struct (rbduckdb_result_ref) until then. The release callback
- * must not call any Ruby API: it can run during GC sweep (via deallocate
+ * must not call any Ruby API: it can run during GC finalization (via deallocate
  * of an unconsumed stream) or from a non-Ruby thread.
  */
 typedef struct {
@@ -34,10 +34,14 @@ static int stream_get_next(struct ArrowArrayStream *stream, struct ArrowArray *o
 static const char *stream_get_last_error(struct ArrowArrayStream *stream);
 static void stream_release(struct ArrowArrayStream *stream);
 
+/*
+ * No RUBY_TYPED_FREE_IMMEDIATELY: deallocate can block, and must not run
+ * during GC sweep: releasing the stream may destroy a streaming result (see result.c).
+ */
 static const rb_data_type_t arrow_array_stream_data_type = {
     "DuckDB/ArrowArrayStream",
     {NULL, deallocate, memsize,},
-    0, 0, RUBY_TYPED_FREE_IMMEDIATELY
+    0, 0, RUBY_TYPED_WB_PROTECTED
 };
 
 static void deallocate(void *ctx) {

data/ext/duckdb/arrow_import.c

@@ -26,7 +26,7 @@ static VALUE connection__arrow_release(VALUE self, VALUE address);
 static const rb_data_type_t arrow_converted_schema_data_type = {
     "DuckDB/ArrowConvertedSchema",
     {NULL, deallocate, memsize,},
-    0, 0, RUBY_TYPED_FREE_IMMEDIATELY
+    0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED
 };
 
 static void deallocate(void *ctx) {

data/ext/duckdb/client_context.c

@@ -10,7 +10,7 @@ static VALUE client_context_connection_id(VALUE self);
 static const rb_data_type_t client_context_data_type = {
     "DuckDB/ClientContext",
     {NULL, deallocate, memsize,},
-    0, 0, RUBY_TYPED_FREE_IMMEDIATELY
+    0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED
 };
 
 static void deallocate(void *ctx) {

data/ext/duckdb/column.c

@@ -12,7 +12,7 @@ static VALUE column_name(VALUE oDuckDBColumn);
 static const rb_data_type_t column_data_type = {
     "DuckDB/Column",
     {NULL, deallocate, memsize,},
-    0, 0, RUBY_TYPED_FREE_IMMEDIATELY
+    0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED
 };
 
 static void deallocate(void *ctx) {

data/ext/duckdb/column.h

@@ -2,7 +2,6 @@
 #define RUBY_DUCKDB_COLUMN_H
 
 struct _rubyDuckDBColumn {
-    VALUE result;
     idx_t col;
 };
 

data/ext/duckdb/config.c

@@ -13,7 +13,7 @@ static VALUE config_set_config(VALUE self, VALUE key, VALUE value);
 static const rb_data_type_t config_data_type = {
     "DuckDB/Config",
     {NULL, deallocate, memsize,},
-    0, 0, RUBY_TYPED_FREE_IMMEDIATELY
+    0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED
 };
 
 static void deallocate(void * ctx) {

data/ext/duckdb/connection.c

@@ -4,6 +4,7 @@ VALUE cDuckDBConnection;
 
 static void deallocate(void *ctx);
 static void mark(void *ctx);
+static void compact(void *ctx);
 static VALUE allocate(VALUE klass);
 static size_t memsize(const void *p);
 static VALUE connection_disconnect(VALUE self);
@@ -19,10 +20,15 @@ static VALUE connection__register_aggregate_function_set(VALUE self, VALUE aggre
 static VALUE connection__register_table_function(VALUE self, VALUE table_function);
 static VALUE connection__get_table_names(VALUE self, VALUE query, VALUE qualified);
 
+/*
+ * No RUBY_TYPED_FREE_IMMEDIATELY: deallocate can block, and must not run
+ * during GC sweep: duckdb_disconnect rolls back an open transaction and may drop the
+ * last reference to the database (checkpoint on shutdown).
+ */
 static const rb_data_type_t connection_data_type = {
     "DuckDB/Connection",
-    {mark, deallocate, memsize,},
-    0, 0, RUBY_TYPED_FREE_IMMEDIATELY
+    {mark, deallocate, memsize, compact},
+    0, 0, RUBY_TYPED_WB_PROTECTED
 };
 
 static void deallocate(void *ctx) {
@@ -34,16 +40,29 @@ static void deallocate(void *ctx) {
 
 static void mark(void *ctx) {
     rubyDuckDBConnection *p = (rubyDuckDBConnection *)ctx;
-    rb_gc_mark(p->registered_functions);
+    rb_gc_mark_movable(p->database);
+}
+
+static void compact(void *ctx) {
+    rubyDuckDBConnection *p = (rubyDuckDBConnection *)ctx;
+    p->database = rb_gc_location(p->database);
 }
 
 static VALUE allocate(VALUE klass) {
     rubyDuckDBConnection *ctx = xcalloc((size_t)1, sizeof(rubyDuckDBConnection));
-    VALUE obj = TypedData_Wrap_Struct(klass, &connection_data_type, ctx);
-    VALUE registered_functions = rb_ary_new();
-    ctx->registered_functions = registered_functions;
-    RB_GC_GUARD(registered_functions);
-    return obj;
+    ctx->database = Qnil;
+    return TypedData_Wrap_Struct(klass, &connection_data_type, ctx);
+}
+
+/* Anchors obj to this connection's database so it outlives the connection. */
+static void connection_retain_registered(VALUE self, VALUE obj) {
+    rubyDuckDBConnection *ctx;
+
+    TypedData_Get_Struct(self, rubyDuckDBConnection, &connection_data_type, ctx);
+    if (NIL_P(ctx->database)) {
+        rb_raise(eDuckDBError, "connection is not associated with a database");
+    }
+    rbduckdb_database_retain(ctx->database, obj);
 }
 
 static size_t memsize(const void *p) {
@@ -69,6 +88,7 @@ VALUE rbduckdb_create_connection(VALUE oDuckDBDatabase) {
     if (duckdb_connect(ctxdb->db, &(ctxcon->con)) == DuckDBError) {
         rb_raise(eDuckDBError, "connection error");
     }
+    RB_OBJ_WRITE(obj, &ctxcon->database, oDuckDBDatabase);
 
     return obj;
 }
@@ -79,8 +99,12 @@ static VALUE connection_disconnect(VALUE self) {
     TypedData_Get_Struct(self, rubyDuckDBConnection, &connection_data_type, ctx);
     duckdb_disconnect(&(ctx->con));
 
-    /* Clear registered functions to release memory */
-    rb_ary_clear(ctx->registered_functions);
+    /*
+     * Registered functions are not released here: the catalog entry is still
+     * reachable from other connections. They are released with the database.
+     * This connection no longer needs the database, so stop retaining it.
+     */
+    RB_OBJ_WRITE(self, &ctx->database, Qnil);
 
     return self;
 }
@@ -145,9 +169,11 @@ static VALUE connection__connect(VALUE self, VALUE oDuckDBDatabase) {
     ctxdb = rbduckdb_get_struct_database(oDuckDBDatabase);
     TypedData_Get_Struct(self, rubyDuckDBConnection, &connection_data_type, ctx);
 
+    /* Set only on success: a failed connect must not re-anchor registrations to the new database. */
     if (duckdb_connect(ctxdb->db, &(ctx->con)) == DuckDBError) {
         rb_raise(eDuckDBError, "connection error");
     }
+    RB_OBJ_WRITE(self, &ctx->database, oDuckDBDatabase);
 
     return self;
 }
@@ -239,8 +265,8 @@ static VALUE connection__register_logical_type(VALUE self, VALUE logical_type) {
         rb_raise(eDuckDBError, "Failed to register logical type");
     }
 
-    /* Keep reference to prevent GC while connection is alive */
-    rb_ary_push(ctxcon->registered_functions, logical_type);
+    /* Keep reference alive as long as the database can call it */
+    connection_retain_registered(self, logical_type);
 
     return self;
 }
@@ -260,8 +286,8 @@ static VALUE connection__register_scalar_function(VALUE self, VALUE scalar_funct
         rb_raise(eDuckDBError, "Failed to register scalar function");
     }
 
-    /* Keep reference to prevent GC while connection is alive */
-    rb_ary_push(ctxcon->registered_functions, scalar_function);
+    /* Keep reference alive as long as the database can call it */
+    connection_retain_registered(self, scalar_function);
 
     return self;
 }
@@ -281,8 +307,8 @@ static VALUE connection__register_scalar_function_set(VALUE self, VALUE scalar_f
         rb_raise(eDuckDBError, "Failed to register scalar function set");
     }
 
-    /* Keep reference to prevent GC while connection is alive */
-    rb_ary_push(ctxcon->registered_functions, scalar_function_set);
+    /* Keep reference alive as long as the database can call it */
+    connection_retain_registered(self, scalar_function_set);
 
     return self;
 }
@@ -302,8 +328,8 @@ static VALUE connection__register_aggregate_function(VALUE self, VALUE aggregate
         rb_raise(eDuckDBError, "Failed to register aggregate function");
     }
 
-    /* Keep reference to prevent GC while connection is alive */
-    rb_ary_push(ctxcon->registered_functions, aggregate_function);
+    /* Keep reference alive as long as the database can call it */
+    connection_retain_registered(self, aggregate_function);
 
     return self;
 }
@@ -323,8 +349,8 @@ static VALUE connection__register_aggregate_function_set(VALUE self, VALUE aggre
         rb_raise(eDuckDBError, "Failed to register aggregate function set");
     }
 
-    /* Keep reference to prevent GC while connection is alive */
-    rb_ary_push(ctxcon->registered_functions, aggregate_function_set);
+    /* Keep reference alive as long as the database can call it */
+    connection_retain_registered(self, aggregate_function_set);
 
     return self;
 }
@@ -343,8 +369,8 @@ static VALUE connection__register_table_function(VALUE self, VALUE table_functio
         rb_raise(eDuckDBError, "Failed to register table function");
     }
 
-    /* Keep reference to prevent GC while connection is alive */
-    rb_ary_push(ctxcon->registered_functions, table_function);
+    /* Keep reference alive as long as the database can call it */
+    connection_retain_registered(self, table_function);
 
     return self;
 }

data/ext/duckdb/connection.h

@@ -3,7 +3,8 @@
 
 struct _rubyDuckDBConnection {
     duckdb_connection con;
-    VALUE registered_functions;
+    /* The DuckDB::Database this connection belongs to. Registrations are anchored there. */
+    VALUE database;
 };
 
 typedef struct _rubyDuckDBConnection rubyDuckDBConnection;

data/ext/duckdb/data_chunk.c

@@ -16,7 +16,7 @@ static VALUE data_chunk__reset(VALUE self);
 static const rb_data_type_t data_chunk_data_type = {
     "DuckDB/DataChunk",
     {NULL, deallocate, memsize,},
-    0, 0, RUBY_TYPED_FREE_IMMEDIATELY
+    0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED
 };
 
 static void deallocate(void *ctx) {

data/ext/duckdb/database.c

@@ -4,6 +4,8 @@ VALUE cDuckDBDatabase;
 
 static void close_database(rubyDuckDB *p);
 static void deallocate(void * ctx);
+static void mark(void *ctx);
+static void compact(void *ctx);
 static VALUE allocate(VALUE klass);
 static size_t memsize(const void *p);
 static duckdb_config create_config_with_ruby_api(void);
@@ -11,16 +13,37 @@ static VALUE database__initialize(VALUE self, VALUE file, VALUE config);
 static VALUE database__connect(VALUE self);
 static VALUE database_close(VALUE self);
 
+/*
+ * No RUBY_TYPED_FREE_IMMEDIATELY: deallocate can block, and must not run
+ * during GC sweep: duckdb_close may checkpoint to disk and joins worker threads.
+ */
 static const rb_data_type_t database_data_type = {
     "DuckDB/Database",
-    {NULL, deallocate, memsize,},
-    0, 0, RUBY_TYPED_FREE_IMMEDIATELY
+    {mark, deallocate, memsize, compact},
+    0, 0, RUBY_TYPED_WB_PROTECTED
 };
 
 static void close_database(rubyDuckDB *p) {
     duckdb_close(&(p->db));
 }
 
+/*
+ * duckdb_close joins DuckDB's worker threads, and a worker tearing down a
+ * pipeline can still run a UDF callback -- the aggregate destroy callback,
+ * say, for a query that failed and left its pipeline queued. That callback is
+ * dispatched to the executor thread, which needs the GVL, so closing while
+ * holding it deadlocks: the worker waits for the executor, the executor waits
+ * for us, and we wait for the worker.
+ *
+ * The GC free function calls close_database directly and cannot do this;
+ * a free function must not release the GVL. A database reaching close via
+ * the GC with callbacks still pending would deadlock the same way.
+ */
+static void *close_database_without_gvl(void *ctx) {
+    close_database((rubyDuckDB *)ctx);
+    return NULL;
+}
+
 static void deallocate(void * ctx) {
     rubyDuckDB *p = (rubyDuckDB *)ctx;
 
@@ -28,13 +51,57 @@ static void deallocate(void * ctx) {
     xfree(p);
 }
 
+static void mark(void *ctx) {
+    rubyDuckDB *p = (rubyDuckDB *)ctx;
+
+    rb_gc_mark_movable(p->registered_functions);
+    rb_gc_mark_movable(p->cached_path);
+    rb_gc_mark_movable(p->cache_wrappers);
+}
+
+static void compact(void *ctx) {
+    rubyDuckDB *p = (rubyDuckDB *)ctx;
+
+    p->registered_functions = rb_gc_location(p->registered_functions);
+    p->cached_path = rb_gc_location(p->cached_path);
+    p->cache_wrappers = rb_gc_location(p->cache_wrappers);
+}
+
 static size_t memsize(const void *p) {
     return sizeof(rubyDuckDB);
 }
 
 static VALUE allocate(VALUE klass) {
     rubyDuckDB *ctx = xcalloc((size_t)1, sizeof(rubyDuckDB));
-    return TypedData_Wrap_Struct(klass, &database_data_type, ctx);
+    VALUE obj = TypedData_Wrap_Struct(klass, &database_data_type, ctx);
+    ctx->cached_path = Qnil;
+    ctx->cache_wrappers = Qnil;
+    RB_OBJ_WRITE(obj, &ctx->registered_functions, rb_ary_new());
+    return obj;
+}
+
+/*
+ * Keeps obj alive as long as this database is alive.
+ * DuckDB registers functions in the database-level catalog and has no
+ * unregister API, so releasing them any earlier leaves a dangling extra_info.
+ */
+void rbduckdb_database_retain(VALUE database, VALUE obj) {
+    rubyDuckDB *ctx;
+
+    TypedData_Get_Struct(database, rubyDuckDB, &database_data_type, ctx);
+    rb_ary_push(ctx->registered_functions, obj);
+}
+
+/*
+ * Records that this database is the InstanceCache's wrapper for path, so the
+ * cache can find it again and #close can remove it from the cache's weak set.
+ */
+void rbduckdb_database_set_cache_entry(VALUE database, VALUE path, VALUE wrappers) {
+    rubyDuckDB *ctx;
+
+    TypedData_Get_Struct(database, rubyDuckDB, &database_data_type, ctx);
+    RB_OBJ_WRITE(database, &ctx->cached_path, rb_str_new_frozen(path));
+    RB_OBJ_WRITE(database, &ctx->cache_wrappers, wrappers);
 }
 
 rubyDuckDB *rbduckdb_get_struct_database(VALUE obj) {
@@ -119,7 +186,16 @@ static VALUE database__connect(VALUE self) {
 static VALUE database_close(VALUE self) {
     rubyDuckDB *ctx;
     TypedData_Get_Struct(self, rubyDuckDB, &database_data_type, ctx);
-    close_database(ctx);
+    /*
+     * A closed wrapper must not be handed to the next get_or_create, so drop it
+     * from the cache's set first. The next call then opens a fresh wrapper.
+     */
+    if (!NIL_P(ctx->cache_wrappers)) {
+        rb_funcall(ctx->cache_wrappers, rb_intern("delete"), 1, self);
+        RB_OBJ_WRITE(self, &ctx->cache_wrappers, Qnil);
+        RB_OBJ_WRITE(self, &ctx->cached_path, Qnil);
+    }
+    rb_thread_call_without_gvl(close_database_without_gvl, ctx, NULL, NULL);
     return self;
 }
 

data/ext/duckdb/database.h

@@ -3,12 +3,24 @@
 
 struct _rubyDuckDB {
     duckdb_database db;
+    /* Functions and logical types registered on any connection to this database */
+    VALUE registered_functions;
+    /*
+     * Set only for a database handed out by DuckDB::InstanceCache: the path it
+     * was opened under, and the cache's weak set of live wrappers. Together they
+     * let the cache hand back this same wrapper for the same path, and let
+     * #close drop this wrapper from the set. Qnil for any other database.
+     */
+    VALUE cached_path;
+    VALUE cache_wrappers;
 };
 
 typedef struct _rubyDuckDB rubyDuckDB;
 
 rubyDuckDB *rbduckdb_get_struct_database(VALUE obj);
 VALUE rbduckdb_create_database_obj(duckdb_database db);
+void rbduckdb_database_retain(VALUE database, VALUE obj);
+void rbduckdb_database_set_cache_entry(VALUE database, VALUE path, VALUE wrappers);
 void rbduckdb_init_database(void);
 
 #endif

data/ext/duckdb/expression.c

@@ -11,7 +11,7 @@ static VALUE expression__fold(VALUE self, VALUE client_context);
 static const rb_data_type_t expression_data_type = {
     "DuckDB/Expression",
     {NULL, deallocate, memsize,},
-    0, 0, RUBY_TYPED_FREE_IMMEDIATELY
+    0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED
 };
 
 static void deallocate(void *ctx) {

data/ext/duckdb/extracted_statements.c

@@ -9,7 +9,7 @@ static size_t memsize(const void *p);
 static const rb_data_type_t extract_statements_data_type = {
     "DuckDB/ExtractedStatements",
     {NULL, deallocate, memsize,},
-    0, 0, RUBY_TYPED_FREE_IMMEDIATELY
+    0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED
 };
 
 static void deallocate(void *ctx);