lmdb 0.7.5 → 0.8.1

data/ext/lmdb_ext/lmdb_ext.h

@@ -1,6 +1,7 @@
 #ifndef _LMDB_EXT_H
 #define _LMDB_EXT_H
 
+#include <errno.h>
 #include "ruby.h"
 #include "lmdb.h"
 
@@ -29,24 +30,28 @@
 #  define RARRAY_AREF(ary,n) (RARRAY_PTR(ary)[n])
 #endif
 
-#define ENVIRONMENT(var, var_env)                       \
-        Environment* var_env;                           \
-        Data_Get_Struct(var, Environment, var_env);     \
+#define ENVIRONMENT(var, type, var_env)                        \
+        Environment* var_env;                                  \
+        TypedData_Get_Struct(var, Environment, type, var_env); \
         environment_check(var_env)
 
-#define DATABASE(var, var_db)                   \
-        Database* var_db;                       \
-        Data_Get_Struct(var, Database, var_db);
+#define DATABASE(var, type, var_db)                  \
+        Database* var_db;                            \
+        TypedData_Get_Struct(var, Database, type, var_db)
 
-#define TRANSACTION(var, var_txn)                       \
+#define TRANSACTION(var, type, var_txn)                 \
         Transaction* var_txn;                           \
-        Data_Get_Struct(var, Transaction, var_txn)
+        TypedData_Get_Struct(var, Transaction, type, var_txn)
 
-#define CURSOR(var, var_cur)                    \
-        Cursor* var_cur;                        \
-        Data_Get_Struct(var, Cursor, var_cur);  \
+#define CURSOR(var, type, var_cur)                   \
+        Cursor* var_cur;                             \
+        TypedData_Get_Struct(var, Cursor, type, var_cur); \
         cursor_check(var_cur)
 
+#define CURSOR_NOCHECK(var, type, var_cur)               \
+        Cursor* var_cur;                                 \
+        TypedData_Get_Struct(var, Cursor, type, var_cur)
+
 /*
   hey yo if you can convince hyc to add a function like
   mdb_txn_get_flags or even mdb_txn_is_rdonly, you could probably get
@@ -92,7 +97,7 @@ typedef struct {
         VALUE self;
         const char* name;
         int argc;
-        const VALUE* argv;
+        VALUE* argv;
 } HelperArgs;
 
 typedef struct {
@@ -118,6 +123,17 @@ static VALUE cEnvironment, cDatabase, cTransaction, cCursor, cError;
 #include "errors.h"
 #undef ERROR
 
+#define GC_MARK(val) \
+    do { if ((val) && !NIL_P(val)) rb_gc_mark_movable(val); } while(0)
+
+#ifdef HAVE_RB_GC_MARK_MOVABLE
+#define GC_MARK_MOVABLE(val) \
+    do { if ((val) && !NIL_P(val)) rb_gc_mark_movable(val); } while(0)
+
+#define GC_LOCATION(val) \
+    do { if ((val) && !NIL_P(val)) (val) = rb_gc_location(val); } while(0)
+#endif
+
 // BEGIN PROTOTYPES
 void Init_lmdb_ext();
 static MDB_txn* active_txn(VALUE self);
@@ -129,10 +145,10 @@ static VALUE cursor_close(VALUE self);
 static VALUE cursor_count(VALUE self);
 static VALUE cursor_delete(int argc, VALUE *argv, VALUE self);
 static VALUE cursor_first(VALUE self);
-static void cursor_free(Cursor* cursor);
 static VALUE cursor_get(VALUE self);
 static VALUE cursor_last(VALUE self);
-static void cursor_mark(Cursor* cursor);
+static void cursor_mark(void* ptr);
+static void cursor_free(void* ptr);
 static VALUE cursor_next(int argc, VALUE* argv, VALUE self);
 static VALUE cursor_prev(VALUE self);
 static VALUE cursor_put(int argc, VALUE* argv, VALUE self);
@@ -143,7 +159,8 @@ static VALUE database_cursor(VALUE self);
 static VALUE database_delete(int argc, VALUE *argv, VALUE self);
 static VALUE database_drop(VALUE self);
 static VALUE database_get(VALUE self, VALUE vkey);
-static void database_mark(Database* database);
+static void database_mark(void* ptr);
+static void database_free(void* ptr);
 static VALUE database_put(int argc, VALUE *argv, VALUE self);
 static VALUE database_stat(VALUE self);
 static VALUE database_get_flags(VALUE self);
@@ -158,9 +175,9 @@ static VALUE environment_copy(VALUE self, VALUE path);
 static VALUE environment_database(int argc, VALUE *argv, VALUE self);
 static VALUE environment_databases(VALUE self);
 static VALUE environment_flags(VALUE self);
-static void environment_free(Environment *environment);
+static void environment_mark(void* ptr);
+static void environment_free(void* ptr);
 static VALUE environment_info(VALUE self);
-static void environment_mark(Environment* environment);
 static VALUE environment_new(int argc, VALUE *argv, VALUE klass);
 static int environment_options(VALUE key, VALUE value, EnvironmentOptions* options);
 static VALUE environment_path(VALUE self);
@@ -174,8 +191,8 @@ static VALUE stat2hash(const MDB_stat* stat);
 static VALUE transaction_abort(VALUE self);
 static VALUE transaction_commit(VALUE self);
 static void transaction_finish(VALUE self, int commit);
-static void transaction_free(Transaction* transaction);
-static void transaction_mark(Transaction* transaction);
+static void transaction_mark(void* ptr);
+static void transaction_free(void* ptr);
 static VALUE with_transaction(VALUE venv, VALUE(*fn)(VALUE), VALUE arg, int flags);
 // END PROTOTYPES
 

data/lib/lmdb/database.rb

@@ -1,4 +1,22 @@
 module LMDB
+  class Environment
+
+    # Start a new transaction if there isn't already one going.
+    #
+    # @param readonly [false, true]
+    # @param block [Proc] the block to run
+    # @yieldparam [LMDB::Transaction] the transaction handle
+    # @yieldreturn [Object] your pick
+    # @return [Object] whatever the block returns
+    #
+    def transaction? readonly = false, &block
+      raise ArgumentError, 'no block for conditional transaction' unless block
+
+      t = active_txn
+      t ? block.call(t) : transaction(!!readonly, &block)
+    end
+  end
+
   class Database
     include Enumerable
 
@@ -10,9 +28,8 @@ module LMDB
     #      key, value = record
     #      puts "at #{key}: #{value}"
     #    end
-    def each
-      maybe_txn true do
-      # env.transaction do
+    def each &block
+      env.transaction? true do
         cursor do |c|
           while i = c.next
             yield(i)
@@ -55,12 +72,12 @@ module LMDB
     # @yield key [String] the next key in the database.
     # @return [Enumerator] in lieu of a block.
     def each_key(&block)
-      return enum_for :each_key unless block_given?
-      maybe_txn true do
-      #env.transaction do
+      return enum_for :each_key unless block
+
+      env.transaction? true do
         cursor do |c|
           while (rec = c.next true)
-            yield rec.first
+            block.call rec.first
           end
         end
       end
@@ -73,24 +90,25 @@ module LMDB
     # @yield value [String] the next value associated with the key.
     # @return [Enumerator] in lieu of a block.
     def each_value(key, &block)
-      return enum_for :each_value, key unless block_given?
+      return enum_for :each_value, key unless block
 
-      value = get(key) or return
-      unless dupsort?
-        yield value
-        return
-      end
+      op = -> txn do
+        value = get(key) or return
+        unless dupsort?
+          block.call value
+          return
+        end
 
-      maybe_txn true do
-      # env.transaction do
         cursor do |c|
           rec = c.set key
           while rec
-            yield rec.last
+            block.call rec.last
             rec = c.next_range key
           end
         end
       end
+
+      env.transaction? true, &op
     end
 
     # Return the cardinality (number of duplicates) of a given
@@ -99,8 +117,7 @@ module LMDB
     # @return [Integer] The number of entries under the key.
     def cardinality(key)
       ret = 0
-      maybe_txn true do
-      # env.transaction do
+      env.transaction? true do
         if get key
           if dupsort?
             cursor do |c|
@@ -118,18 +135,20 @@ module LMDB
     # Test if the database has a given key (or, if opened in
     # +:dupsort+, value)
     def has?(key, value = nil)
-      v = get(key) or return false
-      return true if value.nil? or value.to_s == v
-      return false unless dupsort?
-
-      # warn "checking dupsort value `#{value.inspect}` (#{value.class})"
-
-      ret = false
-      # read-only txn was having trouble being nested inside a read-write
-      # maybe_txn(true) { cursor { |c| ret = !!c.set(key, value) } }
-      env.transaction(true) { cursor { |c| ret = !!c.set(key, value) } }
+      env.transaction? true do
+        if v = get(key)
+          if value.nil? or value.to_s == v
+            true
+          elsif !dupsort?
+            false
+          else
+            ret = false
+            cursor { |c| ret = !!c.set(key, value) }
 
-      ret
+            ret
+          end
+        end
+      end
     end
 
     # Conditionally put a value into the database.
@@ -148,7 +167,7 @@ module LMDB
 
       flags = { (dupsort? ? :nodupdata : :nooverwrite) => true }
 
-      env.transaction do |txn|
+      env.transaction? do |txn|
         put(key, value, **options.merge(flags)) unless has?(key, value)
       end
     end
@@ -164,7 +183,7 @@ module LMDB
     # @return [void]
     #
     def delete?(key, value = nil)
-      env.transaction { |txn| delete(key, value) if has?(key, value) }
+      env.transaction? { |txn| delete(key, value) if has?(key, value) }
     end
 
     # Return how many records there are in this database.
@@ -179,26 +198,5 @@ module LMDB
     def empty?
       stat[:entries] == 0
     end
-
-    private
-
-    # having trouble with read-only transactions embedded in
-    # read-write for some reason; can't pin it down to test it yet so
-    # going to do this (djt; 2020-02-10)
-    #
-    # 2025-11-14: this miiiigght no longer be necessary?? like as of
-    # whenever i implemented that short-circuiting code
-    #
-    def maybe_txn(readonly, &block)
-      if t = env.active_txn
-        # warn "reusing #{t.readonly? ? 'read-only ' : ''}txn #{t.inspect}"
-        yield t
-      else
-        env.transaction !!readonly do |t|
-          # warn "new #{t.readonly? ? 'read-only ' : ''}txn #{t.inspect}"
-          yield t
-        end
-      end
-    end
   end
 end

data/lib/lmdb/version.rb

@@ -1,3 +1,3 @@
 module LMDB
-  VERSION = '0.7.5'.freeze
+  VERSION = '0.8.1'.freeze
 end

data/spec/gc_torture_spec.rb

@@ -0,0 +1,162 @@
+require 'helper'
+require 'lmdb'
+require 'tmpdir'
+require 'fileutils'
+
+RSpec.describe 'LMDB GC safety' do
+  let(:env)  { LMDB.new(path, mapsize: 2**24) }
+  let(:db)   { env.database('test', create: true) }
+
+  before(:each) { db }  # open db before each example
+
+  after(:each) do
+    env.close rescue nil
+    FileUtils.rm_rf path
+  end
+
+  # -----------------------------------------------------------------------
+  # 1. Basic GC.compact safety — no open transactions
+  #    Verifies compaction callbacks update all VALUE references correctly
+  # -----------------------------------------------------------------------
+  it 'survives GC.compact with no open transactions' do
+    env.transaction { db['key'] = 'value' }
+    expect { GC.compact }.not_to raise_error
+    env.transaction(true) { expect(db['key']).to eq 'value' }
+  end
+
+  # -----------------------------------------------------------------------
+  # 2. GC.compact during an open read transaction
+  #    Verifies the transaction object survives compaction
+  # -----------------------------------------------------------------------
+  it 'survives GC.compact with an open read transaction' do
+    env.transaction { db['key'] = 'value' }
+    env.transaction(true) do |txn|
+      expect { GC.compact }.not_to raise_error
+      expect(db['key']).to eq 'value'
+      expect(txn.finished?).to be false
+    end
+  end
+
+  # -----------------------------------------------------------------------
+  # 3. GC.compact between consecutive write transactions
+  #    This is the Store::Digest scenario — write, compact, write again
+  # -----------------------------------------------------------------------
+  it 'allows a new write transaction after GC.compact' do
+    env.transaction { db['key1'] = 'value1' }
+    GC.compact
+    expect {
+      env.transaction { db['key2'] = 'value2' }
+    }.not_to raise_error
+    env.transaction(true) do
+      expect(db['key1']).to eq 'value1'
+      expect(db['key2']).to eq 'value2'
+    end
+  end
+
+  # -----------------------------------------------------------------------
+  # 4. GC.compact after multiple write transactions
+  #    Stress version — compact after several writes
+  # -----------------------------------------------------------------------
+  it 'allows write transactions after repeated GC.compact calls' do
+    5.times do |i|
+      env.transaction { db["key#{i}"] = "value#{i}" }
+      GC.compact
+    end
+    env.transaction(true) do
+      5.times do |i|
+        expect(db["key#{i}"]).to eq "value#{i}"
+      end
+    end
+  end
+
+  # -----------------------------------------------------------------------
+  # 5. GC.compact with a pseudo-transaction (RO nested in RW)
+  #    Verifies pseudo-transaction compaction safety
+  # -----------------------------------------------------------------------
+  it 'survives GC.compact after pseudo-transaction use' do
+    env.transaction do
+      db['key'] = 'value'
+      env.transaction(true) do
+        expect(db['key']).to eq 'value'
+      end
+    end
+    GC.compact
+    expect {
+      env.transaction { db['key2'] = 'value2' }
+    }.not_to raise_error
+  end
+
+  # -----------------------------------------------------------------------
+  # 6. GC.compact with cursor use
+  #    Verifies cursor compaction safety
+  # -----------------------------------------------------------------------
+  it 'survives GC.compact after cursor iteration' do
+    env.transaction do
+      5.times { |i| db["key#{i}"] = "value#{i}" }
+    end
+    env.transaction(true) do
+      db.cursor do |c|
+        c.first
+        GC.compact
+        expect(c.next).not_to be_nil
+      end
+    end
+    GC.compact
+    expect {
+      env.transaction { db['new'] = 'write' }
+    }.not_to raise_error
+  end
+
+  # -----------------------------------------------------------------------
+  # 7. GC.start (sweep without compaction) — baseline
+  #    Should always pass; establishes that non-compacting GC is safe
+  # -----------------------------------------------------------------------
+  it 'survives GC.start (sweep only) between write transactions' do
+    env.transaction { db['key1'] = 'value1' }
+    GC.start
+    expect {
+      env.transaction { db['key2'] = 'value2' }
+    }.not_to raise_error
+  end
+
+  # -----------------------------------------------------------------------
+  # 8. Transaction object survives GC after finishing
+  #    Verifies that a finished transaction's Ruby object can be
+  #    collected without corrupting the environment
+  # -----------------------------------------------------------------------
+  it 'allows GC to collect finished transaction objects safely' do
+    10.times do |i|
+      env.transaction { db["key#{i}"] = "value#{i}" }
+    end
+    # force collection of all those transaction objects
+    GC.compact
+    GC.start
+    expect {
+      env.transaction { db['final'] = 'write' }
+    }.not_to raise_error
+  end
+
+  # -----------------------------------------------------------------------
+  # 9. Many databases, many transactions, GC between each
+  #    Closer to the Store::Digest workload profile
+  # -----------------------------------------------------------------------
+  it 'handles multiple named databases with GC.compact between writes' do
+    dbs = %w[alpha beta gamma delta].map do |name|
+      env.database(name, create: true)
+    end
+    GC.compact
+
+    10.times do |i|
+      env.transaction do
+        dbs.each { |d| d["key#{i}"] = "value#{i}" }
+      end
+      GC.compact
+    end
+
+    env.transaction(true) do
+      dbs.each do |d|
+        10.times { |i| expect(d["key#{i}"]).to eq "value#{i}" }
+      end
+    end
+  end
+end

data/spec/helper.rb

@@ -27,7 +27,7 @@ end
 RSpec.configure do |c|
   c.include LMDB::SpecHelper
   c.after { FileUtils.rm_rf TEMP_ROOT }
-  c.expect_with :rspec do |cc|
-    cc.syntax = :should
-  end
+  # c.expect_with :rspec do |cc|
+  #   cc.syntax = :should
+  # end
 end