lmdb 0.5.3 → 0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c02c1c99b143e331bf5fdbae711a57c4845b32ca07c4c041c9c639b96cbb8388
-  data.tar.gz: ce305105acf85f4161e654dc80ad045baeded4e8a8e39f45fed9f8da68f5c2ce
+  metadata.gz: 2eac165bb0261c0af6813f16df9b725b2ca2a63f4c5de98734473d9976930735
+  data.tar.gz: 3aa0562060e4586dd7c7a44e795adc1449f489f57634787ef5e6480abbd49a48
 SHA512:
-  metadata.gz: 1303411d1515bf39a6406540f3a10d9e12c2bd3fea597d17a9e2b949bcf7707244381467a624edd15fada41ac65ae87721c2e42a9ec4dadb5b37e9f8db2a55b0
-  data.tar.gz: aeaa5861dcbde41e9fec9634f6cae749dce6a1348ab9541f90247c2cff60dcb10180701348757a00ba7fc11e93431ef97cfb3064e5face3cc50c704498ca183d
+  metadata.gz: 22cc9f78002a05aa25fba212e4d3e3e97566ed7a9bd2e0c84996297398131b5afd57da0e1510a27086ac4b1938401d1369393f9b0515bdd044856109510c6432
+  data.tar.gz: eea983ccb0638bed2dbcfcd3e2ea76271072ea91d259b95e392d505c26f184c351f89ae6076025b93f47bfec3e42add3ec41055e5edb67bee5bfec2e8e341a80

data/.gitignore

@@ -11,3 +11,5 @@ doc/
 .yardoc/
 *.gem
 dtrace/
+\#*\#
+.\#*

data/CONTRIBUTORS

@@ -5,3 +5,4 @@ Julien Ammous <schmurfy@gmail.com>
 Nathaniel Pierce <nwpierce@gmail.com>
 Richard Golding <golding@chrysaetos.org>
 Joel VanderWerf <vjoel@users.sourceforge.net>
+Dorian Taylor <code@doriantaylor.com>

data/Rakefile

@@ -7,12 +7,21 @@ PRJ = File.basename(GEMSPEC, ".gemspec")
 require 'bundler/setup'
 require 'rspec/core/rake_task'
 require 'rake/extensiontask'
+require 'ruby_memcheck'
+require 'ruby_memcheck/rspec/rake_task'
+
+RubyMemcheck.config(binary_name: 'lmdb_ext')
 
 RSpec::Core::RakeTask.new :spec
 Rake::ExtensionTask.new :lmdb_ext
 
+
 task :default => [:compile, :spec]
 
+namespace :spec do
+  RubyMemcheck::RSpec::RakeTask.new(valgrind: :compile)
+end
+
 def version
   @version ||= begin
     require "#{PRJ}/version"

data/behaviour.org

@@ -0,0 +1,35 @@
+#+STARTUP: showall hidestars
+* the situtation
+  - lmdb has two kinds of transaction: read-write and read-only
+  - there are two properties associated with transactions:
+    - nesting
+    - multiplicity
+  - *read-write* transactions can nest, but there can only be one stack of read-write transactions per *database*, that is, amongst /all/ threads and all processes attached to the persitent storage.
+  - *read-only* transactions /cannot/ nest (because it's not meaningful for a read-only transaction to nest), but there can be a read-only transaction per thread (and of course multiple threads per process).
+    - (i can't remember if you can have a thread with a read-write *and* read-only transaction going but we can probably assume not)
+  - so therefore we need a way to distinguish between read-write and read-only as well as identify the thread /across processes/ that has the one read-write transaction open.
+* undocumented behaviour
+** ~mdb_txn_begin~
+   - if the environment is read-only and the transaction is read-write, returns ~EACCES~
+   - if there is a parent transaction and the current transaction's flags are ~MDB_RDONLY~ or ~MDB_WRITEMAP~ (?) or ~TXN_BLOCKED~
+     - if the *parent's* transaction is ~MDB_TXN_RDONLY~ (which is the same as ~MDB_RDONLY~), return ~EINVAL~
+       - that's saying "read-only transactions can't be nested"
+     - otherwise, return ~MDB_BAD_TXN~
+       - this is saying "read-only transactions can't be children of read-write parents"
+   - otherwise a few boring scenarios where the function may return ~ENOMEM~
+   - otherwise check ~mdb_cursor_shadow~ or ~mdb_txn_renew0~
+   - XXX does ~mdb_txn_begin~ block when waiting for the read-write??
+* desired behaviour
+** ruby interface
+   - when the ruby programmer opens a read-only transaction within a read-only transaction, this should be a noop
+     - don't push any stack, don't allocate any resources, just do nothing
+   - when the ruby programmer opens a read-only transaction within a read-/write/ transaction, this should raise an exception
+     - in practice there's no harm except that this is more about communicating the right thing to the ruby programmer
+       - do we warn?
+   - problem: there's no way to know (via the lmdb api) if another process has a read-write transaction open
+     - poll?
+       - actually no it probably doesn't matter (the mdb api blocks anyway?)
+       - /actually/ actually, the cursed-ass goto pseudo-loop containing the ~CALL_WITHOUT_GVL~ deals with that
+** internal implementation
+   - a successfully-created read-write transaction has to set ~rw_txn_thread~ to the current thread (unless it is a sub-transaction in which case noop)
+   - when a read-write transaction is committed or aborted, ~rw_txn_thread~ has to be set back to null (unless the transaction has a parent)

data/ext/lmdb_ext/extconf.rb

@@ -1,6 +1,7 @@
 require 'mkmf'
 
 $CFLAGS = '-std=c99 -Wall -g'
+$CFLAGS << ' -fdeclspec' if /darwin/.match? RUBY_PLATFORM
 
 # Embed lmdb if we cannot find it
 if enable_config("bundled-lmdb", false) || !(find_header('lmdb.h') && have_library('lmdb', 'mdb_env_create'))

data/ext/lmdb_ext/lmdb_ext.c

@@ -26,6 +26,7 @@ static void check(int code) {
 
         const char* err = mdb_strerror(code);
         const char* sep = strchr(err, ':');
+        // increment the offset by two in case there is a colon (plus space)
         if (sep)
                 err = sep + 2;
 
@@ -37,7 +38,7 @@ static void check(int code) {
 }
 
 static void transaction_free(Transaction* transaction) {
-        if (transaction->txn) {
+        if (transaction->txn && !NIL_P(transaction->txn)) {
           //int id = (int)mdb_txn_id(transaction->txn);
           //rb_warn(sprintf("Memory leak: Garbage collecting active transaction %d", id));
           rb_warn("Memory leak: Garbage collecting active transaction");
@@ -49,6 +50,7 @@ static void transaction_free(Transaction* transaction) {
 
 static void transaction_mark(Transaction* transaction) {
         rb_gc_mark(transaction->parent);
+        rb_gc_mark(transaction->child);
         rb_gc_mark(transaction->env);
         rb_gc_mark(transaction->cursors);
 }
@@ -130,55 +132,79 @@ static VALUE transaction_env(VALUE self) {
  *   @return [false,true] whether the transaction is read-only.
  */
 static VALUE transaction_is_readonly(VALUE self) {
-  TRANSACTION(self, transaction);
-  //MDB_txn* txn = transaction->txn;
-  return (transaction->flags & MDB_RDONLY) ? Qtrue : Qfalse;
+    TRANSACTION(self, transaction);
+    //MDB_txn* txn = transaction->txn;
+    return (transaction->flags & MDB_RDONLY) ? Qtrue : Qfalse;
 }
 
 
 static void transaction_finish(VALUE self, int commit) {
-        TRANSACTION(self, transaction);
-
-        if (!transaction->txn)
-                rb_raise(cError, "Transaction is terminated");
-
-        if (transaction->thread != rb_thread_current())
-                rb_raise(cError, "Wrong thread");
-
-        // Check nesting
-        VALUE p = environment_active_txn(transaction->env);
-        while (!NIL_P(p) && p != self) {
-                TRANSACTION(p, txn);
-                p = txn->parent;
-        }
-        if (p != self)
-                rb_raise(cError, "Transaction is not active");
-
-        int ret = 0;
-        if (commit)
-                ret = mdb_txn_commit(transaction->txn);
-        else
-                mdb_txn_abort(transaction->txn);
-
-        long i;
-        for (i=0; i<RARRAY_LEN(transaction->cursors); i++) {
-                VALUE cursor = RARRAY_AREF(transaction->cursors, i);
-                cursor_close(cursor);
-        }
-        rb_ary_clear(transaction->cursors);
-
-        // Mark child transactions as closed
-        p = environment_active_txn(transaction->env);
+    TRANSACTION(self, transaction);
+
+    if (!transaction->txn)
+        rb_raise(cError, "Transaction is already terminated");
+
+    if (transaction->thread != rb_thread_current())
+        rb_raise(cError, "The thread closing the transaction "
+                 "is not the one that opened it");
+
+    // ensure the transaction being closed is the active one
+    VALUE p = environment_active_txn(transaction->env);
+    while (!NIL_P(p) && p != self) {
+        TRANSACTION(p, txn);
+        p = txn->parent;
+    }
+    // bail out if the transaction `self` is not the active one
+    if (p != self)
+        rb_raise(cError, "Transaction is not active");
+
+    // now eliminate the cursors
+    long i;
+    for (i=0; i<RARRAY_LEN(transaction->cursors); i++) {
+        VALUE cursor = RARRAY_AREF(transaction->cursors, i);
+        cursor_close(cursor);
+    }
+    rb_ary_clear(transaction->cursors);
+
+    // now actually finish the internal transaction
+    int ret = 0;
+    if (commit)
+        ret = mdb_txn_commit(transaction->txn);
+    else
+        mdb_txn_abort(transaction->txn);
+
+    // eliminate child transactions
+    if (transaction->child) {
+        p = self; // again this is a VALUE
+        Transaction* txn = transaction; // and this is the struct
+
+        // descend into deepest child transaction
+        do {
+            p = txn->child;
+            // this is TRANSACTION minus the declaration
+            Data_Get_Struct(txn->child, Transaction, txn);
+        } while (txn->child);
+
+        // now we ascend back up
         while (p != self) {
-                TRANSACTION(p, txn);
-                txn->txn = 0;
-                p = txn->parent;
+            TRANSACTION(p, txn);
+            txn->txn = 0;
+            p = txn->parent;
         }
-        transaction->txn = 0;
+    }
+    transaction->txn = 0;
 
-        environment_set_active_txn(transaction->env, transaction->thread, transaction->parent);
+    // no more active read-write transaction; unset the registry
+    if (!(transaction->flags & MDB_RDONLY) && !transaction->parent) {
+        ENVIRONMENT(transaction->env, env);
+        env->rw_txn_thread = NULL;
+    }
 
-        check(ret);
+    // now set the active transaction to the parent, if there is one
+    environment_set_active_txn(transaction->env, transaction->thread,
+                               transaction->parent);
+
+    check(ret);
 }
 
 // Ruby 1.8.7 compatibility
@@ -219,65 +245,143 @@ static void stop_txn_begin(void *arg)
         txn_args->stop = 1;
 }
 
+/**
+ * This is the code that opens transactions. Read-write transactions
+ * have to be called outside the GVL because they will block without
+ *
+ *
+ * Here is the basic problem with LMDB transactions:
+ *
+ * - There can only be one read-write transaction per LMDB
+ *   ENVIRONMENT, not process, not thread.
+ *
+ * - Read-write transactions can nest.
+ *
+ * - There can only be one active transaction per thread.
+ *
+ * - Every thread can have an active read-ONLY transaction, but only one.
+ *
+ * - This is because read-only transactions canNOT nest (it is
+ *   meaningless to have a nested read-only transaction)
+ *
+ * - Furthermore it is an error to open a read-only transaction under
+ *   a read-write transaction.
+ *
+ * - Same goes for opening a read-write transaction in the same thread
+ *   as an active read-only transaction.
+ *
+ * Nevertheless, the downstream Ruby user may not be able to
+ * completely control calls to env.transaction (e.g. if they are
+ * wrapping a transaction around a proc or lambda that happens to
+ * contain its own transaction), plus every call in LMDB needs to be
+ * implicitly wrapped in a transaction anyway.
+ *
+ * What this means is that we will, first, have to keep explicit track
+ * of the read-write transaction, if one exists. Second, we will have
+ * to simulate nesting for read-only transactions, so the behaviour in
+ * Ruby is the same as a read-write transaction.
+ *
+ */
+
 static VALUE with_transaction(VALUE venv, VALUE(*fn)(VALUE), VALUE arg, int flags) {
-        ENVIRONMENT(venv, environment);
+    ENVIRONMENT(venv, environment);
 
-        MDB_txn* txn;
-        TxnArgs txn_args;
+    MDB_txn* txn;
+    TxnArgs txn_args;
 
-    retry:
-        txn = NULL;
+    VALUE thread  = rb_thread_current();
+    VALUE vparent = environment_active_txn(venv);
 
-        txn_args.env = environment->env;
-        txn_args.parent = active_txn(venv);
-        txn_args.flags = flags;
-        txn_args.htxn = &txn;
-        txn_args.result = 0;
-        txn_args.stop = 0;
+    Transaction* tparent = NULL;
+    if (vparent && !NIL_P(vparent))
+        Data_Get_Struct(vparent, Transaction, tparent);
 
-        if (flags & MDB_RDONLY) {
-                call_txn_begin(&txn_args);
-        }
-        else {
-                CALL_WITHOUT_GVL(
-                    call_txn_begin, &txn_args,
-                    stop_txn_begin, &txn_args);
-
-                if (txn_args.stop || !txn) {
-                        // !txn is when rb_thread_call_without_gvl2
-                        // returns before calling txn_begin
-                        if (txn) mdb_txn_abort(txn);
-
-                        //rb_warn("got here lol");
-                        rb_thread_check_ints();
-                        goto retry; // in what cases do we get here?
-                }
+    // rb_warn("fart lol");
+
+    // XXX note this is a cursed goto loop that could almost certainly
+    // be rewritten as a do-while
+ retry:
+    txn = NULL;
+
+    txn_args.env    = environment->env;
+    txn_args.parent = active_txn(venv);
+    txn_args.flags  = flags;
+    txn_args.htxn   = &txn;
+    txn_args.result = 0;
+    txn_args.stop   = 0;
+
+    if (flags & MDB_RDONLY) {
+        if (tparent && tparent->flags & MDB_RDONLY)
+            // this is a no-op: put the same actual transaction in a
+            // different wrapper struct
+            txn = txn_args.parent;
+        else
+            // this will return an error if the parent transaction is
+            // read-write, so we don't need to handle the case explicitly
+            call_txn_begin(&txn_args);
+    }
+    else {
+        if (tparent) {
+            // first we have to determine if we're on the same thread
+            // as the parent, which in turn must be the same as the
+            // environment's registry for which thread has the
+            // read-write transaction
+            if (thread != tparent->thread ||
+                thread != environment->rw_txn_thread)
+                rb_raise(cError,
+                         "Attempt to nest transaction on a different thread");
         }
 
-        check(txn_args.result);
-
-        Transaction* transaction;
-        VALUE vtxn = Data_Make_Struct(cTransaction, Transaction, transaction_mark, transaction_free, transaction);
-        transaction->parent = environment_active_txn(venv);
-        transaction->env = venv;
-        transaction->txn = txn;
-        transaction->flags = flags;
-        transaction->thread = rb_thread_current();
-        transaction->cursors = rb_ary_new();
-        environment_set_active_txn(venv, transaction->thread, vtxn);
-
-        int exception;
-        VALUE ret = rb_protect(fn, NIL_P(arg) ? vtxn : arg, &exception);
-
-        if (exception) {
-          //rb_warn("lol got exception");
-                if (vtxn == environment_active_txn(venv))
-                        transaction_abort(vtxn);
-                rb_jump_tag(exception);
+        CALL_WITHOUT_GVL(call_txn_begin, &txn_args, stop_txn_begin, &txn_args);
+
+        if (txn_args.stop || !txn) {
+            // !txn is when rb_thread_call_without_gvl2
+            // returns before calling txn_begin
+            if (txn) {
+                mdb_txn_abort(txn);
+                txn_args.result = 0;
+            }
+
+            //rb_warn("got here lol");
+            rb_thread_check_ints();
+            goto retry; // in what cases do we get here?
         }
+
+        // set the thread
+        environment->rw_txn_thread = thread;
+    }
+
+    // this will raise unless result is zero
+    check(txn_args.result);
+
+    Transaction* transaction;
+    VALUE vtxn = Data_Make_Struct(cTransaction, Transaction, transaction_mark,
+                                  transaction_free, transaction);
+    transaction->parent  = vparent;
+    transaction->env     = venv;
+    transaction->txn     = txn;
+    transaction->flags   = flags;
+    transaction->thread  = rb_thread_current();
+    transaction->cursors = rb_ary_new();
+
+    // set the parent's child to self
+    if (tparent) tparent->child = vtxn;
+
+    environment_set_active_txn(venv, transaction->thread, vtxn);
+
+    // now we run the function in the transaction
+    int exception;
+    VALUE ret = rb_protect(fn, NIL_P(arg) ? vtxn : arg, &exception);
+
+    if (exception) {
+        // rb_warn("lol got exception");
         if (vtxn == environment_active_txn(venv))
-                transaction_commit(vtxn);
-        return ret;
+            transaction_abort(vtxn);
+        rb_jump_tag(exception);
+    }
+    if (vtxn == environment_active_txn(venv))
+        transaction_commit(vtxn);
+    return ret;
 }
 
 static void environment_check(Environment* environment) {
@@ -674,7 +778,7 @@ static MDB_txn* active_txn(VALUE self) {
                 return 0;
         TRANSACTION(vtxn, transaction);
         if (!transaction->txn)
-                rb_raise(cError, "Transaction is terminated");
+                rb_raise(cError, "Transaction is already terminated");
         if (transaction->thread != rb_thread_current())
                 rb_raise(cError, "Wrong thread");
         return transaction->txn;

data/ext/lmdb_ext/lmdb_ext.h

@@ -47,19 +47,35 @@
         Data_Get_Struct(var, Cursor, var_cur);  \
         cursor_check(var_cur)
 
+/*
+  hey yo if you can convince hyc to add a function like
+  mdb_txn_get_flags or even mdb_txn_is_rdonly, you could probably get
+  rid of these wrapper structs and a lot of complexity, cause that is
+  literally the only reason why they're needed
+
+  actually no we would also need to know when a txn has a child and
+  there is also no mdb_txn_get_child or whatever
+*/
+
 typedef struct {
         VALUE    env;
-        VALUE    parent;
+        VALUE    parent; // ignored for ro threads
+        VALUE    child;  // ditto
         VALUE    thread;
         VALUE    cursors;
         MDB_txn* txn;
         unsigned int flags;
 } Transaction;
 
+// we have an extra field `rw_txn_thread` here that acts as the
+// registry for the single read-write transaction. if it's populated,
+// that's the one
+
 typedef struct {
         MDB_env* env;
-        VALUE    thread_txn_hash;
-        VALUE    txn_thread_hash;
+        VALUE    thread_txn_hash; /* transaction -> thread  */
+        VALUE    txn_thread_hash; /* thread      -> transaction */
+        VALUE    rw_txn_thread;   /* the thread with the rw transaction */
 } Environment;
 
 typedef struct {

data/lib/lmdb/database.rb

@@ -11,8 +11,8 @@ module LMDB
     #      puts "at #{key}: #{value}"
     #    end
     def each
-      # maybe_txn true do
-      env.transaction do
+      maybe_txn true do
+      # env.transaction do
         cursor do |c|
           while i = c.next
             yield(i)
@@ -56,8 +56,8 @@ module LMDB
     # @return [Enumerator] in lieu of a block.
     def each_key(&block)
       return enum_for :each_key unless block_given?
-      # maybe_txn true do
-      env.transaction do
+      maybe_txn true do
+      #env.transaction do
         cursor do |c|
           while (rec = c.next true)
             yield rec.first
@@ -81,8 +81,8 @@ module LMDB
         return
       end
 
-      #maybe_txn true do
-      env.transaction do
+      maybe_txn true do
+      # env.transaction do
         cursor do |c|
           method = :set
           while rec = c.send(method, key)
@@ -124,8 +124,9 @@ module LMDB
 
       ret = false
       # read-only txn was having trouble being nested inside a read-write
-      #maybe_txn true do
-      env.transaction do
+      maybe_txn true do
+      # env.transaction true do
+      # env.transaction do
         cursor { |c| ret = !!c.set(key, value) }
       end
       ret

data/lib/lmdb/version.rb

@@ -1,3 +1,3 @@
 module LMDB
-  VERSION = '0.5.3'.freeze
+  VERSION = '0.6'.freeze
 end

data/lmdb.gemspec

@@ -23,4 +23,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'rake', "~> 13.0"
   s.add_development_dependency 'rake-compiler', '~> 1.1'
   s.add_development_dependency 'rspec', "~> 3.0"
+  s.add_development_dependency 'ruby_memcheck', "~> 1.0"
 end

data/spec/helper.rb

@@ -2,6 +2,9 @@ require 'lmdb'
 require 'rspec'
 require 'fileutils'
 
+# for valgrind
+at_exit { GC.start }
+
 SPEC_ROOT = File.dirname(__FILE__)
 TEMP_ROOT = File.join(SPEC_ROOT, 'tmp')
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lmdb
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: '0.6'
 platform: ruby
 authors:
 - Daniel Mendler
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-04-22 00:00:00.000000000 Z
+date: 2022-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: ruby_memcheck
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 description: lmdb is a Ruby binding to OpenLDAP Lightning MDB.
 email: mail@daniel-mendler.de
 executables: []
@@ -66,6 +80,7 @@ files:
 - Gemfile
 - README.md
 - Rakefile
+- behaviour.org
 - ext/lmdb_ext/cursor_delete_flags.h
 - ext/lmdb_ext/cursor_put_flags.h
 - ext/lmdb_ext/dbi_flags.h
@@ -95,7 +110,7 @@ homepage: https://github.com/minad/lmdb
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -110,8 +125,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.3.11
+signing_key:
 specification_version: 4
 summary: Ruby bindings to Lightning MDB
 test_files: