lmdb 0.5.3 → 0.6.1

data/ext/lmdb_ext/lmdb_ext.c

@@ -26,6 +26,7 @@ static void check(int code) {
 
         const char* err = mdb_strerror(code);
         const char* sep = strchr(err, ':');
+        // increment the offset by two in case there is a colon (plus space)
         if (sep)
                 err = sep + 2;
 
@@ -37,20 +38,21 @@ static void check(int code) {
 }
 
 static void transaction_free(Transaction* transaction) {
-        if (transaction->txn) {
-          //int id = (int)mdb_txn_id(transaction->txn);
-          //rb_warn(sprintf("Memory leak: Garbage collecting active transaction %d", id));
-          rb_warn("Memory leak: Garbage collecting active transaction");
-          // transaction_abort(transaction);
-          // mdb_txn_abort(transaction->txn);
-        }
-        free(transaction);
+    if (transaction->txn) {
+        //int id = (int)mdb_txn_id(transaction->txn);
+        //rb_warn(sprintf("Memory leak: Garbage collecting active transaction %d", id));
+        rb_warn("Memory leak: Garbage collecting active transaction");
+        // transaction_abort(transaction);
+        // mdb_txn_abort(transaction->txn);
+    }
+    free(transaction);
 }
 
 static void transaction_mark(Transaction* transaction) {
-        rb_gc_mark(transaction->parent);
-        rb_gc_mark(transaction->env);
-        rb_gc_mark(transaction->cursors);
+    rb_gc_mark(transaction->parent);
+    rb_gc_mark(transaction->child);
+    rb_gc_mark(transaction->env);
+    rb_gc_mark(transaction->cursors);
 }
 
 /**
@@ -130,55 +132,80 @@ static VALUE transaction_env(VALUE self) {
  *   @return [false,true] whether the transaction is read-only.
  */
 static VALUE transaction_is_readonly(VALUE self) {
-  TRANSACTION(self, transaction);
-  //MDB_txn* txn = transaction->txn;
-  return (transaction->flags & MDB_RDONLY) ? Qtrue : Qfalse;
+    TRANSACTION(self, transaction);
+    //MDB_txn* txn = transaction->txn;
+    return (transaction->flags & MDB_RDONLY) ? Qtrue : Qfalse;
 }
 
 
 static void transaction_finish(VALUE self, int commit) {
-        TRANSACTION(self, transaction);
-
-        if (!transaction->txn)
-                rb_raise(cError, "Transaction is terminated");
-
-        if (transaction->thread != rb_thread_current())
-                rb_raise(cError, "Wrong thread");
-
-        // Check nesting
-        VALUE p = environment_active_txn(transaction->env);
-        while (!NIL_P(p) && p != self) {
-                TRANSACTION(p, txn);
-                p = txn->parent;
-        }
-        if (p != self)
-                rb_raise(cError, "Transaction is not active");
-
-        int ret = 0;
-        if (commit)
-                ret = mdb_txn_commit(transaction->txn);
-        else
-                mdb_txn_abort(transaction->txn);
-
-        long i;
-        for (i=0; i<RARRAY_LEN(transaction->cursors); i++) {
-                VALUE cursor = RARRAY_AREF(transaction->cursors, i);
-                cursor_close(cursor);
-        }
-        rb_ary_clear(transaction->cursors);
-
-        // Mark child transactions as closed
-        p = environment_active_txn(transaction->env);
+    TRANSACTION(self, transaction);
+
+    if (!transaction->txn)
+        rb_raise(cError, "Transaction is already terminated");
+
+    if (transaction->thread != rb_thread_current())
+        rb_raise(cError, "The thread closing the transaction "
+                 "is not the one that opened it");
+
+    // ensure the transaction being closed is the active one
+    VALUE p = environment_active_txn(transaction->env);
+    while (!NIL_P(p) && p != self) {
+        TRANSACTION(p, txn);
+        p = txn->parent;
+    }
+    // bail out if the transaction `self` is not the active one
+    if (p != self)
+        rb_raise(cError, "Transaction is not active");
+
+    // now eliminate the cursors
+    long i;
+    for (i=0; i<RARRAY_LEN(transaction->cursors); i++) {
+        VALUE cursor = RARRAY_AREF(transaction->cursors, i);
+        cursor_close(cursor);
+    }
+    rb_ary_clear(transaction->cursors);
+
+    // now actually finish the internal transaction
+    int ret = 0;
+    if (commit)
+        ret = mdb_txn_commit(transaction->txn);
+    else
+        mdb_txn_abort(transaction->txn);
+
+    // eliminate child transactions
+    if (transaction->child) {
+        p = self; // again this is a VALUE
+        Transaction* txn = transaction; // and this is the struct
+
+        // descend into deepest child transaction
+        do {
+            p = txn->child;
+            // this is TRANSACTION minus the declaration
+            Data_Get_Struct(txn->child, Transaction, txn);
+        } while (txn->child);
+
+        // now we ascend back up
         while (p != self) {
-                TRANSACTION(p, txn);
-                txn->txn = 0;
-                p = txn->parent;
+            TRANSACTION(p, txn);
+            txn->txn = 0;
+            p = txn->parent;
         }
-        transaction->txn = 0;
+    }
+    transaction->txn = 0;
 
-        environment_set_active_txn(transaction->env, transaction->thread, transaction->parent);
+    // no more active read-write transaction; unset the registry
+    if (!(transaction->flags & MDB_RDONLY) && !transaction->parent) {
+        ENVIRONMENT(transaction->env, env);
+        // maybe this should be Qnil, i dunno
+        env->rw_txn_thread = (VALUE)NULL;
+    }
 
-        check(ret);
+    // now set the active transaction to the parent, if there is one
+    environment_set_active_txn(transaction->env, transaction->thread,
+                               transaction->parent);
+
+    check(ret);
 }
 
 // Ruby 1.8.7 compatibility
@@ -219,65 +246,143 @@ static void stop_txn_begin(void *arg)
         txn_args->stop = 1;
 }
 
+/**
+ * This is the code that opens transactions. Read-write transactions
+ * have to be called outside the GVL because they will block without
+ *
+ *
+ * Here is the basic problem with LMDB transactions:
+ *
+ * - There can only be one read-write transaction per LMDB
+ *   ENVIRONMENT, not process, not thread.
+ *
+ * - Read-write transactions can nest.
+ *
+ * - There can only be one active transaction per thread.
+ *
+ * - Every thread can have an active read-ONLY transaction, but only one.
+ *
+ * - This is because read-only transactions canNOT nest (it is
+ *   meaningless to have a nested read-only transaction)
+ *
+ * - Furthermore it is an error to open a read-only transaction under
+ *   a read-write transaction.
+ *
+ * - Same goes for opening a read-write transaction in the same thread
+ *   as an active read-only transaction.
+ *
+ * Nevertheless, the downstream Ruby user may not be able to
+ * completely control calls to env.transaction (e.g. if they are
+ * wrapping a transaction around a proc or lambda that happens to
+ * contain its own transaction), plus every call in LMDB needs to be
+ * implicitly wrapped in a transaction anyway.
+ *
+ * What this means is that we will, first, have to keep explicit track
+ * of the read-write transaction, if one exists. Second, we will have
+ * to simulate nesting for read-only transactions, so the behaviour in
+ * Ruby is the same as a read-write transaction.
+ *
+ */
+
 static VALUE with_transaction(VALUE venv, VALUE(*fn)(VALUE), VALUE arg, int flags) {
-        ENVIRONMENT(venv, environment);
+    ENVIRONMENT(venv, environment);
 
-        MDB_txn* txn;
-        TxnArgs txn_args;
+    MDB_txn* txn;
+    TxnArgs txn_args;
 
-    retry:
-        txn = NULL;
+    VALUE thread  = rb_thread_current();
+    VALUE vparent = environment_active_txn(venv);
 
-        txn_args.env = environment->env;
-        txn_args.parent = active_txn(venv);
-        txn_args.flags = flags;
-        txn_args.htxn = &txn;
-        txn_args.result = 0;
-        txn_args.stop = 0;
+    Transaction* tparent = NULL;
+    if (vparent && !NIL_P(vparent))
+        Data_Get_Struct(vparent, Transaction, tparent);
 
-        if (flags & MDB_RDONLY) {
-                call_txn_begin(&txn_args);
-        }
-        else {
-                CALL_WITHOUT_GVL(
-                    call_txn_begin, &txn_args,
-                    stop_txn_begin, &txn_args);
-
-                if (txn_args.stop || !txn) {
-                        // !txn is when rb_thread_call_without_gvl2
-                        // returns before calling txn_begin
-                        if (txn) mdb_txn_abort(txn);
-
-                        //rb_warn("got here lol");
-                        rb_thread_check_ints();
-                        goto retry; // in what cases do we get here?
-                }
+    // rb_warn("fart lol");
+
+    // XXX note this is a cursed goto loop that could almost certainly
+    // be rewritten as a do-while
+ retry:
+    txn = NULL;
+
+    txn_args.env    = environment->env;
+    txn_args.parent = active_txn(venv);
+    txn_args.flags  = flags;
+    txn_args.htxn   = &txn;
+    txn_args.result = 0;
+    txn_args.stop   = 0;
+
+    if (flags & MDB_RDONLY) {
+        if (tparent && tparent->flags & MDB_RDONLY)
+            // this is a no-op: put the same actual transaction in a
+            // different wrapper struct
+            txn = txn_args.parent;
+        else
+            // this will return an error if the parent transaction is
+            // read-write, so we don't need to handle the case explicitly
+            call_txn_begin(&txn_args);
+    }
+    else {
+        if (tparent) {
+            // first we have to determine if we're on the same thread
+            // as the parent, which in turn must be the same as the
+            // environment's registry for which thread has the
+            // read-write transaction
+            if (thread != tparent->thread ||
+                thread != environment->rw_txn_thread)
+                rb_raise(cError,
+                         "Attempt to nest transaction on a different thread");
         }
 
-        check(txn_args.result);
-
-        Transaction* transaction;
-        VALUE vtxn = Data_Make_Struct(cTransaction, Transaction, transaction_mark, transaction_free, transaction);
-        transaction->parent = environment_active_txn(venv);
-        transaction->env = venv;
-        transaction->txn = txn;
-        transaction->flags = flags;
-        transaction->thread = rb_thread_current();
-        transaction->cursors = rb_ary_new();
-        environment_set_active_txn(venv, transaction->thread, vtxn);
-
-        int exception;
-        VALUE ret = rb_protect(fn, NIL_P(arg) ? vtxn : arg, &exception);
-
-        if (exception) {
-          //rb_warn("lol got exception");
-                if (vtxn == environment_active_txn(venv))
-                        transaction_abort(vtxn);
-                rb_jump_tag(exception);
+        CALL_WITHOUT_GVL(call_txn_begin, &txn_args, stop_txn_begin, &txn_args);
+
+        if (txn_args.stop || !txn) {
+            // !txn is when rb_thread_call_without_gvl2
+            // returns before calling txn_begin
+            if (txn) {
+                mdb_txn_abort(txn);
+                txn_args.result = 0;
+            }
+
+            //rb_warn("got here lol");
+            rb_thread_check_ints();
+            goto retry; // in what cases do we get here?
         }
+
+        // set the thread
+        environment->rw_txn_thread = thread;
+    }
+
+    // this will raise unless result is zero
+    check(txn_args.result);
+
+    Transaction* transaction;
+    VALUE vtxn = Data_Make_Struct(cTransaction, Transaction, transaction_mark,
+                                  transaction_free, transaction);
+    transaction->parent  = vparent;
+    transaction->env     = venv;
+    transaction->txn     = txn;
+    transaction->flags   = flags;
+    transaction->thread  = rb_thread_current();
+    transaction->cursors = rb_ary_new();
+
+    // set the parent's child to self
+    if (tparent) tparent->child = vtxn;
+
+    environment_set_active_txn(venv, transaction->thread, vtxn);
+
+    // now we run the function in the transaction
+    int exception;
+    VALUE ret = rb_protect(fn, NIL_P(arg) ? vtxn : arg, &exception);
+
+    if (exception) {
+        // rb_warn("lol got exception");
         if (vtxn == environment_active_txn(venv))
-                transaction_commit(vtxn);
-        return ret;
+            transaction_abort(vtxn);
+        rb_jump_tag(exception);
+    }
+    if (vtxn == environment_active_txn(venv))
+        transaction_commit(vtxn);
+    return ret;
 }
 
 static void environment_check(Environment* environment) {
@@ -496,47 +601,50 @@ static int environment_options(VALUE key, VALUE value, EnvironmentOptions* optio
  *      end
  */
 static VALUE environment_new(int argc, VALUE *argv, VALUE klass) {
-        VALUE path, option_hash;
+    VALUE path, option_hash;
 
 #ifdef RB_SCAN_ARGS_KEYWORDS
-        rb_scan_args_kw(RB_SCAN_ARGS_LAST_HASH_KEYWORDS,
-                        argc, argv, "1:", &path, &option_hash);
+    rb_scan_args_kw(RB_SCAN_ARGS_LAST_HASH_KEYWORDS,
+                    argc, argv, "1:", &path, &option_hash);
 #else
-        rb_scan_args(argc, argv, "1:", &path, &option_hash);
+    rb_scan_args(argc, argv, "1:", &path, &option_hash);
 #endif
 
-        EnvironmentOptions options = {
-                .flags = MDB_NOTLS,
-                .maxreaders = -1,
-                .maxdbs = 128,
-                .mapsize = 0,
-                .mode = 0755,
-        };
-        if (!NIL_P(option_hash))
-                rb_hash_foreach(option_hash, environment_options, (VALUE)&options);
-
-        MDB_env* env;
-        check(mdb_env_create(&env));
-
-        Environment* environment;
-        VALUE venv = Data_Make_Struct(cEnvironment, Environment, environment_mark, environment_free, environment);
-        environment->env = env;
-        environment->thread_txn_hash = rb_hash_new();
-        environment->txn_thread_hash = rb_hash_new();
-
-        if (options.maxreaders > 0)
-                check(mdb_env_set_maxreaders(env, options.maxreaders));
-        if (options.mapsize > 0)
-                check(mdb_env_set_mapsize(env, options.mapsize));
-
-        check(mdb_env_set_maxdbs(env, options.maxdbs <= 0 ? 1 : options.maxdbs));
-        VALUE expanded_path = rb_file_expand_path(path, Qnil);
-        check(mdb_env_open(env, StringValueCStr(expanded_path), options.flags, options.mode));
+    EnvironmentOptions options = {
+        .flags = MDB_NOTLS,
+        .maxreaders = -1,
+        .maxdbs = 128,
+        .mapsize = 0,
+        .mode = 0755,
+    };
+    if (!NIL_P(option_hash))
+        rb_hash_foreach(option_hash, (int (*)(ANYARGS))environment_options,
+                        (VALUE)&options);
+
+    MDB_env* env;
+    check(mdb_env_create(&env));
+
+    Environment* environment;
+    VALUE venv = Data_Make_Struct(cEnvironment, Environment, environment_mark,
+                                  environment_free, environment);
+    environment->env = env;
+    environment->thread_txn_hash = rb_hash_new();
+    environment->txn_thread_hash = rb_hash_new();
+
+    if (options.maxreaders > 0)
+        check(mdb_env_set_maxreaders(env, options.maxreaders));
+    if (options.mapsize > 0)
+        check(mdb_env_set_mapsize(env, options.mapsize));
 
-        if (rb_block_given_p())
-                return rb_ensure(rb_yield, venv, environment_close, venv);
+    check(mdb_env_set_maxdbs(env, options.maxdbs <= 0 ? 1 : options.maxdbs));
+    VALUE expanded_path = rb_file_expand_path(path, Qnil);
+    check(mdb_env_open(env, StringValueCStr(expanded_path), options.flags,
+                       options.mode));
 
-        return venv;
+    if (rb_block_given_p())
+        return rb_ensure(rb_yield, venv, environment_close, venv);
+
+    return venv;
 }
 
 /**
@@ -674,7 +782,7 @@ static MDB_txn* active_txn(VALUE self) {
                 return 0;
         TRANSACTION(vtxn, transaction);
         if (!transaction->txn)
-                rb_raise(cError, "Transaction is terminated");
+                rb_raise(cError, "Transaction is already terminated");
         if (transaction->thread != rb_thread_current())
                 rb_raise(cError, "Wrong thread");
         return transaction->txn;
@@ -773,32 +881,35 @@ static void database_mark(Database* database) {
  *       transaction or a read-only environment.
  */
 static VALUE environment_database(int argc, VALUE *argv, VALUE self) {
-        ENVIRONMENT(self, environment);
-        if (!active_txn(self))
-                return call_with_transaction(self, self, "database", argc, argv, 0);
+    ENVIRONMENT(self, environment);
+    if (!active_txn(self))
+        return call_with_transaction(self, self, "database", argc, argv, 0);
 
-        VALUE name, option_hash;
+    VALUE name, option_hash;
 #ifdef RB_SCAN_ARGS_KEYWORDS
-        rb_scan_args_kw(RB_SCAN_ARGS_KEYWORDS,
-                        argc, argv, "01:", &name, &option_hash);
+    rb_scan_args_kw(RB_SCAN_ARGS_KEYWORDS,
+                    argc, argv, "01:", &name, &option_hash);
 #else
-        rb_scan_args(argc, argv, "01:", &name, &option_hash);
+    rb_scan_args(argc, argv, "01:", &name, &option_hash);
 #endif
 
 
-        int flags = 0;
-        if (!NIL_P(option_hash))
-                rb_hash_foreach(option_hash, database_flags, (VALUE)&flags);
+    int flags = 0;
+    if (!NIL_P(option_hash))
+        rb_hash_foreach(option_hash, (int (*)(ANYARGS))database_flags,
+                        (VALUE)&flags);
 
-        MDB_dbi dbi;
-        check(mdb_dbi_open(need_txn(self), NIL_P(name) ? 0 : StringValueCStr(name), flags, &dbi));
+    MDB_dbi dbi;
+    check(mdb_dbi_open(need_txn(self), NIL_P(name) ? 0 : StringValueCStr(name),
+                       flags, &dbi));
 
-        Database* database;
-        VALUE vdb = Data_Make_Struct(cDatabase, Database, database_mark, free, database);
-        database->dbi = dbi;
-        database->env = self;
+    Database* database;
+    VALUE vdb = Data_Make_Struct(cDatabase, Database, database_mark, free,
+                                 database);
+    database->dbi = dbi;
+    database->env = self;
 
-        return vdb;
+    return vdb;
 }
 
 /**
@@ -960,33 +1071,34 @@ static VALUE database_get(VALUE self, VALUE vkey) {
  *       data.
  */
 static VALUE database_put(int argc, VALUE *argv, VALUE self) {
-        DATABASE(self, database);
-        if (!active_txn(database->env))
-                return call_with_transaction(database->env, self, "put", argc, argv, 0);
+    DATABASE(self, database);
+    if (!active_txn(database->env))
+        return call_with_transaction(database->env, self, "put", argc, argv, 0);
 
-        VALUE vkey, vval, option_hash = Qnil;
+    VALUE vkey, vval, option_hash = Qnil;
 #ifdef RB_SCAN_ARGS_KEYWORDS
-        rb_scan_args_kw(RB_SCAN_ARGS_LAST_HASH_KEYWORDS,
-                        argc, argv, "20:", &vkey, &vval, &option_hash);
+    rb_scan_args_kw(RB_SCAN_ARGS_LAST_HASH_KEYWORDS,
+                    argc, argv, "20:", &vkey, &vval, &option_hash);
 #else
-        rb_scan_args(argc, argv, "20:", &vkey, &vval, &option_hash);
+    rb_scan_args(argc, argv, "20:", &vkey, &vval, &option_hash);
 #endif
 
-        int flags = 0;
-        if (!NIL_P(option_hash))
-                rb_hash_foreach(option_hash, database_put_flags, (VALUE)&flags);
+    int flags = 0;
+    if (!NIL_P(option_hash))
+        rb_hash_foreach(option_hash, (int (*)(ANYARGS))database_put_flags,
+                        (VALUE)&flags);
 
-        vkey = StringValue(vkey);
-        vval = StringValue(vval);
+    vkey = StringValue(vkey);
+    vval = StringValue(vval);
 
-        MDB_val key, value;
-        key.mv_size = RSTRING_LEN(vkey);
-        key.mv_data = RSTRING_PTR(vkey);
-        value.mv_size = RSTRING_LEN(vval);
-        value.mv_data = RSTRING_PTR(vval);
+    MDB_val key, value;
+    key.mv_size = RSTRING_LEN(vkey);
+    key.mv_data = RSTRING_PTR(vkey);
+    value.mv_size = RSTRING_LEN(vval);
+    value.mv_data = RSTRING_PTR(vval);
 
-        check(mdb_put(need_txn(database->env), database->dbi, &key, &value, flags));
-        return Qnil;
+    check(mdb_put(need_txn(database->env), database->dbi, &key, &value, flags));
+    return Qnil;
 }
 
 /**
@@ -1350,31 +1462,32 @@ static VALUE cursor_get(VALUE self) {
  *       data.
  */
 static VALUE cursor_put(int argc, VALUE* argv, VALUE self) {
-        CURSOR(self, cursor);
+    CURSOR(self, cursor);
 
-        VALUE vkey, vval, option_hash;
+    VALUE vkey, vval, option_hash;
 #ifdef RB_SCAN_ARGS_KEYWORDS
-        rb_scan_args_kw(RB_SCAN_ARGS_LAST_HASH_KEYWORDS,
-                        argc, argv, "2:", &vkey, &vval, &option_hash);
+    rb_scan_args_kw(RB_SCAN_ARGS_LAST_HASH_KEYWORDS,
+                    argc, argv, "2:", &vkey, &vval, &option_hash);
 #else
-        rb_scan_args(argc, argv, "2:", &vkey, &vval, &option_hash);
+    rb_scan_args(argc, argv, "2:", &vkey, &vval, &option_hash);
 #endif
 
-        int flags = 0;
-        if (!NIL_P(option_hash))
-                rb_hash_foreach(option_hash, cursor_put_flags, (VALUE)&flags);
+    int flags = 0;
+    if (!NIL_P(option_hash))
+        rb_hash_foreach(option_hash, (int (*)(ANYARGS))cursor_put_flags,
+                        (VALUE)&flags);
 
-        vkey = StringValue(vkey);
-        vval = StringValue(vval);
+    vkey = StringValue(vkey);
+    vval = StringValue(vval);
 
-        MDB_val key, value;
-        key.mv_size = RSTRING_LEN(vkey);
-        key.mv_data = RSTRING_PTR(vkey);
-        value.mv_size = RSTRING_LEN(vval);
-        value.mv_data = RSTRING_PTR(vval);
+    MDB_val key, value;
+    key.mv_size = RSTRING_LEN(vkey);
+    key.mv_data = RSTRING_PTR(vkey);
+    value.mv_size = RSTRING_LEN(vval);
+    value.mv_data = RSTRING_PTR(vval);
 
-        check(mdb_cursor_put(cursor->cur, &key, &value, flags));
-        return Qnil;
+    check(mdb_cursor_put(cursor->cur, &key, &value, flags));
+    return Qnil;
 }
 
 #define METHOD cursor_delete_flags
@@ -1392,22 +1505,23 @@ static VALUE cursor_put(int argc, VALUE* argv, VALUE self) {
  *        if the database was opened with +:dupsort+.
  */
 static VALUE cursor_delete(int argc, VALUE *argv, VALUE self) {
-        CURSOR(self, cursor);
+    CURSOR(self, cursor);
 
-        VALUE option_hash;
+    VALUE option_hash;
 #ifdef RB_SCAN_ARGS_KEYWORDS
-        rb_scan_args_kw(RB_SCAN_ARGS_LAST_HASH_KEYWORDS,
-                        argc, argv, ":", &option_hash);
+    rb_scan_args_kw(RB_SCAN_ARGS_LAST_HASH_KEYWORDS,
+                    argc, argv, ":", &option_hash);
 #else
-        rb_scan_args(argc, argv, ":", &option_hash);
+    rb_scan_args(argc, argv, ":", &option_hash);
 #endif
 
-        int flags = 0;
-        if (!NIL_P(option_hash))
-                rb_hash_foreach(option_hash, cursor_delete_flags, (VALUE)&flags);
+    int flags = 0;
+    if (!NIL_P(option_hash))
+        rb_hash_foreach(option_hash, (int (*)(ANYARGS))cursor_delete_flags,
+                        (VALUE)&flags);
 
-        check(mdb_cursor_del(cursor->cur, flags));
-        return Qnil;
+    check(mdb_cursor_del(cursor->cur, flags));
+    return Qnil;
 }
 
 /**