localmemcache 0.3.0 → 0.4.0

data/src/localmemcache.h

@@ -69,7 +69,7 @@
  *
  *  == Clearing memory pools
  *
- *  Removing memory pools can be done with LocalMemCache.clear(options). 
+ *  Removing memory pools can be done with LocalMemCache.drop(options). 
  *
  *  == Environment
  *  
@@ -136,6 +136,11 @@ int local_memcache_set(local_memcache_t *lmc, const char *key, size_t n_key,
  */
 int local_memcache_delete(local_memcache_t *lmc, char *key, size_t n_key);
 
+/*
+ * Clears content of hashtable.
+ */
+int local_memcache_clear(local_memcache_t *lmc);
+
 /*
  * Releases memory pool handle.
  */
@@ -164,14 +169,23 @@ int local_memcache_free(local_memcache_t *lmc, lmc_error_t *e);
  * triggering the automatic recovery.)
  *
  */
-int local_memcache_iterate(local_memcache_t *lmc, void *ctx, 
+int local_memcache_iterate(local_memcache_t *lmc, void *ctx, size_t *ofs,
     LMC_ITERATOR_P(iter));
 
 /*
- * Deletes a memory pool.  If repair is 1, locked semaphores are
+ *  Retrieves random pair from hashtable.
+ *
+ *  It will return a newly allocated strings for r_key and r_value which you
+ *  will need to free() after use.
+ */
+int local_memcache_random_pair_new(local_memcache_t *lmc, 
+    char **r_key, size_t *n_key, char **r_value, size_t *n_value);
+
+/*
+ * Deletes a memory pool.  If force is 1, locked semaphores are
  * removed as well.
  *
- * WARNING: Do only call this method with the repair option if you are sure
+ * WARNING: Do only call this method with the force option if you are sure
  * that you really want to remove this memory pool and no more processes are
  * still using it.
  *
@@ -183,8 +197,8 @@ int local_memcache_iterate(local_memcache_t *lmc, void *ctx,
  * The memory pool must be specified by either setting the filename or
  * namespace parameter.  
  */
-int local_memcache_clear_namespace(const char *namespace, const char *filename,
-    int repair, lmc_error_t *e);
+int local_memcache_drop_namespace(const char *namespace, const char *filename,
+    int force, lmc_error_t *e);
 /*
  * Tries to repair a corrupt namespace.  Usually one doesn't call this method
  * directly, it's invoked automatically when operations time out.
@@ -199,6 +213,10 @@ int local_memcache_check_namespace(const char *namespace, const char *filename,
 const char *__local_memcache_get(local_memcache_t *lmc, 
     const char *key, size_t n_key, size_t *n_value);
 
+/* internal, do not use */
+int __local_memcache_random_pair(local_memcache_t *lmc, 
+    char **r_key, size_t *n_key, char **r_value, size_t *n_value);
+
 /* internal, do not use */
 int lmc_unlock_shm_region(const char *who, local_memcache_t *lmc);
 #endif

data/src/ruby-binding/localmemcache.rb

@@ -14,6 +14,7 @@ class LocalMemCache
   class ShmLockFailed < LocalMemCacheError; end
   class ShmUnlockFailed < LocalMemCacheError; end
   class MemoryPoolClosed < LocalMemCacheError; end
+  class DBVersionNotSupported < LocalMemCacheError; end
 
   #  Creates a new handle for accessing a shared memory region.
   # 
@@ -41,35 +42,19 @@ class LocalMemCache
     _new(o);
   end
 
-  # NOTE: This method is deprecated, use LocalMemCache.clear(*args) instead.
-  #
-  # Deletes a memory pool.  If the repair flag is set, locked semaphores are
-  # removed as well.  
-  #
-  # If you delete a pool and other processes still have handles open on it, the
-  # status of these handles becomes undefined.  There's no way for a process to
-  # know when a handle is not valid anymore, so only delete a memory pool if
-  # you are sure that all handles are closed.
-  #
-  # WARNING: Do only call this method with the repair=true flag if you are sure
-  # that you really want to remove this memory pool and no more processes are
-  # still using it.
-  def self.clear_namespace(namespace, repair = false) 
-    clear :namespace => namespace.to_s, :repair => repair
+  #  <code>SharedObjectStorage</code> inherits from class LocalMemCache but
+  #  stores Ruby objects as values instead of just strings (It still uses
+  #  strings for the keys, though).
+  class SharedObjectStorage < LocalMemCache
+    def []=(key,val) super(key, Marshal.dump(val)) end
+    def [](key) v = super(key); v.nil? ? nil : Marshal.load(v) end
+    def each_pair(&block) 
+      super {|k, mv| block.call(k, Marshal.load(mv)) }
+    end
+    def random_pair
+      rp = super
+      rp.nil? ? nil : [rp.first, Marshal.load(rp.last)]
+    end
   end
 
-  # NOTE: This method is deprecated, use LocalMemCache.check(*args) instead.
-  #
-  # Tries to repair a corrupt namespace.  Usually one doesn't call this method
-  # directly, it's invoked automatically when operations time out.
-  # 
-  # valid options are 
-  # [:namespace] 
-  # [:filename] 
-  # 
-  # The memory pool must be specified by either setting the :filename or
-  # :namespace option.  The default for :repair is false.
-  def self.check_namespace(namespace) 
-    check :namespace => namespace.to_s
-  end
 end

data/src/ruby-binding/rblocalmemcache.c

@@ -65,7 +65,7 @@ static VALUE LocalMemCache;
 static VALUE lmc_rb_sym_namespace;
 static VALUE lmc_rb_sym_filename;
 static VALUE lmc_rb_sym_size_mb;
-static VALUE lmc_rb_sym_repair;
+static VALUE lmc_rb_sym_force;
 
 /* :nodoc: */
 void __rb_lmc_raise_exception(const char *error_type, const char *m) {
@@ -124,12 +124,12 @@ local_memcache_t *get_LocalMemCache(VALUE obj) {
 }
 
 /*
- * call-seq: LocalMemCache.clear(*args)
+ * call-seq: LocalMemCache.drop(*args)
  *
- * Deletes a memory pool.  If the :repair option is set, locked semaphores are
+ * Deletes a memory pool.  If the :force option is set, locked semaphores are
  * removed as well.
  *
- * WARNING: Do only call this method with the :repair option if you are sure
+ * WARNING: Do only call this method with the :force option if you are sure
  * that you really want to remove this memory pool and no more processes are
  * still using it.
  *
@@ -138,21 +138,21 @@ local_memcache_t *get_LocalMemCache(VALUE obj) {
  * know when a handle is not valid anymore, so only delete a memory pool if
  * you are sure that all handles are closed.
  *
- * valid options for clear are 
+ * valid options for drop are 
  * [:namespace] 
  * [:filename] 
- * [:repair] 
+ * [:force] 
  *
  * The memory pool must be specified by either setting the :filename or
- * :namespace option.  The default for :repair is false.
+ * :namespace option.  The default for :force is false.
  */
-static VALUE LocalMemCache__clear(VALUE klass, VALUE o) {
+static VALUE LocalMemCache__drop(VALUE klass, VALUE o) {
   lmc_check_dict(o);
   lmc_error_t e;
-  if (!local_memcache_clear_namespace(
+  if (!local_memcache_drop_namespace(
       rstring_ptr_null(rb_hash_aref(o, lmc_rb_sym_namespace)), 
       rstring_ptr_null(rb_hash_aref(o, lmc_rb_sym_filename)),
-      bool_value(rb_hash_aref(o, lmc_rb_sym_repair)), &e)) {
+      bool_value(rb_hash_aref(o, lmc_rb_sym_force)), &e)) {
     rb_lmc_raise_exception(&e); 
   }
   return Qnil;
@@ -212,6 +212,27 @@ static VALUE LocalMemCache__get(VALUE obj, VALUE key) {
   return rr;
 }
 
+/* 
+ *  call-seq:
+ *     lmc.random_pair()   ->  [key, value] or nil
+ *
+ *  Retrieves random pair from hashtable.
+ */
+static VALUE LocalMemCache__random_pair(VALUE obj) {
+  char *k, *v;
+  size_t n_k, n_v;
+  VALUE r = Qnil;
+  if (__local_memcache_random_pair(get_LocalMemCache(obj), &k, &n_k, &v, 
+      &n_v)) {
+    r = rb_ary_new();
+    rb_ary_push(r, lmc_ruby_string2(k, n_k));
+    rb_ary_push(r, lmc_ruby_string2(v, n_v));
+  }
+  lmc_unlock_shm_region("local_memcache_random_pair", 
+      get_LocalMemCache(obj));
+  return r;
+}
+
 /* 
  *  call-seq:
  *     lmc.set(key, value)   ->   Qnil
@@ -229,6 +250,19 @@ static VALUE LocalMemCache__set(VALUE obj, VALUE key, VALUE value) {
   return Qnil;
 }
 
+
+/*
+ *  call-seq: 
+ *     lmc.clear -> Qnil
+ *
+ *  Clears content of hashtable.
+ */
+static VALUE LocalMemCache__clear(VALUE obj) {
+  local_memcache_t *lmc = get_LocalMemCache(obj);
+  if (!local_memcache_clear(lmc)) rb_lmc_raise_exception(&lmc->error); 
+  return Qnil;
+}
+
 /* 
  *  call-seq:
  *     lmc.delete(key)   ->   Qnil
@@ -274,8 +308,12 @@ static VALUE __LocalMemCache__keys(VALUE d) {
   VALUE r = rb_ary_entry(d, 1);
   lmc_ruby_iter_collect_keys data;
   data.ary = r;
-  int success = local_memcache_iterate(get_LocalMemCache(obj), 
-      (void *) &data, lmc_ruby_iter);
+  int success = 2;
+  size_t ofs = 0;
+  while (success == 2) {
+    success = local_memcache_iterate(get_LocalMemCache(obj), 
+        (void *) &data, &ofs, lmc_ruby_iter);
+  }
   if (!success) { return Qnil; }
   return Qnil;
 }
@@ -299,6 +337,68 @@ static VALUE LocalMemCache__keys(VALUE obj) {
   return rb_ary_entry(d, 1);
 }
 
+typedef struct {
+  VALUE ary;
+} lmc_ruby_iter_collect_pairs_t;
+
+/* :nodoc: */
+int lmc_ruby_iter_collect_pairs(void *ctx, const char* key, const char* value) {
+  lmc_ruby_iter_collect_pairs_t *data = ctx;
+  rb_ary_push(data->ary, rb_assoc_new(lmc_ruby_string(key), 
+      lmc_ruby_string(value)));
+  return 1;
+}
+
+/* :nodoc: */
+static VALUE __LocalMemCache__each_pair(VALUE d) {
+  VALUE obj = rb_ary_entry(d, 0);
+  int success = 2;
+  size_t ofs = 0;
+  while (success == 2) {
+    VALUE r = rb_ary_new();
+    lmc_ruby_iter_collect_pairs_t data;
+    data.ary = r;
+    success = local_memcache_iterate(get_LocalMemCache(obj), 
+        (void *) &data, &ofs, lmc_ruby_iter_collect_pairs);
+    long i;
+    for (i = 0; i < RARRAY(r)->len; i++) {
+      rb_yield(RARRAY(r)->ptr[i]);
+    }
+  }
+  if (!success) { return Qnil; }
+  return Qnil;
+}
+
+/* 
+ *  call-seq:
+ *     lmc.each_pair {|k, v|  block } -> nil
+ *
+ *  Iterates over hashtable.
+ */
+static VALUE LocalMemCache__each_pair(VALUE obj) {
+  VALUE d = rb_ary_new();
+  rb_ary_push(d, obj);
+  int error = 0;
+  rb_protect(__LocalMemCache__each_pair, d, &error);
+  if (error) {
+    lmc_unlock_shm_region("local_memcache_iterate", get_LocalMemCache(obj));
+    rb_exc_raise(ruby_errinfo);
+  }
+  return Qnil;
+}
+
+/* 
+ *  call-seq:
+ *     lmc.size -> number
+ *
+ *  Number of pairs in the hashtable.
+ */
+static VALUE LocalMemCache__size(VALUE obj) {
+  local_memcache_t *lmc = get_LocalMemCache(obj);
+  ht_hash_t *ht = lmc->base + lmc->va_hash;
+  return rb_int2big(ht->size);
+}
+
 /*
  * Document-class: LocalMemCache
  * 
@@ -337,7 +437,7 @@ static VALUE LocalMemCache__keys(VALUE obj) {
  *
  *  == Clearing memory pools
  *
- *  Removing memory pools can be done with LocalMemCache.clear(options). 
+ *  Removing memory pools can be done with LocalMemCache.drop(options). 
  *
  *  == Environment
  *  
@@ -345,12 +445,17 @@ static VALUE LocalMemCache__keys(VALUE obj) {
  *  reside in /var/tmp/localmemcache.  This can be overriden by setting the
  *  LMC_NAMESPACES_ROOT_PATH variable in the environment.
  *
+ *  == Storing Ruby Objects
+ *
+ *  If you want to store Ruby objects instead of just strings, consider 
+ *  using LocalMemCache::SharedObjectStorage.
+ *
  */
 void Init_rblocalmemcache() {
   LocalMemCache = rb_define_class("LocalMemCache", rb_cObject);
   rb_define_singleton_method(LocalMemCache, "_new", LocalMemCache__new2, 1);
-  rb_define_singleton_method(LocalMemCache, "clear", 
-      LocalMemCache__clear, 1);
+  rb_define_singleton_method(LocalMemCache, "drop", 
+      LocalMemCache__drop, 1);
   rb_define_singleton_method(LocalMemCache, "check", 
       LocalMemCache__check, 1);
   rb_define_singleton_method(LocalMemCache, "disable_test_crash", 
@@ -361,12 +466,18 @@ void Init_rblocalmemcache() {
   rb_define_method(LocalMemCache, "[]", LocalMemCache__get, 1);
   rb_define_method(LocalMemCache, "delete", LocalMemCache__delete, 1);
   rb_define_method(LocalMemCache, "set", LocalMemCache__set, 2);
+  rb_define_method(LocalMemCache, "clear", LocalMemCache__clear, 0);
   rb_define_method(LocalMemCache, "[]=", LocalMemCache__set, 2);
   rb_define_method(LocalMemCache, "keys", LocalMemCache__keys, 0);
+  rb_define_method(LocalMemCache, "each_pair", LocalMemCache__each_pair, 0);
+  rb_define_method(LocalMemCache, "random_pair", LocalMemCache__random_pair, 
+      0);
   rb_define_method(LocalMemCache, "close", LocalMemCache__close, 0);
+  rb_define_method(LocalMemCache, "size", LocalMemCache__size, 0);
 
   lmc_rb_sym_namespace = ID2SYM(rb_intern("namespace"));
   lmc_rb_sym_filename = ID2SYM(rb_intern("filename"));
   lmc_rb_sym_size_mb = ID2SYM(rb_intern("size_mb"));
-  lmc_rb_sym_repair = ID2SYM(rb_intern("repair"));
+  lmc_rb_sym_force = ID2SYM(rb_intern("force"));
+  rb_require("localmemcache.rb");
 }

data/src/tests/allocfailure.rb

@@ -3,7 +3,7 @@ $DIR=File.dirname(__FILE__)
 
 require 'localmemcache'
 
-LocalMemCache.clear_namespace("alloc-failure-test", true);
+LocalMemCache.drop :namespace => "alloc-failure-test", :force => true
 
 
   LocalMemCache.enable_test_crash

data/src/tests/bench.rb

@@ -6,7 +6,7 @@ require 'localmemcache'
 
 Bacon.summary_on_exit
 
-LocalMemCache.clear_namespace("speed-comparison");
+LocalMemCache.drop :namespace => "speed-comparison", :force => true
 $lm2 = LocalMemCache.new :namespace=>"speed-comparison"
 
 def compare_speed(n)

data/src/tests/bench_keys.rb

@@ -6,7 +6,7 @@ require 'localmemcache'
 
 Bacon.summary_on_exit
 
-LocalMemCache.clear_namespace("speed-comparison");
+LocalMemCache.drop :namespace => "speed-comparison", :force => true
 $lm2 = LocalMemCache.new :namespace=>"speed-comparison"
 
 

data/src/tests/crash.rb

@@ -3,7 +3,7 @@ $DIR=File.dirname(__FILE__)
 
 require 'localmemcache'
 
-LocalMemCache.clear_namespace("crash-t", true);
+LocalMemCache.drop :namespace => "crash-t", :force => true
 #exit
 #puts "c"
 #LocalMemCache.check_namespace("crash-t");

data/src/tests/lmc.rb

@@ -6,10 +6,10 @@ require 'localmemcache'
 
 Bacon.summary_on_exit
 
-LocalMemCache.clear_namespace("test", true)
-$lm = LocalMemCache.new :namespace=>"test"
+LocalMemCache.drop :namespace => "test", :force => true
+$lm = LocalMemCache.new :namespace=>"test", :size_mb => 20
 
-LocalMemCache.clear_namespace("test-small", true)
+LocalMemCache.drop :namespace => "test-small", :force => true
 $lms = LocalMemCache.new :namespace=>"test-small", :size_mb => 0.20;
 
 describe 'LocalMemCache' do
@@ -37,11 +37,22 @@ describe 'LocalMemCache' do
     $lm.keys().size.should.equal 2
   end
 
+  it 'should support each_pair' do 
+    $lm.each_pair {|k, v| }
+  end
+
   it 'should support \0 in values and keys' do
     $lm["null"] = "foo\0goo"
     $lm["null"].should.equal "foo\0goo"
   end
 
+  it 'should support random_pair' do
+    $lm.random_pair.size.should.equal 2
+    LocalMemCache.drop :namespace => :empty, :force => true
+    ll = LocalMemCache.new :namespace => :empty
+    ll.random_pair.should.be.nil
+  end
+
   it 'should throw an exception when accessing a closed pool' do
     $lm.close
     should.raise(LocalMemCache::MemoryPoolClosed) { $lm.keys }
@@ -52,23 +63,68 @@ describe 'LocalMemCache' do
     should.raise(LocalMemCache::MemoryPoolFull) { $lms["two"] = "b" * 8000000; }
   end
 
+
+  it 'should support clearing of the hashtable' do
+    ($lms.size > 0).should.be.true
+    $lms.clear
+    $lms.size.should.equal 0
+  end
+
+  it 'should support size' do
+    LocalMemCache.drop :namespace =>"size-test", :force=>true
+    ll = LocalMemCache.new :namespace => "size-test", :size_mb => 0.2
+    ll.size.should.equal 0
+    ll[:one] = 1
+    ll.size.should.equal 1
+    ll.delete(:one)
+    ll.size.should.equal 0
+    ll[:foo] = 1
+    ll.size.should.equal 1
+    ll[:goo] = 2
+    ll.size.should.equal 2
+    ll.clear
+    ll.size.should.equal 0
+  end
+
+
   it 'should support checking of namespaces' do 
-    LocalMemCache.check_namespace("test")
+    LocalMemCache.check :namespace => "test"
   end
 
-  it 'should support clearing of namespaces' do
-    LocalMemCache.clear_namespace("test")
+
+  it 'should support dropping of namespaces' do
+    LocalMemCache.drop :namespace => "test"
   end
 
   it 'should support filename parameters' do
-    lm = LocalMemCache.new :filename => ".tmp.a.lmc"
+    LocalMemCache.drop :filename => ".tmp.a.lmc", :force => true
+    lm = LocalMemCache.new :filename => ".tmp.a.lmc", :size_mb => 0.20
     lm[:boo] = 1
-    lm.keys.size.should.equal 1
+    lm.size.should.equal 1
     File.exists?(".tmp.a.lmc").should.be.true
     LocalMemCache.check :filename => ".tmp.a.lmc"
-    LocalMemCache.clear :filename => ".tmp.a.lmc"
+    LocalMemCache.drop :filename => ".tmp.a.lmc"
   end
 
+end
 
+LocalMemCache.drop :namespace => "test-shared-os", :force => true
+$lmsh = LocalMemCache::SharedObjectStorage.new :namespace=>"test-shared-os", 
+    :size_mb => 20
+
+describe 'LocalMemCache::SharedObjectStorage' do
+  it 'should allow to set and query for ruby objects' do
+    $lmsh["non-existant"].should.be.nil
+    $lmsh["array"] = [:foo, :boo]
+    $lmsh["array"].should.be.kind_of? Array
+  end
+
+  it 'support iteration' do
+    $lmsh.each_pair {|k, v| v.should.be.kind_of? Array }
+  end
+
+  it 'support random_pair' do
+    $lmsh.random_pair.last.should.be.kind_of? Array
+  end
 end