localmemcache 0.4.2 → 0.4.3

data/.gitignore

@@ -1,5 +1,6 @@
 *.o
 *.a
+.tmp*
 core
 vgcore*
 core*

data/Rakefile

@@ -25,11 +25,7 @@ task :changelog do
 end
 
 task :sanity_test do
-  sh "./configure && make -C src clean && make -C src && " +
-      "(cd src/ruby-binding; ruby extconf.rb) && " +
-      "make -C src/ruby-binding/ && " +
-      "(cd src/tests; ruby extconf.rb) && " +
-      "make -C src/tests/ && ./src/tests/lmc "
+  sh "./src/tests/sanity-test"
 end
 
 task :site_rdoc do

data/THANKS

@@ -1 +1,2 @@
-Armin Roehrl: For his thoughts and support
+* Armin Roehrl: For his thoughts and support
+* Paul Mineiro: Bug reports and improving iteration speed

data/VERSION

@@ -1 +1 @@
-0.4.2
+0.4.3

data/site/index.html

@@ -17,7 +17,35 @@ speed</b>.
 Since version 0.3.0 it supports <b>persistence</b>, also making it <b>a fast
 alternative to GDBM, Berkeley DB, and Tokyo Cabinet</b>.
 
-<h2>Version 0.4.2: Improving Append Performance (2009-08-10)</h2>
+<h2>Version 0.4.3: Improving Iteration and bugfixes (2009-10-02)</h2>
+
+Fixes by <b>Paul Mineiro</b> (thanks!):
+<li><b>iteration speedup</b>: each_pair is now a lot faster when dealing with a
+large dictionary.  Eg: For 400k elements it's now about <b>14 times
+faster</b>.</li>
+<li><b>fix for leak</b> in local_memcache_free()</li>
+<li><b>C API</b>: Some C API functions were not yet covered by tests.  This
+has been fixed.</li>
+<p>
+
+Other fixes:
+<li><b>autorepair</b> fixed: Autorepair of namespaces did sometimes wrongly
+report corrupt namespaces.
+<li><b>:min_alloc_size</b> now also works on Ruby 1.9.</li>
+
+
+<script type="text/javascript">
+// <![CDATA[
+function show_complete_history() {
+  var e = document.getElementById('old-news')
+  e.style.display = e.style.display == "none" ? "block" : "none";
+  return false;
+}
+// ]]
+</script>
+
+<a href="#" onclick="show_complete_history()" />Previous Releases</a>
+<div id='old-news' style='display:none'><h2>Version 0.4.2: Improving Append Performance (2009-08-10)</h2>
 
 In 0.4.2 the <b>:min_alloc_size</b> parameter was introduced to help with use
 cases that intend to use a hash table with growing values. This is
@@ -26,6 +54,7 @@ with a large list of unusable free blocks. By setting the
 <b>:min_alloc_size</b> parameter you help the allocator to plan better
 ahead. (<a href="http://localmemcache.rubyforge.org/doc/classes/LocalMemCache.html#M000001">more</a>)
 
+</div>
 
 <h2>Key features as of 0.4.0 (2009-05-16)</h2>
 <li><a href="#performance">blazingly fast</a></li>
@@ -41,7 +70,7 @@ ahead. (<a href="http://localmemcache.rubyforge.org/doc/classes/LocalMemCache.ht
 Note for <b>OS X</b>: <b>OS X</b> disqualifies as <b>HFS+</b> doesn't
 have sparse files and <b>sem_timedwait</b>() and <b>sem_getvalue</b>() aren't
 supported as well.<br>
-Note for <b>FreeBSD</b>: It has been reported that Localmemcache
+Note for <b>FreeBSD</b>: It has been reported that localmemcache
 sometimes hangs there, it is not yet clear what the problem is.
 
 <h2>Install</h2>
@@ -96,7 +125,7 @@ Ruby Hash of Strings:    <b>4,963.313</b> ms
 
 <p>
 
-So, on my machine, using <b>Localmemcache</b> 0.4.0 to store key-value
+So, on my machine, using <b>localmemcache</b> 0.4.0 to store key-value
 data on disk is about <b>10</b>% slower than keeping them in memory in a
 Ruby hash of strings.  It's about <b>40</b>% faster than <b>Tokyo Cabinet</b>
 (which offers features similar to <b>LocalMemCache</b>).
@@ -109,15 +138,23 @@ can be retrieved by executing</p>
 <pre><code>git clone git://github.com/sck/localmemcache.git
 </code></pre>
 
+<h2>Caveats</h2>
+
+<li>Localmemcache's .lmc files are not binary compatible, they are essentially
+memory mapped c structs</li>
+<li>Because of the convenient auto repair feature after a lock timeout,
+localmemcache is allergic to SIGSTOP (If you manage to SIGSTOP a process
+while localmemcache is currently possessing a lock, that is)</li>
+
 <h2>Who uses Localmemcache?</h2>
 
-<a href="http://www.personifi.com">Personifi</a> use Localmemcache to
+<a href="http://www.personifi.com">Personifi</a> use localmemcache to
 serve billions of hits each month.  Armin Roehrl: "we use
 localmemcache because it solves one problem very well and we love it!"
 
 <h2>Tips for backups</h2>
 
-Note that you cannot copy Localmemcache's .lmc files while other
+Note that you cannot copy localmemcache's .lmc files while other
 processes are making changes to the data, this will likely result in a
 corrupt backup.  So you need to make sure that none of your processes are
 writing during the time you do an backup.  As for copying sparse files,

data/src/lmc_hashtable.c

@@ -185,25 +185,28 @@ int ht_delete(void *base, va_ht_hash_t va_ht, const char *key, size_t n_key) {
 }
 
 int ht_hash_iterate(void *base, va_ht_hash_t va_ht, void *ctx, 
-    size_t *ofs, LMC_ITERATOR_P(iter)) {
+    ht_iter_status_t *s, LMC_ITERATOR_P(iter)) {
   va_ht_hash_entry_t va_hr;
   ht_hash_entry_t *hr = &lmc_null_node;
   ht_hash_t *ht = base + va_ht;
   size_t k;
   size_t item_c = 0;
   size_t slice_counter = 0;
-  for (k = 0; k < LMC_HT_BUCKETS; k++) {
+  for (k = s->bucket_ofs; k < LMC_HT_BUCKETS; k++) {
     for (va_hr = ht->va_buckets[k]; va_hr != 0 && hr != NULL; 
         va_hr = hr->va_next) {
       hr = va_hr ? base + va_hr : 0;
-      if (*ofs < ++item_c) {
+      if (s->ofs < ++item_c) {
         iter(ctx, base + hr->va_key, base + hr->va_value);
         if (++slice_counter > 999) {
-          *ofs = item_c;
+          s->ofs = item_c;
           return 2;
         }
       }
     }
+    ++(s->bucket_ofs);
+    s->ofs = 0;
+    item_c = 0;
   }
   return 1;
 }

data/src/lmc_hashtable.h

@@ -16,6 +16,11 @@ typedef struct {
   va_string_t va_value;
 } ht_hash_entry_t;
 
+typedef struct {
+  size_t ofs;
+  size_t bucket_ofs;
+} ht_iter_status_t;
+
 #define LMC_HT_BUCKETS 20731
 #define LMC_ITERATOR_P(n) int ((n)) \
     (void *ctx, const char *key, const char *value)
@@ -35,7 +40,7 @@ const char *ht_get(void *base, va_ht_hash_t va_ht, const char *key, size_t n_key
     size_t *n_value); 
 int ht_delete(void *base, va_ht_hash_t va_ht, const char *key, size_t n_key);
 int ht_hash_destroy(void *base, va_ht_hash_t ht);
-int ht_hash_iterate(void *base, va_ht_hash_t ht, void *ctx, size_t *ofs,
+int ht_hash_iterate(void *base, va_ht_hash_t ht, void *ctx, ht_iter_status_t *s,
     LMC_ITERATOR_P(iter));
 int ht_random_pair(void *base, va_ht_hash_t va_ht, char **r_key, 
     size_t *n_key, char **r_value, size_t *n_value);

data/src/lmc_lock.c

@@ -32,13 +32,19 @@ lmc_lock_t *lmc_lock_init(const char *ns, int init, lmc_error_t *e) {
   return l;
 }
 
+void lmc_lock_free(lmc_lock_t* l) {
+  if (!l) return;
+  if (l->sem) { sem_close(l->sem); }
+  free(l);
+}
+
 int lmc_clear_namespace_lock(const char *ns) {
   char namespace[1024];
   lmc_namespacify(namespace, ns);
   lmc_error_t e;
   lmc_lock_t *l = lmc_lock_init(namespace, 1, &e);
   lmc_lock_repair(l);
-  free(l);
+  lmc_lock_free(l);
   return 1;
 }
 
@@ -72,7 +78,7 @@ int lmc_sem_timed_wait_mandatory(lmc_lock_t* l) {
 #else
   struct timespec ts;
   clock_gettime(CLOCK_REALTIME, &ts);
-  ts.tv_sec += 20;
+  ts.tv_sec += 60;
   return sem_timedwait(l->sem, &ts);
 #endif
 }

data/src/lmc_lock.h

@@ -13,6 +13,7 @@ typedef struct {
 } lmc_lock_t;
 
 lmc_lock_t *lmc_lock_init(const char *namespace, int init, lmc_error_t *e);
+void lmc_lock_free(lmc_lock_t* l);
 int lmc_lock_obtain(const char *where, lmc_lock_t* l, lmc_error_t *e);
 int lmc_lock_obtain_mandatory(const char *where, lmc_lock_t* l, lmc_error_t *e);
 int lmc_lock_release(const char *where, lmc_lock_t* l, lmc_error_t *e);

data/src/lmc_valloc.c

@@ -360,11 +360,11 @@ void lmc_free(void *base, size_t chunk) {
   __lmc_free(base, va_used_chunk, uc_size);
 }
 
-int lmc_um_getbit(char *bf, int i) {
+int lmc_um_getbit(char *bf, size_t i) {
   bf += i / 8; return (*bf & (1 << (i % 8))) != 0;
 }
 
-void lmc_um_setbit(char *bf, int i, int v) {
+void lmc_um_setbit(char *bf, size_t i, size_t v) {
   bf += i / 8;
   if (v) *bf |= 1 << (i % 8);    
   else *bf &= ~(1 << (i % 8));  
@@ -382,7 +382,8 @@ int lmc_um_find_leaks(void *base, char *bf) {
   for (i = 0; i < md->total_size; ++i) { 
     if (!gap) {
       size_t *b = (void *)bf + i / 8;
-      while (*b == m && i < md->total_size - sizeof(size_t)) { 
+      size_t ee = md->total_size - sizeof(size_t) * 8;
+      while (*b == m && i < ee) { 
         i += sizeof(size_t) * 8; b++; 
       }
     }
@@ -415,10 +416,13 @@ int lmc_um_check_unmarked(void *base, char *bf, size_t va, size_t size) {
   size_t end = va + size;
   for (i = va; i < va + size; ++i) { 
     size_t *b = (void *)bf + i / 8;
-    while (*b == n && i < end - sizeof(size_t)) { 
+    size_t ee = end - sizeof(size_t) * 8;
+    while (*b == n && i < ee) { 
       i += sizeof(size_t) * 8; b++; 
     }
-    if (lmc_um_getbit(bf, i) != 0) { return 0; }
+    if (lmc_um_getbit(bf, i) != 0) { 
+      return 0; 
+    }
   }
   return 1;
 }
@@ -428,11 +432,15 @@ int lmc_um_mark(void *base, char *bf, size_t va, size_t size) {
   lmc_mem_descriptor_t *md = base;
   if ((va > sizeof(lmc_mem_descriptor_t)) &&
       (!lmc_is_va_valid(base, va) || !lmc_is_va_valid(base, va + size))) {
-    printf("[localmemcache] Error: VA start out of range: "
+    fprintf(stderr, "[localmemcache] Error: VA start out of range: "
         "va: %zd - %zd max %zd!\n", va, va + size, md->total_size);
     return 0;
   }
-  if (!lmc_um_check_unmarked(base, bf, va, size)) return 0;
+  if (!lmc_um_check_unmarked(base, bf, va, size)) {
+    fprintf(stderr, "[localmemcache] Error: Part of a block to be "
+        "marked used is used already (va: %zd s: %zd) !\n", va, size);
+    return 0;
+  }
   for (i = va; i < va + size; ++i) { 
     if (i % 8 == 0) {
       size_t b_start = i / 8;
@@ -450,7 +458,8 @@ int lmc_um_mark(void *base, char *bf, size_t va, size_t size) {
 int lmc_um_mark_allocated(void *base, char *bf, size_t va) {
   size_t real_va = va - sizeof(size_t);
   size_t s = *(size_t *)(base + real_va);
-  return lmc_um_mark(base, bf, real_va, s);
+  int r =  lmc_um_mark(base, bf, real_va, s);
+  return r;
 }
 
 char *lmc_um_new_mem_usage_bitmap(void *base) {

data/src/localmemcache.c

@@ -56,6 +56,8 @@ int lmc_namespace_or_filename(char *result, const char* ons, const char *ofn,
   return 0;
 }
 
+void lmc_checkize(char *result, char *s) { snprintf(result, 1023, "%s-check", s); }
+
 int local_memcache_drop_namespace(const char *namespace, const char *filename,
     int force, lmc_error_t *e) {
   char clean_ns[1024];
@@ -65,13 +67,13 @@ int local_memcache_drop_namespace(const char *namespace, const char *filename,
   if (force) {
     lmc_lock_t *l = lmc_lock_init((char *)clean_ns, 1, e);
     lmc_lock_repair(l);
-    free(l);
+    lmc_lock_free(l);
     char check_lock_name[1024];
-    snprintf((char *)&check_lock_name, 1023, "%s-check", (char *)clean_ns);
+    lmc_checkize(check_lock_name, clean_ns);
     lmc_lock_t *check_l;
     check_l =  lmc_lock_init(check_lock_name, 1, e);
     lmc_lock_repair(check_l);
-    free(check_l);
+    lmc_lock_free(check_l);
   }
   return 1;
 }
@@ -159,9 +161,45 @@ local_memcache_t *local_memcache_create(const char *namespace,
   return __local_memcache_create((char *)clean_ns, si, min_alloc_size, 0, 0, e);
 }
 
+int lmc_lock_shm_region(const char *who, local_memcache_t *lmc) {
+  int r;
+  int retry_counter = 0;
+retry:
+  if (retry_counter++ > 10) {
+    fprintf(stderr, "[localmemcache] Too many retries: "
+        "Cannot repair namespace '%s'\n", lmc->namespace);
+    return 0;
+  }
+  r = lmc_lock_obtain(who, lmc->lock, &lmc->error);
+  if (!r && (strcmp(lmc->error.error_type, "LockTimedOut") == 0)) {
+    if (__local_memcache_check_namespace(lmc->namespace, 
+        &lmc->error))  goto retry;
+    fprintf(stderr, "[localmemcache] Cannot repair namespace '%s'\n", 
+        lmc->namespace);
+  }
+  if (!r) return 0;
+  if (!lmc_set_lock_flag(lmc->base, &lmc->error)) {
+    lmc_lock_release(who, lmc->lock, &lmc->error);
+    return 0;
+  }
+  return 1;
+}
+
+int __local_memcache_free(local_memcache_t *lmc, lmc_error_t *e, int lock) {
+  if (lock && !lmc_lock_shm_region("local_memcache_free", lmc)) return 0;
+  int r = ht_hash_destroy(lmc->base, lmc->va_hash);
+  if (lock && !lmc_unlock_shm_region("local_memcache_free", lmc)) return 0;
+  lmc_shm_destroy(lmc->shm, e);
+  free(lmc->namespace);
+  lmc_lock_free(lmc->lock);
+  free(lmc);
+  return r;
+}
+
 int __local_memcache_check_namespace(const char *clean_ns, lmc_error_t *e) {
+  int repair_failed = 0;
   char check_lock_name[1024];
-  snprintf((char *)check_lock_name, 1023, "%s-check", (char *)clean_ns);
+  lmc_checkize(check_lock_name, (char *)clean_ns);
 
   if (!lmc_does_namespace_exist((char *)clean_ns)) { 
     lmc_clear_namespace_lock(check_lock_name);
@@ -216,17 +254,21 @@ release:
 release_but_no_lock_correction:
   local_memcache_free(lmc, e);
   lmc_lock_release("local_memcache_check_namespace", check_l, e);
-  free(check_l);
+  lmc_lock_free(check_l);
   return 1;
 failed:
+  repair_failed = 1;
   lmc_handle_error_with_err_string("local_memcache_check_namespace",
       "Unable to recover namespace", "RecoveryFailed", e);
-  local_memcache_free(lmc, e);
+  __local_memcache_free(lmc, e, 0);
   lmc_lock_release("local_memcache_check_namespace", check_l, e);
+  fprintf(stderr, "[localmemcache] Recovery failed!\n");
 check_lock_failed:
-  free(check_l);
-  fprintf(stderr, "[localmemcache] Failed to repair namespace '%s'\n", 
-      clean_ns);
+  lmc_lock_free(check_l);
+  if (!repair_failed) {
+    fprintf(stderr, "[localmemcache] Failed to obtain the 'check lock' to repair "
+        "namespace '%s'\n", clean_ns);
+  }
   return 0;
 }
 
@@ -238,30 +280,6 @@ int local_memcache_check_namespace(const char *namespace, const char *filename,
   return __local_memcache_check_namespace(clean_ns, e);
 }
 
-int lmc_lock_shm_region(const char *who, local_memcache_t *lmc) {
-  int r;
-  int retry_counter = 0;
-retry:
-  if (retry_counter++ > 10) {
-    fprintf(stderr, "[localmemcache] Too many retries: "
-        "Cannot repair namespace '%s'\n", lmc->namespace);
-    return 0;
-  }
-  r = lmc_lock_obtain(who, lmc->lock, &lmc->error);
-  if (!r && (strcmp(lmc->error.error_type, "LockTimedOut") == 0)) {
-    if (__local_memcache_check_namespace(lmc->namespace, 
-        &lmc->error))  goto retry;
-    fprintf(stderr, "[localmemcache] Cannot repair namespace '%s'\n", 
-        lmc->namespace);
-  }
-  if (!r) return 0;
-  if (!lmc_set_lock_flag(lmc->base, &lmc->error)) {
-    lmc_lock_release(who, lmc->lock, &lmc->error);
-    return 0;
-  }
-  return 1;
-}
-
 int lmc_unlock_shm_region(const char *who, local_memcache_t *lmc) {
   int r = 1;
   if (!lmc_release_lock_flag(lmc->base, &lmc->error)) r = 0;
@@ -279,8 +297,11 @@ const char *__local_memcache_get(local_memcache_t *lmc,
 char *local_memcache_get_new(local_memcache_t *lmc, 
     const char *key, size_t n_key, size_t *n_value) {
   const char *r = __local_memcache_get(lmc, key, n_key, n_value);
-  char *new_s = malloc(*n_value);
-  memcpy(new_s, r, *n_value);
+  char *new_s = 0;
+  if (r) {
+    new_s = malloc(*n_value);
+    memcpy(new_s, r, *n_value);
+  }
   if (!lmc_unlock_shm_region("local_memcache_get_new", lmc)) return 0;
   return new_s;
 }
@@ -296,11 +317,13 @@ int local_memcache_random_pair_new(local_memcache_t *lmc,
     char **r_key, size_t *n_key, char **r_value, size_t *n_value) {
   char *k;
   char *v;
+  (*r_key) = 0;
+  (*r_value) = 0;
   if (__local_memcache_random_pair(lmc, &k, n_key, &v, n_value)) {
-    char *new_k = malloc(*n_key);
-    memcpy(new_k, k, *n_key);
-    char *new_v = malloc(*n_value);
-    memcpy(new_v, v, *n_value);
+    (*r_key) = malloc(*n_key);
+    memcpy((*r_key), k, *n_key);
+    (*r_value) = malloc(*n_value);
+    memcpy((*r_value), v, *n_value);
   }
   if (!lmc_unlock_shm_region("local_memcache_random_pair", lmc)) return 0;
   return 1;
@@ -334,19 +357,21 @@ int local_memcache_delete(local_memcache_t *lmc, char *key, size_t n_key) {
 }
 
 int local_memcache_free(local_memcache_t *lmc, lmc_error_t *e) {
-  if (!lmc_lock_shm_region("local_memcache_free", lmc)) return 0;
-  int r = ht_hash_destroy(lmc->base, lmc->va_hash);
-  if (!lmc_unlock_shm_region("local_memcache_free", lmc)) return 0;
-  lmc_shm_destroy(lmc->shm, e);
-  free(lmc->namespace);
-  free(lmc->lock);
-  return r;
+  return __local_memcache_free(lmc, e, 1);
 }
 
 int local_memcache_iterate(local_memcache_t *lmc, void *ctx, 
-    size_t *ofs, LMC_ITERATOR_P(iter)) {
+    ht_iter_status_t *s, LMC_ITERATOR_P(iter)) {
   if (!lmc_lock_shm_region("local_memcache_iterate", lmc)) return 0;
-  int r = ht_hash_iterate(lmc->base, lmc->va_hash, ctx, ofs, iter);
+  int r = ht_hash_iterate(lmc->base, lmc->va_hash, ctx, s, iter);
   if (!lmc_unlock_shm_region("local_memcache_iterate", lmc)) return 0;
   return r;
 }
+
+int local_memcache_check_consistency(local_memcache_t *lmc, lmc_error_t *e) {
+  lmc_mem_descriptor_t *md = lmc->base;
+  if (!lmc_lock_shm_region("local_memcache_check_consistency", lmc)) return 0;
+  int r = ht_check_memory(lmc->base, md->va_hash);
+  if (!lmc_unlock_shm_region("local_memcache_check_consistency", lmc)) return 0;
+  return r;
+}

data/src/localmemcache.h

@@ -170,7 +170,9 @@ int local_memcache_free(local_memcache_t *lmc, lmc_error_t *e);
  *
  *   local_memcache_t *lmc;
  *   collector_t c;
- *   local_memcache_iterate(lmc, (void *) &c, my_collect);
+ *   ht_iter_status_t s;
+ *   memset(s, sizeof(ht_iter_status_t));
+ *   local_memcache_iterate(lmc, (void *) &c, &s, my_collect);
  *
  * The memory pool will be locked while iteration takes place, so try to make
  * sure you can iterate within under 2 seconds otherwise other waiting
@@ -178,7 +180,7 @@ int local_memcache_free(local_memcache_t *lmc, lmc_error_t *e);
  * triggering the automatic recovery.)
  *
  */
-int local_memcache_iterate(local_memcache_t *lmc, void *ctx, size_t *ofs,
+int local_memcache_iterate(local_memcache_t *lmc, void *ctx, ht_iter_status_t *s,
     LMC_ITERATOR_P(iter));
 
 /*
@@ -218,6 +220,7 @@ int local_memcache_drop_namespace(const char *namespace, const char *filename,
 int local_memcache_check_namespace(const char *namespace, const char *filename, 
     lmc_error_t *e);
 
+
 /* internal, do not use */
 const char *__local_memcache_get(local_memcache_t *lmc, 
     const char *key, size_t n_key, size_t *n_value);
@@ -228,4 +231,8 @@ int __local_memcache_random_pair(local_memcache_t *lmc,
 
 /* internal, do not use */
 int lmc_unlock_shm_region(const char *who, local_memcache_t *lmc);
+
+/* internal, do not use */
+int local_memcache_check_consistency(local_memcache_t *lmc, lmc_error_t *e);
+
 #endif

data/src/ruby-binding/rblocalmemcache.c

@@ -29,7 +29,7 @@
 #endif
 
 /* :nodoc: */
-long long_value(VALUE i) { return NUM2LONG(rb_Integer(i)); }
+long long_value(VALUE i) { return NIL_P(i) ? 0 : NUM2LONG(rb_Integer(i)); }
 /* :nodoc: */
 double double_value(VALUE i) { return NUM2DBL(i); }
 /* :nodoc: */
@@ -322,10 +322,11 @@ static VALUE __LocalMemCache__keys(VALUE d) {
   lmc_ruby_iter_collect_keys data;
   data.ary = r;
   int success = 2;
-  size_t ofs = 0;
+  ht_iter_status_t s;
+  memset(&s, 0x0, sizeof(ht_iter_status_t));
   while (success == 2) {
     success = local_memcache_iterate(get_LocalMemCache(obj), 
-        (void *) &data, &ofs, lmc_ruby_iter);
+        (void *) &data, &s, lmc_ruby_iter);
   }
   if (!success) { return Qnil; }
   return Qnil;
@@ -366,13 +367,14 @@ int lmc_ruby_iter_collect_pairs(void *ctx, const char* key, const char* value) {
 static VALUE __LocalMemCache__each_pair(VALUE d) {
   VALUE obj = rb_ary_entry(d, 0);
   int success = 2;
-  size_t ofs = 0;
+  ht_iter_status_t s;
+  memset(&s, 0x0, sizeof(ht_iter_status_t));
   while (success == 2) {
     VALUE r = rb_ary_new();
     lmc_ruby_iter_collect_pairs_t data;
     data.ary = r;
     success = local_memcache_iterate(get_LocalMemCache(obj), 
-        (void *) &data, &ofs, lmc_ruby_iter_collect_pairs);
+        (void *) &data, &s, lmc_ruby_iter_collect_pairs);
     long i;
     for (i = 0; i < RARRAY_LEN(r); i++) {
       rb_yield(RARRAY_PTR(r)[i]);
@@ -412,6 +414,17 @@ static VALUE LocalMemCache__size(VALUE obj) {
   return rb_int2big(ht->size);
 }
 
+/* 
+ * internal, do not use
+ */
+static VALUE LocalMemCache__check_consistency(VALUE obj) {
+  lmc_error_t e;
+  rb_lmc_handle_t *h;
+  Data_Get_Struct(obj, rb_lmc_handle_t, h);
+  return local_memcache_check_consistency(rb_lmc_check_handle_access(h), &e) ? 
+      Qtrue : Qfalse;
+}
+
 /*
  * Document-class: LocalMemCache
  * 
@@ -487,6 +500,8 @@ void Init_rblocalmemcache() {
       0);
   rb_define_method(LocalMemCache, "close", LocalMemCache__close, 0);
   rb_define_method(LocalMemCache, "size", LocalMemCache__size, 0);
+  rb_define_method(LocalMemCache, "check_consistency", 
+      LocalMemCache__check_consistency, 0);
 
   lmc_rb_sym_namespace = ID2SYM(rb_intern("namespace"));
   lmc_rb_sym_filename = ID2SYM(rb_intern("filename"));

data/src/tests/bench.rb

@@ -49,13 +49,3 @@ def measure_time(c, &block)
 end
 
 compare_speed(2_000_000)
-#test_gdbm(2_000_000)
-
-#$stdout.write "ht shm setting x 20000: "
-#tmeasure (2_000_000) { 
-#  v = $lm2.get("f").to_i + 1
-#  #puts "v:#{v}"
-#  $lm2.set("f", v) 
-#}
-#puts "foo: #{$lm2.get("f")}"
- 

data/src/tests/capi.c

@@ -0,0 +1,79 @@
+#include <stdio.h>
+#include <localmemcache.h>
+
+#define fail_if(t,m) \
+  if ((t)) { printf("FAILED: %s\n", (m)); goto test_failed; } \
+  else { printf("  - %s\n", (m)); }
+
+#define TEST_START(m) \
+  printf("%s\n", (m)); \
+  {
+
+#define TEST_END \
+  }
+
+int main() {
+  lmc_error_t e;
+  if (!local_memcache_drop_namespace("capitest", 0, 1, &e)) {
+    fprintf(stderr, "drop failed: %s\n", e.error_str);
+    return 1;
+  }
+  local_memcache_t *lmc = local_memcache_create("capitest", 0, 0, 0, &e);
+  if (!lmc) {
+    fprintf(stderr, "Couldn't create localmemcache: %s\n", e.error_str);
+    return 1;
+  }
+  if (!local_memcache_set(lmc, "foo", 3, "1", 1)) goto failed;
+
+  TEST_START("local_memcache_get_new")
+    size_t n_value;
+    char *value = local_memcache_get_new(lmc, "foo", 3, &n_value);
+    fail_if(!value, "value")
+    fail_if(n_value != 1, "n_value");
+    fail_if(value[0] != '1', "value[0]");
+    free(value);
+
+    char *value2 = local_memcache_get_new(lmc, "unknown", 7, &n_value);
+    fail_if(value2, "value2");
+  TEST_END
+
+  TEST_START("random_pair")
+    char *rk, *rv;
+    size_t n_rk, n_rv;
+    int r = local_memcache_random_pair_new(lmc, &rk, &n_rk, &rv, &n_rv);
+    fail_if(!r, "local_memcache_random_pair_new");
+    fail_if(n_rk != 3, "n_rk");
+    fail_if(rk[0] != 'f', "rk0");
+    fail_if(n_rv != 1, "n_rv");
+    fail_if(rv[0] != '1', "rv0");
+    free(rk);
+    free(rv);
+  TEST_END
+
+  if (!local_memcache_delete(lmc, "foo", 3)) goto failed;
+
+  TEST_START("random_pair(empty)")
+    char *rk, *rv;
+    size_t n_rk, n_rv;
+    int r = local_memcache_random_pair_new(lmc, &rk, &n_rk, &rv, &n_rv);
+    fail_if(!r, "local_memcache_random_pair_new:empty");
+    fail_if(rk, "rk:empty")
+    fail_if(rv, "rv:empty")
+  TEST_END
+
+  printf("Ok\n");
+
+test_failed:
+  if (!local_memcache_free(lmc, &e)) {
+    fprintf(stderr, "Failed to release localmemcache: %s\n", e.error_str);
+    return 1;
+  }
+
+
+  return 0;
+
+failed:
+  fprintf(stderr, "FAILED: %s\n", lmc->error.error_str);
+  return 1;
+
+}

data/src/tests/capi.sh

@@ -0,0 +1,6 @@
+#! /bin/sh
+
+gcc -g -o capi.bin capi.c -I ../ -L ../ -llmc -lrt && \
+    valgrind -v --leak-check=full ./capi.bin 2>.tmp.valgrind
+
+grep "\\.c" .tmp.valgrind

data/src/tests/crash

@@ -5,7 +5,7 @@ script=$DIR/crash.rb
 
 export CFLAGS=-DDO_TEST_CRASH
 make -C .. clean && make -C .. && make -C ../ruby-binding && 
-ruby extconf.rb && make clean && make
+ruby extconf.rb && make clean && make || exit 1
 
 #ulimit -c 0
 

data/src/tests/crash-small

@@ -0,0 +1,19 @@
+#! /bin/sh
+D=`dirname $0`
+DIR=`cd $D; pwd`
+script=$DIR/crash-small.rb
+
+export CFLAGS=-DDO_TEST_CRASH
+make -C .. clean && make -C .. && make -C ../ruby-binding && 
+ruby extconf.rb && make clean && make || exit 1
+
+#ulimit -c 0
+
+if test "x$1" = "x-d"; then
+  irb -r $script
+else
+  #valgrind --leak-check=full --tool=memcheck ruby $script
+  ruby $script
+fi
+
+#ulimit -c unlimited

data/src/tests/crash-small.rb

@@ -0,0 +1,11 @@
+$DIR=File.dirname(__FILE__)
+['.', '..', '../ruby-binding/'].each {|p| $:.unshift File.join($DIR, p) }
+
+require 'localmemcache'
+
+LocalMemCache.drop :namespace => "crash-small-t", :force => true
+$lm = LocalMemCache.new :namespace=>"crash-small-t"
+
+$lm["one"] = "1"
+$lm.check_consistency
+

data/src/tests/crash.rb

@@ -4,10 +4,6 @@ $DIR=File.dirname(__FILE__)
 require 'localmemcache'
 
 LocalMemCache.drop :namespace => "crash-t", :force => true
-#exit
-#puts "c"
-#LocalMemCache.check_namespace("crash-t");
-
 
 $pids = []
 5.times { $pids << fork {
@@ -23,7 +19,7 @@ $pids = []
   puts "#{$$} Worker finished"
 }}
 
-40.times {
+10.times {
 $pid = fork {
   LocalMemCache.enable_test_crash
   $lm2 = LocalMemCache.new :namespace=>"crash-t"

data/src/tests/iter

@@ -0,0 +1,11 @@
+#! /bin/sh
+D=`dirname $0`
+DIR=`cd $D; pwd`
+script=$DIR/iter.rb
+
+if test "x$1" = "x-d"; then
+  irb -r $script
+else
+  #valgrind --leak-check=full --tool=memcheck ruby $script
+  ruby $script
+fi

data/src/tests/iter.rb

@@ -0,0 +1,41 @@
+$DIR=File.dirname(__FILE__)
+['.', '..', '../ruby-binding/'].each {|p| $:.unshift File.join($DIR, p) }
+
+require 'bacon'
+require 'localmemcache'
+
+Bacon.summary_on_exit
+
+LocalMemCache.drop :namespace => "iteration", :force => true
+$l = LocalMemCache.new :namespace=>"iteration"
+
+def compare_speed(n)
+  
+  puts "lmc: filling dict"
+  measure_time(1) { n.times {|i| $l[i] = i } }
+  puts "lmc: iterating"
+  c = 0
+  measure_time(1) {
+    $l.each_pair {|k,v| c += 1 }
+  }
+  raise "lmc: error: c(#{c})!= n(#{n})" if c != n
+  
+  puts "rhash: filling dict"
+  $hh = {}
+  measure_time(1) { n.times {|i| $hh[i] = i } }
+  puts "rhash: iterating"
+  c = 0
+  measure_time(1) {
+    $hh.each {|k,v| c += 1 }
+  }
+  raise "rhash: error: c!= n" if c != n
+end
+
+def measure_time(c, &block)
+  _then = Time.now
+  c.times { block.call }
+  now = Time.now
+  puts "#{(now - _then)*1000} ms"
+end
+
+compare_speed(400_000)

data/src/tests/lmc.rb

@@ -53,6 +53,18 @@ describe 'LocalMemCache' do
     ll.random_pair.should.be.nil
   end
 
+  it 'should allow be consistent' do
+    $lm.check_consistency.should.be.true
+  end
+
+  it 'should support iteration' do 
+    s = $lm.size
+    s.should.equal 3
+    c = 0
+    $lm.each_pair {|a,b| c += 1 }
+    s.should.equal c
+  end
+
   it 'should throw an exception when accessing a closed pool' do
     $lm.close
     should.raise(LocalMemCache::MemoryPoolClosed) { $lm.keys }
@@ -87,6 +99,7 @@ describe 'LocalMemCache' do
   end
 
 
+
   it 'should support checking of namespaces' do 
     LocalMemCache.check :namespace => "test"
   end

data/src/tests/parallelwrite

@@ -11,5 +11,5 @@ if test "x$1" = "x-d"; then
   irb -r $script
 else
   #valgrind --leak-check=full --tool=memcheck ruby $script
-  ruby $script
+  time ruby $script
 fi

data/src/tests/run-all-tests

@@ -0,0 +1,26 @@
+#! /bin/sh
+
+
+echo "SHM"
+./shm
+echo "LMC"
+./lmc
+echo "capi"
+./capi.sh
+
+echo "bench"
+./bench
+echo "bench-append"
+./bench-append
+
+echo "ttlmc"
+./ttlmc
+echo "ttrandom_pair"
+./ttrandom_pair
+echo "ttalloc"
+./ttalloc
+
+echo "parallelwrite"
+./parallelwrite
+echo "crash"
+./crash

data/src/tests/sanity-test

@@ -0,0 +1,7 @@
+#! /bin/sh
+
+./configure && make -C src clean && make -C src && \
+  (cd src/ruby-binding; ruby extconf.rb) && \
+  make -C src/ruby-binding/ && \
+  (cd src/tests; ruby extconf.rb) && \
+  make -C src/tests/ && ./src/tests/lmc 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: localmemcache
 version: !ruby/object:Gem::Version 
-  version: 0.4.2
+  version: 0.4.3
 platform: ruby
 authors: 
 - Sven C. Koehler
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-08-10 00:00:00 +00:00
+date: 2009-10-02 00:00:00 +00:00
 default_executable: 
 dependencies: []
 
@@ -131,15 +131,23 @@ files:
 - src/tests/bench.rb
 - src/tests/bench_keys
 - src/tests/bench_keys.rb
+- src/tests/capi.c
+- src/tests/capi.sh
 - src/tests/crash
+- src/tests/crash-small
+- src/tests/crash-small.rb
 - src/tests/crash.rb
 - src/tests/extconf.rb
+- src/tests/iter
+- src/tests/iter.rb
 - src/tests/lmc
 - src/tests/lmc.rb
 - src/tests/lmctestapi.c
 - src/tests/parallelwrite
 - src/tests/parallelwrite.rb
+- src/tests/run-all-tests
 - src/tests/runtest.sh
+- src/tests/sanity-test
 - src/tests/shm
 - src/tests/shm.rb
 - src/tests/torture.rb