localmemcache 0.2.2 → 0.3.0

data/site/doc/rdoc-style.css

@@ -0,0 +1,208 @@
+
+body {
+    font-family: Verdana,Arial,Helvetica,sans-serif;
+    font-size:   90%;
+    margin: 0;
+    margin-left: 40px;
+    padding: 0;
+    background: white;
+}
+
+h1,h2,h3,h4 { margin: 0; color: #efefef; background: transparent; }
+h1 { font-size: 150%; }
+h2,h3,h4 { margin-top: 1em; }
+
+a { background: #eef; color: #039; text-decoration: none; }
+a:hover { background: #039; color: #eef; }
+
+/* Override the base stylesheet's Anchor inside a table cell */
+td > a {
+  background: transparent;
+  color: #039;
+  text-decoration: none;
+}
+
+/* and inside a section title */
+.section-title > a {
+  background: transparent;
+  color: #eee;
+  text-decoration: none;
+}
+
+/* === Structural elements =================================== */
+
+div#index {
+    margin: 0;
+    margin-left: -40px;
+    padding: 0;
+    font-size: 90%;
+}
+
+
+div#index a {
+    margin-left: 0.7em;
+}
+
+div#index .section-bar {
+   margin-left: 0px;
+   padding-left: 0.7em;
+   background: #ccc;
+   font-size: small;
+}
+
+
+div#classHeader, div#fileHeader {
+    width: auto;
+    color: white;
+    padding: 0.5em 1.5em 0.5em 1.5em;
+    margin: 0;
+    margin-left: -40px;
+    border-bottom: 3px solid #006;
+}
+
+div#classHeader a, div#fileHeader a {
+    background: inherit;
+    color: white;
+}
+
+div#classHeader td, div#fileHeader td {
+    background: inherit;
+    color: white;
+}
+
+
+div#fileHeader {
+    background: #057;
+}
+
+div#classHeader {
+    background: #048;
+}
+
+
+.class-name-in-header {
+  font-size:  180%;
+  font-weight: bold;
+}
+
+
+div#bodyContent {
+    padding: 0 1.5em 0 1.5em;
+}
+
+div#description {
+    padding: 0.5em 1.5em;
+    background: #efefef;
+    border: 1px dotted #999;
+}
+
+div#description h1,h2,h3,h4,h5,h6 {
+    color: #125;;
+    background: transparent;
+}
+
+div#validator-badges {
+    text-align: center;
+}
+div#validator-badges img { border: 0; }
+
+div#copyright {
+    color: #333;
+    background: #efefef;
+    font: 0.75em sans-serif;
+    margin-top: 5em;
+    margin-bottom: 0;
+    padding: 0.5em 2em;
+}
+
+
+/* === Classes =================================== */
+
+table.header-table {
+    color: white;
+    font-size: small;
+}
+
+.type-note {
+    font-size: small;
+    color: #DEDEDE;
+}
+
+.xxsection-bar {
+    background: #eee;
+    color: #333;
+    padding: 3px;
+}
+
+.section-bar {
+   color: #333;
+   border-bottom: 1px solid #999;
+    margin-left: -20px;
+}
+
+
+.section-title {
+    background: #79a;
+    color: #eee;
+    padding: 3px;
+    margin-top: 2em;
+    margin-left: -30px;
+    border: 1px solid #999;
+}
+
+.top-aligned-row {  vertical-align: top }
+.bottom-aligned-row { vertical-align: bottom }
+
+/* --- Context section classes ----------------------- */
+
+.context-row { }
+.context-item-name { font-family: monospace; font-weight: bold; color: black; }
+.context-item-value { font-size: small; color: #448; }
+.context-item-desc { color: #333; padding-left: 2em; }
+
+/* --- Method classes -------------------------- */
+.method-detail {
+    background: #efefef;
+    padding: 0;
+    margin-top: 0.5em;
+    margin-bottom: 1em;
+    border: 1px dotted #ccc;
+}
+.method-heading {
+  color: black;
+  background: #ccc;
+  border-bottom: 1px solid #666;
+  padding: 0.2em 0.5em 0 0.5em;
+}
+.method-signature { color: black; background: inherit; }
+.method-name { font-weight: bold; }
+.method-args { font-style: italic; }
+.method-description { padding: 0 0.5em 0 0.5em; }
+
+/* --- Source code sections -------------------- */
+
+a.source-toggle { font-size: 90%; }
+div.method-source-code {
+    background: #262626;
+    color: #ffdead;
+    margin: 1em;
+    padding: 0.5em;
+    border: 1px dashed #999;
+    overflow: hidden;
+}
+
+div.method-source-code pre { color: #ffdead; overflow: hidden; }
+
+/* --- Ruby keyword styles --------------------- */
+
+.standalone-code { background: #221111; color: #ffdead; overflow: hidden; }
+
+.ruby-constant  { color: #7fffd4; background: transparent; }
+.ruby-keyword { color: #00ffff; background: transparent; }
+.ruby-ivar    { color: #eedd82; background: transparent; }
+.ruby-operator  { color: #00ffee; background: transparent; }
+.ruby-identifier { color: #ffdead; background: transparent; }
+.ruby-node    { color: #ffa07a; background: transparent; }
+.ruby-comment { color: #b22222; font-weight: bold; background: transparent; }
+.ruby-regexp  { color: #ffa07a; background: transparent; }
+.ruby-value   { color: #7fffd4; background: transparent; }

data/site/index.html

@@ -7,44 +7,29 @@
   <body>  
 
 <div id="content">
-<h1>The beauty of memcached.&nbsp;&nbsp;For local data.&nbsp;&nbsp;Blazingly fast.</h1>
+<h1>A persistent key-value database based on mmap()'ed shared memory</h1>
 
 <p><b>Localmemcache</b> is a library for C and ruby that aims to provide
 an interface similar to memcached but for accessing local data instead of
 remote data.  It's based on <b>mmap()</b>'ed shared memory for maximum speed.
-
-<h2>2009-04-05: Changes for 0.2.1</h2>
-
-<li>Fixed a bug that prevented setting values in the hashtable</li>
-<li>Accessing a closed memory cache does no longer result in a crash</li>
-<li><b>Speed improvements</b>: On my machine localmemcache is now only
-about <b>20</b>% slower than Ruby's hash (<b>0.2.0</b> was about 
-<b>40</b>% slower)</li>
-<li><b>OS X is now officially not supported</b> as it lacks sem_timedwait and
-sem_getvalue (You still can force a build but it won't be able to
-recover from crashes.)</li>
-<li>The environment variable <b>LMC_NAMESPACES_ROOT_PATH</b> can now be
-used to override the default, which is <b>/var/tmp/localmemcache</b></li>
-
-
-<h2>2009-03-30: New features in 0.2.0</h2>
-
-<li><b>Logging</b>: In case your application is terminated while
-accessing the shared memory (eg by <b>kill -9</b>), it is now able to
-<b>restore the integrity of your data</b>.</li>
-<li><b>\0 character</b> can now be used in <b>keys</b> and
-<b>values</b></li>
-<li>The ruby binding now features a <b>keys()</b> method.</b>
-<li>Added a <b>C API</b>.  See <a href="http://github.com/sck/localmemcache/blob/8c753f74c53b107d271975bedcc1f91a3dbd6961/src/localmemcache.h">localmemcache.h</a></li>
-
-<h2>Supported Systems</h2>
-
-<li>Unix (for <b>mmap())</b></li>
-<li>OS X is currently not supported because it doesn't have
-sem_timedwait() and sem_getvalue()</li>
-<li>A CPU architecture with more than <b>32 bits</b> is recommended, since
-otherwise you might run out of virtual address space when you use larger
-shared memory segments.</li>
+Since version 0.3.0 it supports persistence, also making it a fast
+alternative to GDBM and Berkeley DB.
+
+<h2>Key features as of 0.3.0 (2009-04-17)</h2>
+<li><a href="#performance">blazingly fast</a></li>
+<li>persistent</li>
+<li>parallel writes are supported by default</li>
+<li>uses transactions internally to avoid data corruption</li>
+<li>lightweight: the core library is just about <b>1400</b> lines of <b>C</b> code</li>
+
+<h2>Requirements</h2>
+<li>a &gt;=64bit Unix</li>
+<li>a file system that offers <a href="http://en.wikipedia.org/wiki/Sparse_file">sparse files</a></li>
+Note for <b>OS X</b>: <b>OS X</b> disqualifies as <b>HFS+</b> doesn't
+have sparse files and <b>sem_timedwait</b>() and <b>sem_getvalue</b>() aren't
+supported as well.
+Note for <b>FreeBSD</b>: It has been reported that Localmemcache
+sometimes hangs there, it is not yet clear what the problem is.
 
 <h2>Install</h2>
 
@@ -53,19 +38,24 @@ shared memory segments.</li>
 <pre><code>gem install localmemcache
 </code></pre>
 
+
 If you just want to use the <b>C API</b>, download the .tar.gz from <a href="http://rubyforge.org/frs/?group_id=7925">here</a>.
 
 <h2>Using</h2>
+<b>API</b>:&nbsp;<a href="http://github.com/sck/localmemcache/blob/e980ba949034a349dbf0e6f8f9ef77bbe3e3e314/src/localmemcache.h">C</a>|<a href="http://localmemcache.rubyforge.org/doc/">Ruby</a><br>
 <p><pre><code>require 'localmemcache'
-$lm = LocalMemCache.new :namespace => :viewcounters
+# 1. the memcached way
+# $lm = LocalMemCache.new :namespace => :viewcounters
+# 2. the GDBM way
+$lm = LocalMemCache.new :filename => "./viewcounters.lmc"
 $lm[:foo] = 1
 $lm[:foo]
 $lm.delete(:foo)
 </code></pre> 
-(C version of this example: <a href="http://github.com/sck/localmemcache/blob/8c753f74c53b107d271975bedcc1f91a3dbd6961/example/hello.c">hello.c</a>)
+(C version of this example: <a href="http://github.com/sck/localmemcache/blob/e980ba949034a349dbf0e6f8f9ef77bbe3e3e314/example/hello.c">hello.c</a>)
 </p>
-<a href="http://github.com/sck/localmemcache/tree/master">Read more</a>.
 
+<a name="performance" id="performance"></a>
 <h2>Performance</h2>
 
 <p>Here's a quick speed comparison, made on an 
@@ -79,14 +69,22 @@ Ruby benchmark pseudo code:
 </code></pre>
 
 <pre>
-MemCache:            <b>253,326.122</b> ms
-LocalMemCache 0.2.1: <b>5,799.225</b> ms
-Ruby's Hash:         <b>4,963.313</b> ms
+Tokyo Tyrant:          <b>298,226.197</b> ms
+MemCache:              <b>253,326.122</b> ms
+GDBM:                   <b>24,226.116</b> ms
+Localmemcache 0.2.2:     <b>5,799.225</b> ms
+Localmemcache 0.3.0:     <b>5,300.055</b> ms
+Ruby Hash of Strings:    <b>4,963.313</b> ms
 </pre>
+(<a href="http://github.com/sck/localmemcache/tree/e980ba949034a349dbf0e6f8f9ef77bbe3e3e314/bench">Code of the benchmarks used</a>)
 
-So, on my machine <b>Localmemcache</b> 0.2.1 is about <b>43</b> times
-faster than using memcached locally, and about <b>20</b>% slower than
-Ruby's hash.
+<p>
+
+So, on my machine, using <b>Localmemcache</b> 0.3.0 to store key-value
+data on disk is about <b>10</b>% slower than keeping them in memory in a
+Ruby hash of strings.   Also keep in mind that of the above mentioned
+software components <b>Tokyo Tyrant</b> is the only one that offers
+persistence and parallel writes like <b>Localmemcache</b> does.
 
 <h2>Source code</h2>
 
@@ -96,10 +94,16 @@ can be retrieved by executing</p>
 <pre><code>git clone git://github.com/sck/localmemcache.git
 </code></pre>
 
-<h2>Read on</h2>
+<h2>Tips for backups</h1>
+
+Note that you cannot copy Localmemcache's .lmc files while other
+processes are making changes to the data, this will likely result in a
+corrupt backup.  So you need to make sure that none of your processes are
+writing during the time you do an backup.  As for copying sparse files,
+<b>cp</b> recognizes them automatically, with <b>tar</b> you need to use
+the <b>-S</b> option.
 
-A bit of documenation on how <b>localmemcache</b> <a
-href="http://github.com/sck/localmemcache/blob/96b5e0863e1da7f17e249a6c8884984e0c23ee2f/INTERNALS">works</a>.
+<h2><a href="http://localmemcache.rubyforge.org/doc/classes/LocalMemCache.html">Read on / RDoc</a></h2>
 
 <h2>License</h2>
 

data/src/lmc_common.c

@@ -7,6 +7,7 @@
 #include <sys/types.h>
 #include <unistd.h>
 #include <stdlib.h>
+#include <string.h>
 
 int lmc_test_crash_enabled = 0;
 int lmc_showed_status = 0;
@@ -41,3 +42,24 @@ size_t lmc_test_valloc_fail(const char *file, int line, const char *function,
   return lmc_valloc(base, s);
 }
 
+void lmc_clean_string(char *result, const char *source) {
+  size_t n = strlen(source);
+  if (n > 256) { n = 256; }
+  const char *s = source;
+  char *d = result;
+  char ch;
+  for (; n--; d++, s++) {
+    ch = *s;
+    if ((ch >= 'a' && ch <= 'z') || 
+        (ch >= 'A' && ch <= 'Z')) {
+      *d = ch;
+    } else {
+      *d = '-';
+    }
+  }
+  *d = 0x0;
+}
+
+int lmc_is_filename(const char *s) {
+  return strlen(s) > 1 && (s[0] == '/' || (s[0] == '.' && s[1] == '/'));
+}

data/src/lmc_common.h

@@ -5,6 +5,8 @@
 #ifndef _LMC_COMMON_H_INCLUDED_
 #define _LMC_COMMON_H_INCLUDED_
 
+#define LMC_DB_VERSION 0
+
 extern int lmc_test_crash_enabled;
 #ifdef DO_TEST_CRASH
 #define LMC_TEST_CRASH lmc_test_crash(__FILE__, __LINE__, __FUNCTION__);
@@ -21,6 +23,8 @@ extern int lmc_test_crash_enabled;
 void lmc_test_crash(const char* file, int line, const char *function);
 size_t lmc_test_valloc_fail(const char *file, int line, const char *function,
     void *base, size_t s);
+void lmc_clean_string(char *result, const char *source);
+int lmc_is_filename(const char *s);
 
 #endif
 

data/src/lmc_hashtable.h

@@ -16,7 +16,7 @@ typedef struct {
   va_string_t va_value;
 } ht_hash_entry_t;
 
-#define LMC_HT_BUCKETS 499
+#define LMC_HT_BUCKETS 20731
 #define LMC_ITERATOR_P(n) int ((n)) \
     (void *ctx, const char *key, const char *value)
 

data/src/lmc_lock.c

@@ -1,3 +1,7 @@
+/*
+ * Copyright (c) 2009, Sven C. Koehler
+ */
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <fcntl.h>
@@ -8,11 +12,19 @@
 #include <unistd.h>
 #include <time.h>
 #include "lmc_lock.h"
+#include "lmc_common.h"
+
+void lmc_namespacify(char *result, const char *s) {
+  if (lmc_is_filename(s)) { lmc_clean_string(result, s); } 
+  else { strcpy(result, s); }
+}
 
-lmc_lock_t *lmc_lock_init(const char *namespace, int init, lmc_error_t *e) {
+lmc_lock_t *lmc_lock_init(const char *ns, int init, lmc_error_t *e) {
+  char namespace[1024];
+  lmc_namespacify(namespace, ns);
   lmc_lock_t *l = malloc(sizeof(lmc_lock_t));
   if (!l) return NULL;
-  strncpy((char *)&l->namespace, namespace, 1023);
+  snprintf((char *)&l->namespace, 1023, "%s", namespace);
   
   lmc_handle_error((l->sem = sem_open(l->namespace, O_CREAT, 0600, init)) == 
       NULL, "sem_open", "LockError", e);
@@ -20,7 +32,9 @@ lmc_lock_t *lmc_lock_init(const char *namespace, int init, lmc_error_t *e) {
   return l;
 }
 
-int lmc_clear_namespace_lock(const char *namespace) {
+int lmc_clear_namespace_lock(const char *ns) {
+  char namespace[1024];
+  lmc_namespacify(namespace, ns);
   lmc_error_t e;
   lmc_lock_t *l = lmc_lock_init(namespace, 1, &e);
   lmc_lock_repair(l);

data/src/lmc_shm.c

@@ -11,6 +11,7 @@
 #include <fcntl.h>
 #include <sys/mman.h>
 
+#include "lmc_common.h"
 #include "lmc_shm.h"
 
 const char *lmc_namespace_root_path() {
@@ -37,7 +38,8 @@ void lmc_shm_ensure_root_path() {
 }
 
 void lmc_file_path_for_namespace(char *result, const char *ns) {
-  snprintf(result, 1023, "%s/%s.lmc", lmc_namespace_root_path(), ns);
+  if (lmc_is_filename(ns)) { snprintf(result, 1023, "%s", ns); } 
+  else { snprintf(result, 1023, "%s/%s.lmc", lmc_namespace_root_path(), ns); }
 }
 
 int lmc_does_namespace_exist(const char *ns) {
@@ -78,7 +80,7 @@ lmc_shm_t *lmc_shm_create(const char* namespace, size_t size, int use_persistenc
     STD_OUT_OF_MEMORY_ERROR("lmc_shm_create");
     return NULL; 
   }
-  strncpy((char *)&mc->namespace, namespace, 1023);
+  snprintf((char *)&mc->namespace, 1023, "%s", namespace);
   mc->use_persistence = 0;
   mc->size = size;
 

data/src/lmc_valloc.c

@@ -69,7 +69,7 @@ lmc_mem_status_t lmc_status(void *base, char *where) {
   ms.total_mem = md->total_size;
   while (c) { 
     if (!lmc_is_va_valid(base, (void *)c - base)) {
-      printf("lmc: [%s] invalid pointer detected: %zd...\n", where, 
+      printf("[localmemcache] [%s] invalid pointer detected: %zd...\n", where, 
           (void *)c - base);
       lmc_dump(base);
       abort();
@@ -108,7 +108,8 @@ void lmc_init_memory(void *ptr, size_t size) {
   size_t s = size - sizeof(lmc_mem_descriptor_t);
   md->first_free = sizeof(lmc_mem_descriptor_t);
   md->magic = 0xF00D;
-  md->locked = 0;
+  md->version = LMC_DB_VERSION;
+  md->locked = 1;
   md->total_size = s;
   lmc_mem_chunk_descriptor_t *c = ptr + sizeof(lmc_mem_descriptor_t);
   c->next = 0;
@@ -319,7 +320,7 @@ void lmc_free(void *base, size_t chunk) {
   if (chunk == 0) { return; }
   if (!(chunk >= sizeof(lmc_mem_descriptor_t) + sizeof(size_t)) ||
       !lmc_is_va_valid(base, chunk)) {
-    printf("lmc_free: Invalid pointer: %zd\n", chunk);
+    fprintf(stderr, "[localmemcache] lmc_free: Invalid pointer: %zd\n", chunk);
     return;
   }
   size_t va_used_chunk = chunk - sizeof(size_t);
@@ -396,8 +397,8 @@ int lmc_um_mark(void *base, char *bf, size_t va, size_t size) {
   lmc_mem_descriptor_t *md = base;
   if ((va > sizeof(lmc_mem_descriptor_t)) &&
       (!lmc_is_va_valid(base, va) || !lmc_is_va_valid(base, va + size))) {
-    printf("lmc: Error: VA start out of range: va: %zd - %zd max %zd!\n", 
-        va, va + size, md->total_size);
+    printf("[localmemcache] Error: VA start out of range: "
+        "va: %zd - %zd max %zd!\n", va, va + size, md->total_size);
     return 0;
   }
   if (!lmc_um_check_unmarked(base, bf, va, size)) return 0;

data/src/lmc_valloc.h

@@ -26,6 +26,7 @@ typedef struct {
   size_t magic;
   size_t va_hash;
   int locked;
+  size_t version;
   lmc_log_descriptor_t log;
 } lmc_mem_descriptor_t;