blurrily 0.0.1 → 0.1.0

data/ext/blurrily/search_tree.c

@@ -0,0 +1,66 @@
+#include <stdlib.h>
+#include <inttypes.h>
+#include "blurrily.h"
+#include "ruby.h"
+
+/******************************************************************************/
+
+typedef struct blurrily_refs_t {
+  VALUE hash;
+} blurrily_refs_t;
+
+/******************************************************************************/
+
+int blurrily_refs_new(blurrily_refs_t** refs_ptr)
+{
+  blurrily_refs_t* refs = NULL;
+
+  refs = (blurrily_refs_t*) malloc(sizeof(blurrily_refs_t));
+  if (!refs) return -1;
+
+  refs->hash = rb_hash_new();
+  *refs_ptr = refs;
+  return 0;
+}
+
+/******************************************************************************/
+
+void blurrily_refs_mark(blurrily_refs_t* refs)
+{
+  rb_gc_mark(refs->hash);
+  return;
+}
+
+/******************************************************************************/
+
+void blurrily_refs_free(blurrily_refs_t** refs_ptr)
+{
+  blurrily_refs_t* refs = *refs_ptr;
+
+  refs->hash = Qnil;
+  free(refs);
+  *refs_ptr = NULL;
+  return;
+}
+
+/******************************************************************************/
+
+void blurrily_refs_add(blurrily_refs_t* refs, uint32_t ref)
+{
+  (void) rb_hash_aset(refs->hash, UINT2NUM(ref), Qtrue);
+  return;
+}
+
+/******************************************************************************/
+
+void blurrily_refs_remove(blurrily_refs_t* refs, uint32_t ref)
+{
+  (void) rb_hash_aset(refs->hash, UINT2NUM(ref), Qnil);
+}
+
+/******************************************************************************/
+
+int blurrily_refs_test(blurrily_refs_t* refs, uint32_t ref)
+{
+  return rb_hash_aref(refs->hash, UINT2NUM(ref)) == Qtrue ? 1 : 0;
+}

data/ext/blurrily/search_tree.h

@@ -0,0 +1,30 @@
+/*
+  
+  search_tree.h --
+
+  List of all references that's fast to query for existence.
+
+*/
+#include <inttypes.h>
+
+
+typedef struct blurrily_refs_t blurrily_refs_t;
+
+
+/* Allocate a search tree */
+int blurrily_refs_new(blurrily_refs_t** refs_ptr);
+
+/* Destroy a search tree */
+void blurrily_refs_free(blurrily_refs_t** refs_ptr);
+
+/* Mark with Ruby's GC */
+void blurrily_refs_mark(blurrily_refs_t* refs);
+
+/* Add a reference */
+void blurrily_refs_add(blurrily_refs_t* refs, uint32_t ref);
+
+/* Remove a reference */
+void blurrily_refs_remove(blurrily_refs_t* refs, uint32_t ref);
+
+/* Test for a reference (1 = present, 0 = absent) */
+int blurrily_refs_test(blurrily_refs_t* refs, uint32_t ref);

data/ext/blurrily/storage.c

@@ -4,6 +4,7 @@
 #include <assert.h>
 #include <fcntl.h>
 #include <sys/mman.h>
+#include <sys/errno.h>
 #include <unistd.h>
 #include <sys/stat.h>
 
@@ -21,14 +22,13 @@
 #endif
 
 #include "storage.h"
-
-#include "log.h"
+#include "search_tree.h"
 
 /******************************************************************************/
 
 #define PAGE_SIZE                   4096
 #define TRIGRAM_COUNT               (TRIGRAM_BASE * TRIGRAM_BASE * TRIGRAM_BASE)
-#define TRIGRAM_ENTRIES_START_SIZE  PAGE_SIZE/8
+#define TRIGRAM_ENTRIES_START_SIZE  PAGE_SIZE/sizeof(trigram_entry_t)
 
 /******************************************************************************/
 
@@ -50,7 +50,7 @@ struct PACKED_STRUCT trigram_entries_t
   uint32_t         used;
 
   trigram_entry_t* entries;         /* set when the structure is in memory */
-  size_t           entries_offset;  /* set when the structure is on disk */
+  off_t            entries_offset;  /* set when the structure is on disk */
 
   uint8_t          dirty;           /* not optimised (presorted) yet */
 };
@@ -68,7 +68,7 @@ struct PACKED_STRUCT trigram_map_t
   uint32_t          total_references;
   uint32_t          total_trigrams;
   size_t            mapped_size;        /* when mapped from disk, the number of bytes mapped */
-  int               mapped_fd;          /* when mapped from disk, the file descriptor */
+  blurrily_refs_t*  refs;
   
   trigram_entries_t map[TRIGRAM_COUNT]; /* this whole structure is ~500KB */
 };
@@ -88,6 +88,17 @@ static int fake_mergesort(void *base, size_t nel, size_t width, int (*compar)(co
 
 /******************************************************************************/
 
+#define SMALLOC(_NELEM,_TYPE) (_TYPE*) smalloc(_NELEM, sizeof(_TYPE))
+
+static void* smalloc(size_t nelem, size_t length)
+{
+  void* result = malloc(nelem * length);
+  if (result) memset(result, 0xAA, nelem * length);
+  return result;
+}
+
+/******************************************************************************/
+
 /* 1 -> little endian, 2 -> big endian */
 static uint8_t get_big_endian()
 {
@@ -171,24 +182,23 @@ int blurrily_storage_new(trigram_map* haystack_ptr)
   int                 k        = 0;
 
   LOG("blurrily_storage_new\n");
-  haystack = (trigram_map) malloc(sizeof(trigram_map_t));
+  haystack = SMALLOC(1, trigram_map_t);
   if (haystack == NULL) return -1;
 
-  memset(haystack, 0x00, sizeof(trigram_map_t));
-
   memcpy(haystack->magic, "trigra", 6);
   haystack->big_endian   = get_big_endian();
   haystack->pointer_size = get_pointer_size();
 
   haystack->mapped_size      = 0; /* not mapped, as we just created it in memory */
-  haystack->mapped_fd        = 0;
   haystack->total_references = 0;
   haystack->total_trigrams   = 0;
+  haystack->refs             = NULL;
   for(k = 0, ptr = haystack->map ; k < TRIGRAM_COUNT ; ++k, ++ptr) {
     ptr->buckets = 0;
     ptr->used    = 0;
     ptr->dirty   = 0;
     ptr->entries = (trigram_entry_t*)NULL;
+    ptr->entries_offset = 0;
   }
 
   *haystack_ptr = haystack;
@@ -212,25 +222,46 @@ int blurrily_storage_load(trigram_map* haystack, const char* path)
   res = fstat(fd, &metadata);
   if (res < 0) goto cleanup;
 
+  /* check this file is at least lng enough to have a header */
+  if (metadata.st_size < (off_t) sizeof(trigram_map_t)) {
+    errno = EPROTO;
+    res = -1;
+    goto cleanup;
+  }
+
   header = (trigram_map) mmap(NULL, metadata.st_size, PROT_READ|PROT_WRITE, MAP_PRIVATE, fd, 0);
-  assert(header != NULL);
+  if (header == MAP_FAILED) {
+    res = -1;
+    header = NULL;
+    goto cleanup;
+  }
+
+  /* fd not needed once mapping established */
+  res = close(fd);
+  if (res < 0) goto cleanup;
+  fd = -1;
 
   /* check magic */
-  /* TODO */
+  res = memcmp(header->magic, "trigra", 6);
+  if (res != 0 || header->big_endian != get_big_endian() || header->pointer_size != get_pointer_size()) {
+    errno = EPROTO;
+    res = -1;
+    goto cleanup;
+  }
 
   /* fix header data */
   header->mapped_size = metadata.st_size;
-  header->mapped_fd   = fd;
   origin = (uint8_t*)header;
   for (int k = 0; k < TRIGRAM_COUNT; ++k) {
     trigram_entries_t* map = header->map + k;
     if (map->entries_offset == 0) continue;
     map->entries = (trigram_entry_t*) (origin + map->entries_offset);
-    map->entries_offset = 0;
   }
   *haystack = header;
 
 cleanup:
+  if (fd > 0) (void) close(fd);
+  if (res < 0 && header != NULL) (void) munmap(header, metadata.st_size);
   return res;
 }
 
@@ -239,29 +270,28 @@ cleanup:
 int blurrily_storage_close(trigram_map* haystack_ptr)
 {
   trigram_map         haystack = *haystack_ptr;
-  int                 res      = -1;
+  int                 res      = 0;
+  trigram_entries_t*  ptr = haystack->map;
 
   LOG("blurrily_storage_close\n");
 
-  if (haystack->mapped_size) {
-    int fd = haystack->mapped_fd;
+  for(int k = 0 ; k < TRIGRAM_COUNT ; ++k) {
+    if (ptr->entries_offset == 0) free(ptr->entries);
+    ++ptr;
+  }
 
-    res = munmap(haystack, haystack->mapped_size);
-    assert(res >= 0);
+  if (haystack->refs) blurrily_refs_free(&haystack->refs);
 
-    res = close(fd);
-    assert(res >= 0);
+  if (haystack->mapped_size) {
+    res = munmap(haystack, haystack->mapped_size);
+    if (res < 0) goto cleanup;
   } else {
-    trigram_entries_t*  ptr = haystack->map;
-    for(int k = 0 ; k < TRIGRAM_COUNT ; ++k) {
-      free(ptr->entries);
-      ++ptr;
-    }
     free(haystack);
   }
 
+cleanup:
   *haystack_ptr = NULL;
-  return 0;
+  return res;
 }
 
 /******************************************************************************/
@@ -269,7 +299,7 @@ int blurrily_storage_close(trigram_map* haystack_ptr)
 int blurrily_storage_save(trigram_map haystack, const char* path)
 {
   int         fd          = -1;
-  int         res         = -1;
+  int         res         = 0;
   uint8_t*    ptr         = (uint8_t*)NULL;
   size_t      total_size  = 0;
   size_t      offset      = 0;
@@ -282,7 +312,7 @@ int blurrily_storage_save(trigram_map haystack, const char* path)
   }
 
   /* path for temporary file */
-  snprintf(path_tmp, PATH_MAX, "%s.tmp", path);
+  snprintf(path_tmp, PATH_MAX, "%s.tmp.%ld", path, random());
 
   /* compute storage space required */
   total_size += round_to_page(sizeof(trigram_map_t));
@@ -293,16 +323,19 @@ int blurrily_storage_save(trigram_map haystack, const char* path)
 
   /* open and map file */
   fd = open(path_tmp, O_RDWR | O_CREAT | O_TRUNC, 0644);
-  assert(fd >= 0);
+  if (fd < 0) goto cleanup;
 
   res = ftruncate(fd, total_size);
-  assert(res >= 0);
+  if (res < 0) goto cleanup;
 
   ptr = mmap(NULL, total_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
-  assert(ptr != NULL);
+  if (ptr == MAP_FAILED) { res = -1 ; goto cleanup ; }
+
+  (void) close(fd);
+  fd = -1;
 
   /* flush data */
-  memset(ptr, 0x00, total_size);
+  memset(ptr, 0xFF, total_size);
 
   /* copy header & clean copy */
   memcpy(ptr, (void*)haystack, sizeof(trigram_map_t));
@@ -310,7 +343,7 @@ int blurrily_storage_save(trigram_map haystack, const char* path)
   header = (trigram_map)ptr;
 
   header->mapped_size = 0;
-  header->mapped_fd   = 0;
+  header->refs        = NULL;
 
   /* copy each map, set offset in header */
   for (int k = 0; k < TRIGRAM_COUNT; ++k) {
@@ -330,17 +363,34 @@ int blurrily_storage_save(trigram_map haystack, const char* path)
   }
   assert(offset == total_size);
 
-  res = munmap(ptr, total_size);
-  assert(res >= 0);
-
-  res = close(fd);
-  assert(res >= 0);
+cleanup:
+  if (ptr != NULL && total_size > 0) {
+    res = munmap(ptr, total_size);
+  }
 
   /* commit by renaming the file */
-  res = rename(path_tmp, path);
-  assert(res >= 0);
+  if (res >= 0 && path) {
+    res = rename(path_tmp, path);
+  }
 
-  return 0;
+  return res;
+}
+
+/******************************************************************************/
+
+void add_all_refs(trigram_map haystack)
+{
+  assert(haystack->refs != NULL);
+
+  for (int k = 0; k < TRIGRAM_COUNT; ++k) {
+    trigram_entries_t* map = haystack->map + k;
+    trigram_entry_t*   ptr = map->entries;
+    assert(map->used <= map->buckets);
+    for (uint32_t j = 0; j < map->used; ++j, ++ptr) {
+      uint32_t ref = ptr->reference;
+      blurrily_refs_add(haystack->refs, ref);
+    }
+  }
 }
 
 /******************************************************************************/
@@ -348,13 +398,19 @@ int blurrily_storage_save(trigram_map haystack, const char* path)
 int blurrily_storage_put(trigram_map haystack, const char* needle, uint32_t reference, uint32_t weight)
 {
   int        nb_trigrams  = -1;
-  int        length       = strlen(needle);
+  size_t     length       = strlen(needle);
   trigram_t* trigrams     = (trigram_t*)NULL;
 
-  trigrams = (trigram_t*)malloc((length+1) * sizeof(trigram_t));
+  if (!haystack->refs) {
+    blurrily_refs_new(&haystack->refs);
+    add_all_refs(haystack);
+  }
+  if (blurrily_refs_test(haystack->refs, reference)) return 0;
+  if (weight <= 0) weight = (uint32_t) length;
+
+  trigrams = SMALLOC(length+1, trigram_t);
   nb_trigrams = blurrily_tokeniser_parse_string(needle, trigrams);
 
-  if (weight <= 0) weight = length;
 
   for (int k = 0; k < nb_trigrams; ++k) {
     trigram_t          t       = trigrams[k];
@@ -369,33 +425,51 @@ int blurrily_storage_put(trigram_map haystack, const char* needle, uint32_t refe
       LOG("- alloc for %d\n", t);
 
       map->buckets = TRIGRAM_ENTRIES_START_SIZE;
-      map->entries = (trigram_entry_t*) calloc(map->buckets, sizeof(trigram_entry_t));
+      map->entries = SMALLOC(map->buckets, trigram_entry_t);
     }
-    if (map->used == map->buckets) {
-      uint32_t new_buckets = map->buckets * 4/3;
+    else if (map->used == map->buckets) {
+      uint32_t         new_buckets = map->buckets * 4/3;
       trigram_entry_t* new_entries = NULL;
       LOG("- realloc for %d\n", t);
 
       /* copy old data, free old pointer, zero extra space */
-      new_entries = malloc(new_buckets * sizeof(trigram_entry_t));
+      new_entries = SMALLOC(new_buckets, trigram_entry_t);
       assert(new_entries != NULL);
       memcpy(new_entries, map->entries, map->buckets * sizeof(trigram_entry_t));
-      free(map->entries);
-      memset(new_entries + map->buckets, 0x00, (new_buckets - map->buckets) * sizeof(trigram_entry_t));
+      /* scribble the rest of the map*/      
+      // memset(new_entries + map->buckets, 0xFF, (new_buckets - map->buckets) * sizeof(trigram_entry_t));
+
+      #ifndef NDEBUG
+        /* scribble old data */
+        memset(map->entries, 0xFF, map->buckets * sizeof(trigram_entry_t));
+      #endif
+
+      if (map->entries_offset) {
+        /* old data was on disk, just mark it as no longer on disk */
+        map->entries_offset = 0;
+      } else {
+        /* free old data */
+        free(map->entries);
+      }
+
       /* swap fields */
       map->buckets = new_buckets;
       map->entries = new_entries;
     }
+
+    /* insert new entry */
+    assert(map->used < map->buckets);
     map->entries[map->used] = entry;
-    
     map->used += 1;
     map->dirty = 1;
   }
   haystack->total_trigrams   += nb_trigrams;
   haystack->total_references += 1;
 
+  blurrily_refs_add(haystack->refs, reference);
+
   free((void*)trigrams);
-  return 0;
+  return nb_trigrams;
 }
 
 /******************************************************************************/
@@ -403,7 +477,7 @@ int blurrily_storage_put(trigram_map haystack, const char* needle, uint32_t refe
 int blurrily_storage_find(trigram_map haystack, const char* needle, uint16_t limit, trigram_match results)
 {
   int              nb_trigrams = -1;
-  int              length      = strlen(needle);
+  size_t           length      = strlen(needle);
   trigram_t*       trigrams    = (trigram_t*)NULL;
   int              nb_entries  = -1;
   trigram_entry_t* entries     = NULL;
@@ -414,7 +488,7 @@ int blurrily_storage_find(trigram_map haystack, const char* needle, uint16_t lim
   uint32_t         last_ref    = (uint32_t)-1;
   int              nb_results  = 0;
 
-  trigrams = (trigram_t*)malloc((length+1) * sizeof(trigram_t));
+  trigrams = SMALLOC(length+1, trigram_t);
   nb_trigrams = blurrily_tokeniser_parse_string(needle, trigrams);
   if (nb_trigrams == 0) goto cleanup;
 
@@ -429,7 +503,7 @@ int blurrily_storage_find(trigram_map haystack, const char* needle, uint16_t lim
   if (nb_entries == 0) goto cleanup;
 
   /* allocate sorting memory */
-  entries = (trigram_entry_t*) malloc(nb_entries * sizeof(trigram_entry_t));
+  entries = SMALLOC(nb_entries, trigram_entry_t);
   assert(entries != NULL);
   LOG("allocated space for %zd trigrams entries\n", nb_entries);
 
@@ -464,7 +538,7 @@ int blurrily_storage_find(trigram_map haystack, const char* needle, uint16_t lim
   LOG("total %zd distinct matches\n", nb_matches);
 
   /* allocate maches result */
-  matches = (trigram_match_t*) calloc(nb_matches, sizeof(trigram_match_t));
+  matches = SMALLOC(nb_matches, trigram_match_t);
   assert(matches != NULL);
 
   /* reduction, counting matches per reference */
@@ -519,15 +593,18 @@ int blurrily_storage_delete(trigram_map haystack, uint32_t reference)
       entry = map->entries + j;
       if (entry->reference != reference) continue;
 
+      /* swap with the last entry */
       *entry = map->entries[map->used - 1];
+      memset(map->entries + map->used - 1, 0xFF, sizeof(trigram_entry_t));
+
       map->used -= 1;
 
       ++trigrams_deleted;
       --j;
     }
   }
-  haystack->total_trigrams   -= trigrams_deleted;
-  haystack->total_references -= 1;
+  haystack->total_trigrams -= trigrams_deleted;
+  if (trigrams_deleted > 0) haystack->total_references -= 1;
   return trigrams_deleted;
 }
 
@@ -539,3 +616,11 @@ int blurrily_storage_stats(trigram_map haystack, trigram_stat_t* stats)
   stats->trigrams   = haystack->total_trigrams;
   return 0;
 }
+
+/******************************************************************************/
+
+void blurrily_storage_mark(trigram_map haystack)
+{
+  if (haystack->refs) blurrily_refs_mark(haystack->refs);
+  return;
+}