blurrily 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ed39eb955b4d71f3b924a16be4430046ba1d02ab
+  data.tar.gz: 1c5a5b42b6877ad3d66928a0fe0520ea73defa9b
+SHA512:
+  metadata.gz: 54fdb049c894470cf18afdafe18053607e1b4336b6f7353866ae8d81115e87a97ed6f5273270d930a88c292bf02a361868280997b6dbe5668c894aa456745950
+  data.tar.gz: b8c280aa93d062a9a89fbda80cdf3365efcb34ed3e3c28d8dadf6c9b9ee5deba389a3a0233c788e8e182894b1067254ec9a9ef4ae80e7c1676a60edd6cd50e83

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 HouseTrip Ltd.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,36 @@
+# Blurrily — Fast fuzzy text search
+
+[![Build Status](https://travis-ci.org/mezis/blurrily.png?branch=master)](https://travis-ci.org/mezis/blurrily)
+[![Dependency Status](https://gemnasium.com/mezis/blurrily.png)](https://gemnasium.com/mezis/blurrily)
+[![Code Climate](https://codeclimate.com/github/mezis/blurrily.png)](https://codeclimate.com/github/mezis/blurrily)
+
+This will be a C version of [fuzzily](http://github.com/mezis/fuzzily), a
+Ruby gem to perform fuzzy text searching.
+
+WORK IN PROGRESS.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'blurrily'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install blurrily
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/ext/blurrily/blurrily.h

@@ -0,0 +1,2 @@
+#define PACKED_STRUCT __attribute__ ((__packed__))
+#define UNUSED(_IDENT) _IDENT __attribute__ ((unused))

data/ext/blurrily/extconf.rb

@@ -0,0 +1,17 @@
+require 'mkmf'
+
+PLATFORM = `uname`.strip.upcase
+SHARED_FLAGS = "-DPLATFORM_#{PLATFORM} --std=c99 -Wall -Wextra -Werror"
+
+case PLATFORM
+when 'LINUX'
+  SHARED_FLAGS += ' -D_XOPEN_SOURCE=500' # for ftruncate to be present
+end
+
+# production
+$CFLAGS += " #{SHARED_FLAGS} -O3 -fno-fast-math"
+
+# development
+# $CFLAGS += " #{SHARED_FLAGS} -O0 -g"
+
+create_makefile('blurrily/map_ext')

data/ext/blurrily/log.h

@@ -0,0 +1,5 @@
+#ifdef DEBUG
+#define LOG(...) fprintf(stderr, __VA_ARGS__)
+#else
+#define LOG(...)
+#endif

data/ext/blurrily/map_ext.c

@@ -0,0 +1,174 @@
+#include <ruby.h>
+#include <assert.h>
+#include "storage.h"
+#include "blurrily.h"
+
+/******************************************************************************/
+
+static void blurrily_free(void* haystack)
+{
+  int res = -1;
+
+  res = blurrily_storage_close((trigram_map*) &haystack);
+  assert(res >= 0);
+}
+
+/******************************************************************************/
+
+static VALUE blurrily_new(VALUE class) {
+  VALUE       wrapper  = Qnil;
+  trigram_map haystack = (trigram_map)NULL;
+  int         res      = -1;
+
+  res = blurrily_storage_new(&haystack);
+  if (res < 0) { rb_sys_fail(NULL); return Qnil; }
+
+  wrapper = Data_Wrap_Struct(class, 0, blurrily_free, (void*)haystack);
+  rb_obj_call_init(wrapper, 0, NULL);
+  return wrapper;
+}
+
+/******************************************************************************/
+
+static VALUE blurrily_load(VALUE class, VALUE rb_path) {
+  char*       path     = StringValuePtr(rb_path);
+  VALUE       wrapper  = Qnil;
+  trigram_map haystack = (trigram_map)NULL;
+  int         res      = -1;
+
+  res = blurrily_storage_load(&haystack, path);
+  if (res < 0) { rb_sys_fail(NULL); return Qnil; }
+
+  wrapper = Data_Wrap_Struct(class, 0, blurrily_free, (void*)haystack);
+  rb_obj_call_init(wrapper, 0, NULL);
+  return wrapper;
+}
+
+/******************************************************************************/
+
+static VALUE blurrily_initialize(VALUE UNUSED(self)) {
+  return Qtrue;
+}
+
+/******************************************************************************/
+
+static VALUE blurrily_put(VALUE self, VALUE rb_needle, VALUE rb_reference, VALUE rb_weight) {
+  trigram_map  haystack  = (trigram_map)NULL;
+  int          res       = -1;
+  char*        needle    = StringValuePtr(rb_needle);
+  uint32_t     reference = NUM2UINT(rb_reference);
+  uint32_t     weight    = NUM2UINT(rb_weight);
+
+  Data_Get_Struct(self, struct trigram_map_t, haystack);
+
+  res = blurrily_storage_put(haystack, needle, reference, weight);
+  assert(res >= 0);
+
+  return Qnil;
+}
+
+/******************************************************************************/
+
+static VALUE blurrily_delete(VALUE self, VALUE rb_reference) {
+  trigram_map  haystack  = (trigram_map)NULL;
+  uint32_t     reference = NUM2UINT(rb_reference);
+  int          res       = -1;
+
+  Data_Get_Struct(self, struct trigram_map_t, haystack);
+
+  res = blurrily_storage_delete(haystack, reference);
+  assert(res >= 0);
+
+  return INT2NUM(res);
+}
+
+/******************************************************************************/
+
+static VALUE blurrily_save(VALUE self, VALUE rb_path) {
+  trigram_map  haystack  = (trigram_map)NULL;
+  int          res       = -1;
+  const char*  path      = StringValuePtr(rb_path);
+
+  Data_Get_Struct(self, struct trigram_map_t, haystack);
+
+  res = blurrily_storage_save(haystack, path);
+  assert(res >= 0);
+
+  return Qnil;
+}
+
+/******************************************************************************/
+
+static VALUE blurrily_find(VALUE self, VALUE rb_needle, VALUE rb_limit) {
+  trigram_map   haystack   = (trigram_map)NULL;
+  int           res        = -1;
+  const char*   needle     = StringValuePtr(rb_needle);
+  int           limit      = NUM2UINT(rb_limit);
+  trigram_match matches    = NULL;
+  VALUE         rb_matches = Qnil;
+
+  if (limit <= 0) { limit = 10 ; }
+  matches = (trigram_match) malloc(limit * sizeof(trigram_match_t));
+
+  Data_Get_Struct(self, struct trigram_map_t, haystack);
+
+  res = blurrily_storage_find(haystack, needle, limit, matches);
+  assert(res >= 0);
+
+  /* wrap the matches into a Ruby array */
+  rb_matches = rb_ary_new();
+  for (int k = 0; k < res; ++k) {
+    VALUE rb_match = rb_ary_new();
+    rb_ary_push(rb_match, rb_uint_new(matches[k].reference));
+    rb_ary_push(rb_match, rb_uint_new(matches[k].matches));
+    rb_ary_push(rb_match, rb_uint_new(matches[k].weight));
+    rb_ary_push(rb_matches, rb_match);
+  }
+  return rb_matches;
+}
+
+
+/******************************************************************************/
+
+static VALUE blurrily_stats(VALUE self)
+{
+  trigram_map     haystack = (trigram_map)NULL;
+  trigram_stat_t  stats;
+  VALUE           result   = rb_hash_new();
+  int             res      = -1;
+
+  Data_Get_Struct(self, struct trigram_map_t, haystack);
+
+  res = blurrily_storage_stats(haystack, &stats);
+  assert(res >= 0);
+
+  (void) rb_hash_aset(result, ID2SYM(rb_intern("references")), UINT2NUM(stats.references));
+  (void) rb_hash_aset(result, ID2SYM(rb_intern("trigrams")),   UINT2NUM(stats.trigrams));
+
+  return result;
+}
+
+/******************************************************************************/
+
+void Init_map_ext(void) {
+  VALUE module = Qnil;
+  VALUE klass  = Qnil;
+
+  /* assume we haven't yet defined blurrily */
+  module = rb_define_module("Blurrily");
+  assert(module != Qnil);
+
+  klass = rb_define_class_under(module, "Map", rb_cObject);
+  assert(klass != Qnil);
+
+  rb_define_singleton_method(klass, "new",  blurrily_new,  0);
+  rb_define_singleton_method(klass, "load", blurrily_load, 1);
+
+  rb_define_method(klass, "initialize", blurrily_initialize, 0);
+  rb_define_method(klass, "put",        blurrily_put,        3);
+  rb_define_method(klass, "delete",     blurrily_delete,     1);
+  rb_define_method(klass, "save",       blurrily_save,       1);
+  rb_define_method(klass, "find",       blurrily_find,       2);
+  rb_define_method(klass, "stats",      blurrily_stats,      0);
+  return;
+}

data/ext/blurrily/storage.c

@@ -0,0 +1,541 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <sys/stat.h>
+
+#ifdef PLATFORM_LINUX
+  #include <linux/limits.h>
+  #define MERGESORT fake_mergesort
+#else
+  #include <limits.h>
+  #define MERGESORT mergesort
+#endif
+
+#ifndef PATH_MAX
+  /* safe default ... */
+  #define PATH_MAX 1024
+#endif
+
+#include "storage.h"
+
+#include "log.h"
+
+/******************************************************************************/
+
+#define PAGE_SIZE                   4096
+#define TRIGRAM_COUNT               (TRIGRAM_BASE * TRIGRAM_BASE * TRIGRAM_BASE)
+#define TRIGRAM_ENTRIES_START_SIZE  PAGE_SIZE/8
+
+/******************************************************************************/
+
+/* one trigram entry -- client reference and sorting weight */
+struct PACKED_STRUCT trigram_entry_t
+{
+  uint32_t reference;
+  uint32_t weight;
+};
+typedef struct trigram_entry_t trigram_entry_t;
+
+
+/* collection of entries for a given trigram */
+/* <entries> points to an array of <buckets> entries */
+/* of which <used> are filled */
+struct PACKED_STRUCT trigram_entries_t
+{
+  uint32_t         buckets;
+  uint32_t         used;
+
+  trigram_entry_t* entries;         /* set when the structure is in memory */
+  size_t           entries_offset;  /* set when the structure is on disk */
+
+  uint8_t          dirty;           /* not optimised (presorted) yet */
+};
+typedef struct trigram_entries_t trigram_entries_t;
+
+
+/* hash map of all possible trigrams to collection of entries */
+/* there are 28^3 = 19,683 possible trigrams */
+struct PACKED_STRUCT trigram_map_t
+{
+  char              magic[6];           /* the string "trigra" */
+  uint8_t           big_endian;
+  uint8_t           pointer_size;
+
+  uint32_t          total_references;
+  uint32_t          total_trigrams;
+  size_t            mapped_size;        /* when mapped from disk, the number of bytes mapped */
+  int               mapped_fd;          /* when mapped from disk, the file descriptor */
+  
+  trigram_entries_t map[TRIGRAM_COUNT]; /* this whole structure is ~500KB */
+};
+typedef struct trigram_map_t trigram_map_t;
+
+/******************************************************************************/
+
+#ifdef PLATFORM_LINUX
+/* fake version of mergesort(3) implemented with qsort(3) as Linux lacks */
+/* the specific variants */
+static int fake_mergesort(void *base, size_t nel, size_t width, int (*compar)(const void *, const void *))
+{
+  qsort(base, nel, width, compar);
+  return 0;
+}
+#endif
+
+/******************************************************************************/
+
+/* 1 -> little endian, 2 -> big endian */
+static uint8_t get_big_endian()
+{
+  uint32_t magic = 0xAA0000BB;
+  uint8_t  head  = *((uint8_t*) &magic);
+
+  return (head == 0xBB) ? 1 : 2;
+}
+
+/******************************************************************************/
+
+/* 4  or 8 (bytes) */
+static uint8_t get_pointer_size()
+{
+  return (uint8_t) sizeof(void*);
+}
+
+/******************************************************************************/
+
+static int compare_entries(const void* left_p, const void* right_p)
+{
+  trigram_entry_t* left  = (trigram_entry_t*)left_p;
+  trigram_entry_t* right = (trigram_entry_t*)right_p;
+  return (int)left->reference - (int)right->reference;
+}
+
+/* compares matches on #matches (descending) then weight (ascending) */
+static int compare_matches(const void* left_p, const void* right_p)
+{
+  trigram_match_t* left  = (trigram_match_t*)left_p;
+  trigram_match_t* right = (trigram_match_t*)right_p;
+  /* int delta = (int)left->matches - (int)right->matches; */
+  int delta = (int)right->matches - (int)left->matches;
+
+  return (delta != 0) ? delta : ((int)left->weight - (int)right->weight);
+
+}
+
+/******************************************************************************/
+
+static void sort_map_if_dirty(trigram_entries_t* map)
+{ 
+  int res = -1;
+  if (! map->dirty) return;
+
+  res = MERGESORT(map->entries, map->used, sizeof(trigram_entry_t), &compare_entries);
+  assert(res >= 0);
+  map->dirty = 0;
+}
+
+/******************************************************************************/
+
+static size_t round_to_page(size_t value)
+{
+  if (value % PAGE_SIZE == 0) return value;
+  return (value / PAGE_SIZE + 1) * PAGE_SIZE;
+}
+
+/******************************************************************************/
+
+static size_t get_map_size(trigram_map haystack, int index)
+{
+  return haystack->map[index].buckets * sizeof(trigram_entry_t);
+}
+
+/******************************************************************************/
+
+static void free_if(void* ptr)
+{
+  if (ptr == NULL) return;
+  free(ptr);
+  return;
+}
+
+/******************************************************************************/
+
+int blurrily_storage_new(trigram_map* haystack_ptr)
+{
+  trigram_map         haystack = (trigram_map)NULL;
+  trigram_entries_t*  ptr      = NULL;
+  int                 k        = 0;
+
+  LOG("blurrily_storage_new\n");
+  haystack = (trigram_map) malloc(sizeof(trigram_map_t));
+  if (haystack == NULL) return -1;
+
+  memset(haystack, 0x00, sizeof(trigram_map_t));
+
+  memcpy(haystack->magic, "trigra", 6);
+  haystack->big_endian   = get_big_endian();
+  haystack->pointer_size = get_pointer_size();
+
+  haystack->mapped_size      = 0; /* not mapped, as we just created it in memory */
+  haystack->mapped_fd        = 0;
+  haystack->total_references = 0;
+  haystack->total_trigrams   = 0;
+  for(k = 0, ptr = haystack->map ; k < TRIGRAM_COUNT ; ++k, ++ptr) {
+    ptr->buckets = 0;
+    ptr->used    = 0;
+    ptr->dirty   = 0;
+    ptr->entries = (trigram_entry_t*)NULL;
+  }
+
+  *haystack_ptr = haystack;
+  return 0;
+}
+
+/******************************************************************************/
+
+int blurrily_storage_load(trigram_map* haystack, const char* path)
+{
+  int         fd          = -1;
+  int         res         = -1;
+  trigram_map header      = NULL;
+  uint8_t*    origin      = NULL;
+  struct stat metadata;
+
+  /* open and map file */
+  res = fd = open(path, O_RDONLY);
+  if (res < 0) goto cleanup;
+
+  res = fstat(fd, &metadata);
+  if (res < 0) goto cleanup;
+
+  header = (trigram_map) mmap(NULL, metadata.st_size, PROT_READ|PROT_WRITE, MAP_PRIVATE, fd, 0);
+  assert(header != NULL);
+
+  /* check magic */
+  /* TODO */
+
+  /* fix header data */
+  header->mapped_size = metadata.st_size;
+  header->mapped_fd   = fd;
+  origin = (uint8_t*)header;
+  for (int k = 0; k < TRIGRAM_COUNT; ++k) {
+    trigram_entries_t* map = header->map + k;
+    if (map->entries_offset == 0) continue;
+    map->entries = (trigram_entry_t*) (origin + map->entries_offset);
+    map->entries_offset = 0;
+  }
+  *haystack = header;
+
+cleanup:
+  return res;
+}
+
+/******************************************************************************/
+
+int blurrily_storage_close(trigram_map* haystack_ptr)
+{
+  trigram_map         haystack = *haystack_ptr;
+  int                 res      = -1;
+
+  LOG("blurrily_storage_close\n");
+
+  if (haystack->mapped_size) {
+    int fd = haystack->mapped_fd;
+
+    res = munmap(haystack, haystack->mapped_size);
+    assert(res >= 0);
+
+    res = close(fd);
+    assert(res >= 0);
+  } else {
+    trigram_entries_t*  ptr = haystack->map;
+    for(int k = 0 ; k < TRIGRAM_COUNT ; ++k) {
+      free(ptr->entries);
+      ++ptr;
+    }
+    free(haystack);
+  }
+
+  *haystack_ptr = NULL;
+  return 0;
+}
+
+/******************************************************************************/
+
+int blurrily_storage_save(trigram_map haystack, const char* path)
+{
+  int         fd          = -1;
+  int         res         = -1;
+  uint8_t*    ptr         = (uint8_t*)NULL;
+  size_t      total_size  = 0;
+  size_t      offset      = 0;
+  trigram_map header      = NULL;
+  char        path_tmp[PATH_MAX];
+
+  /* cleanup maps in memory */
+  for (int k = 0; k < TRIGRAM_COUNT; ++k) {
+    sort_map_if_dirty(haystack->map + k);
+  }
+
+  /* path for temporary file */
+  snprintf(path_tmp, PATH_MAX, "%s.tmp", path);
+
+  /* compute storage space required */
+  total_size += round_to_page(sizeof(trigram_map_t));
+
+  for (int k = 0; k < TRIGRAM_COUNT; ++k) {
+    total_size += round_to_page(get_map_size(haystack, k));
+  }
+
+  /* open and map file */
+  fd = open(path_tmp, O_RDWR | O_CREAT | O_TRUNC, 0644);
+  assert(fd >= 0);
+
+  res = ftruncate(fd, total_size);
+  assert(res >= 0);
+
+  ptr = mmap(NULL, total_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
+  assert(ptr != NULL);
+
+  /* flush data */
+  memset(ptr, 0x00, total_size);
+
+  /* copy header & clean copy */
+  memcpy(ptr, (void*)haystack, sizeof(trigram_map_t));
+  offset += round_to_page(sizeof(trigram_map_t));
+  header = (trigram_map)ptr;
+
+  header->mapped_size = 0;
+  header->mapped_fd   = 0;
+
+  /* copy each map, set offset in header */
+  for (int k = 0; k < TRIGRAM_COUNT; ++k) {
+    size_t block_size = get_map_size(haystack, k);
+
+    if (block_size > 0) {
+      memcpy(ptr+offset, haystack->map[k].entries, block_size);
+
+      header->map[k].entries        = NULL;
+      header->map[k].entries_offset = offset;
+
+      offset += round_to_page(block_size);
+    } else {
+      header->map[k].entries        = NULL;
+      header->map[k].entries_offset = 0;
+    }
+  }
+  assert(offset == total_size);
+
+  res = munmap(ptr, total_size);
+  assert(res >= 0);
+
+  res = close(fd);
+  assert(res >= 0);
+
+  /* commit by renaming the file */
+  res = rename(path_tmp, path);
+  assert(res >= 0);
+
+  return 0;
+}
+
+/******************************************************************************/
+
+int blurrily_storage_put(trigram_map haystack, const char* needle, uint32_t reference, uint32_t weight)
+{
+  int        nb_trigrams  = -1;
+  int        length       = strlen(needle);
+  trigram_t* trigrams     = (trigram_t*)NULL;
+
+  trigrams = (trigram_t*)malloc((length+1) * sizeof(trigram_t));
+  nb_trigrams = blurrily_tokeniser_parse_string(needle, trigrams);
+
+  if (weight <= 0) weight = length;
+
+  for (int k = 0; k < nb_trigrams; ++k) {
+    trigram_t          t       = trigrams[k];
+    trigram_entries_t* map     = &haystack->map[t];
+    trigram_entry_t    entry   = { reference, weight };
+
+    assert(t < TRIGRAM_COUNT);
+    assert(map-> used <= map-> buckets);
+
+    /* allocate more space as needed (exponential growth) */
+    if (map->buckets == 0) {
+      LOG("- alloc for %d\n", t);
+
+      map->buckets = TRIGRAM_ENTRIES_START_SIZE;
+      map->entries = (trigram_entry_t*) calloc(map->buckets, sizeof(trigram_entry_t));
+    }
+    if (map->used == map->buckets) {
+      uint32_t new_buckets = map->buckets * 4/3;
+      trigram_entry_t* new_entries = NULL;
+      LOG("- realloc for %d\n", t);
+
+      /* copy old data, free old pointer, zero extra space */
+      new_entries = malloc(new_buckets * sizeof(trigram_entry_t));
+      assert(new_entries != NULL);
+      memcpy(new_entries, map->entries, map->buckets * sizeof(trigram_entry_t));
+      free(map->entries);
+      memset(new_entries + map->buckets, 0x00, (new_buckets - map->buckets) * sizeof(trigram_entry_t));
+      /* swap fields */
+      map->buckets = new_buckets;
+      map->entries = new_entries;
+    }
+    map->entries[map->used] = entry;
+    
+    map->used += 1;
+    map->dirty = 1;
+  }
+  haystack->total_trigrams   += nb_trigrams;
+  haystack->total_references += 1;
+
+  free((void*)trigrams);
+  return 0;
+}
+
+/******************************************************************************/
+
+int blurrily_storage_find(trigram_map haystack, const char* needle, uint16_t limit, trigram_match results)
+{
+  int              nb_trigrams = -1;
+  int              length      = strlen(needle);
+  trigram_t*       trigrams    = (trigram_t*)NULL;
+  int              nb_entries  = -1;
+  trigram_entry_t* entries     = NULL;
+  trigram_entry_t* entry_ptr   = NULL;
+  int              nb_matches  = -1;
+  trigram_match_t* matches     = NULL;
+  trigram_match_t* match_ptr   = NULL;
+  uint32_t         last_ref    = (uint32_t)-1;
+  int              nb_results  = 0;
+
+  trigrams = (trigram_t*)malloc((length+1) * sizeof(trigram_t));
+  nb_trigrams = blurrily_tokeniser_parse_string(needle, trigrams);
+  if (nb_trigrams == 0) goto cleanup;
+
+  LOG("%d trigrams in '%s'\n", nb_trigrams, needle);
+
+  /* measure size required for sorting */
+  nb_entries = 0;
+  for (int k = 0; k < nb_trigrams; ++k) {
+    trigram_t t = trigrams[k];
+    nb_entries += haystack->map[t].used;
+  }
+  if (nb_entries == 0) goto cleanup;
+
+  /* allocate sorting memory */
+  entries = (trigram_entry_t*) malloc(nb_entries * sizeof(trigram_entry_t));
+  assert(entries != NULL);
+  LOG("allocated space for %zd trigrams entries\n", nb_entries);
+
+  /* copy data for sorting */
+  entry_ptr = entries;
+  for (int k = 0; k < nb_trigrams; ++k) {
+    trigram_t t       = trigrams[k];
+    size_t    buckets = haystack->map[t].used;
+
+    sort_map_if_dirty(haystack->map + t);
+    memcpy(entry_ptr, haystack->map[t].entries, buckets * sizeof(trigram_entry_t));
+    entry_ptr += buckets;
+  }
+  assert(entry_ptr == entries + nb_entries);
+
+  /* sort data */
+  MERGESORT(entries, nb_entries, sizeof(trigram_entry_t), &compare_entries);
+  LOG("sorting entries\n");
+
+  /* count distinct matches */
+  entry_ptr  = entries;
+  last_ref   = -1;
+  nb_matches = 0;
+  for (int k = 0; k < nb_entries; ++k) {
+    if (entry_ptr->reference != last_ref) {
+      last_ref = entry_ptr->reference;
+      ++nb_matches;
+    }
+    ++entry_ptr;    
+  }
+  assert(entry_ptr == entries + nb_entries);
+  LOG("total %zd distinct matches\n", nb_matches);
+
+  /* allocate maches result */
+  matches = (trigram_match_t*) calloc(nb_matches, sizeof(trigram_match_t));
+  assert(matches != NULL);
+
+  /* reduction, counting matches per reference */
+  entry_ptr = entries;
+  match_ptr = matches;
+  match_ptr->matches   = 0;
+  match_ptr->reference = entry_ptr->reference; /* setup the first match to */
+  match_ptr->weight    = entry_ptr->weight;    /* simplify the loop */
+  for (int k = 0; k < nb_entries; ++k) {
+    if (entry_ptr->reference != match_ptr->reference) {
+      ++match_ptr;
+      match_ptr->reference = entry_ptr->reference;
+      match_ptr->weight    = entry_ptr->weight;
+      match_ptr->matches   = 1;
+    } else {
+      match_ptr->matches  += 1;
+    }
+    assert((int) match_ptr->matches <= nb_trigrams);
+    ++entry_ptr;    
+  }
+  assert(match_ptr == matches + nb_matches - 1);
+  assert(entry_ptr == entries + nb_entries);
+
+  /* sort by weight (qsort) */
+  qsort(matches, nb_matches, sizeof(trigram_match_t), &compare_matches);
+
+  /* output results */
+  nb_results = (limit < nb_matches) ? limit : nb_matches;
+  for (int k = 0; k < nb_results; ++k) {
+    results[k] = matches[k];
+    LOG("match %d: reference %d, matchiness %d, weight %d\n", k, matches[k].reference, matches[k].matches, matches[k].weight);
+  }
+
+cleanup:
+  free_if(entries);
+  free_if(matches);
+  free_if(trigrams);
+  return nb_results;
+}
+
+/******************************************************************************/
+
+int blurrily_storage_delete(trigram_map haystack, uint32_t reference)
+{
+  int trigrams_deleted = 0;
+
+  for (int k = 0; k < TRIGRAM_COUNT; ++k) {
+    trigram_entries_t* map       = haystack->map + k;
+    trigram_entry_t*   entry     = NULL;
+
+    for (unsigned int j = 0; j < map->used; ++j) {
+      entry = map->entries + j;
+      if (entry->reference != reference) continue;
+
+      *entry = map->entries[map->used - 1];
+      map->used -= 1;
+
+      ++trigrams_deleted;
+      --j;
+    }
+  }
+  haystack->total_trigrams   -= trigrams_deleted;
+  haystack->total_references -= 1;
+  return trigrams_deleted;
+}
+
+/******************************************************************************/
+
+int blurrily_storage_stats(trigram_map haystack, trigram_stat_t* stats)
+{
+  stats->references = haystack->total_references;
+  stats->trigrams   = haystack->total_trigrams;
+  return 0;
+}

data/ext/blurrily/storage.h

@@ -0,0 +1,109 @@
+/*
+  
+  storage.h --
+
+  Trigram map creation, persistence, and qurying.
+
+*/
+#include <inttypes.h>
+#include "tokeniser.h"
+#include "blurrily.h"
+
+struct trigram_map_t;
+typedef struct trigram_map_t* trigram_map;
+
+struct PACKED_STRUCT trigram_match_t {
+  uint32_t reference;
+  uint32_t matches;
+  uint32_t weight;
+};
+typedef struct trigram_match_t trigram_match_t;
+typedef struct trigram_match_t* trigram_match;
+
+typedef struct trigram_stat_t {
+  uint32_t references;
+  uint32_t trigrams;
+
+} trigram_stat_t;
+
+
+/* 
+  Create a new trigram map, resident in memory.
+*/
+int blurrily_storage_new(trigram_map* haystack);
+
+/* 
+  Load an existing trigram map from disk.
+*/
+int blurrily_storage_load(trigram_map* haystack, const char* path);
+
+/* 
+  Release resources claimed by <new> or <open>.
+*/
+int blurrily_storage_close(trigram_map* haystack);
+
+/* 
+  Persist to disk what <blurrily_storage_new> or <blurrily_storage_open>
+  gave you.
+*/
+int blurrily_storage_save(trigram_map haystack, const char* path);
+
+/*
+  Add a new string to the map. <reference> is your identifier for that
+  string, <weight> will be using to discriminate entries that match "as
+  well" when searching.
+
+  If <weight> is zero, it will be replaced by the number of characters in
+  the <needle>.
+
+  Returns positive on success, negative on failure.
+*/
+int blurrily_storage_put(trigram_map haystack, const char* needle, uint32_t reference, uint32_t weight);
+
+/*
+  Check the map for an existing <reference>.
+
+  Returns < 0 on error, 0 if the reference is not found, the number of trigrams
+  for that reference otherwise.
+
+  If <weight> is not NULL, will be set to the weight value passed to the put
+  method on return (is the reference is found).
+
+  If <trigrams> is not NULL, it should point an array <nb_trigrams> long,
+  and up to <nb_trigrams> will be copied into it matching the <needle>
+  originally passed to the put method.
+
+  Not that this is a O(n) method: the whole map will be read.
+*/
+// int blurrily_storage_get(trigram_map haystack, uint32_t reference, uint32_t* weight, int nb_trigrams, trigram_t* trigrams);
+
+/*
+  Remove a <reference> from the map.
+
+  Note that this is very innefective.
+
+  Returns positive on success, negative on failure.
+*/
+int blurrily_storage_delete(trigram_map haystack, uint32_t reference);
+
+/*
+  Return at most <limit> entries matching <needle> from the <haystack>.
+
+  Results are written to <results>. The first results are the ones entries
+  sharing the most trigrams with the <needle>. Amongst entries with the same
+  number of matches, the lightest ones (lowest <weight>) will be returned
+  first.
+
+  <results> should be allocated by the caller.
+
+  Returns number of matches on success, negative on failure.
+*/
+int blurrily_storage_find(trigram_map haystack, const char* needle, uint16_t limit, trigram_match results);
+
+/*
+  Copies metadata into <stats>
+
+  Returns positive on success, negative on failure.
+*/
+int blurrily_storage_stats(trigram_map haystack, trigram_stat_t* stats);
+

data/ext/blurrily/tokeniser.c

@@ -0,0 +1,127 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <math.h>
+#include "tokeniser.h"
+#include "log.h"
+#include "blurrily.h"
+
+
+/******************************************************************************/
+
+static int ipow(int a, int b)
+{
+  int result = 1;
+  
+  while (b-- > 0) result = result * a;
+  return result;
+}
+
+/******************************************************************************/
+
+static void string_to_code(const char* input, trigram_t *output)
+{
+  trigram_t result = 0;
+
+  for (int k = 0 ; k < 3; ++k) {
+    if (input[k] == '*' || input[k] < 'a' || input[k] > 'z') continue;
+    result += ipow(TRIGRAM_BASE, k) * (input[k] - 'a' + 1);
+  }
+
+  *output = result;
+}
+
+/******************************************************************************/
+
+static void code_to_string(trigram_t input, char* output)
+{
+  for (int k = 0 ; k < 3; ++k) {
+    uint16_t elem = input / ipow(TRIGRAM_BASE, k) % TRIGRAM_BASE;
+    if (elem == 0) {
+      output[k] = '*';
+    } else {
+      output[k] = ('a' + elem - 1);
+    }
+  }
+  output[3] = 0;
+}
+
+/******************************************************************************/
+
+static int blurrily_compare_trigrams(const void* left_p, const void* right_p)
+{
+  trigram_t* left  = (trigram_t*)left_p;
+  trigram_t* right = (trigram_t*)right_p;
+  return (int)*left - (int)*right;
+}
+
+/******************************************************************************/
+
+int blurrily_tokeniser_parse_string(const char* input, trigram_t* output)
+{
+  int   length     = strlen(input);
+  char* normalized = (char*) malloc(length+5);
+  int   duplicates = 0;
+
+  snprintf(normalized, length+4, "**%s*", input);
+
+  /* replace spaces with '*' */
+  for (int k = 0; k < length+3; ++k) {
+    if (normalized[k] == ' ') normalized[k] = '*';
+  }
+
+  /* compute trigrams */
+  for (int k = 0; k <= length; ++k) {
+    string_to_code(normalized+k, output+k);
+  }
+
+  /* print results */
+  LOG("-- normalization\n");
+  LOG("%s -> %s\n", input, normalized);
+  LOG("-- tokenisation\n");
+  for (int k = 0; k <= length; ++k) {
+    char res[4];
+
+    code_to_string(output[k], res);
+
+    LOG("%c%c%c -> %d -> %s\n",
+      normalized[k], normalized[k+1], normalized[k+2],
+      output[k], res
+    );
+  }
+
+  /* sort */
+  qsort((void*)output, length+1, sizeof(trigram_t), &blurrily_compare_trigrams);
+
+  /* remove duplicates */
+  for (int k = 1; k <= length; ++k) {
+    trigram_t* previous = output + k - 1;
+    trigram_t* current  = output + k;
+
+    if (*previous == *current) {
+      *previous = 32768;
+      ++duplicates;
+    }
+  }
+
+  /* compact */
+  qsort((void*)output, length+1, sizeof(trigram_t), &blurrily_compare_trigrams);
+
+  /* print again */
+  LOG("-- after sort/compact\n");
+  for (int k = 0; k <= length-duplicates; ++k) {
+    char res[4];
+    code_to_string(output[k], res);
+    LOG("%d -> %s\n", output[k], res);
+  }
+
+  free((void*)normalized);
+  return length+1 - duplicates;
+}
+
+/******************************************************************************/
+
+int blurrily_tokeniser_trigram(trigram_t UNUSED(input), char* UNUSED(output))
+{
+  return 0;
+}

data/ext/blurrily/tokeniser.h

@@ -0,0 +1,41 @@
+/*
+  
+  tokeniser.h --
+
+  Split a string into an array of trigrams.
+
+  The input string should be only lowercase latin letters and spaces
+  (convert using iconv).
+
+  Each trigram is a three-symbol tuple consisting of latters and the
+  "epsilon" character used to represent spaces and beginning-of-word/end-of-
+  word anchors.
+
+  Each trigram is represented by a 16-bit integer.
+
+*/
+#include <inttypes.h>
+
+#define TRIGRAM_BASE 28
+
+typedef uint16_t trigram_t;
+
+/* 
+  Parse the <input> string and store the result in <ouput>.
+  <output> must be allocated by the caller and provide at least as many slots
+  as characters in <input>, plus one.
+  (not all will be necessarily be filled)
+
+  Returns the number of trigrams on success, a negative number on failure.
+*/
+int blurrily_tokeniser_parse_string(const char* input, trigram_t* output);
+
+
+/*
+  Given an <input> returns a string representation of the trigram in <output>.
+  <output> must be allocated by caller and will always be exactly 3
+  <characters plus NULL.
+
+  Returns positive on success, negative on failure.
+*/
+int blurrily_tokeniser_trigram(trigram_t input, char* output);

data/lib/blurrily.rb

@@ -0,0 +1,3 @@
+require "blurrily/map_ext"
+require "blurrily/map"
+require "blurrily/version"

data/lib/blurrily/map.rb

@@ -0,0 +1,34 @@
+require 'blurrily/map_ext'
+require 'active_support/all' # fixme: we only need enough to get mb_chars and alias_method_chain in
+
+module Blurrily
+  Map.class_eval do
+
+    def put_with_string_normalize(needle, reference, weight=0)
+      needle = normalize_string needle
+      put_without_string_normalize(needle, reference, weight)
+    end
+
+    alias_method_chain :put, :string_normalize
+
+
+    def find_with_string_normalize(needle, limit=10)
+      needle = normalize_string needle
+      find_without_string_normalize(needle, limit)
+    end
+
+    alias_method_chain :find, :string_normalize
+
+
+    private
+
+    def normalize_string(needle)
+      result = needle.downcase
+      unless result =~ /^([a-z ])+$/
+        result = result.mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/,'').to_s.gsub(/[^a-z]/,' ')
+      end
+      result.gsub(/\s+/,' ').strip
+    end
+
+  end
+end

data/lib/blurrily/version.rb

@@ -0,0 +1,3 @@
+module Blurrily
+  VERSION = "0.0.1"
+end

metadata

@@ -0,0 +1,199 @@
+--- !ruby/object:Gem::Specification
+name: blurrily
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Julien Letessier
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-03-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-nav
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-doc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Native fuzzy string search
+email:
+- julien.letessier@gmail.com
+executables: []
+extensions:
+- ext/blurrily/extconf.rb
+extra_rdoc_files: []
+files:
+- lib/blurrily/map.rb
+- lib/blurrily/server.rb
+- lib/blurrily/version.rb
+- lib/blurrily.rb
+- ext/blurrily/map_ext.c
+- ext/blurrily/storage.c
+- ext/blurrily/tokeniser.c
+- ext/blurrily/blurrily.h
+- ext/blurrily/log.h
+- ext/blurrily/storage.h
+- ext/blurrily/tokeniser.h
+- ext/blurrily/extconf.rb
+- README.md
+- LICENSE.txt
+homepage: http://github.com/mezis/blurrily
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.0
+signing_key: 
+specification_version: 4
+summary: Native fuzzy string search
+test_files: []
+has_rdoc: