blurrily 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ed39eb955b4d71f3b924a16be4430046ba1d02ab
-  data.tar.gz: 1c5a5b42b6877ad3d66928a0fe0520ea73defa9b
+  metadata.gz: 5278062ebce0b77e8f18ffebdefa1639cafc8b59
+  data.tar.gz: 5ce39cb3bc008428947ff9ab234d8aba07455241
 SHA512:
-  metadata.gz: 54fdb049c894470cf18afdafe18053607e1b4336b6f7353866ae8d81115e87a97ed6f5273270d930a88c292bf02a361868280997b6dbe5668c894aa456745950
-  data.tar.gz: b8c280aa93d062a9a89fbda80cdf3365efcb34ed3e3c28d8dadf6c9b9ee5deba389a3a0233c788e8e182894b1067254ec9a9ef4ae80e7c1676a60edd6cd50e83
+  metadata.gz: 28fdbf9009c005523e30c450fabb461fff877a4c86d37fe83096998f5ab1addd2d0d032c378f5333e553e27fe28aac9a500508967e64a7f0c33ecdc9c960fe06
+  data.tar.gz: 5e9894f29ce68dfc5ceb222e38f8a05c138f5c7cdc770b8a6c0940b765d9ae4d983cc7ee3e74449b94136b696e65dec666de85cfa1aa2f8b2d38970b6add1fad

data/README.md

@@ -1,13 +1,27 @@
-# Blurrily — Fast fuzzy text search
+# Blurrily — Millisecond fuzzy string matching
 
 [![Build Status](https://travis-ci.org/mezis/blurrily.png?branch=master)](https://travis-ci.org/mezis/blurrily)
 [![Dependency Status](https://gemnasium.com/mezis/blurrily.png)](https://gemnasium.com/mezis/blurrily)
 [![Code Climate](https://codeclimate.com/github/mezis/blurrily.png)](https://codeclimate.com/github/mezis/blurrily)
 
-This will be a C version of [fuzzily](http://github.com/mezis/fuzzily), a
-Ruby gem to perform fuzzy text searching.
+> Show me photos of **Marakech** !
+>
+> Here aresome photos of **Marrakesh**, Morroco.
+> Did you mean **Martanesh**, Albania, **Marakkanam**, India, or **Marasheshty**, Romania?
+
+Blurrily find missplet or partial needles in a haystack of strings, quickly.
+It scales well: its response time is typically 1-2ms on user-input datasets
+and 75-100ms on pathological datasets ([more](#Benchmarks)).
+
+Blurrily is compatible and tested with all MRI Rubies from 1.8.7 to 2.0.0.
+It is tested on Linux 2.6 (32bit and 64bit) and MacOS X 10.8.
+
+Blurrily uses a tweaked [trigram](http://en.wikipedia.org/wiki/N-gram)-based
+approach to find good matches. If you're using ActiveRecord and looking for
+a lightweight (albeit much slower), in-process, Rails-friendly version of
+this, check out [fuzzily](http://github.com/mezis/fuzzily), a Ruby gem to
+perform fuzzy text searching in ActiveRecord.
 
-WORK IN PROGRESS.
 
 ## Installation
 
@@ -15,17 +29,171 @@ Add this line to your application's Gemfile:
 
     gem 'blurrily'
 
-And then execute:
-
-    $ bundle
-
 Or install it yourself as:
 
     $ gem install blurrily
 
 ## Usage
 
-TODO: Write usage instructions here
+You can use blurrily as a client/server combination (recommended in
+production), or use the internals standalone.
+
+See the [API Documentation](http://rubydoc.info/github/mezis/blurrily/frames)
+for more details.
+
+### Client/server
+
+Fire up a blurrily server:
+
+    $ blurrily
+
+Open up a console and connect:
+  
+  	$ irb -rubygems
+  	> require 'blurrily/client'
+  	> client = Blurrily::Client.new
+
+Store a needle with a reference:
+
+    > client.put('London', 1337)
+
+Recover a reference form the haystack:
+
+    > client.find('lonndon')
+    #=> [1337]
+
+### Standalone
+
+Create the in-memory database:
+
+    > map = Blurrily::Map.new
+
+Store a needle with a reference:
+
+    > map.put('London', 1337)
+
+Recover a reference form the haystack:
+
+    > map.find('lonndon')
+    #=> [1337]
+
+Save the database to disk:
+
+    > map.save('/var/db/data.trigrams')
+
+Load a previously saved database:
+
+    > map = Blurrily::Map.load('/var/db/data.trigrams')
+
+
+## Caveats
+
+### Diacritics, non-latin languages
+
+Blurrily forms trigrams from the 26 latin letters and a stop character (used
+to model start-of-string and separation between words in multi-word
+strings).
+
+This means that case and diacritrics are completely ignored by Blurrily. For
+instance, *Puy-de-Dôme* is strictly equivalent to *puy de dome*.
+
+It also means that any non-latin input will probably result in garbase data
+and garbage results (although it won't crash).
+
+### Multi-word needles and edge stickyness.
+
+Multi-word needles (say, *New York*) are supported.
+
+The engine always favours matches that begin and end similarly to the
+needle, with a bias to the beginning of the strings.
+
+Thsi is because internally, the string *New York* is turned into this
+sequence of trigrams: `**n`, `*ne`, `new`, `ew*`, `w*y`, `*yo`, `yor`,
+`ork`, `rk*`.
+
+## Production notes
+
+### Memory usage
+
+Blurrily does not store your original strings but rather a flat map of
+references and weights for each trigram in your input strings.
+
+In practice any database will use up a base 560KB for the index header, plus
+128 bits per trigram.
+
+As a rule of thumb idea memory usages is 40MB + 8 times the size of your
+input data, and 50% extra on top during bulk imports (lots of writes to the
+database).
+
+For instance, `/usr/share/dict/words` is a list of 235k English words, and
+weighs 2.5MB. Importing the whole list uses up 75MB of memory, 51MB of which
+are the database.
+
+Note that once a databse has been written to disk and loaded from disk,
+memory usage is minimal (560KB per database) as the database file is memory
+mapped. For performance you do need as much free memory as the database
+size.
+
+### Disk usage
+
+Disk usage is almost exactly like memory usage, since database files are
+nothing more than a memory dump.
+
+In the `/usr/share/dict/words` example, on-disk size is 51MB.
+For the whole list of Geonames places, on-disk size is 1.1GB.
+
+### Read v write
+
+Writing to blurrily (with `#put`) is fairly expensive—it's a search engine
+after all, optimized for intensive reads.
+
+Supporting writes means the engine needs to keep a hash table of all
+references around, typically weighing 50% of your total input. This is build
+lazily while writing however; so if you load a database from disk and only
+ever read, you will not incur the memory penalty.
+
+### Saving & backing up
+
+Blurrily saves atomically (writing to a separate file, then using rename(2)
+to overwrite the old file), meaning you should never lose data.
+
+The server does this for you every 60 seconds and when quitting. If using
+`Blurrily::Map` directly, remember that a map loaded from disk is more
+memory efficient that a map in memory, so if your workload is read-heavy,
+you should `.load` after each `#save`.
+
+Backing up comes with a caveat: database files are only portable across
+architectures if endianness and pointer size are the same (tested between
+darwin-x86_64 and linux-amd64).
+
+Database files are very compressible; `bzip2` typically shrinks them to 20%
+of their original size.
+
+
+## Benchmarks
+
+Blurrily is wicked fast, often 100x faster than it's ancestor,
+[fuzzily](http://github.com/mezis/fuzzily). This is because it's a close-to-
+the-metal, single-purpose index using almost exclusively libc primitives. On
+the inside the only expensive operations it performs are
+
+- memcpy(2) lots of data around (selection);
+- mergesort(3) to aggregate/count similar entries (reduction);
+- qsort(3) to order by counts (sort).
+
+It tends to be faster with large datasets on BSD than on Linux because the
+former has fast quicksort and mergesort, wheras the latter only has `qsort`,
+a slower, catch-all sorter. In complexity terms this is because FIND tends
+to be *O(n)* on BSD and *O(n ln n)* on Linux.
+
+Enough talk, here are the graphs. The `LOAD` and `PUT` operations are O(1)
+and take respectively ~10ms and ~100µs on any platform, so  they aren't
+graphed here.
+
+- [FIND latency](/doc/bench-find.png)
+- [SAVE latency](/doc/bench-save.png)
+- [DELETE latency](/doc/bench-delete.png)
+
 
 ## Contributing
 

data/bin/blurrily

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+$PROGRAM_NAME = 'blurrily'
+
+require "blurrily/server"
+require 'optparse'
+require 'ostruct'
+
+options = OpenStruct.new
+
+# Defaults
+options.port = 12021
+options.directory  = '.'
+options.host = '0.0.0.0'
+
+parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on("-p", "--port <PORT>", "Bind to PORT, defaults to 12021") do |port|
+    puts 'Port has to be numeric value' and exit unless port =~ /\d+/
+    options.port = port.to_i
+  end
+
+  opts.on("-d", "--directory <DIRECTORY>", "Work in DIRECTORY, defaults to .") do |directory|
+    options.directory = directory
+  end
+
+  opts.on("-b", "--bind <ADDRESS>", "Bind to ADDRESS, defaults to 0.0.0.0") do |address|
+    options.host = address || '0.0.0.0'
+  end
+
+  opts.on("-V", "--version", "Output version") do |address|
+    puts Blurrily::VERSION
+    exit
+  end
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end
+
+parser.parse!(ARGV)
+Blurrily::Server.new(:host => options.host, :port => options.port, :directory => options.directory).start

data/ext/blurrily/blurrily.h

@@ -1,2 +1,16 @@
+/*
+
+  blurrily.h --
+
+  Helper macros
+
+*/
+
 #define PACKED_STRUCT __attribute__ ((__packed__))
 #define UNUSED(_IDENT) _IDENT __attribute__ ((unused))
+
+#ifdef DEBUG
+  #define LOG(...) fprintf(stderr, __VA_ARGS__)
+#else
+  #define LOG(...)
+#endif

data/ext/blurrily/extconf.rb

@@ -5,7 +5,11 @@ SHARED_FLAGS = "-DPLATFORM_#{PLATFORM} --std=c99 -Wall -Wextra -Werror"
 
 case PLATFORM
 when 'LINUX'
-  SHARED_FLAGS += ' -D_XOPEN_SOURCE=500' # for ftruncate to be present
+  # make sure ftruncate is available
+  SHARED_FLAGS << ' -D_XOPEN_SOURCE=700'
+  SHARED_FLAGS << ' -D_GNU_SOURCE=1'
+  # make sure off_t is 64 bit long
+  SHARED_FLAGS << ' -D_FILE_OFFSET_BITS=64'
 end
 
 # production

data/ext/blurrily/map_ext.c

@@ -3,18 +3,44 @@
 #include "storage.h"
 #include "blurrily.h"
 
+static VALUE eClosedError = Qnil;
+static VALUE eBlurrilyModule = Qnil;
+
+/******************************************************************************/
+
+static int raise_if_closed(VALUE self)
+{
+  if (rb_ivar_get(self, rb_intern("@closed")) != Qtrue) return 0;
+  rb_raise(eClosedError, "Map was freed");
+  return 1;
+}
+
+static void mark_as_closed(VALUE self)
+{
+  rb_ivar_set(self, rb_intern("@closed"), Qtrue);
+}
+
 /******************************************************************************/
 
 static void blurrily_free(void* haystack)
 {
   int res = -1;
 
+  if (haystack == NULL) return;
   res = blurrily_storage_close((trigram_map*) &haystack);
   assert(res >= 0);
 }
 
 /******************************************************************************/
 
+static void blurrily_mark(void* haystack)
+{
+  if (haystack == NULL) return;
+  blurrily_storage_mark((trigram_map) haystack);
+}
+
+/******************************************************************************/
+
 static VALUE blurrily_new(VALUE class) {
   VALUE       wrapper  = Qnil;
   trigram_map haystack = (trigram_map)NULL;
@@ -23,7 +49,7 @@ static VALUE blurrily_new(VALUE class) {
   res = blurrily_storage_new(&haystack);
   if (res < 0) { rb_sys_fail(NULL); return Qnil; }
 
-  wrapper = Data_Wrap_Struct(class, 0, blurrily_free, (void*)haystack);
+  wrapper = Data_Wrap_Struct(class, blurrily_mark, blurrily_free, (void*)haystack);
   rb_obj_call_init(wrapper, 0, NULL);
   return wrapper;
 }
@@ -39,7 +65,7 @@ static VALUE blurrily_load(VALUE class, VALUE rb_path) {
   res = blurrily_storage_load(&haystack, path);
   if (res < 0) { rb_sys_fail(NULL); return Qnil; }
 
-  wrapper = Data_Wrap_Struct(class, 0, blurrily_free, (void*)haystack);
+  wrapper = Data_Wrap_Struct(class, blurrily_mark, blurrily_free, (void*)haystack);
   rb_obj_call_init(wrapper, 0, NULL);
   return wrapper;
 }
@@ -59,12 +85,13 @@ static VALUE blurrily_put(VALUE self, VALUE rb_needle, VALUE rb_reference, VALUE
   uint32_t     reference = NUM2UINT(rb_reference);
   uint32_t     weight    = NUM2UINT(rb_weight);
 
+  if (raise_if_closed(self)) return Qnil;
   Data_Get_Struct(self, struct trigram_map_t, haystack);
 
   res = blurrily_storage_put(haystack, needle, reference, weight);
   assert(res >= 0);
 
-  return Qnil;
+  return INT2NUM(res);
 }
 
 /******************************************************************************/
@@ -74,6 +101,7 @@ static VALUE blurrily_delete(VALUE self, VALUE rb_reference) {
   uint32_t     reference = NUM2UINT(rb_reference);
   int          res       = -1;
 
+  if (raise_if_closed(self)) return Qnil;
   Data_Get_Struct(self, struct trigram_map_t, haystack);
 
   res = blurrily_storage_delete(haystack, reference);
@@ -89,10 +117,11 @@ static VALUE blurrily_save(VALUE self, VALUE rb_path) {
   int          res       = -1;
   const char*  path      = StringValuePtr(rb_path);
 
+  if (raise_if_closed(self)) return Qnil;
   Data_Get_Struct(self, struct trigram_map_t, haystack);
 
   res = blurrily_storage_save(haystack, path);
-  assert(res >= 0);
+  if (res < 0) rb_sys_fail(NULL);
 
   return Qnil;
 }
@@ -107,11 +136,16 @@ static VALUE blurrily_find(VALUE self, VALUE rb_needle, VALUE rb_limit) {
   trigram_match matches    = NULL;
   VALUE         rb_matches = Qnil;
 
-  if (limit <= 0) { limit = 10 ; }
-  matches = (trigram_match) malloc(limit * sizeof(trigram_match_t));
-
+  if (raise_if_closed(self)) return Qnil;
   Data_Get_Struct(self, struct trigram_map_t, haystack);
 
+  if (limit <= 0) {
+    // rb_limit = rb_const_get(eBlurrilyModule, rb_intern('LIMIT_DEFAULT'));
+    rb_limit = rb_const_get(eBlurrilyModule, rb_intern("LIMIT_DEFAULT"));
+    limit = NUM2UINT(rb_limit);
+  }
+  matches = (trigram_match) malloc(limit * sizeof(trigram_match_t));
+
   res = blurrily_storage_find(haystack, needle, limit, matches);
   assert(res >= 0);
 
@@ -137,6 +171,7 @@ static VALUE blurrily_stats(VALUE self)
   VALUE           result   = rb_hash_new();
   int             res      = -1;
 
+  if (raise_if_closed(self)) return Qnil;
   Data_Get_Struct(self, struct trigram_map_t, haystack);
 
   res = blurrily_storage_stats(haystack, &stats);
@@ -150,15 +185,35 @@ static VALUE blurrily_stats(VALUE self)
 
 /******************************************************************************/
 
+static VALUE blurrily_close(VALUE self)
+{
+  trigram_map     haystack = (trigram_map)NULL;
+  int             res      = -1;
+
+  if (raise_if_closed(self)) return Qnil;
+  Data_Get_Struct(self, struct trigram_map_t, haystack);
+
+  res = blurrily_storage_close(&haystack);
+  if (res < 0) rb_sys_fail(NULL);
+
+  DATA_PTR(self) = NULL;
+  mark_as_closed(self);
+  return Qnil;
+}
+
+/******************************************************************************/
+
 void Init_map_ext(void) {
-  VALUE module = Qnil;
   VALUE klass  = Qnil;
 
   /* assume we haven't yet defined blurrily */
-  module = rb_define_module("Blurrily");
-  assert(module != Qnil);
+  eBlurrilyModule = rb_define_module("Blurrily");
+  assert(eBlurrilyModule != Qnil);
+
+  klass = rb_define_class_under(eBlurrilyModule, "Map", rb_cObject);
+  assert(klass != Qnil);
 
-  klass = rb_define_class_under(module, "Map", rb_cObject);
+  eClosedError = rb_define_class_under(klass, "ClosedError", rb_eRuntimeError);
   assert(klass != Qnil);
 
   rb_define_singleton_method(klass, "new",  blurrily_new,  0);
@@ -170,5 +225,6 @@ void Init_map_ext(void) {
   rb_define_method(klass, "save",       blurrily_save,       1);
   rb_define_method(klass, "find",       blurrily_find,       2);
   rb_define_method(klass, "stats",      blurrily_stats,      0);
+  rb_define_method(klass, "close",      blurrily_close,      0);
   return;
 }