fuzzzy 0.0.1

data/lib/fuzzzy/index.rb

@@ -0,0 +1,17 @@
+module Fuzzzy
+  module Index
+    def _indexer method
+      @indexer ||= {}
+      @indexer[method] ||= class_for(:indexer, method).new
+    end
+
+    def _searcher method
+      @searcher ||= {}
+      @searcher[method] ||= class_for(:searcher, method).new
+    end
+
+    def class_for type, method
+      "fuzzzy/#{method}/#{type}".classify.constantize
+    end
+  end
+end

data/lib/fuzzzy/methods/indexer.rb

@@ -0,0 +1,13 @@
+module Fuzzzy
+  module Indexer
+    def delete_dictionary id
+      redis.del(dictionary_key(id))
+      redis.hincrby(counter_key, index_name, -1)
+    end
+
+    def save_dictionary id, string
+      redis.set(dictionary_key(id), query_index_string)
+      redis.hincrby(counter_key, index_name, 1)
+    end
+  end
+end

data/lib/fuzzzy/methods/method_base.rb

@@ -0,0 +1,32 @@
+module Fuzzzy
+  class MethodBase
+    include Redis
+
+    attr_reader :context
+
+    def with_context cntx
+      @context = cntx.dup and yield if cntx
+    rescue => e
+      raise e
+    ensure
+      @context = nil
+    end
+
+    def index_name
+      context[:index_name]
+    end
+    
+    def prepare_string string
+      return '' unless string
+      str = string.dup.downcase
+      str = context[:filter].call(str) if context[:filter] && context[:filter].respond_to?(:call)
+      str = (str.split - stopwords).join(' ') if context[:strip_stopwords]
+      str
+    end
+    
+    def stopwords
+      return context[:strip_stopwords] if context[:strip_stopwords] && context[:strip_stopwords].is_a?(Array) 
+      Fuzzzy.stopwords
+    end
+  end
+end

data/lib/fuzzzy/methods/ngram/base.rb

@@ -0,0 +1,15 @@
+module Fuzzzy
+  module Ngram
+    class Base < MethodBase
+      def ngrams string=nil
+        string ||= query_index_string
+        return [string] if string.size < 3
+        context[string] ||= (0..string.length-3).map{|idx| string[idx,3] }
+      end
+      
+      def index_type
+        'ngram_i'
+      end
+    end
+  end
+end

data/lib/fuzzzy/methods/ngram/indexer.rb

@@ -0,0 +1,37 @@
+module Fuzzzy
+  module Ngram
+    class Indexer < Base
+      include Fuzzzy::Indexer
+      def query_index_string
+        context[:prepared_dictionary_string] ||= prepare_string(context[:dictionary_string])
+      end
+
+      def create_index cntx
+        with_context(cntx) do
+          return if query_index_string.empty?
+
+          delete_index
+
+          ngrams.each_with_index do |ngram, index|
+            redis.sadd(index_key(ngram, index), context[:id])
+          end
+
+          save_dictionary(context[:id], query_index_string)
+        end
+      end
+
+      def delete_index cntx=nil
+        block = lambda do
+          if older_string = redis.get(dictionary_key(context[:id]))
+            ngrams(older_string).each_with_index do |ngram, index|
+              redis.srem(index_key(ngram, index), context[:id])
+            end
+
+            delete_dictionary(context[:id])
+          end
+        end
+        cntx ? with_context(cntx, &block) : block.call
+      end
+    end
+  end
+end

data/lib/fuzzzy/methods/ngram/searcher.rb

@@ -0,0 +1,95 @@
+module Fuzzzy
+  module Ngram
+    class Searcher < Base
+      # Ruby implementation:
+      # def segment_points index
+      #   right = distance + index
+      #   left = index > distance ? (index - distance) : 0
+      #   i = left
+      #   while i <= right do
+      #     yield i
+      #     i += 1
+      #   end
+      # end
+      SEGMENT_POINTS = <<-EOC
+VALUE 
+_segment_points(VALUE self, VALUE _index) 
+{
+  int index, distance, left, right, i;
+
+  index = NUM2INT(_index);
+  distance = NUM2INT(rb_funcall(self, rb_intern("distance"), 0));
+  right = index + distance;
+
+  if(index > distance) {
+    left = index - distance;
+  } else {
+    left = 0;
+  }
+
+  for(i = left; i <= right; i++) {
+    rb_yield(INT2NUM(i));
+  }
+
+  return Qnil;
+}
+EOC
+
+      inline do |builder|
+        builder.c_raw(SEGMENT_POINTS, :method_name => 'segment_points', :arity => 1)
+      end
+
+      def search cntx
+        with_context(cntx) do
+          return [] if query_index_string.empty?
+          if ids = redis.sunion(*index_keys)
+            ids.each do |id|
+              string = redis.get(dictionary_key(id))
+              dist = Levenshtein.distance(query_index_string, string)
+              result << {
+                :id => id,
+                :distance => dist,
+                :alpha => string
+              } if dist <= distance
+            end
+            result.sort_by!{|item|item[sort_by]} if sort_by
+            result.map!{|item|item[:id]} unless with_cache?
+            result
+          else
+            []
+          end
+        end
+      end
+
+      def index_keys
+        keys = []
+        ngrams.each_with_index do |ngram, index|
+          segment_points(index) do |i|
+            keys << index_key(ngram, i)
+          end
+        end
+        keys
+      end
+
+      def distance
+        context[:distance] ||= 0
+      end
+
+      def result
+        context[:result] ||= []
+      end
+
+      def with_cache?
+        context[:with_cache] ||= false
+      end
+
+      def sort_by
+        context[:sort_by]
+      end
+
+      def query_index_string
+        context[:prepared_query] ||= prepare_string(context[:query])
+      end
+    end
+  end
+end

data/lib/fuzzzy/methods/soundex/base.rb

@@ -0,0 +1,13 @@
+module Fuzzzy
+  module Soundex
+    class Base < MethodBase
+      def soundex string=nil
+        context[:soundex] ||= Text::Soundex.soundex(string || query_index_string).to_s
+      end
+
+      def index_type
+        'soundex_i'
+      end
+    end
+  end
+end

data/lib/fuzzzy/methods/soundex/indexer.rb

@@ -0,0 +1,30 @@
+module Fuzzzy
+  module Soundex
+    class Indexer < Base
+      include Fuzzzy::Indexer
+      def query_index_string
+        context[:prepared_dictionary_string] ||= prepare_string(context[:dictionary_string])
+      end
+
+      def create_index cntx
+        with_context(cntx) do
+          return if query_index_string.empty?
+
+          delete_index
+          redis.sadd(index_key(soundex), context[:id])
+          save_dictionary(context[:id], query_index_string)
+        end
+      end
+
+      def delete_index cntx=nil
+        block = lambda do
+          if older_string = redis.get(dictionary_key(context[:id]))
+            redis.srem(index_key(soundex(older_string)), context[:id])
+            delete_dictionary(context[:id])
+          end
+        end
+        cntx ? with_context(cntx, &block) : block.call
+      end
+    end
+  end
+end

data/lib/fuzzzy/methods/soundex/searcher.rb

@@ -0,0 +1,36 @@
+module Fuzzzy
+  module Soundex
+    class Searcher < Base
+      def search context
+        with_context(context) do
+          return if query_index_string.empty?
+
+          if ids = redis.smembers(index_key(soundex))
+            result = ids.map do |id|
+              string = redis.get(dictionary_key(id))
+              {
+                :id => id,
+                :distance => Levenshtein.distance(query_index_string, string),
+                :alpha => string
+              }
+            end
+
+            result.sort_by!{|item|item[sort_by]} if sort_by
+            result.reject!{|item|item[:distance] > context[:distance]} if context[:distance]
+            result.map{|item|item[:id]}
+          else
+            []
+          end
+        end
+      end
+
+      def sort_by
+        context[:sort_by]
+      end
+
+      def query_index_string
+        context[:prepared_query] ||= prepare_string(context[:query])
+      end
+    end
+  end
+end

data/lib/fuzzzy/orm/mongoid/index.rb

@@ -0,0 +1,86 @@
+module Fuzzzy
+  module Mongoid
+    module Index
+      extend ActiveSupport::Concern
+
+      included do
+        extend Fuzzzy::Index
+        class_attribute :fuzzzy_indexes
+        around_save :create_fuzzzy_indexes
+        around_update :create_fuzzzy_indexes
+        around_destroy :delete_fuzzzy_indexes
+      end
+
+      module ClassMethods 
+        def define_fuzzzy_index field, options={}
+          self.fuzzzy_indexes ||= {}
+          options[:index_name] = index_name(field)
+          self.fuzzzy_indexes[field.to_sym] = default_options.merge(options)
+        end
+
+        def clear_fuzzzy_index field
+          self.fuzzzy_indexes.delete(field.to_sym)
+        end
+
+        def has_fuzzzy_indexes?
+          !!self.fuzzzy_indexes
+        end
+
+        def default_options
+          {:method => :soundex}
+        end
+
+        def indexer method
+          return nil unless has_fuzzzy_indexes?
+          _indexer(method)
+        end
+
+        def searcher method
+          return nil unless has_fuzzzy_indexes?
+          _searcher(method)
+        end
+
+        def search_by field, query, context={}
+          index_context = self.fuzzzy_indexes[field.to_sym].dup
+          raise "You have not fuzzy index for '#{field}' field" unless index_context
+
+          index_context[:query] = query
+          index_context.merge!(context)
+          ids = searcher(index_context[:method]).search(index_context)
+
+          (only_ids? ? ids : scoped.find(ids)) if ids
+        end
+
+        def only_ids?
+          index_context[:only_ids] && !index_context[:with_cache]
+        end
+
+        def index_name field
+          "#{self.name.downcase}:#{field}"
+        end
+      end
+
+      def delete_fuzzzy_indexes &block
+        change_indexes(:delete_index, &block)
+      end
+
+      def create_fuzzzy_indexes &block
+        change_indexes(:create_index, &block)
+      end
+
+      def change_indexes command, condition=nil
+        self.class.fuzzzy_indexes.each do |(field, opts)|
+          change_field_index(command, field, opts.dup) if command == :delete_index ||
+            self.changed.include?(field.to_s)
+        end
+      end
+
+      def change_field_index command, field, options
+        self.class.indexer(options[:method]).send(command, options.merge(
+          :id => self.id,
+          :dictionary_string => self.send(field)
+        ))
+      end
+    end
+  end
+end

data/lib/fuzzzy/redis.rb

@@ -0,0 +1,61 @@
+require 'inline'
+module Fuzzzy
+  module Redis
+    # Ruby implementation is:
+    # def index_key key, key2=nil
+    #   key = "#{key}:#key2" if key2
+    #   "#{shared_key}:#{index_key}:#{key}"
+    # end
+    INDEX_KEY = <<-EOC
+VALUE 
+_index_key(int argc, VALUE *argv, VALUE self) 
+{
+  VALUE type = rb_funcall(self, rb_intern("index_type"), 0);
+  VALUE shared_key = rb_funcall(self, rb_intern("shared_key"), 0);
+  char sep[2] = ":";
+  VALUE key, key2, result;
+  char * buf;
+  unsigned long long length;
+
+  if(rb_scan_args(argc, argv, "11", &key, &key2) == 2) {
+    key = rb_str_dup(key);
+    rb_str_cat(key, sep, 1);
+    rb_str_concat(key, rb_funcall(key2, rb_intern("to_s"), 0));
+  }
+
+  length = RSTRING_LEN(shared_key) + RSTRING_LEN(type) + RSTRING_LEN(key) + 4;
+  buf = malloc(length);
+  snprintf(buf, length, "%s:%s:%s", RSTRING_PTR(shared_key), RSTRING_PTR(type), RSTRING_PTR(key));
+  result = rb_str_new2(buf);
+  free(buf);
+
+  return result;
+}
+EOC
+    
+    inline do |builder|
+      builder.add_compile_flags('-std=c99')
+      builder.c_raw(INDEX_KEY, :method_name => 'index_key', :arity => -1)
+    end
+    
+    def redis
+      Fuzzzy.redis
+    end
+
+    def shared_key
+      context[:shared_key] ||= "fuzzzy:#{index_name}"
+    end
+
+    def dictionary_key id
+      "#{shared_key}:dictionary:#{id}"
+    end
+    
+    def counter_key
+      "fuzzzy:indexes:info"
+    end
+    
+    def self.counter_key
+      "fuzzzy:indexes:info"
+    end
+  end
+end