fuzzzy 0.0.1

data/lib/fuzzzy/server/http.rb

@@ -0,0 +1,99 @@
+require 'pp'
+module Fuzzzy
+  module Server
+    class HTTP < Grape::API
+      class ParamsError < StandardError
+      end
+
+      format :json
+      default_format :json
+      error_format :json
+      version 'v1', :using => :path
+
+      rescue_from :all do |e|
+        rack_response({:error => e.class.name, :message => e.message}.to_json)
+      end
+
+      helpers do
+        include Index
+
+        def search
+          context = search_context
+          check_context!(:query, context)
+          context[:distance] = context[:distance].to_i if context[:distance]
+          _searcher(context[:index_method]).search(context)
+        end
+
+        def check_context! *keys
+          context = keys.pop
+          ([:index_name, :index_method] + keys).each do |key|
+            raise ParamsError.new("Parameter :#{key} not found") if context[key].nil?
+          end
+        end
+
+        def construct_context *keys
+          context = {}
+          ([:index_name, :index_method] + keys).each do |key|
+            context[key] = params[key] if params[key]
+          end
+          context
+        end
+
+        def index_context
+          construct_context(:id, :dictionary_string)
+        end
+
+        def search_context
+          construct_context(:query, :distance, :sort_by, :with_cache)
+        end
+      end
+
+      namespace :info do
+        http_basic do |u, p| 
+          u == 'admin' && p == 'password'
+        end
+
+        get do
+          info = {
+            :ruby => RUBY_VERSION,
+            :environment => Fuzzzy.env,
+            :redis => Fuzzzy.redis.client.id,
+            :root_dir => Fuzzzy.root.to_s
+          }
+          info[:stopwords] = Fuzzzy.stopwords if params[:show_stopwords]
+          info
+        end
+
+        get 'indexes' do
+          indexes_info = {
+            :redis_size => Fuzzzy.redis.info['used_memory_human'],
+            :indexes => Fuzzzy.redis.hgetall(Fuzzzy::Redis.counter_key)
+          }
+        end
+      end
+
+      resource :indexes do
+        # curl  /v1/indexes?index_name=city:name&index_method=ngram&query=search%20string
+        get do
+          search
+        end
+
+        post '/search' do
+          search
+        end
+
+        post do
+          context = index_context
+          check_context!(:id, :dictionary_string, context)
+          _indexer(context[:index_method]).create_index(context)
+        end
+
+        delete do
+          context = index_context
+          check_context!(:id, context)
+          _indexer(context[:index_method]).delete_index(context)
+        end
+      end
+    end
+  end
+end

data/lib/fuzzzy/version.rb

@@ -0,0 +1,3 @@
+module Fuzzzy
+  VERSION = '0.0.1'
+end

data/spec/config/mongoid.yml

@@ -0,0 +1,7 @@
+development:
+  host: localhost
+  database: moneybot_dev
+
+test:
+  host: localhost
+  database: moneybot_test

data/spec/models/city.rb

@@ -0,0 +1,8 @@
+class City
+  include Mongoid::Document
+  include Fuzzzy::Mongoid::Index
+
+  field :name, :type => String
+  field :country, :type => String
+
+end

data/spec/models/indexed_city.rb

@@ -0,0 +1,9 @@
+class IndexedCity
+  include Mongoid::Document
+  include Fuzzzy::Mongoid::Index
+
+  field :name, :type => String
+  field :country, :type => String
+
+  define_fuzzzy_index :name
+end

data/spec/ngram/indexer_spec.rb

@@ -0,0 +1,142 @@
+require 'spec_helper'
+
+describe Fuzzzy::Ngram::Indexer do
+  let(:indexer){Fuzzzy::Ngram::Indexer.new}
+  let(:context){{
+    :index_name => 'city:name',
+    :method => :ngram,
+    :dictionary_string => dictionary_string,
+    :id => id
+  }}
+  let(:dictionary_string){'moscow'}
+  let(:id){'12345'}
+  let(:counter_key){Fuzzzy::Redis.counter_key}
+  
+  before do
+    keys = Fuzzzy.redis.keys("*")
+    Fuzzzy.redis.del(*keys) if keys.length > 0
+  end
+  
+  describe '#ngrams' do
+    specify do
+      indexer.with_context(context) do
+        indexer.ngrams('mo').should == ['mo']
+      end
+    end
+    specify do
+      indexer.with_context(context) do
+        indexer.ngrams('mos').should == ['mos']
+      end
+    end
+    specify do
+      indexer.with_context(context) do
+        indexer.ngrams.should == ['mos', 'osc', 'sco', 'cow']
+      end
+    end
+  end
+  
+  describe '#create_index' do
+    let(:keys){Fuzzzy.redis.keys}
+    let(:dictionary_keys){["fuzzzy:city:name:dictionary:#{id}"]}
+    
+    before do
+      indexer.create_index(context)
+    end
+    
+    specify{keys.size.should == 6}
+    specify do
+      keys.should =~ [
+        'fuzzzy:city:name:ngram_i:mos:0',
+        'fuzzzy:city:name:ngram_i:osc:1',
+        'fuzzzy:city:name:ngram_i:sco:2',
+        'fuzzzy:city:name:ngram_i:cow:3',
+      ] + dictionary_keys + [counter_key]
+    end
+    specify do
+      Fuzzzy.redis.mget(*dictionary_keys).should == ['moscow']
+    end
+    specify do
+      Fuzzzy.redis.sunion(*(keys - dictionary_keys - [counter_key])).should == [id]
+    end
+    specify{Fuzzzy.redis.hgetall(counter_key).should == {'city:name' => '1'}}
+    
+    context 'with empty string' do
+      let(:dictionary_string){''}
+      
+      specify{keys.size.should == 0}
+    end
+    
+    context 'with nulled string' do
+      let(:dictionary_string){nil}
+      
+      specify{keys.size.should == 0}
+    end
+    
+    context 'with multiple calls' do
+      let(:another_id){'11111'}
+      let(:dictionary_keys){[
+        "fuzzzy:city:name:dictionary:#{id}",
+        "fuzzzy:city:name:dictionary:#{another_id}"
+      ]}
+      
+      before do
+        indexer.create_index(context.merge(
+          :dictionary_string => 'Mostyn',
+          :id => another_id
+        ))
+      end
+      
+      specify{keys.size.should == 10}
+      specify do
+        keys.should =~ [
+          'fuzzzy:city:name:ngram_i:mos:0',
+          'fuzzzy:city:name:ngram_i:osc:1',
+          'fuzzzy:city:name:ngram_i:sco:2',
+          'fuzzzy:city:name:ngram_i:cow:3',
+          'fuzzzy:city:name:ngram_i:ost:1',
+          'fuzzzy:city:name:ngram_i:sty:2',
+          'fuzzzy:city:name:ngram_i:tyn:3',
+        ] + dictionary_keys + [counter_key]
+      end
+      specify do
+        Fuzzzy.redis.mget(*dictionary_keys).should == [dictionary_string, 'mostyn']
+      end
+      specify do
+        Fuzzzy.redis.sunion(*(keys - dictionary_keys - [counter_key])).should =~ [id, another_id]
+      end
+      specify{Fuzzzy.redis.hgetall(counter_key).should == {'city:name' => '2'}}
+    end
+  end
+  
+  describe '#delete_index' do
+    let(:keys){Fuzzzy.redis.keys}
+    let(:another_id){'11111'}
+    let(:dictionary_keys){["fuzzzy:city:name:dictionary:#{another_id}"]}
+    
+    before do
+      indexer.create_index(context)
+      indexer.create_index(context.merge(
+        :dictionary_string => 'Mostyn',
+        :id => another_id
+      ))
+      indexer.delete_index(context)
+    end
+    
+    specify{keys.size.should == 6}
+    specify do
+      keys.should =~ [
+        'fuzzzy:city:name:ngram_i:mos:0',
+        'fuzzzy:city:name:ngram_i:ost:1',
+        'fuzzzy:city:name:ngram_i:sty:2',
+        'fuzzzy:city:name:ngram_i:tyn:3',
+      ] + dictionary_keys + [counter_key]
+    end
+    specify do
+      Fuzzzy.redis.mget(*dictionary_keys).should == ['mostyn']
+    end
+    specify do
+      Fuzzzy.redis.sunion(*(keys - dictionary_keys - [counter_key])).should =~ [another_id]
+    end
+    specify{Fuzzzy.redis.hgetall(counter_key).should == {'city:name' => '1'}}
+  end
+end

data/spec/ngram/searcher_spec.rb

@@ -0,0 +1,194 @@
+require 'spec_helper'
+
+describe Fuzzzy::Ngram::Searcher do
+  let(:indexer){Fuzzzy::Ngram::Indexer.new}
+  let(:index_context){{
+    :index_name => 'city:name',
+    :method => :ngram
+  }}
+  let(:searcher){Fuzzzy::Ngram::Searcher.new}
+  let(:context){index_context.merge(:query => query_string, :distance => 1)}
+  
+  before do
+    keys = Fuzzzy.redis.keys("*")
+    Fuzzzy.redis.del(*keys) if keys.length > 0
+  end
+  
+  describe '#segment_points' do
+    context 'when distance = 0' do
+      let(:result){[]}
+      let(:sample){[
+        [0],  # mos
+        [1],  # osk
+        [2],  # sko
+        [3],  # kow
+      ]}
+      before do
+        searcher.with_context(:distance => 0, :query => 'moscow') do
+          searcher.segment_points(index) do |i|
+            result << i
+          end
+        end
+      end
+    
+      (0...4).each do |idx|
+        context "and index = #{idx}" do
+          let(:index){idx}
+          specify{result.should == sample[index]}
+        end
+      end
+    end
+    
+    context 'when distance = 1' do
+      let(:result){[]}
+      let(:sample){[
+        [0, 1],     # mos
+        [0, 1, 2],  # osk
+        [1, 2, 3],  # sko
+        [2, 3, 4],  # kow
+      ]}
+      before do
+        searcher.with_context(:distance => 1, :query => 'moscow') do
+          searcher.segment_points(index) do |i|
+            result << i
+          end
+        end
+      end
+    
+      (0...4).each do |idx|
+        context "and index = #{idx}" do
+          let(:index){idx}
+          specify{result.should == sample[index]}
+        end
+      end
+    end
+    
+    context 'when distance = 3' do
+      let(:result){[]}
+      let(:sample){[
+        [0, 1, 2, 3],         # mos
+        [0, 1, 2, 3, 4],      # osk
+        [0, 1, 2, 3, 4, 5],   # sko
+        [0, 1, 2, 3, 4, 5, 6] # kow
+      ]}
+      before do
+        searcher.with_context(:distance => 3, :query => 'moscow') do
+          searcher.segment_points(index) do |i|
+            result << i
+          end
+        end
+      end
+    
+      (0...4).each do |idx|
+        context "and index = #{idx}" do
+          let(:index){idx}
+          specify{result.should == sample[index]}
+        end
+      end
+    end
+    
+    context 'when distance = 3 and long word' do
+      let(:result){[]}
+      let(:sample){[
+        [0, 1, 2, 3],            # lev
+        [0, 1, 2, 3, 4],         # eve
+        [0, 1, 2, 3, 4, 5],      # ven
+        [0, 1, 2, 3, 4, 5,  6],  # ens
+        [1, 2, 3, 4, 5, 6,  7],  # nsh
+        [2, 3, 4, 5, 6, 7,  8],  # sht
+        [3, 4, 5, 6, 7, 8,  9],  # hte
+        [4, 5, 6, 7, 8, 9,  10], # tei
+        [5, 6, 7, 8, 9, 10, 11]  # ein
+      ]}
+      before do
+        searcher.with_context(:distance => 3, :query => 'levenshtein') do
+          searcher.segment_points(index) do |i|
+            result << i
+          end
+        end
+      end
+    
+      (0...9).each do |idx|
+        context "and index = #{idx}" do
+          let(:index){idx}
+          specify{result.should == sample[index]}
+        end
+      end
+    end
+  end
+  
+  describe '#index_keys' do
+    let(:query_string){'mascow'}
+    specify do
+      searcher.with_context(context) do
+        searcher.index_keys.should =~ [
+          searcher.index_key('mas', 0),
+          searcher.index_key('mas', 1),
+          searcher.index_key('asc', 0),
+          searcher.index_key('asc', 1),
+          searcher.index_key('asc', 2),
+          searcher.index_key('sco', 1),
+          searcher.index_key('sco', 2),
+          searcher.index_key('sco', 3),
+          searcher.index_key('cow', 2),
+          searcher.index_key('cow', 3),
+          searcher.index_key('cow', 4)
+        ]
+      end
+    end
+  end
+  
+  describe '#search' do
+    context 'single word - #1' do
+      before do
+        indexer.create_index(index_context.merge(
+          :dictionary_string => dictionary_string,
+          :id => id
+        ))
+      end
+      
+      let(:query_string){'mascow'}
+      let(:dictionary_string){'moscow'}
+      let(:id){'12345'}
+      
+      specify{searcher.search(context).should == [id]}
+    end
+    
+    context 'single word - #2' do
+      before do
+        indexer.create_index(index_context.merge(
+          :dictionary_string => dictionary_string,
+          :id => id
+        ))
+      end
+      
+      let(:query_string){'jenergija'}
+      let(:dictionary_string){'energiya'}
+      let(:id){'12345'}
+      
+      specify{searcher.search(context.merge(
+        :distance => 2
+      )).should == [id]}
+    end
+    
+    context 'single word - #2' do
+      before do
+        indexer.create_index(index_context.merge(
+          :dictionary_string => dictionary_string,
+          :id => id
+        ))
+      end
+      
+      let(:query_string){'rhus'}
+      let(:dictionary_string){'Aarhus'}
+      let(:id){'12345'}
+      
+      specify{searcher.search(context.merge(
+        :distance => 2
+      )).should == [id]}
+    end
+    
+    context 'many words' do
+    end
+  end
+end