fuzzzy 0.0.1

data/benchmark/fuzzzy_benchmark.rb

@@ -0,0 +1,128 @@
+require 'csv'
+require 'benchmark'
+require 'ruby-prof'
+
+module FuzzzyBenchmark
+  extend Fuzzzy::Index
+
+  module_function
+  def benchmark meth, contexts, index_cntx={}, times=100, &block
+    @search_method = meth
+    @times = times
+    prepare_indexes(default_context.merge(index_cntx))
+
+    Benchmark.bm do |benchmark|
+      contexts.each do |context|
+        report(benchmark, context.merge(index_cntx))
+      end
+
+      yield(benchmark, self) if block_given?
+    end
+
+  ensure
+    Fuzzzy.redis.flushdb
+    @search_method = nil
+  end
+
+  def profile meth, context, index_cntx={}, times=100, &block
+    @search_method = meth
+    @times = times
+    prepare_indexes(default_context.merge(index_cntx))
+
+    RubyProf.start
+
+    @times.times do
+      searcher.search(context)
+    end
+
+    result = RubyProf.stop
+
+    flat_printer = RubyProf::FlatPrinter.new(result)
+    callstack_printer = RubyProf::CallStackPrinter.new(result)
+    graph_printer = RubyProf::GraphHtmlPrinter.new(result)
+
+    File.open(Fuzzzy.root.join('benchmark', 'reports', "#{search_method}_graph.html"), 'w') do |file| 
+      graph_printer.print(file)
+    end
+    File.open(Fuzzzy.root.join('benchmark', 'reports', "#{search_method}_callstack.html"), 'w') do |file| 
+      callstack_printer.print(file)
+    end
+    File.open(Fuzzzy.root.join('benchmark', 'reports', "#{search_method}_flat.txt"), 'w') do |file| 
+      flat_printer.print(file)
+    end
+  ensure
+    Fuzzzy.redis.flushdb
+    @search_method = nil
+  end
+
+  def report bench, context
+    context = default_context.merge(context)
+    result, strings = get_result(context)
+
+    puts "Execute #{@times} times"
+    puts "query: '#{context[:query]}', result: '#{result}' => #{strings}"
+    bench.report(context[:title] || '') do
+      @times.times do
+        searcher.search(context)
+      end
+    end
+    puts ''
+  end
+
+  def get_result cntx
+    result = searcher.search(cntx)
+    strings = result.map{|id|Fuzzzy.redis.get('fuzzzy:city:name:dictionary:' + id)}
+    [result, strings]
+  end
+
+  def prepare_indexes cntx
+    puts "Create index for #{search_method}:"
+    puts "#{fixtures.size} names"
+
+    start = Time.now
+    fixtures.each do |source|
+      indexer.create_index(cntx.merge(
+        :dictionary_string => source[:name].downcase,
+        :id => source[:id]
+      ))
+    end
+
+    puts "#{Time.now - start} sec."
+    puts "size - #{Fuzzzy.redis.info['used_memory_human']}"
+  end
+
+  def default_context
+    {
+      :index_name => 'city:name',
+      :method => search_method
+    }
+  end
+
+  def indexer
+    _indexer(search_method)
+  end
+
+  def searcher
+    _searcher(search_method)
+  end
+
+  def fixtures
+    @fixtures ||= begin
+      result = []
+      CSV.foreach(Fuzzzy.root.join('benchmark', 'data', 'cities.csv').to_s,
+        :headers => true, :encoding => 'utf-8'
+      ) do |row|
+        result << row.to_hash.symbolize_keys
+      end
+      result
+    end
+  end
+
+  def search_method
+    @search_method
+  end
+
+  def search_method= meth
+    @search_method = meth
+  end
+end

data/benchmark/test_ngram.rb

@@ -0,0 +1,36 @@
+$LOAD_PATH.unshift File.expand_path("../../benchmark", __FILE__)
+require File.expand_path('../../lib/fuzzzy', __FILE__)
+require 'fuzzzy_benchmark'
+
+puts 'Without stripping stopwords'
+FuzzzyBenchmark.benchmark(:ngram, [{
+  :query => 'eastleigh naersouthempton',
+  :distance => 4
+}, {
+  :query => 'alixandropolis',
+  :distance => 2
+}, {
+  :query => 'jenan',
+  :distance => 1
+}])
+
+puts 'With stripping stopwords'
+FuzzzyBenchmark.benchmark(:ngram, [{
+  :query => 'eastleigh naersouthempton',
+  :distance => 4,
+  :strip_stopwords => true
+}, {
+  :query => 'alixandropolis',
+  :distance => 2,
+  :strip_stopwords => true
+}, {
+  :query => 'jenan',
+  :distance => 1,
+  :strip_stopwords => true
+}], {:strip_stopwords => true})
+
+FuzzzyBenchmark.profile(:ngram, {
+  :query => 'eastleigh naersouthempton',
+  :distance => 4,
+  :strip_stopwords => true
+}, {:strip_stopwords => true}, 1000)

data/benchmark/test_soundex.rb

@@ -0,0 +1,36 @@
+$LOAD_PATH.unshift File.expand_path("../../benchmark", __FILE__)
+require File.expand_path('../../lib/fuzzzy', __FILE__)
+require 'fuzzzy_benchmark'
+
+puts 'Without stripping stopwords'
+FuzzzyBenchmark.benchmark(:soundex, [{
+  :query => 'eastleigh naersouthempton',
+  :distance => 4
+}, {
+  :query => 'alixandropolis',
+  :distance => 2
+}, {
+  :query => 'jenan',
+  :distance => 1
+}])
+
+puts 'With stripping stopwords'
+FuzzzyBenchmark.benchmark(:soundex, [{
+  :query => 'eastleigh naersouthempton',
+  :distance => 4,
+  :strip_stopwords => true
+}, {
+  :query => 'alixandropolis',
+  :distance => 2,
+  :strip_stopwords => true
+}, {
+  :query => 'jenan',
+  :distance => 1,
+  :strip_stopwords => true
+}], {:strip_stopwords => true})
+
+FuzzzyBenchmark.profile(:soundex, {
+  :query => 'eastleigh naersouthempton',
+  :distance => 4,
+  :strip_stopwords => true
+}, {:strip_stopwords => true}, 1000)

data/config.ru

@@ -0,0 +1,11 @@
+$LOAD_PATH.unshift File.expand_path("../..", __FILE__)
+require './lib/fuzzzy'
+require 'grape'
+
+Fuzzzy.redis = Redis.new(
+  :host => (ENV['REDIS_HOST'] || 'localhost'),
+  :port => (ENV['REDIS_PORT'] || 6379),
+  :database => (ENV['REDIS_DB'] || 0)
+)
+
+run Fuzzzy::Server::HTTP

data/dictionary/en_stopwords.yml

@@ -0,0 +1,175 @@
+--- 
+- a
+- about
+- above
+- after
+- again
+- against
+- all
+- am
+- an
+- and
+- any
+- are
+- aren't
+- as
+- at
+- be
+- because
+- been
+- before
+- being
+- below
+- between
+- both
+- but
+- by
+- can't
+- cannot
+- could
+- couldn't
+- did
+- didn't
+- do
+- does
+- doesn't
+- doing
+- don't
+- down
+- during
+- each
+- few
+- for
+- from
+- further
+- had
+- hadn't
+- has
+- hasn't
+- have
+- haven't
+- having
+- he
+- he'd
+- he'll
+- he's
+- her
+- here
+- here's
+- hers
+- herself
+- him
+- himself
+- his
+- how
+- how's
+- i
+- i'd
+- i'll
+- i'm
+- i've
+- if
+- in
+- into
+- is
+- isn't
+- it
+- it's
+- its
+- itself
+- let's
+- me
+- more
+- most
+- mustn't
+- my
+- myself
+- "no"
+- nor
+- not
+- of
+- "off"
+- "on"
+- once
+- only
+- or
+- other
+- ought
+- our
+- ours
+- ourselves
+- out
+- over
+- own
+- same
+- shan't
+- she
+- she'd
+- she'll
+- she's
+- should
+- shouldn't
+- so
+- some
+- such
+- than
+- that
+- that's
+- the
+- their
+- theirs
+- them
+- themselves
+- then
+- there
+- there's
+- these
+- they
+- they'd
+- they'll
+- they're
+- they've
+- this
+- those
+- through
+- to
+- too
+- under
+- until
+- up
+- very
+- was
+- wasn't
+- we
+- we'd
+- we'll
+- we're
+- we've
+- were
+- weren't
+- what
+- what's
+- when
+- when's
+- where
+- where's
+- which
+- while
+- who
+- who's
+- whom
+- why
+- why's
+- with
+- won't
+- would
+- wouldn't
+- you
+- you'd
+- you'll
+- you're
+- you've
+- your
+- yours
+- yourself
+- yourselves

data/fuzzzy.gemspec

@@ -0,0 +1,44 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "fuzzzy/version"
+
+Gem::Specification.new do |s|
+  s.name        = "fuzzzy"
+  s.version     = Fuzzzy::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Undr"]
+  s.email       = ["lilipoper@gmail.com"]
+  s.homepage    = "http://github.com/undr/fuzzzy"
+  s.summary     = %q{Fuzzy Search client and server}
+  s.description = %q{Fuzzy Search client and server}
+
+  s.rubyforge_project = "fuzzzy"
+  
+  s.add_development_dependency "rspec", ">= 2"
+  s.add_development_dependency "yard", "~> 0.6.0"
+  s.add_development_dependency "ruby-debug19"
+  s.add_development_dependency "bson_ext"
+  s.add_development_dependency "mongoid"
+  s.add_development_dependency "pry"
+  s.add_development_dependency 'ruby-prof'
+  
+  s.add_dependency "bundler"
+  s.add_dependency "rake"
+  s.add_dependency "activesupport"
+  s.add_dependency "eventmachine"
+  s.add_dependency "yajl-ruby"
+  s.add_dependency "levenshtein-ffi"
+  s.add_dependency "text"
+  s.add_dependency "grape"
+  s.add_dependency "hiredis"
+  s.add_dependency "redis"
+  s.add_dependency "daemons"
+  s.add_dependency "ZenTest", "4.5.0"
+  s.add_dependency 'RubyInline'
+  
+  
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/lib/fuzzzy.rb

@@ -0,0 +1,129 @@
+require 'rubygems'
+require 'bundler'
+Bundler.require :default, (ENV['RACK_ENV'] || 'development')
+require 'yaml'
+require 'logger'
+require 'active_support'
+require 'yajl'
+require 'levenshtein-ffi'
+require 'text'
+require 'redis/connection/hiredis'
+require 'redis'
+
+module Fuzzzy
+  extend self
+  extend ActiveSupport::Autoload
+
+  autoload :Redis
+  autoload :Index
+
+  autoload :Indexer, 'fuzzzy/methods/indexer'
+  autoload :MethodBase, 'fuzzzy/methods/method_base'
+
+  module Soundex
+    extend ActiveSupport::Autoload
+
+    autoload :Base, 'fuzzzy/methods/soundex/base'
+    autoload :Indexer, 'fuzzzy/methods/soundex/indexer'
+    autoload :Searcher, 'fuzzzy/methods/soundex/searcher'
+  end
+  
+  module Ngram
+    extend ActiveSupport::Autoload
+
+    autoload :Base, 'fuzzzy/methods/ngram/base'
+    autoload :Indexer, 'fuzzzy/methods/ngram/indexer'
+    autoload :Searcher, 'fuzzzy/methods/ngram/searcher'
+  end
+
+  module Server
+    extend ActiveSupport::Autoload
+
+    autoload :HTTP, 'fuzzzy/server/http'
+  end
+
+  if defined?(Mongoid)
+    module Mongoid
+      extend ActiveSupport::Autoload
+
+      autoload :Index, 'fuzzzy/orm/mongoid/index'
+    end
+  end
+  
+  # Fuzzzy.configure do |config|
+  #   config.logger = Logger.new($stdout)
+  #   config.redis = ::Redis.new(
+  #     :host => 'localhost',
+  #     :port => 6379,
+  #     :database => 0
+  #   )
+  #   config.stopwords = %w{the stopwords list}
+  # end
+  def configure
+    yield self
+  end
+
+  def logger
+    @logger = default_logger unless defined?(@logger)
+    @logger
+  end
+
+  def logger=(logger)
+    case logger
+      when Logger then @logger = logger
+      when false, nil then @logger = nil
+    end
+  end
+
+  def redis
+    @redis ||= ::Redis.new(
+      :host => 'localhost',
+      :port => 6379,
+      :database => 0
+    )
+  end
+  
+  def redis= connection
+    @redis = connection
+  end
+  
+  def stopwords
+    @stopwords ||= default_stopwords
+  end
+  
+  def stopwords= value
+    @stopwords = load_stopwords(value).uniq
+  end
+  
+  def load_stopwords options
+    if options.is_a?(Hash)
+      stops = load_stopwords(options[:stopwords])
+      options[:default] ? (stops + default_stopwords) : stops
+    elsif options.is_a?(Array)
+      options
+    elsif options.is_a?(String) || options.is_a?(Pathname)
+      YAML.load_file(options)
+    else
+      []
+    end
+  end
+  
+  def default_stopwords
+    @default_stopwords ||= load_stopwords(Fuzzzy.root.join('dictionary', 'en_stopwords.yml').to_s)
+  end
+  
+  def env
+    return Rails.env if defined?(Rails)
+    return Sinatra::Base.environment.to_s if defined?(Sinatra)
+    ENV["RACK_ENV"] || 'development'
+  end
+  
+  def root
+    @root ||= Pathname.new(File.expand_path('.'))
+  end
+  
+  protected
+  def default_logger
+    defined?(Rails) && Rails.respond_to?(:logger) ? Rails.logger : ::Logger.new($stdout)
+  end
+end