picky 0.0.0 → 0.0.2

data/lib/tasks/solr.rake

@@ -0,0 +1,36 @@
+# sunspot-solr start --solr-home=solr --data-directory=index/development/solr --pid-dir=solr/pids --log-file=log/solr.log
+
+namespace :solr do
+  
+  namespace :schema do
+    task :generate => :application do
+      generator = Solr::SchemaGenerator.new Indexes.configuration
+      generator.generate
+    end
+  end
+  
+  
+  task :index => :application do
+    Rake::Task[:"solr:start"].invoke
+    sleep 3
+    Indexes.index_solr
+  end
+  
+  
+  def action name
+    `sunspot-solr #{name} --solr-home=solr --data-directory=index/#{SEARCH_ENVIRONMENT}/solr --pid-dir=solr/pids --log-file=log/solr.log`
+  end
+  task :start => :application do
+    Rake::Task['solr:schema:generate'].invoke
+    action :start
+  end
+  task :stop => :application do
+    action :stop
+  end
+  task :restart => :application do
+    action :stop
+    sleep 2
+    action :start
+  end
+  
+end

data/lib/tasks/spec.rake

@@ -0,0 +1,11 @@
+require 'spec'
+require 'spec/rake/spectask'
+
+task :default => :spec
+
+desc "Run all specs in spec directory (excluding plugin specs)"
+Spec::Rake::SpecTask.new(:spec) do |t|
+  spec_root = File.join(File.dirname(__FILE__), '..', '..', 'spec')
+  t.spec_opts = ['--options', "\"#{File.join(spec_root, 'spec.opts')}\""]
+  t.spec_files = FileList[File.join(spec_root, '**', '*_spec.rb')]
+end

data/lib/tasks/statistics.rake

@@ -0,0 +1,13 @@
+namespace :statistics do
+
+  desc "start the server"
+  task :start => :application do
+    Statistics.start unless SEARCH_ENVIRONMENT == 'test'
+  end
+  
+  desc "stop the server"
+  task :stop => :application do
+    Statistics.stop unless SEARCH_ENVIRONMENT == 'test'
+  end
+  
+end

data/lib/tasks/try.rake

@@ -0,0 +1,29 @@
+# Tasks for manually testing your engine configuration.
+#
+namespace :try do
+  
+  desc "Try how a given word would be tokenized when indexing (type:field optional)."
+  task :index, [:text, :type_and_field] => :application do |_, options|
+    text, type_and_field = options.text, options.type_and_field
+    
+    tokenizer = type_and_field ? Indexes.find(*type_and_field.split(':')).tokenizer : Tokenizers::Index.new
+    
+    puts "\"#{text}\" is index tokenized as #{tokenizer.tokenize(text).to_a}"
+  end
+  
+  desc "Try how a given word would be tokenized when querying."
+  task :query, [:text] => :application do |_, options|
+    text = options.text
+    
+    puts "\"#{text}\" is query tokenized as #{Tokenizers::Query.new.tokenize(text).to_a}"
+  end
+  
+  desc "Try the given text with both the index and the query (type:field optional)."
+  task :both, [:text, :type_and_field] => :application do |_, options|
+    text, type_and_field = options.text, options.type_and_field
+    
+    Rake::Task[:"try:index"].invoke text, type_and_field
+    Rake::Task[:"try:query"].invoke text
+  end
+  
+end

data/prototype_project/Gemfile

@@ -0,0 +1,23 @@
+source :gemcutter
+
+# Gems required by Picky.
+#
+gem 'picky'
+gem 'bundler',          '>=0.9.26'
+gem 'rack',             '1.2.1'
+gem 'rack-mount',       '0.6.9'
+gem 'text',             '0.2.0'
+
+# Should be optional.
+#
+gem 'activesupport',    '2.3.8', :require => 'active_support'
+gem 'activerecord',     '2.3.8', :require => 'active_record'
+gem 'rsolr',            '0.12.1'
+gem 'sunspot',          '1.1.0'
+gem 'rack_fast_escape', '2009.06.24'
+gem 'rspec'
+
+# Required by your project.
+#
+gem 'unicorn'
+gem 'mysql'

data/prototype_project/Rakefile

@@ -0,0 +1 @@
+require 'picky-tasks'

data/prototype_project/app/README

@@ -0,0 +1,6 @@
+All your application definitions go here.
+
+/app needs a few files defined by picky:
+  * application.rb - how to handle indexing, queries, routing.
+  * application.ru - rackup.
+  * logging.rb

data/prototype_project/app/application.rb

@@ -0,0 +1,50 @@
+# encoding: utf-8
+#
+# This is your application.
+#
+# Have fun with Picky!
+#
+class PickySearch < Application # The App Constant needs to be identical in application.ru.
+  
+  # This is an example with books that you can adapt.
+  #
+  # Note: Much more is possible, but let's start out easy.
+  #
+  # Ask me if you have questions!
+  #
+  
+  indexes do
+    illegal_characters(/[^äöüa-zA-Z0-9\s\/\-\"\&\.]/)
+    stopwords(/\b(und|der|die|das|mit|im|ein|des|dem|the|of)\b/)
+    split_text_on(/[\s\/\-\"\&\.]/)
+      
+    type :books,
+         Sources::DB.new(
+           'SELECT id, title, author, isbn13 as isbn FROM books',
+           DB.configure(:file => 'app/db.yml')
+         ),
+         field(:title,  :qualifiers => [:t, :title, :titre], :similarity => Similarity::DoubleLevenshtone.new(3)), # Up to three similar title word indexed.
+         field(:author, :qualifiers => [:s, :author, :auteur]),
+         field(:isbn,   :qualifiers => [:i, :isbn])
+  end
+  
+  queries do
+    maximum_tokens 5
+    # Note that Picky needs the following characters to
+    # pass through, as they are control characters: *"~:
+    #
+    illegal_characters(/[^a-zA-Z0-9\s\/\-\,\&äöü\"\~\*\:]/)
+    stopwords(/\b(und|der|die|das|mit|ein|des|dem|the|of)\b/)
+    split_text_on(/[\s\/\-\,\&]+/)
+    
+    # Set some weights according to the position. Note that the order is important.
+    #
+    options = { :weights => Query::Weights.new([:title] => 6, [:author, :title] => 3) }
+    
+    route %r{^/books/full}, Query::Full.new(Indexes[:books], options)
+    route %r{^/books/live}, Query::Live.new(Indexes[:books], options)
+    
+    root 200
+  end
+  
+end

data/prototype_project/app/application.ru

@@ -0,0 +1,29 @@
+# This file has been generated by Picky.
+#
+
+# Require the gem. This loads the search framework.
+#
+require 'picky'
+
+# Load your application. This requires the files in lib TODO
+#
+Loader.load_application
+
+# Load the data. This loads data from cache files e.g. "some_index/*_index.dump" into constant Indexes::SomeIndex.
+#
+Indexes.load_from_cache
+
+# Use Harakiri middleware to kill unicorn child after X requests.
+#
+# See http://vimeo.com/12614970 for more info.
+#
+# Note: Comment this.
+#
+    Rack::Harakiri.after = 50
+use Rack::Harakiri
+
+# Start the application and start accepting requests.
+#
+# Note: Needs to be the same name as in application.rb.
+#
+run PickySearch

data/prototype_project/app/db.yml

@@ -0,0 +1,10 @@
+# Just an example, see application.rb on how this file is used.
+#
+# Note: ActiveRecord is used.
+#
+adapter: mysql
+host: localhost
+username: root
+password:
+database: table_with_search_data
+encoding: utf8

data/prototype_project/app/logging.rb

@@ -0,0 +1,20 @@
+# Standard logging.
+#
+require 'logger'
+PickyLog = Loggers::Search.new ::Logger.new(File.expand_path(File.join(SEARCH_ROOT, 'log/search.log')))
+
+# Example with using the syslog logger.
+# Falling back to the standard log if it isn't available.
+# (For example, because it is used locally and syslog is
+# only available on the servers)
+#
+# begin
+#   log_program_name = 'search/query'
+#   logger           = SyslogLogger.new log_program_name
+#   PickyLog        = Loggers::Search.new logger
+#   puts "Logging on syslog #{log_program_name}."
+# rescue StandardError
+#   puts "Could not connect to the syslog, using the normal log."
+#   require 'logger'
+#   PickyLog        = Loggers::Search.new ::Logger.new(File.join(SEARCH_ROOT, 'log/search.log'))
+# end

data/prototype_project/app/unicorn.ru

@@ -0,0 +1,10 @@
+worker_processes 2
+listen           '/tmp/unicorn.sock', :backlog => 1
+listen           9292, :tcp_nopush => true
+timeout          10
+pid              'tmp/pids/unicorn.pid'
+preload_app      true
+
+after_fork do |_, _|
+  GC.disable
+end

data/prototype_project/log/README

@@ -0,0 +1 @@
+Logs go here by default.

data/prototype_project/script/console

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+#
+irb = 'irb'
+
+require 'optparse'
+options = { :sandbox => false, :irb => irb }
+OptionParser.new do |opt|
+  opt.banner = "Usage: console [environment] [options]"
+  opt.on("--irb=[#{irb}]", 'Invoke a different irb.') { |v| options[:irb] = v }
+  opt.parse!(ARGV)
+end
+
+libs =  " -r irb/completion"
+libs << %( -r "picky" )
+
+ENV['SEARCH_ENV'] = case ARGV.first
+  when "p"; "production"
+  when "d"; "development"
+  when "t"; "test"
+  else
+    ARGV.first || ENV['SEARCH_ENV'] || 'development'
+end
+
+puts "Use \x1b[1;30mLoader.load_application\x1b[m to load app."
+puts "Use \x1b[1;30mIndexes.load_from_cache\x1b[m after that to load indexes."
+puts "Copy the following line to do just that:"
+puts "\x1b[1;30mLoader.load_application; Indexes.load_from_cache; nil\x1b[m"
+puts ""
+puts "Now you can for example create a query instance."
+puts "\x1b[1;30mfull_books = Query::Full.new(Indexes[:books]); nil\x1b[m"
+puts "and search on it"
+puts "\x1b[1;30mfull_books.search_with_text 'bla'\x1b[m"
+exec "#{options[:irb]} #{libs} --simple-prompt"

data/spec/ext/performant_spec.rb

@@ -0,0 +1,64 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe Performant::Array do
+
+  describe "memory_efficient_intersect" do
+    before(:each) do
+      GC.disable
+    end
+    after(:each) do
+      GC.enable
+      GC.start
+    end
+    it "should intersect empty arrays correctly" do
+      arys = [[3,4], [1,2,3], []]
+
+      Performant::Array.memory_efficient_intersect(arys.sort_by(&:size)).should == []
+    end
+    it "should handle intermediate empty results correctly" do
+      arys = [[5,4], [1,2,3], [3,4,5,8,9]]
+
+      Performant::Array.memory_efficient_intersect(arys.sort_by(&:size)).should == []
+    end
+    it "should intersect correctly" do
+      arys = [[3,4], [1,2,3], [3,4,5,8,9]]
+
+      Performant::Array.memory_efficient_intersect(arys.sort_by(&:size)).should == [3]
+    end
+    it "should intersect correctly again" do
+      arys = [[3,4,5,6,7], [1,2,3,5,6,7], [3,4,5,6,7,8,9]]
+
+      Performant::Array.memory_efficient_intersect(arys.sort_by(&:size)).should == [3,5,6,7]
+    end
+    it "should intersect many arrays" do
+      arys = [[3,4,5,6,7], [1,2,3,5,6,7], [3,4,5,6,7,8,9], [1,2,3,4,5,6,7,8,9,10], [2,3,5,6,7,19], [1,2,3,4,5,6,7,8,9,10], [2,3,5,6,7,19]]
+
+      Performant::Array.memory_efficient_intersect(arys.sort_by(&:size)).should == [3,5,6,7]
+    end
+    it "should handle random arrays" do
+      proto = Array.new(100, 3_500_000)
+      arys = [proto.map { |e| rand e }, proto.map { |e| rand e }, proto.map { |e| rand e }]
+
+      Performant::Array.memory_efficient_intersect(arys.sort_by(&:size)).should == arys.inject(arys.shift.dup) { |total, ary| total & arys }
+    end
+    it "should be optimal for 2 small arrays of 50/10_000" do
+      arys = [(1..50).to_a, (10_000..20_000).to_a << 7]
+
+      # brute force
+      Benchmark.realtime do
+        Performant::Array.memory_efficient_intersect(arys.sort_by(&:size))
+      end.should <= 0.001
+    end
+    it "should be optimal for 2 small arrays of 50/10_000" do
+      arys = [(1..50).to_a, (10_000..20_000).to_a << 7]
+
+      # &
+      Benchmark.realtime do
+        arys.inject(arys.shift.dup) do |total, ary|
+          total & arys
+        end
+      end.should <= 0.0015
+    end
+  end
+
+end

data/spec/lib/application_spec.rb

@@ -0,0 +1,61 @@
+# encoding: utf-8
+#
+require 'spec_helper'
+
+describe Application do
+  
+  describe 'routing' do
+    it 'should be there' do
+      lambda { Application.routing }.should_not raise_error
+    end
+    it "should return a new Routing instance" do
+      Application.routing.should be_kind_of(Routing)
+    end
+    it "should cache the instance" do
+      Application.routing.should == Application.routing
+    end
+  end
+  
+  describe 'call' do
+    before(:each) do
+      @routes = stub :routes
+      Application.stub! :routing => @routes
+    end
+    it 'should delegate' do
+      @routes.should_receive(:call).once.with :env
+      
+      Application.call :env
+    end
+  end
+  
+  describe "indexes" do
+    
+  end
+  describe "indexes_configuration" do
+    it 'should be there' do
+      lambda { Application.indexes_configuration }.should_not raise_error
+    end
+    it "should return a new Routing instance" do
+      Application.indexes_configuration.should be_kind_of(Configuration::Indexes)
+    end
+    it "should cache the instance" do
+      Application.indexes_configuration.should == Application.indexes_configuration
+    end
+  end
+  
+  describe "queries" do
+    
+  end
+  describe "queries_configuration" do
+    it 'should be there' do
+      lambda { Application.queries_configuration }.should_not raise_error
+    end
+    it "should return a new Routing instance" do
+      Application.queries_configuration.should be_kind_of(Configuration::Queries)
+    end
+    it "should cache the instance" do
+      Application.queries_configuration.should == Application.queries_configuration
+    end
+  end
+  
+end

data/spec/lib/cacher/partial/subtoken_spec.rb

@@ -0,0 +1,89 @@
+require 'spec_helper'
+
+describe Cacher::Partial::Subtoken do
+  
+  context 'default down_to' do
+    before(:each) do
+      @cacher = Cacher::Partial::Subtoken.new
+    end
+    describe 'down_to' do
+      it 'should return the right value' do
+        @cacher.down_to.should == 1
+      end
+    end
+    describe 'generate_from' do
+      it 'should generate the right index' do
+        @cacher.generate_from( :florian => [1], :flavia => [2] ).should == {
+          :florian => [1],
+          :floria => [1],
+          :flori => [1],
+          :flor => [1],
+          :flo => [1],
+          :fl => [1, 2],
+          :f => [1, 2], 
+          :flavia => [2],
+          :flavi => [2],
+          :flav => [2],
+          :fla => [2]
+        }
+      end
+    end
+  end
+  context 'down_to set' do
+    context 'default starting_at' do
+      before(:each) do
+        @cacher = Cacher::Partial::Subtoken.new :down_to => 4
+      end
+      describe 'starting_at' do
+        it 'should return the right value' do
+          @cacher.starting_at.should == 0
+        end
+      end
+      describe 'down_to' do
+        it 'should return the right value' do
+          @cacher.down_to.should == 4
+        end
+      end
+      describe 'generate_from' do
+        it 'should generate the right index' do
+          @cacher.generate_from( :florian => [1], :flavia => [2] ).should == {
+            :florian => [1],
+            :floria => [1],
+            :flori => [1],
+            :flor => [1],
+            :flavia => [2],
+            :flavi => [2],
+            :flav => [2]
+          }
+        end
+      end
+    end
+    context 'starting_at -1' do
+      before(:each) do
+        @cacher = Cacher::Partial::Subtoken.new :down_to => 4, :starting_at => -1
+      end
+      describe 'starting_at' do
+        it 'should return the right value' do
+          @cacher.starting_at.should == -2
+        end
+      end
+      describe 'down_to' do
+        it 'should return the right value' do
+          @cacher.down_to.should == 4
+        end
+      end
+      describe 'generate_from' do
+        it 'should generate the right index' do
+          @cacher.generate_from( :florian => [1], :flavia => [2] ).should == {
+            :floria => [1],
+            :flori => [1],
+            :flor => [1],
+            :flavi => [2],
+            :flav => [2]
+          }
+        end
+      end
+    end
+  end
+  
+end

data/spec/lib/cacher/partial_generator_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+
+describe Cacher::PartialGenerator do
+
+  context 'integration' do
+    it 'should generate the correct values with a given strategy' do
+      generator = Cacher::PartialGenerator.new :meier => [1], :maier => [2]
+      
+      generator.generate(Cacher::Partial::Subtoken.new).should == {
+        :meier => [1],
+        :meie => [1],
+        :mei => [1],
+        :me => [1],
+        :m => [1, 2],
+        :maier => [2],
+        :maie => [2],
+        :mai => [2],
+        :ma => [2]
+      }
+    end
+    it 'should generate the correct values with a given specific strategy' do
+      generator = Cacher::PartialGenerator.new :meier => [1], :maier => [2]
+      
+      generator.generate(Cacher::Partial::Subtoken.new(:down_to => 3)).should == {
+        :meier => [1],
+        :meie => [1],
+        :mei => [1],
+        :maier => [2],
+        :maie => [2],
+        :mai => [2]
+      }
+    end
+  end
+
+end

data/spec/lib/cacher/similarity/double_levenshtone_spec.rb

@@ -0,0 +1,60 @@
+# encoding: utf-8
+#
+require 'spec_helper'
+
+describe Cacher::Similarity::DoubleLevenshtone do
+
+  before(:each) do
+    @similarity = Cacher::Similarity::DoubleLevenshtone.new
+  end
+
+  def self.it_should_encode text, expected
+    it "should encode #{text.inspect} correctly" do
+      @similarity.encoded(text).should == expected
+    end
+  end
+  def self.it_should_generate_from index, expected
+    it "should generate #{expected.inspect} correctly from #{index.inspect}" do
+      @similarity.generate_from(index).should == expected
+    end
+  end
+
+  it_should_encode :meier,       :MR
+  it_should_encode :grossberger, :KRSP
+  it_should_encode :hadelbla,    :HTLP
+
+  it_should_generate_from({}, {})
+  it_should_generate_from({ :maier => nil, :meier => nil }, :MR => [:maier, :meier]) # should be correctly ordered
+  it_should_generate_from({ :maier => nil, :meier => nil, :hallaballa => nil }, :MR => [:maier, :meier], :HLPL => [:hallaballa])
+  it_should_generate_from({ :susan => nil, :susanne => nil, :bruderer => nil }, :SSN => [:susan, :susanne], :PRTR => [:bruderer])
+
+  describe 'with reduced amount' do
+    before(:each) do
+      @similarity = Cacher::Similarity::DoubleLevenshtone.new(1)
+    end
+    it_should_generate_from({ :maier => nil, :meier => nil }, :MR => [:maier])
+    it_should_generate_from({ :susan => nil, :susanne => nil, :bruderer => nil }, :SSN => [:susan], :PRTR => [:bruderer])
+  end
+
+  describe 'hashify' do
+    it 'should turn an empty list into an empty hash' do
+      @similarity.send(:hashify, []).should == {}
+    end
+    it 'should turn the list into an unordered similarity' do
+      @similarity.send(:hashify, [:meier, :maier]).should == { :MR => [:meier, :maier] }
+    end
+    it 'should turn the list into a encoded hash' do
+      @similarity.send(:hashify, [:meier, :maier, :peter]).should == { :MR => [:meier, :maier], :PTR => [:peter] }
+    end
+  end
+
+  context 'integration' do
+    it 'should return the right ordered array' do
+      index = @similarity.generate_from :meier => nil, :maier => nil, :mairai => nil, :mair => nil, :meira => nil
+      code  = @similarity.encoded :maier
+
+      index[code].should == [:mair, :maier, :meier, :meira, :mairai]
+    end
+  end
+
+end

data/spec/lib/cacher/similarity/none_spec.rb

@@ -0,0 +1,23 @@
+# encoding: utf-8
+#
+require 'spec_helper'
+
+describe Cacher::Similarity::None do
+
+  before(:each) do
+    @similarity = Cacher::Similarity::None.new
+  end
+
+  describe 'encode' do
+    it 'should always return nil' do
+      @similarity.encoded(:whatever).should == nil
+    end
+  end
+
+  describe 'generate_from' do
+    it 'should return an empty hash, always' do
+      @similarity.generate_from(:anything).should == {}
+    end
+  end
+
+end

data/spec/lib/cacher/similarity_generator_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+describe Cacher::SimilarityGenerator do
+
+  context 'integration' do
+    it 'should generate the correct values' do
+      generator = Cacher::SimilarityGenerator.new :anything_really
+
+      generator.generate.should == {}
+    end
+    it 'should generate the correct values with a given strategy' do
+      generator = Cacher::SimilarityGenerator.new :meier => nil,
+                                                  :maier => nil,
+                                                  :mayer => nil,
+                                                  :meyer => nil,
+                                                  :peter => nil
+
+      generator.generate(Cacher::Similarity::DoubleLevenshtone.new).should == { :MR => [:meier, :maier, :mayer, :meyer], :PTR => [:peter] }
+    end
+  end
+
+end