picky 1.5.2 → 1.5.3

data/lib/tasks/index.rake

@@ -1,7 +1,7 @@
-# Global.
+# Indexing tasks.
 #
 namespace :index do
-  
+
   desc "Takes a snapshot, indexes, and caches in random order."
   task :randomly => :application do
     Indexes.index true
@@ -10,23 +10,21 @@ namespace :index do
   task :ordered => :application do
     Indexes.index false
   end
-  
+
   # desc "Generates the index snapshots."
+  #
+  # Note: Hidden since it is only needed by pro users.
+  #
+  # desc "Generate the data snapshots (intermediate table on a DB source)"
   task :generate_snapshots => :application do
     Indexes.take_snapshot
   end
-  
+
   desc "Generates a specific index from index snapshots (category optional)."
   task :specific, [:index, :category] => :application do |_, options|
     index, category = options.index, options.category
     Indexes.generate_index_only index.to_sym, category && category.to_sym
     Indexes.generate_cache_only index.to_sym, category && category.to_sym
   end
-  
-  desc 'Checks the index files for files that are suspiciously small or missing.'
-  task :check => :application do
-    Indexes.check_caches
-    puts "All indexes look ok."
-  end
-  
+
 end

data/lib/tasks/routes.rake

@@ -1,7 +1,8 @@
+# Routing tasks.
+#
 desc "Shows the available URL paths"
 task :routes => :application do
-  puts "Note: Anchored (\u2713) regexps are faster, e.g. /\\A.*\\Z/ or /^.*$/.\n\n"
   Application.apps.each do |app|
-    p app
+    puts app.to_routes
   end
 end

data/lib/tasks/shortcuts.rake

@@ -1,13 +1,25 @@
+# Shortcut tasks.
+#
+
 desc "Generate the index (random order)."
 task :index => :application do
   Rake::Task[:'index:randomly'].invoke
 end
 
-desc "Try the given text in the indexer/query (index:category optional)."
-task :try, [:text, :index_and_category] => :application do |_, options|
-  text, index_and_category = options.text, options.index_and_category
-  
-  Rake::Task[:'try:both'].invoke text, index_and_category
+desc "Try the given text in the indexer/query (index and category optional)."
+task :try, [:text, :index, :category] => :application do |_, options|
+  text, index, category = options.text, options.index, options.category
+
+  Rake::Task[:'try:both'].invoke text, index, category
+end
+
+desc "Application summary."
+task :stats do
+  Rake::Task[:'stats:app'].invoke
+end
+desc "Analyze your indexes (needs rake index)."
+task :analyze do
+  Rake::Task[:'stats:analyze'].invoke
 end
 
 desc "Start the server."

data/lib/tasks/statistics.rake

@@ -1,13 +1,21 @@
-namespace :statistics do
-
-  # desc "start the server"
-  # task :start => :application do
-  #   Statistics.start unless PICKY_ENVIRONMENT == 'test'
-  # end
-  
-  # desc "stop the server"
-  # task :stop => :application do
-  #   Statistics.stop unless PICKY_ENVIRONMENT == 'test'
-  # end
-  
+# Statistics tasks.
+#
+namespace :stats do
+  task :prepare => :application do
+    require File.expand_path('../../picky/statistics', __FILE__)
+    statistics = Statistics.instance
+  end
+  task :app => :prepare do
+    Statistics.instance.application
+    puts Statistics.instance
+  end
+  task :analyze => :prepare do
+    begin
+      Statistics.instance.analyze
+    rescue StandardError
+      puts "\n\033[31mNote: rake analyze needs prepared indexes. Run rake index first.\033[m\n\n"
+      raise
+    end
+    puts Statistics.instance
+  end
 end

data/lib/tasks/try.rake

@@ -1,29 +1,29 @@
 # Tasks for manually testing your engine configuration.
 #
 namespace :try do
-  
+
   # desc "Try how a given word would be tokenized when indexing (type:category optional)."
-  task :index, [:text, :index_and_category] => :application do |_, options|
-    text, index_and_category = options.text, options.index_and_category
-    
-    tokenizer = index_and_category ? Indexes.find(*index_and_category.split(':')).tokenizer : Internals::Tokenizers::Index.default
-    
+  task :index, [:text, :index, :category] => :application do |_, options|
+    text, index, category = options.text, options.index, options.category
+
+    tokenizer = index && category ? Indexes.find(index, category).tokenizer : Internals::Tokenizers::Index.default
+
     puts "\"#{text}\" is saved in the index as             #{tokenizer.tokenize(text.dup).to_a}"
   end
-  
+
   # desc "Try how a given word would be tokenized when querying."
   task :query, [:text] => :application do |_, options|
     text = options.text
-    
+
     puts "\"#{text}\" as a query will be preprocessed into #{Internals::Tokenizers::Query.default.tokenize(text.dup).to_a.map(&:to_s).map(&:to_sym)}"
   end
-  
+
   # desc "Try the given text with both the index and the query (type:category optional)."
-  task :both, [:text, :index_and_category] => :application do |_, options|
-    text, index_and_category = options.text, options.index_and_category
-    
-    Rake::Task[:"try:index"].invoke text, index_and_category
+  task :both, [:text, :index, :category] => :application do |_, options|
+    text, index, category = options.text, options.index, options.category
+
+    Rake::Task[:"try:index"].invoke text, index, category
     Rake::Task[:"try:query"].invoke text
   end
-  
+
 end

data/spec/lib/application_spec.rb

@@ -8,7 +8,7 @@ describe Application do
     it "should run ok" do
       lambda {
         class MinimalTestApplication < Application
-          books = index :books, Sources::DB.new('SELECT id, title FROM books', :file => 'app/db.yml')
+          books = Index::Memory.new :books, Sources::DB.new('SELECT id, title FROM books', :file => 'app/db.yml')
           books.define_category :title
           
           full = Query::Full.new books
@@ -43,7 +43,7 @@ describe Application do
                            substitutes_characters_with: CharacterSubstituters::WestEuropean.new,
                            maximum_tokens: 5
           
-          books_index = index :books,
+          books_index = Index::Memory.new :books,
                               Sources::DB.new('SELECT id, title, author, isbn13 as isbn FROM books', :file => 'app/db.yml')
           books_index.define_category :title,
                                       similarity: Similarity::DoubleLevenshtone.new(3) # Up to three similar title word indexed.
@@ -51,7 +51,7 @@ describe Application do
           books_index.define_category :isbn,
                                       partial: Partial::None.new # Partially searching on an ISBN makes not much sense.
           
-          geo_index = index :geo, Sources::CSV.new(:location, :north, :east, file: 'data/ch.csv', col_sep: ',')
+          geo_index = Index::Memory.new :geo, Sources::CSV.new(:location, :north, :east, file: 'data/ch.csv', col_sep: ',')
           geo_index.define_category :location
           geo_index.define_ranged_category(:north1, 1, precision: 3, from: :north)
                    .define_ranged_category(:east1,  1, precision: 3, from: :east)

data/spec/lib/index/base_spec.rb

@@ -4,22 +4,44 @@ require 'spec_helper'
 
 describe Index::Base do
   
+  let(:some_source) { stub :source, :harvest => nil }
+  
   context 'initializer' do
     it 'works' do
-      lambda { described_class.new :some_index_name, :some_source }.should_not raise_error
+      expect { described_class.new :some_index_name, some_source }.to_not raise_error
+    end
+    it 'fails correctly' do
+      expect { described_class.new 0, some_source }.to raise_error(<<-ERROR
+The index identifier (you gave "0") for Index::Memory/Index::Redis should be a String/Symbol,
+Examples:
+  Index::Memory.new(:my_cool_index, ...) # Recommended
+  Index::Redis.new("a-redis-index", ...)
+ERROR
+)
+    end
+    it 'fails correctly' do
+      expect { described_class.new :some_index_name, :some_source }.to raise_error(<<-ERROR
+The index "some_index_name" should use a data source that responds to the method #harvest, which yields(id, text).
+Or it could use one of the built-in sources:
+  Sources::DB,
+  Sources::CSV,
+  Sources::Delicious,
+  Sources::Couch
+ERROR
+)
     end
     it 'registers with the indexes' do
       @api = described_class.allocate
       
       ::Indexes.should_receive(:register).once.with @api
       
-      @api.send :initialize, :some_index_name, :some_source
+      @api.send :initialize, :some_index_name, some_source
     end
   end
   
   context 'unit' do
     before(:each) do
-      @api = described_class.new :some_index_name, :some_source
+      @api = described_class.new :some_index_name, some_source
     end
 
     describe 'define_category' do

data/spec/lib/internals/extensions/object_spec.rb

@@ -2,32 +2,58 @@ require 'spec_helper'
 
 describe Object do
   
-  before(:each) do
-    @object = Object.new
-  end
-  
-  describe "exclaim" do
-    it "delegates to puts" do
-      @object.should_receive(:puts).once.with :bla
-      
-      @object.exclaim :bla
+  context 'basic object' do
+    let(:object) { described_class.new }
+    
+    describe "exclaim" do
+      it "delegates to puts" do
+        object.should_receive(:puts).once.with :bla
+
+        object.exclaim :bla
+      end
+    end
+
+    describe "timed_exclaim" do
+      it "should exclaim right" do
+        Time.stub! :now => Time.parse('07-03-1977 12:34:56')
+        object.should_receive(:exclaim).once.with "12:34:56: bla"
+
+        object.timed_exclaim 'bla'
+      end
+    end
+
+    describe 'warn_gem_missing' do
+      it 'should warn right' do
+        object.should_receive(:warn).once.with "gnorf gem missing!\nTo use gnarble gnarf, you need to:\n  1. Add the following line to Gemfile:\n     gem 'gnorf'\n  2. Then, run:\n     bundle update\n"
+
+        object.warn_gem_missing 'gnorf', 'gnarble gnarf'
+      end
     end
   end
   
-  describe "timed_exclaim" do
-    it "should exclaim right" do
-      Time.stub! :now => Time.parse('07-03-1977 12:34:56')
-      @object.should_receive(:exclaim).once.with "12:34:56: bla"
+  describe 'indented_to_s' do
+    describe String do
+      let(:object) { described_class.new("Hello\nTest") }
       
-      @object.timed_exclaim 'bla'
+      it 'indents a default amount' do
+        object.indented_to_s.should == "  Hello\n  Test"
+      end
+      it 'indents twice' do
+        object.indented_to_s.indented_to_s.should == "    Hello\n    Test"
+      end
+      it 'indents correctly' do
+        object.indented_to_s(3).should == "   Hello\n   Test"
+      end
     end
-  end
-  
-  describe 'puts_gem_missing' do
-    it 'should puts right' do
-      @object.should_receive(:puts).once.with "gnorf gem missing!\nTo use gnarble gnarf, you need to:\n  1. Add the following line to Gemfile:\n     gem 'gnorf'\n  2. Then, run:\n     bundle update\n"
+    describe Array do
+      let(:object) { described_class.new(["Hello", "Test"]) }
       
-      @object.puts_gem_missing 'gnorf', 'gnarble gnarf'
+      it 'indents a default amount' do
+        object.indented_to_s.should == "  Hello\n  Test"
+      end
+      it 'indents twice' do
+        object.indented_to_s.indented_to_s.should == "    Hello\n    Test"
+      end
     end
   end
   

data/spec/lib/internals/frontend_adapters/rack_spec.rb

@@ -192,17 +192,17 @@ describe Internals::FrontendAdapters::Rack do
         Internals::Adapters::Rack.stub! :app_for => :some_query_app
       end
       it 'should add the right route' do
-        @routes.should_receive(:add_route).once.with :some_query_app, { :request_method => "GET", :path_info => /some_url/ }
+        @routes.should_receive(:add_route).once.with :some_query_app, { :request_method => "GET", :path_info => /some_url/ }, {}, "some_query"
         
         @rack_adapter.route_one %r{some_url}, :some_query, {}
       end
       it 'should add the right route' do
-        @routes.should_receive(:add_route).once.with :some_query_app, { :request_method => "GET", :path_info => /some_url/ }
+        @routes.should_receive(:add_route).once.with :some_query_app, { :request_method => "GET", :path_info => /some_url/ }, {}, "some_query"
         
         @rack_adapter.route_one 'some_url', :some_query, {}
       end
       it 'should add the right route' do
-        @routes.should_receive(:add_route).once.with :some_query_app, { :request_method => "GET", :glarf => :blarf, :path_info => /some_url/ }
+        @routes.should_receive(:add_route).once.with :some_query_app, { :request_method => "GET", :glarf => :blarf, :path_info => /some_url/ }, {}, "some_query"
         
         @rack_adapter.route_one 'some_url', :some_query, { :glarf => :blarf }
       end

data/spec/lib/internals/index/redis/basic_spec.rb

@@ -0,0 +1,67 @@
+require 'spec_helper'
+
+describe Internals::Index::Redis::Basic do
+  
+  let(:redis) { described_class.new "some_namespace" }
+
+  describe 'load, retrieve, backup, delete' do
+    it 'is nothing they do (at least on the backend)' do
+      redis.should_receive(:backend).never
+      
+      redis.load
+      redis.retrieve
+      redis.backup
+      redis.delete
+    end
+  end
+  
+  describe 'cache_small?' do
+    context 'size 0' do
+      before(:each) do
+        redis.stub! :size => 0
+      end
+      it 'is small' do
+        redis.cache_small?.should == true
+      end
+    end
+    context 'size 1' do
+      before(:each) do
+        redis.stub! :size => 1
+      end
+      it 'is not small' do
+        redis.cache_small?.should == false
+      end
+    end
+  end
+  
+  describe 'cache_ok?' do
+    context 'size 0' do
+      before(:each) do
+        redis.stub! :size => 0
+      end
+      it 'is not ok' do
+        redis.cache_ok?.should == false
+      end
+    end
+    context 'size 1' do
+      before(:each) do
+        redis.stub! :size => 1
+      end
+      it 'is ok' do
+        redis.cache_ok?.should == true
+      end
+    end
+  end
+  
+  describe "size" do
+    it 'delegates to the backend' do
+      backend = stub :backend
+      redis.stub! :backend => backend
+      
+      backend.should_receive(:dbsize).once.with
+      
+      redis.size
+    end
+  end
+  
+end

data/spec/lib/internals/indexers/serial_spec.rb

@@ -40,7 +40,7 @@ describe Indexers::Serial do
   
   describe "indexing_message" do
     it "informs the user about what it is going to index" do
-      @indexer.should_receive(:timed_exclaim).once.with 'INDEX some_index some_category'
+      @indexer.should_receive(:timed_exclaim).once.with '"some_index:some_category": Starting indexing.'
       
       @indexer.indexing_message
     end

data/spec/lib/internals/results/base_spec.rb

@@ -26,18 +26,6 @@ describe Internals::Results::Base do
     end
   end
   
-  describe "random_ids" do
-    before(:each) do
-      @allocations = stub :allocations
-      @results = described_class.new :unimportant, @allocations
-    end
-    it "delegates" do
-      @allocations.should_receive(:random_ids).once.with :anything
-      
-      @results.random_ids :anything
-    end
-  end
-  
   describe 'to_log' do
     before(:each) do
       time = stub :time, :to_s => '0-08-16 10:07:33'

data/spec/lib/internals/tokenizers/base_spec.rb

@@ -12,11 +12,38 @@ describe Internals::Tokenizers::Base do
     it 'rejects tokens that are called :hello' do
       tokenizer.reject([:hel, :hell, :hello]).should == [:hel, :hell]
     end
+    describe 'to_s' do
+      it 'does something' do
+        tokenizer.to_s.should == <<-EXPECTED
+Removes characters:        -
+Stopwords:                 -
+Splits text on:            /\\s/
+Removes chars after split: -
+Normalizes words:          -
+Rejects tokens?            Yes, see line 8 in app/application.rb
+Substitutes chars?         -
+EXPECTED
+      end
+    end
   end
   
   context 'with normal instance' do
     let(:tokenizer) { described_class.new }
-
+    
+        describe 'to_s' do
+          it 'does something' do
+            tokenizer.to_s.should == <<-EXPECTED
+Removes characters:        -
+Stopwords:                 -
+Splits text on:            /\\s/
+Removes chars after split: -
+Normalizes words:          -
+Rejects tokens?            -
+Substitutes chars?         -
+EXPECTED
+          end
+        end
+    
     describe 'reject_token_if' do
       it 'rejects empty tokens by default' do
         tokenizer.reject(['a', nil, '', 'b']).should == ['a', 'b']
@@ -32,6 +59,9 @@ describe Internals::Tokenizers::Base do
       it "doesn't substitute if there is no substituter" do
         tokenizer.substitute_characters('abcdefghijklmnopqrstuvwxyzäöü').should == 'abcdefghijklmnopqrstuvwxyzäöü'
       end
+      it 'raises if nothing with #substitute is given' do
+        expect { tokenizer.substitutes_characters_with Object.new }.to raise_error("The substitutes_characters_with option needs a character substituter, which responds to #substitute.")
+      end
       it "uses the substituter to replace characters" do
         tokenizer.substitutes_characters_with CharacterSubstituters::WestEuropean.new
 
@@ -45,6 +75,9 @@ describe Internals::Tokenizers::Base do
     end
 
     describe "removes_characters_after_splitting" do
+      it 'handles broken arguments' do
+        expect { tokenizer.removes_characters_after_splitting("gnorf") }.to raise_error(ArgumentError)
+      end
       context "without removes_characters_after_splitting called" do
         it "has remove_after_normalizing_illegals" do
           expect { tokenizer.remove_after_normalizing_illegals('any') }.to_not raise_error
@@ -69,6 +102,9 @@ describe Internals::Tokenizers::Base do
     end
 
     describe "normalizes_words" do
+      it 'handles broken arguments' do
+        expect { tokenizer.normalizes_words(:not_an_array) }.to raise_error(ArgumentError)
+      end
       context "without normalizes_words called" do
         it "has normalize_with_patterns" do
           expect { tokenizer.normalize_with_patterns('any') }.to_not raise_error
@@ -96,6 +132,12 @@ describe Internals::Tokenizers::Base do
     end
 
     describe "splits_text_on" do
+      it 'handles nonbroken arguments' do
+        expect { tokenizer.splits_text_on("hello") }.to_not raise_error(ArgumentError)
+      end
+      it 'handles broken arguments' do
+        expect { tokenizer.splits_text_on(:gnorf) }.to raise_error(ArgumentError)
+      end
       context "without splits_text_on called" do
         it "has split" do
           lambda { tokenizer.split('any') }.should_not raise_error
@@ -121,6 +163,9 @@ describe Internals::Tokenizers::Base do
     end
 
     describe "removes_characters" do
+      it 'handles broken arguments' do
+        expect { tokenizer.removes_characters("hello") }.to raise_error(ArgumentError)
+      end
       context "without removes_characters called" do
         it "has remove_illegals" do
           expect { tokenizer.remove_illegals('any') }.to_not raise_error
@@ -145,6 +190,9 @@ describe Internals::Tokenizers::Base do
     end
 
     describe 'stopwords' do
+      it 'handles broken arguments' do
+        expect { tokenizer.stopwords("hello") }.to raise_error(ArgumentError)
+      end
       context 'without stopwords given' do
         it 'should define a method remove_stopwords' do
           lambda { tokenizer.remove_stopwords('from this text') }.should_not raise_error

data/spec/lib/query/allocations_spec.rb

@@ -179,62 +179,6 @@ describe Internals::Query::Allocations do
     end
   end
 
-  describe 'random_ids' do
-    context 'enough ids' do
-      before(:each) do
-        @allocation1 = stub :allocation1, :ids => [1, 2, 3]
-        @allocation2 = stub :allocation2, :ids => [4, 5, 6, 7]
-        @allocations = described_class.new [@allocation1, @allocation2]
-      end
-      it 'should return one random id from the first allocations by default' do
-        @allocations.random_ids.size.should == 1
-      end
-      it 'should return multiple random ids from the first allocation' do
-        @allocations.random_ids(5).size.should == 3
-      end
-      it "should return one random id from the first allocation's ids" do
-        (1..7).to_a.should include(@allocations.random_ids.first)
-      end
-      it 'should not contain the same id twice' do
-        20.times do
-          @allocations.random_ids(2).uniq.size.should_not == 1
-        end
-      end
-    end
-    context 'just one id' do
-      before(:each) do
-        @allocation1 = stub :allocation1, :ids => [1]
-        @allocation2 = stub :allocation2, :ids => []
-        @allocations = described_class.new [@allocation1, @allocation2]
-      end
-      it 'should return one random id from its allocations by default' do
-        @allocations.random_ids.size.should == 1
-      end
-      it 'should return multiple random ids from its allocations' do
-        @allocations.random_ids(5).size.should == 1
-      end
-      it 'should return one random id from the allocations ids' do
-        @allocations.random_ids.should == [1]
-      end
-    end
-    context 'no id' do
-      before(:each) do
-        @allocation1 = stub :allocation1, :ids => []
-        @allocation2 = stub :allocation2, :ids => []
-        @allocations = described_class.new [@allocation1, @allocation2]
-      end
-      it 'should return one random id from its allocations by default' do
-        @allocations.random_ids.size.should == 0
-      end
-      it 'should return no random id from its allocations' do
-        @allocations.random_ids(5).size.should == 0
-      end
-      it 'should return no random id from the allocations ids' do
-        @allocations.random_ids.should == []
-      end
-    end
-  end
-
   describe 'to_result' do
     context 'all allocations have results' do
       before(:each) do