picky 3.6.16 → 4.0.0pre1

data/lib/picky/results/exact_first.rb

@@ -0,0 +1,53 @@
+# encoding: utf-8
+#
+module Picky
+
+  class Results
+
+    # This index combines an exact and partial index.
+    # It serves to order the results such that exact hits are found first.
+    #
+    module ExactFirst
+
+      # Installs the exact first on the given category
+      # or on the categories of the index, if an index is given.
+      #
+      # THINK Can we unextend in the case it is an index?
+      #
+      def self.extended index_or_category
+        if index_or_category.respond_to? :categories
+          extend_each_of index_or_category.categories
+          index_or_category
+        end
+      end
+      def self.extend_each_of categories
+        categories.categories.each { |category| category.extend self }
+      end
+
+      # Overrides the original method.
+      #
+      def ids token
+        text = token.text
+        if token.partial?
+          exact.ids(text) | partial.ids(text)
+        else
+          exact.ids text
+        end
+      end
+
+      # Overrides the original method.
+      #
+      def weight token
+        text = token.text
+        if token.partial?
+          [exact.weight(text), partial.weight(text)].compact.max
+        else
+          exact.weight text
+        end
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/scheduler.rb

@@ -0,0 +1,43 @@
+module Picky
+
+  class Scheduler
+
+    attr_reader :parallel
+
+    def initialize options = {}
+      @parallel = options[:parallel]
+
+      configure
+    end
+
+    def configure
+      if fork?
+        def schedule &block
+          scheduler.schedule &block
+        end
+
+        def finish
+          scheduler.join
+        end
+
+        def scheduler
+          @scheduler ||= Procrastinate::Scheduler.start Procrastinate::SpawnStrategy::Default.new(2)
+        end
+      else
+        def schedule
+          yield
+        end
+
+        def finish
+          # Don't do anything.
+        end
+      end
+    end
+
+    def fork?
+      parallel && Process.respond_to?(:fork)
+    end
+
+  end
+
+end

data/lib/picky/search.rb

@@ -27,8 +27,6 @@ module Picky
     # Takes:
     # * A number of indexes
     #
-    # TODO Add reduce_allocations_to_amount (rename).
-    #
     # It is also possible to define the tokenizer and weights like so.
     # Example:
     #   search = Search.new(index1, index2, index3) do

data/lib/picky/sources/csv.rb

@@ -62,9 +62,8 @@ module Picky
       #
       def harvest category
         index = category_names.index category.from
-        get_data do |ary|
-          indexed_id = ary.shift
-          text       = ary[index]
+        get_data do |indexed_id, *ary|
+          text = ary[index]
           next unless text
           text.force_encoding 'utf-8' # TODO Still needed?
           yield indexed_id, text

data/lib/picky/sources/db.rb

@@ -56,8 +56,9 @@ module Picky
       end
 
       # Creates a database adapter for use with this source.
+      #
       def create_database_adapter # :nodoc:
-        # TODO Do not use ActiveRecord directly. Use set_table_name etc.
+        # THINK Do not use ActiveRecord directly? Use set_table_name etc?
         #
         adapter_class = Class.new ActiveRecord::Base
         adapter_class.abstract_class = true
@@ -96,7 +97,7 @@ module Picky
       # Uses CREATE TABLE AS with the given SELECT statement to create a snapshot of the data.
       #
       def take_snapshot index
-        timed_exclaim %Q{"#{index.identifier}": Taking snapshot of database data.}
+        timed_exclaim %Q{  "#{index.identifier}": Taking snapshot of database data.}
 
         origin = snapshot_table_name index.name
         on_database = database.connection
@@ -143,7 +144,7 @@ module Picky
       def get_data category, offset, &block # :nodoc:
         select_statement = harvest_statement_with_offset category, offset
 
-        # TODO Rewrite ASAP.
+        # THINK Not really nice like this. Rewrite if possible.
         #
         if database.connection.adapter_name == "PostgreSQL"
           id_key   = 'id'

data/lib/picky/sources/mongo.rb

@@ -61,7 +61,7 @@ module Picky
   			JSON.parse(resp)['rows'].each do |row|
   			  text = row[collection].to_s
   			  next unless text
-  				index_key = row.delete(@@id_key) # TODO Still works, I removed .values
+  				index_key = row.delete @@id_key # Note: I removed .values
   				yield index_key, text
   			end
   		end

data/lib/picky/tokenizer.rb

@@ -20,10 +20,6 @@ module Picky
       @query_default ||= new
     end
 
-    # TODO Move EMPTY_STRING top level.
-    #
-    EMPTY_STRING = ''.freeze
-
     def to_s
       reject_condition_location = @reject_condition.to_s[/:(\d+) \(lambda\)/, 1]
       <<-TOKENIZER

data/lib/picky/wrappers/bundle/location.rb

@@ -52,7 +52,7 @@ module Picky
             bundle.load
 
             # TODO Symbols (of the location_anchor!!!).
-            #      It should always be a Symbol.
+            # It should always be a Symbol.
             #
             @calculation.anchor = bundle['location_anchor'] && bundle['location_anchor'].to_f || raise("Configuration 'location_anchor' for #{bundle.identifier} missing. Did you run rake index already?")
           end

data/lib/picky.rb

@@ -4,9 +4,10 @@ module Picky
   #
   require 'active_support/core_ext'
   require 'text'
-  require 'yajl' # TODO Maybe replace by multi_json.
+  require 'yajl' # THINK Maybe replace by multi_json?
   require 'rack' # TODO Remove.
   require 'rack_fast_escape' # TODO Remove.
+  require 'procrastinate'
 
   # Require the constants.
   #
@@ -19,7 +20,6 @@ module Picky
   # Load the framework
   #
   Loader.load_framework
-  puts "Loaded picky with environment '#{PICKY_ENVIRONMENT}' in #{PICKY_ROOT} on Ruby #{RUBY_VERSION}."
 
   # Check if delegators need to be installed.
   #

data/lib/tasks/index.rake

@@ -1,22 +1,21 @@
 # Indexing tasks.
 #
-desc "Generate the index (index, category optional)."
-task :index, [:index, :category] => :application do |_, options|
-  index, category = options.index, options.category
-
-  specific = Picky::Indexes
-  specific = specific[index]    if index
-  specific = specific[category] if category
-  specific.index
-end
+desc "Generate the index in parallel (index, category optional)."
+task :index, [:index, :category] => :'index:parallel'
 
 namespace :index do
 
-  task :randomly => :application do
-    Picky::Indexes.index true
-  end
-  task :ordered => :application do
-    Picky::Indexes.index false
+  [:parallel, :serial].each do |kind|
+    desc "Generate the index in #{kind} (index, category optional)."
+    task kind, [:index, :category] => :application do |_, options|
+      index, category = options.index, options.category
+
+      specific = Picky::Indexes
+      specific = specific[index]    if index
+      specific = specific[category] if category
+
+      specific.index Picky::Scheduler.new(kind => true)
+    end
   end
 
 end

data/spec/functional/backends/file_spec.rb

@@ -5,8 +5,6 @@ require 'spec_helper'
 # Describes a Picky index that uses the File backend
 # for data storage.
 #
-# TODO Everything should just fail.
-#
 describe Picky::Backends::File do
 
   class Book
@@ -49,7 +47,7 @@ describe Picky::Backends::File do
     it 'handles removing with more than one entry' do
       data.add Book.new(2, 'title', 'author')
 
-      books.search('title').ids.should == ['2', '1'] # TODO Should be ['2', '1']
+      books.search('title').ids.should == ['2', '1']
 
       data.remove '1'
 
@@ -59,7 +57,7 @@ describe Picky::Backends::File do
       data.add Book.new(2, 'title', 'author')
       data.add Book.new(3, 'title', 'author')
 
-      books.search('title').ids.should == ['3', '2', '1'] # TODO Should be ['3', '2', '1']
+      books.search('title').ids.should == ['3', '2', '1']
 
       data.remove '1'
 

data/spec/functional/backends/memory_spec.rb

@@ -47,7 +47,7 @@ describe Picky::Backends::Memory do
     it 'handles removing with more than one entry' do
       data.add Book.new(2, 'title', 'author')
 
-      books.search('title').ids.should == ['2', '1'] # TODO Should be ['2', '1']
+      books.search('title').ids.should == ['2', '1']
 
       data.remove '1'
 
@@ -57,7 +57,7 @@ describe Picky::Backends::Memory do
       data.add Book.new(2, 'title', 'author')
       data.add Book.new(3, 'title', 'author')
 
-      books.search('title').ids.should == ['3', '2', '1'] # TODO Should be ['3', '2', '1']
+      books.search('title').ids.should == ['3', '2', '1']
 
       data.remove '1'
 

data/spec/functional/backends/redis_spec.rb

@@ -57,7 +57,7 @@ describe Picky::Backends::Redis do
       data.add Book.new(2, 'title', 'author')
       data.add Book.new(3, 'title', 'author')
 
-      books.search('title').ids.should == ['3', '2', '1'] # TODO Should be ['3', '2', '1']
+      books.search('title').ids.should == ['3', '2', '1']
 
       data.remove '1'
 

data/spec/functional/exact_first_spec.rb

@@ -23,7 +23,27 @@ describe "exact first" do
     normal = Picky::Search.new index
     normal.search("disco").ids.should == [2, 1] # 2 was added later.
 
-    index = Picky::Wrappers::Category::ExactFirst.wrap index
+    index.extend Picky::Results::ExactFirst
+
+    exact_first = Picky::Search.new index
+    exact_first.search("disco").ids.should == [1, 2] # Exact first.
+    exact_first.search("disc").ids.should  == [2, 1] # Not exact, so not first.
+  end
+
+  it 'handles extending single categories' do
+    index    = Picky::Index.new :exact_first
+    category = index.category :text, partial: Picky::Partial::Substring.new(from: 1)
+
+    require 'ostruct'
+    exact   = OpenStruct.new id: 1, text: "disco"
+    partial = OpenStruct.new id: 2, text: "discofox"
+    index.add exact
+    index.add partial
+
+    normal = Picky::Search.new index
+    normal.search("disco").ids.should == [2, 1] # 2 was added later.
+
+    category.extend Picky::Results::ExactFirst
 
     exact_first = Picky::Search.new index
     exact_first.search("disco").ids.should == [1, 2] # Exact first.
@@ -41,14 +61,14 @@ describe "exact first" do
       category :text, partial: Picky::Partial::Substring.new(from: 1)
     end
     normal = Picky::Search.new data
-    Picky::Indexes.index_for_tests
+    Picky::Indexes.index
 
     normal.search("disco").ids.should == [1, 2] # Ordering with which it was added.
 
-    data = Picky::Wrappers::Category::ExactFirst.wrap data
+    data.extend Picky::Results::ExactFirst
     exact_first = Picky::Search.new data
 
-    Picky::Indexes.index_for_tests
+    Picky::Indexes.index
 
     exact_first.search("disco").ids.should == [2, 1] # Exact first.
     exact_first.search("disc").ids.should  == [1, 2] # Not exact, so not first.

data/spec/functional/realtime_spec.rb

@@ -26,7 +26,7 @@ describe "Realtime Indexing" do
     end
 
     context 'dumping and loading' do
-      it "doesn't find books anymore after dumping and reloading and updating" do
+      it "doesn't find books anymore after dumping and loading and updating" do
         index.replace Book.new(2, "Title New", "Author New")
 
         books.search("title").ids.should == [2, 1]
@@ -243,7 +243,9 @@ describe "Realtime Indexing" do
         index.replace Book.new(2, "Title New", "Author New")
         index.add Book.new(3, "TTL", "AUTHR")
 
-        books.search('author:Athr~').ids.should == [2, 1, 3] # TODO Is that what I'd expect?
+        # Note: [2, 1] are in one allocation, [3] in the other.
+        #
+        books.search('author:Athr~').ids.should == [2, 1, 3]
       end
     end
   end
@@ -465,7 +467,9 @@ describe "Realtime Indexing" do
         index.replace Book.new("two", "Title New", "Author New")
         index.add Book.new("three", "TTL", "AUTHR")
 
-        books.search('author:Athr~').ids.should == [:two, :one, :three] # TODO Is that what I'd expect?
+        # Note: Allocations are [:two, :one], then [:three].
+        #
+        books.search('author:Athr~').ids.should == [:two, :one, :three]
       end
     end
   end

data/spec/lib/application_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 describe Picky::Application do
-  
+
   describe "integration" do
     it "should run ok" do
       lambda {
@@ -14,10 +14,10 @@ describe Picky::Application do
                     :file => 'app/db.yml'
                   )
                   end
-          books.define_category :title
-          
+          books.category :title
+
           rack_adapter.stub! :exclaim # Stopping it from exclaiming.
-          
+
           route %r{^/books} => Picky::Search.new(books)
         end
         Picky::Tokenizer.index_default.tokenize 'some text'
@@ -35,29 +35,29 @@ describe Picky::Application do
                     removes_characters_after_splitting: /[\.]/,
                     normalizes_words:                   [[/\$(\w+)/i, '\1 dollars']],
                     rejects_token_if:                   lambda { |token| token.blank? || token == :amistad }
-                           
+
           searching removes_characters: /[^a-zA-Z0-9äöü\s\/\-\,\&\"\~\*\:]/,
                     stopwords:          /\b(and|the|of|it|in|for)\b/,
                     splits_text_on:     /[\s\/\-\,\&]+/,
                     normalizes_words:   [[/Deoxyribonucleic Acid/i, 'DNA']],
-                    
+
                     substitutes_characters_with: Picky::CharacterSubstituters::WestEuropean.new,
                     maximum_tokens: 5
-          
+
           books_index = Picky::Index.new :books do
             source Picky::Sources::DB.new(
               'SELECT id, title, author, isbn13 as isbn FROM books',
               :file => 'app/db.yml'
             )
           end
-          
-          books_index.define_category :title,
-                                      similarity: Picky::Similarity::DoubleMetaphone.new(3) # Up to three similar title word indexed.
-          books_index.define_category :author,
-                                      similarity: Picky::Similarity::Soundex.new(2)
-          books_index.define_category :isbn,
+
+          books_index.category :title,
+                               similarity: Picky::Similarity::DoubleMetaphone.new(3) # Up to three similar title word indexed.
+          books_index.category :author,
+                               similarity: Picky::Similarity::Soundex.new(2)
+          books_index.category :isbn,
                                       partial: Picky::Partial::None.new # Partially searching on an ISBN makes not much sense.
-          
+
           geo_index = Picky::Index.new :geo do
             source          Picky::Sources::CSV.new(:location, :north, :east, file: 'data/ch.csv', col_sep: ',')
             indexing        removes_characters: /[^a-z]/
@@ -66,11 +66,11 @@ describe Picky::Application do
             ranged_category :north1, 1, precision: 3, from: :north
             ranged_category :east1,  1, precision: 3, from: :east
           end
-          
+
           rack_adapter.stub! :exclaim # Stopping it from exclaiming.
-          
+
           route %r{^/books} => Picky::Search.new(books_index)
-          
+
           buks_search = Picky::Search.new(books_index) do
             searching removes_characters: /[buks]/
             ignore    :author
@@ -80,39 +80,39 @@ describe Picky::Application do
       }.should_not raise_error
     end
   end
-  
+
   describe 'finalize' do
     before(:each) do
       described_class.stub! :check
     end
     it 'checks if all is ok' do
       described_class.should_receive(:check).once.with
-      
+
       described_class.finalize
     end
     it 'tells the rack adapter to finalize' do
       described_class.rack_adapter.should_receive(:finalize).once.with
-      
+
       described_class.finalize
     end
   end
-  
+
   describe 'check' do
     it 'does something' do
       described_class.should_receive(:warn).once.with "\nWARNING: No routes defined for application configuration in Class.\n\n"
-      
+
       described_class.check
     end
   end
-  
+
   describe 'delegation' do
     it "should delegate route" do
       described_class.rack_adapter.should_receive(:route).once.with :path => :query
-      
+
       described_class.route :path => :query
     end
   end
-  
+
   describe 'rack_adapter' do
     it 'should be there' do
       lambda { described_class.rack_adapter }.should_not raise_error
@@ -124,11 +124,11 @@ describe Picky::Application do
       described_class.rack_adapter.should == described_class.rack_adapter
     end
   end
-  
+
   describe 'route' do
     it 'is delegated' do
       described_class.rack_adapter.should_receive(:route).once.with :some_options
-      
+
       described_class.route(:some_options)
     end
     it 'raises on block' do
@@ -139,7 +139,7 @@ describe Picky::Application do
       }.to raise_error("Warning: block passed into #route method, not into Search.new!")
     end
   end
-  
+
   describe 'call' do
     before(:each) do
       @routes = stub :routes
@@ -147,9 +147,9 @@ describe Picky::Application do
     end
     it 'should delegate' do
       @routes.should_receive(:call).once.with :env
-      
+
       described_class.call :env
     end
   end
-  
+
 end

data/spec/lib/backends/backend_spec.rb

@@ -2,37 +2,35 @@ require 'spec_helper'
 
 describe Picky::Backends::Backend do
 
-  context 'without options' do
-    let(:backend) { described_class.new }
+  let(:backend) { described_class.new }
 
-    describe 'extract_lambda_or' do
-      it 'returns a given non-lambda' do
-        backend.extract_lambda_or(:thing).should == :thing
-      end
-      it 'calls a given lambda with the given args' do
-        lam = ->() do
-          :lam
-        end
-        
-        backend.extract_lambda_or(lam).should == :lam
+  describe 'extract_lambda_or' do
+    it 'returns a given non-lambda' do
+      backend.extract_lambda_or(:thing).should == :thing
+    end
+    it 'calls a given lambda with the given args' do
+      lam = ->() do
+        :lam
       end
-      it 'calls a given lambda with the given args' do
-        lam = ->(arg1) do
-          arg1.should == 1
-          :lam
-        end
-        
-        backend.extract_lambda_or(lam, 1).should == :lam
+
+      backend.extract_lambda_or(lam).should == :lam
+    end
+    it 'calls a given lambda with the given args' do
+      lam = ->(arg1) do
+        arg1.should == 1
+        :lam
       end
-      it 'calls a given lambda with the given args' do
-        lam = ->(arg1, arg2) do
-          arg1.should == 1
-          arg2.should == 2
-          :lam
-        end
-        
-        backend.extract_lambda_or(lam, 1, 2).should == :lam
+
+      backend.extract_lambda_or(lam, 1).should == :lam
+    end
+    it 'calls a given lambda with the given args' do
+      lam = ->(arg1, arg2) do
+        arg1.should == 1
+        arg2.should == 2
+        :lam
       end
+
+      backend.extract_lambda_or(lam, 1, 2).should == :lam
     end
   end
 

data/spec/lib/backends/{memory → prepared}/text_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe Picky::Backends::Memory::Text do
+describe Picky::Backends::Prepared::Text do
 
   let(:text) { described_class.new "some_cache_path" }
 

data/spec/lib/category_indexing_spec.rb

@@ -127,7 +127,7 @@ describe Picky::Category do
         category.stub! :indexer => @indexer
       end
       it "tells the indexer to index" do
-        @indexer.should_receive(:index).once
+        @indexer.should_receive(:prepare).once
 
         category.prepare
       end

data/spec/lib/extensions/symbol_spec.rb

@@ -13,7 +13,7 @@ describe Symbol do
       performance_of { @token.each_subtoken { |subtoken| } }.should < 0.00065
     end
     it 'is fast enough' do
-      performance_of { @token.each_intoken { |intoken| } }.should < 0.025 # TODO Slow!
+      performance_of { @token.each_intoken { |intoken| } }.should < 0.025 # THINK Is this too slow?
     end
   end