picky 4.17.1 → 4.18.0

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    YTkyZGMwZGMyNjFiNTVlMjU2Mjg5ZjhmMGY2MzM2N2NlY2M5YmEzMw==
-  data.tar.gz: !binary |-
-    YmYzMWQ5OTM0YzdiZDVlYjMyMzliZWY5ZDYzZWU4M2JiNmJkNDEwMg==
-!binary "U0hBNTEy":
-  metadata.gz: !binary |-
-    ZmE5NDZlZjVhN2I2OGM2YzUyMmM0ZWFjMGU4Mjc5NzA0MmEwMmU5MTg1ZDg0
-    ZWJiYzhiZTc4MDgzMjY3MGZlM2U3N2QxOTMyZWY2NzQzN2UwZmVkYWRjZWVl
-    YzdkNGFjYjYwZTM5ODdlOGMzZGY4MjljNjk5ZDYzZmU3MzcwMGE=
-  data.tar.gz: !binary |-
-    ZDY1OThhODliOWI4M2VmOTM3MWYzZTVmNjlmMDM4NzhiNmZkY2UzOTdkYTJl
-    MTA0N2U0MjVhYTU4YTYxZDI2ZDBiYjMxNGYzODFkNGUyM2VjMGJiZTI5NDZl
-    OGVlZWJlODhiNzY5YmJjODk3NjMzMTQ5NjBkYWM5MTk0ZmYyOTI=
+SHA1:
+  metadata.gz: 5771adfbe24b649d5377b9d1233783ff5edb0c1e
+  data.tar.gz: 140aca46019f8b09496b508c9842a3ac84f2a933
+SHA512:
+  metadata.gz: 07d12c4437ba880e486d73626873f1e1fc16e26512a37d564a172cb23bf517ad2c1503d841d0bb3dcd8ef9653a2f776979a2467761a01c412df2acf5a6837547
+  data.tar.gz: bf488e2fac14e6d1f4a26a354a8579e5931b18779c3e115e92062008bc1535f0e6ebdcbb21bfa60758be8df5557c64779a420957ef4c3b106ace7afe12b6a384

data/lib/picky/backends/backend.rb

@@ -48,13 +48,13 @@ module Picky
         json bundle.index_path(:realtime)
       end
 
-      # Returns the total score of the combinations.
-      #
-      # Default implementation. Override to speed up.
-      #
-      def weight combinations
-        combinations.score
-      end
+      # # Returns the total score of the combinations.
+      # #
+      # # Default implementation. Override to speed up.
+      # #
+      # def score combinations
+      #   combinations.score
+      # end
 
       # Returns the result ids for the allocation.
       #
@@ -72,17 +72,11 @@ module Picky
       # unfortunately.
       #
       def ids combinations, _, _
-        # Get the ids for each combination.
-        #
-        id_arrays = combinations.inject([]) do |total, combination|
-          total << combination.ids
-        end
-
-        # Call the optimized C algorithm.
+        # Get the ids for each combination and pass to the optimized C algorithm.
         #
         # Note: It orders the passed arrays by size.
         #
-        Performant::Array.memory_efficient_intersect id_arrays
+        Performant::Array.memory_efficient_intersect combinations.map { |combination| combination.ids }
       end
 
       #

data/lib/picky/backends/prepared/text.rb

@@ -49,7 +49,7 @@ module Picky
         #
         # Yields an id string and a token.
         #
-        # TODO Think about the comma - what if you have commas in the id?
+        # TODO Think about the comma - what if you have commas in the id? Use CSV?
         #
         def retrieve
           id    = nil

data/lib/picky/backends/sqlite/basic.rb

@@ -15,8 +15,6 @@ module Picky
           @empty      = options[:empty]
           @initial    = options[:initial]
           @realtime   = options[:realtime]
-
-          lazily_initialize_client
         
           # Note: If on OSX, too many files get opened during
           #       the specs -> ulimit -n 3000
@@ -46,9 +44,10 @@ module Picky
           db.execute 'delete from key_value'
         end
         
-        # TODO Replace with db method?
+        # Lazily creates SQLite client.
+        # Note: Perhaps it would be advisable to create only one, when initialising.
         #
-        def lazily_initialize_client
+        def db
           @db ||= (create_directory cache_path; SQLite3::Database.new cache_path)
         end
 
@@ -67,13 +66,7 @@ module Picky
         end
 
         def reset
-          # TODO Still necessary?
-          #
-          create_directory cache_path
-          lazily_initialize_client
-
           truncate_db
-
           self
         end
 

data/lib/picky/bundle.rb

@@ -145,10 +145,10 @@ module Picky
 
     # If a key format is set, use it, else forward to the category.
     #
-    # TODO Removed optimisation: @key_format ||= @category.key_format
+    # TODO Remove optimisation? @category.key_format
     #
     def key_format
-      @category.key_format
+      @key_format ||= @category.key_format
     end
 
     # Path and partial filename of a specific subindex.

data/lib/picky/category.rb

@@ -7,7 +7,6 @@ module Picky
     attr_accessor :exact,
                   :partial
     attr_reader :name,
-                :prepared,
                 :backend
     attr_writer :source
 
@@ -71,22 +70,22 @@ module Picky
 
       @exact     = exact_for weights, similarity, options
       @partial   = partial_for @exact, partial, weights, options
-
-      @prepared  = Backends::Prepared::Text.new prepared_index_path
     end
     # Since the options hash might contain options that do not exist,
     # we should warn people if they use the wrong options.
     # (Problem is that if the option is not found, then Picky will use the default)
     #
-    # TODO Rewrite it such that this does not need to be maintained separately.
+    # TODO Rewrite it such that this does not need to be maintained separately (and gets available options automatically).
     #
     @@known_keys = [:indexing, :partial, :qualifier, :qualifiers, :ranging, :similarity, :source, :tokenize, :tokenizer, :weight]
     def warn_if_unknown options
-      warn <<-WARNING if options && (options.keys - @@known_keys).size > 0
+      if options && (options.keys - @@known_keys).size > 0
+        warn <<-WARNING
 
 Warning: Category options #{options} for category #{name} contain an unknown option.
          Working options are: #@@known_keys.
 WARNING
+      end
     end
     def weights_from options
       Generators::Weights.from options[:weight], index_name, name
@@ -112,6 +111,12 @@ WARNING
       end
     end
     
+    # Lazily create a prepared index proxy.
+    #
+    def prepared
+      @prepared ||= Backends::Prepared::Text.new prepared_index_path
+    end
+    
     # Indexes and loads the category.
     #
     def reindex

data/lib/picky/category_indexed.rb

@@ -52,7 +52,8 @@ module Picky
     # Returns the right index bundle for this token.
     #
     def bundle_for token
-      token.partial? ? partial : exact
+      # token.partial? ? partial : exact
+      token.select_bundle exact, partial
     end
 
   end

data/lib/picky/category_realtime.rb

@@ -74,9 +74,16 @@ module Picky
       
       format = self.key_format?
       tokens.each { |text| add_tokenized_token id, text, where, format }
-    rescue NoMethodError
-      # TODO Improve error message by pointing out what exactly goes wrong: thing xy does not have an #each method.
-      raise %Q{You probably set tokenize: false on category "#{name}". It will need an Enumerator of previously tokenized tokens.}
+    rescue NoMethodError => e
+      show_informative_add_text_error_message_for e
+    end
+    
+    def show_informative_add_text_error_message_for e
+      if e.name == :each
+        raise %Q{#{e.message}. You probably set tokenize: false on category "#{name}". It will need an Enumerator of previously tokenized tokens.}
+      else
+        raise e
+      end
     end
 
     #

data/lib/picky/helpers/measuring.rb

@@ -7,14 +7,12 @@ module Picky
 
       # Returns a duration in seconds.
       #
-      def timed *args, &block
-        block_to_be_measured = lambda &block
+      def timed
+        time_begin = Time.new
 
-        time_begin = Time.now.to_f
+        yield
 
-        block_to_be_measured.call *args
-
-        Time.now.to_f - time_begin
+        (Time.new - time_begin).to_f
       end
 
     end

data/lib/picky/index_indexing.rb

@@ -93,8 +93,6 @@ module Picky
     #
     # Parameter is a method name to use on the key (e.g. :to_i, :to_s, :strip, :split).
     #
-    # TODO Rename id_format?
-    #
     def key_format key_format = nil
       key_format ? (@key_format = key_format) : @key_format
     end

data/lib/picky/indexes.rb

@@ -11,7 +11,7 @@ module Picky
     attr_reader :indexes,
                 :index_mapping
 
-    forward :size, :each, :to => :indexes
+    forward :size, :each, :map, :to => :indexes
     each_forward :reindex, :to => :indexes
     instance_forward :clear,
                      :clear_indexes,
@@ -47,7 +47,7 @@ module Picky
     # Registers an index with the indexes.
     #
     def register index
-      # TODO DO not store duplicate indexes.
+      # TODO Do not store duplicate indexes.
       #
       # self.indexes.delete_if { |existing| existing.name == index.name }
       self.indexes << index

data/lib/picky/query/allocation.rb

@@ -40,8 +40,9 @@ module Picky
         @score ||= if @combinations.empty?
           0 # Optimization.
         else
-          @backend.weight(@combinations) + @combinations.boost_for(weights)
-        end 
+          # TODO Was @backend.score(@combinations) - indirection for maximum flexibility.
+          @combinations.score + weights.boost_for(@combinations)
+        end
       end
 
       # Asks the backend for the (intersected) ids.

data/lib/picky/query/allocations.rb

@@ -60,7 +60,7 @@ module Picky
       #
       # TODO Rewrite, speed up.
       #
-      def remove_allocations qualifiers_array = []
+      def remove_allocations qualifiers_array
         return if qualifiers_array.empty?
         @allocations.select! do |allocation|
           allocation_qualifiers = allocation.combinations.to_qualifiers.clustered_uniq
@@ -77,7 +77,7 @@ module Picky
       #
       # TODO Rewrite, speed up.
       #
-      def keep_allocations qualifiers_array = []
+      def keep_allocations qualifiers_array
         return if qualifiers_array.empty?
         @allocations.select! do |allocation|
           allocation_qualifiers = allocation.combinations.to_qualifiers.clustered_uniq

data/lib/picky/query/boosts.rb

@@ -59,6 +59,8 @@ module Picky
       # Note: Cache this if more complicated weighings become necessary.
       # Note: Maybe make combinations comparable to Symbols?
       #
+      # TODO Push into categories? Store boosts in categories?
+      #
       def boost_for combinations
         boost_for_categories combinations.map { |combination| combination.category_name }
       end

data/lib/picky/query/combination.rb

@@ -5,7 +5,7 @@ module Picky
     # Describes the Combination of:
     #  * a token
     #  * a category
-    #  * the weight of the token in the category (cached)
+    #  * the weight of the token in the category (cached from earlier)
     #
     # An Allocation consists of an ordered number of Combinations.
     #

data/lib/picky/query/combinations.rb

@@ -16,6 +16,7 @@ module Picky
 
       forward :empty?,
               :inject,
+              :map,
               :to => :@combinations
 
       def initialize combinations = []
@@ -29,9 +30,6 @@ module Picky
       def score
         @combinations.inject(0) { |total, combination| total + combination.weight }
       end
-      def boost_for weights
-        weights.boost_for @combinations
-      end
 
       # Filters the tokens and categories such that categories
       # that are passed in, are removed.

data/lib/picky/query/indexes.rb

@@ -183,7 +183,7 @@ module Picky
         # If an element has size 0, this means one of the
         # tokens could not be allocated.
         #
-        return [] if possible_combinations.any?(&:empty?)
+        return [] if possible_combinations.any? { |possible_combination| possible_combination.empty? }
 
         # Generate the first multiplicator "with which" (well, not quite) to multiply the smallest amount of combinations.
         #

data/lib/picky/query/token.rb

@@ -41,8 +41,8 @@ module Picky
       end
       def process
         qualify
-        partialize
         similarize
+        partialize
         rangify
         remove_illegals
         self
@@ -64,8 +64,8 @@ module Picky
       #
       # TODO Do we really need to set the predefined categories on the token?
       #
-      def predefined_categories mapper
-        @predefined_categories || extract_predefined(mapper)
+      def predefined_categories mapper = nil
+        @predefined_categories || mapper && extract_predefined(mapper)
       end
       def extract_predefined mapper
         user_qualified = categorize_with mapper, @qualifiers
@@ -76,6 +76,12 @@ module Picky
           mapper.map qualifier
         end.compact
       end
+      
+      # Selects the bundle to be used.
+      #
+      def select_bundle exact, partial
+        @partial ? partial : exact
+      end
 
       # Partial is a conditional setter.
       #
@@ -90,8 +96,11 @@ module Picky
       #
       # It can't be similar and partial at the same time.
       #
+      # Note: @partial is calculated at processing time (see Token#process).
+      #
       def partial?
-        !@similar && @partial
+        # Was: !@similar && @partial
+        @partial
       end
 
       # If the text ends with *, partialize it. If with ",
@@ -106,15 +115,21 @@ module Picky
       @@no_partial = /\"\z/
       @@partial    = /\*\z/
       def partialize
-        self.partial = false or return unless @text !~ @@no_partial
-        self.partial = true unless @text !~ @@partial
+        # A token is partial? only if it not similar
+        # and is partial.
+        #
+        # It can't be similar and partial at the same time.
+        #
+        self.partial = false or return if @similar
+        self.partial = false or return if @text =~ @@no_partial
+        self.partial = true if @text =~ @@partial # TODO Move this one line up since it occurs more often?
       end
       # Define a character which stops a token from
       # being a partial token, even if it is the last token.
       #
       # Default is '"'.
       #
-      # This is used in a regexp (%r{#{char}\z}) for String#!~, 
+      # This is used in a regexp (%r{#{char}\z}) for String#=~, 
       # so escape the character.
       #
       # Example:
@@ -124,12 +139,13 @@ module Picky
       def self.no_partial_character= character
         @@no_partial_character = character
         @@no_partial = %r{#{character}\z}
+        redefine_illegals
       end
       # Define a character which makes a token a partial token.
       #
       # Default is '*'.
       #
-      # This is used in a regexp (%r{#{char}\z}) for String#!~, 
+      # This is used in a regexp (%r{#{char}\z}) for String#=~, 
       # so escape the character.
       #
       # Example:
@@ -151,15 +167,15 @@ module Picky
       @@no_similar = %r{#@@no_similar_character\z}
       @@similar    = %r{#@@similar_character\z}
       def similarize
-        self.similar = false or return unless @text !~ @@no_similar
-        self.similar = true unless @text !~ @@similar
+        self.similar = false or return if @text =~ @@no_similar
+        self.similar = true if @text =~ @@similar
       end
       # Define a character which stops a token from
       # being a similar token, even if it is the last token.
       #
       # Default is '"'.
       #
-      # This is used in a regexp (%r{#{char}\z}) for String#!~, 
+      # This is used in a regexp (%r{#{char}\z}) for String#=~, 
       # so escape the character.
       #
       # Example:
@@ -169,12 +185,13 @@ module Picky
       def self.no_similar_character= character
         @@no_similar_character = character
         @@no_similar = %r{#{character}\z}
+        redefine_illegals
       end
       # Define a character which makes a token a similar token.
       #
       # Default is '~'.
       #
-      # This is used in a regexp (%r{#{char}\z}) for String#!~, 
+      # This is used in a regexp (%r{#{char}\z}) for String#=~, 
       # so escape the character.
       #
       # Example:
@@ -200,9 +217,7 @@ module Picky
         @@range_character = character
       end
       def rangify
-        if @text.include? @@range_character
-          @range = @text.split(@@range_character, 2)
-        end
+        @range = @text.split(@@range_character, 2) if @text.include? @@range_character
       end
       def range
         @range
@@ -222,7 +237,7 @@ module Picky
         @text.gsub! @@illegals, EMPTY_STRING unless @text == EMPTY_STRING
       end
       def self.redefine_illegals
-        @@illegals = %r{[#@@no_similar_character#@@partial_character#@@similar_character]}
+        @@illegals = %r{[#@@no_similar_character#@@similar_character#@@no_partial_character#@@partial_character]}
       end
       redefine_illegals
       
@@ -236,7 +251,9 @@ module Picky
         similar? ? categories.similar_possible_for(self) : categories.possible_for(self)
       end
       
-      # TODO
+      # If the Token has weight for the given category,
+      # it will return a new combination for the tuple
+      # (self, category, weight).
       #
       def combination_for category
         weight = category.weight self
@@ -259,12 +276,16 @@ module Picky
       @@qualifier_text_delimiter = ':'
       @@qualifiers_delimiter     = ','
       def qualify
-        @qualifiers, @text = (@text || EMPTY_STRING).split(@@qualifier_text_delimiter, 2)
-        if @text
-          @qualifiers = @qualifiers.split @@qualifiers_delimiter
-        else
-          @text = @qualifiers || EMPTY_STRING
-          @qualifiers = nil
+        # TODO Is this actually an optimization?
+        # Check using include? + split, and split alone.
+        #
+        if @text.include? @@qualifier_text_delimiter
+          @qualifiers, @text = @text.split @@qualifier_text_delimiter, 2
+          if @text
+            @qualifiers = @qualifiers.split @@qualifiers_delimiter
+          else
+            @text, @qualifiers = @qualifiers, nil
+          end
         end
       end
       # Define a character which separates the qualifier

data/lib/picky/search.rb

@@ -265,6 +265,8 @@ module Picky
 
     # Gets sorted allocations for the tokens.
     #
+    # TODO Remove and just call prepared (and rename to sorted)?
+    #
     def sorted_allocations tokens, amount = nil
       indexes.prepared_allocations_for tokens, boosts, amount
     end

data/lib/picky/search_facets.rb

@@ -36,7 +36,9 @@ module Picky
       # Pre-tokenize key token – replace text below.
       # Note: The original is not important.
       #
-      # TODO Don't use predefined.
+      # TODO Don't use predefined. Perhaps do:
+      # key_token = Query::Token.new ''
+      # key_token.predefined_categories = [index[category_identifier]]
       #
       key_token = Query::Token.new '', nil, predefined_categories
       

data/lib/picky/tokenizer.rb

@@ -212,7 +212,7 @@ A short overview:
   stopwords                   /regexp/
   splits_text_on              /regexp/ or "String", default /\s/
   normalizes_words            [[/replace (this)/, 'with this \\1'], ...]
-  rejects_token_if            Proc/lambda, default :blank?.to_proc
+  rejects_token_if            Proc/lambda, default :empty?.to_proc
   substitutes_characters_with Picky::CharacterSubstituter or responds to #substitute(String)
   stems_with                  Instance responds to #stem(String)
   case_sensitive              true/false
@@ -222,7 +222,7 @@ ERROR
     def default_options
       {
         splits_text_on: /\s/,
-        rejects_token_if: :blank?.to_proc
+        rejects_token_if: :empty?.to_proc
       }
     end
 
@@ -234,7 +234,7 @@ ERROR
     #
     def tokenize text
       text   = preprocess text.to_s # processing the text
-      return empty_tokens if text.blank?
+      return empty_tokens if text.empty? # TODO blank?
       words  = pretokenize text # splitting and preparations for tokenizing
       return empty_tokens if words.empty?
       tokens = tokens_for words # creating tokens / strings

data/spec/functional/allocations_uniq_by_definition_spec.rb

@@ -7,30 +7,35 @@ require 'spec_helper'
 #
 describe 'uniqueness of allocations' do
 
-  it 'is already uniq' do
-    index = Picky::Index.new :already_uniq do
-      category :category1
-      category :category2
-      category :category3
-    end
+  context 'is already uniq' do
+    let(:index) do
+      index = Picky::Index.new :already_uniq do
+        category :category1
+        category :category2
+        category :category3
+      end
 
-    thing = Struct.new(:id, :category1, :category2, :category3)
-    index.add thing.new(1, 'text1', 'text2', 'text3')
-    
-    try = Picky::Search.new index do
-      max_allocations 100
+      thing = Struct.new(:id, :category1, :category2, :category3)
+      index.add thing.new(1, 'text1', 'text2', 'text3')
+      
+      index
+    end
+    let(:try) do
+      Picky::Search.new index do
+        max_allocations 100
+      end
     end
     
     # Picky finds three categories.
     #
-    try.search('text*').ids.should == [1,1,1]
+    it { try.search('text*').ids.should == [1,1,1] }
     
     # Picky finds 9 possible allocations.
     #
-    try.search('text* text*').ids.should == [1,1,1]*3
+    it { try.search('text* text*').ids.should == [1,1,1]*3 }
     
     # Picky finds 27 possible allocations.
     #
-    try.search('text* text* text*', 100).ids.should == [1,1,1]*3*3
+    it { try.search('text* text* text*', 100).ids.should == [1,1,1]*3*3 }
   end
 end

data/spec/functional/arrays_as_ids_spec.rb

@@ -1,17 +1,17 @@
 # encoding: utf-8
 #
 require 'spec_helper'
+require 'ostruct'
 
 describe "Array IDs" do
+  
+  let(:index) { Picky::Index.new :arrays } 
+  let(:try) { Picky::Search.new index }
 
   # This tests the weights option.
   #
   it 'can use Arrays as IDs' do
-    index = Picky::Index.new :arrays do
-      category :text1
-    end
-
-    require 'ostruct'
+    index.category :text1
 
     thing = OpenStruct.new id: ['id1', 'thing1'], text1: "ohai"
     other = OpenStruct.new id: ['id2', 'thing2'], text1: "ohai kthxbye"
@@ -19,8 +19,6 @@ describe "Array IDs" do
     index.add thing
     index.add other
 
-    try = Picky::Search.new index
-
     try.search("text1:ohai").ids.should == [
       ["id2", "thing2"],
       ["id1", "thing1"]
@@ -30,22 +28,15 @@ describe "Array IDs" do
   # This tests the weights option.
   #
   it 'can use split as key_format' do
-    index = Picky::Index.new :arrays do
-      key_format :split
-      
-      category :text1
-    end
-
-    require 'ostruct'
-    
+    index.key_format :split
+    index.category :text1
+
     thing = OpenStruct.new id: "id1 thing1", text1: "ohai"
     other = OpenStruct.new id: "id2 thing2", text1: "ohai kthxbye"
 
     index.add thing
     index.add other
 
-    try = Picky::Search.new index
-
     try.search("text1:ohai").ids.should == [
       ["id2", "thing2"],
       ["id1", "thing1"]