picky 3.2.0 → 3.3.0

data/lib/picky/bundle_realtime.rb

@@ -0,0 +1,80 @@
+module Picky
+
+  class Bundle
+
+    # Removes the given id from the indexes.
+    #
+    def remove id
+      # Is it anywhere?
+      #
+      syms = @realtime_mapping[id]
+      return unless syms
+
+      syms.each do |sym|
+        ids = @inverted[sym]
+        ids.delete id
+
+        encoded = self.similarity_strategy.encoded sym
+
+        if ids.empty?
+          @inverted.delete   sym
+          @weights.delete    sym
+          # Since no element uses this sym anymore, we can delete the similarity for it.
+          # TODO Not really. Since multiple syms can point to the same encoded.
+          @similarity.delete encoded
+        else
+          @weights[sym] = self.weights_strategy.weight_for ids.size
+        end
+      end
+
+      @realtime_mapping.delete id
+    end
+
+    # Returns a reference to the array where the id has been added.
+    #
+    def add id, sym
+      ary = @inverted[sym]
+
+      syms = @realtime_mapping[id]
+      syms = (@realtime_mapping[id] = []) unless syms # TODO Nicefy.
+
+      # Inverted.
+      #
+      ids = if syms.include? sym
+        ids = @inverted[sym]
+        ids.delete  id # Move id
+        ids.unshift id # to front
+      else
+        syms << sym
+        ids = @inverted[sym] ||= []
+        ids.unshift id
+      end
+
+      # Weights.
+      #
+      @weights[sym] = self.weights_strategy.weight_for ids.size
+
+      # Similarity.
+      #
+      if encoded = self.similarity_strategy.encoded(sym)
+        similarity = @similarity[encoded] ||= []
+        if similarity.include? sym
+          similarity.delete sym  # Not completely correct, as others will also be affected, but meh.
+          similarity.unshift sym #
+        else
+          similarity.unshift sym
+        end
+      end
+    end
+
+    # Partializes the text and then adds each.
+    #
+    def add_partialized id, text
+      self.partial_strategy.each_partial text do |partial_text|
+        add id, partial_text
+      end
+    end
+
+  end
+
+end

data/lib/picky/categories.rb

@@ -9,8 +9,12 @@ module Picky
              :map,
              :to => :categories
 
-    each_delegate :reindex,
+    each_delegate :cache,
+                  :dump,
                   :each_category,
+                  :empty,
+                  :index,
+                  :reindex,
                   :to => :categories
 
     # A list of indexed categories.

data/lib/picky/category.rb

@@ -2,7 +2,9 @@ module Picky
 
   class Category
 
-    attr_reader :name
+    attr_reader :name,
+                :exact,
+                :partial
 
     # Mandatory params:
     #  * name: Category name to use as identifier and file names.
@@ -38,20 +40,12 @@ module Picky
       no_partial    = Generators::Partial::None.new
       no_similarity = Generators::Similarity::None.new
 
-      @indexing_exact   = Indexing::Bundle.new  :exact,  self, index.backend, weights, no_partial, similarity, options
-      @indexing_partial = Indexing::Bundle.new :partial, self, index.backend, weights, partial, no_similarity, options
-
-      # Indexed.
-      #
-      @indexed_exact  = Indexed::Bundle.new :exact, self, index.backend, weights, no_partial, similarity
+      @exact = Bundle.new :exact, self, index.backend, weights, no_partial, similarity, options
       if partial.use_exact_for_partial?
-        @indexed_partial  = @indexed_exact
+        @partial = Wrappers::Bundle::ExactPartial.new @exact
       else
-        @indexed_partial  = Indexed::Bundle.new :partial, self, index.backend, weights, partial, no_similarity
+        @partial = Bundle.new :partial, self, index.backend, weights, partial, no_similarity, options
       end
-
-      # @exact   = exact_lambda.call(@exact, @partial)   if exact_lambda   = options[:exact_lambda]
-      # @partial = partial_lambda.call(@exact, @partial) if partial_lambda = options[:partial_lambda]
     end
 
     # Indexes and reloads the category.
@@ -61,6 +55,11 @@ module Picky
       reload
     end
 
+    def dump
+      exact.dump
+      partial.dump
+    end
+
     # Index name.
     #
     def index_name
@@ -98,17 +97,10 @@ module Picky
       @prepared_index_file ||= Backends::Memory::Text.new prepared_index_path
       @prepared_index_file.open &block
     end
-    # Creates the index directory including all necessary paths above it.
-    #
-    # Note: Interface method called by any indexers.
-    #
-    def prepare_index_directory
-      FileUtils.mkdir_p index_directory
-    end
 
     # The index directory for this category.
     #
-    # TODO Push down into files?
+    # TODO Push down into files? Yes.
     #
     def index_directory
       @index_directory ||= "#{PICKY_ROOT}/index/#{PICKY_ENVIRONMENT}/#{@index.name}"

data/lib/picky/category_indexed.rb

@@ -4,15 +4,12 @@ module Picky
   #
   class Category
 
-    attr_reader :indexed_exact,
-                :indexed_partial
-
     # Loads the index from cache.
     #
     def load_from_cache
       timed_exclaim %Q{"#{identifier}": Loading index from cache.}
-      indexed_exact.load
-      indexed_partial.load
+      exact.load
+      partial.load
     end
     alias reload load_from_cache
 
@@ -31,7 +28,7 @@ module Picky
     # Returns the right index bundle for this token.
     #
     def bundle_for token
-      token.partial? ? indexed_partial : indexed_exact
+      token.partial? ? partial : exact
     end
 
     # Returns a combination for the token,

data/lib/picky/category_indexing.rb

@@ -4,8 +4,8 @@ module Picky
   #
   class Category
 
-    attr_reader :indexing_exact,
-                :indexing_partial
+    attr_reader :exact,
+                :partial
 
     # Prepares and caches this category.
     #
@@ -19,10 +19,15 @@ module Picky
     # Indexes, creates the "prepared_..." file.
     #
     def prepare
+      empty
       with_data_snapshot do
         indexer.index [self]
       end
     end
+    def empty
+      exact.empty
+      partial.empty_configuration
+    end
 
     # Take a data snapshot if the source offers it.
     #
@@ -42,23 +47,19 @@ module Picky
       generate_caches_from_source
       generate_partial
       generate_caches_from_memory
-      dump_caches
+      dump
       timed_exclaim %Q{"#{identifier}": Caching finished.}
     end
     # Generate the cache data.
     #
     def generate_caches_from_source
-      indexing_exact.generate_caches_from_source
+      exact.generate_caches_from_source
     end
     def generate_partial
-      indexing_partial.generate_partial_from indexing_exact.inverted
+      partial.generate_partial_from exact.inverted
     end
     def generate_caches_from_memory
-      indexing_partial.generate_caches_from_memory
-    end
-    def dump_caches
-      indexing_exact.dump
-      indexing_partial.dump
+      partial.generate_caches_from_memory
     end
 
     # Return an appropriate source.
@@ -111,16 +112,16 @@ module Picky
     #
     def check
       timed_exclaim "Checking #{identifier}."
-      indexing_exact.raise_unless_cache_exists
-      indexing_partial.raise_unless_cache_exists
+      exact.raise_unless_cache_exists
+      partial.raise_unless_cache_exists
     end
 
     # Deletes the caches.
     #
     def clear
       timed_exclaim "Deleting #{identifier}."
-      indexing_exact.delete
-      indexing_partial.delete
+      exact.delete
+      partial.delete
     end
 
     # Backup the caches.
@@ -128,8 +129,8 @@ module Picky
     #
     def backup
       timed_exclaim "Backing up #{identifier}."
-      indexing_exact.backup
-      indexing_partial.backup
+      exact.backup
+      partial.backup
     end
 
     # Restore the caches.
@@ -137,8 +138,8 @@ module Picky
     #
     def restore
       timed_exclaim "Restoring #{identifier}."
-      indexing_exact.restore
-      indexing_partial.restore
+      exact.restore
+      partial.restore
     end
 
   end

data/lib/picky/category_realtime.rb

@@ -6,15 +6,15 @@ module Picky
     # given id.
     #
     def remove id
-      indexed_exact.remove id
-      indexed_partial.remove id
+      exact.remove id
+      partial.remove id
     end
 
     # Adds and indexes this category of the
     # given object.
     #
     def add object
-      tokens, _ = tokenizer.tokenize object.send(from).to_s
+      tokens, _ = tokenizer.tokenize object.send(from)
       add_tokenized object.id, tokens
     end
 
@@ -33,13 +33,8 @@ module Picky
       tokens.each do |text|
         next unless text
         text = text.to_sym
-        indexed_exact.add id, text
-
-        # TODO Refactor. Push into indexed_partial?
-        #
-        indexed_partial.partial_strategy.each_partial text do |partial_text|
-          indexed_partial.add id, partial_text
-        end
+        exact.add id, text
+        partial.add_partialized id, text
       end
     end
 

data/lib/picky/extensions/symbol.rb

@@ -19,7 +19,7 @@ class Symbol # :nodoc:all
 
   # :keys.each_intoken         # => yields each of [:keys, :key, :eys, :ke, :ey, :ys, :k, :e, :y, :s]
   # :keys.each_intoken(2)      # => yields each of [:keys, :key, :eys, :ke, :ey, :ys]
-  # :keys.each_intoken(2, 3)   # => yields each of [:keys, :key, :eys]
+  # :keys.each_intoken(2, 3)   # => yields each of [:key, :eys, :ke, :ey, :ys]
   # :keys.each_intoken(10, 12) # => yields nothing (min larger than sym)
   #
   def each_intoken min_length = 1, max_length = -1

data/lib/picky/generators/partial/default.rb

@@ -2,7 +2,7 @@ module Picky
 
   module Generators
     module Partial
-      Default = Substring.new from: -3, to: -1
+      Default = Postfix.new from: -3
     end
   end
 

data/lib/picky/generators/partial/postfix.rb

@@ -0,0 +1,30 @@
+module Picky
+
+  module Generators
+
+    module Partial
+
+      class Postfix < Substring
+
+        # The from option signifies where in the symbol it
+        # will start in generating the subtokens.
+        #
+        # Examples:
+        #
+        # With :hello
+        # * from: 1 # => [:hello, :hell, :hel, :he, :h]
+        # * from: 4 # => [:hello, :hell]
+        #
+        def initialize options = {}
+          options[:to] = -1
+
+          super options
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/generators/partial/substring.rb

@@ -18,8 +18,14 @@ module Picky
               token.each_subtoken @from, &block
             end
           else
-            def each_subtoken token, &block
-              token[0..@to].intern.each_subtoken @from, &block
+            if @from < 0 && @to < 0
+              def each_subtoken token, &block
+                token[0..@to].intern.each_subtoken @from - @to - 1, &block
+              end
+            else
+              def each_subtoken token, &block
+                token[0..@to].intern.each_subtoken @from, &block
+              end
             end
           end
 

data/lib/picky/index.rb

@@ -90,6 +90,7 @@ module Picky
                 :categories
 
     delegate :[],
+             :dump,
              :each_category,
              :to => :categories
 
@@ -102,7 +103,7 @@ module Picky
     # * source: Where the data comes from, e.g. Sources::CSV.new(...). Optional, can be defined in the block using #source.
     # * result_identifier: Use if you'd like a different identifier/name in the results than the name of the index.
     # * after_indexing: As of this writing only used in the db source. Executes the given after_indexing as SQL after the indexing process.
-    # * tokenizer: Call and pass either a tokenizer (responds to #tokenize) or the options for a tokenizer..
+    # * indexing: Call and pass either a tokenizer (responds to #tokenize) or the options for a tokenizer..
     # * key_format: Call and pass in a format method for the ids (default is #to_i).
     #
     # Example:
@@ -231,8 +232,7 @@ module Picky
       options = { partial: Partial::None.new }.merge options
 
       define_category category_name, options do |category|
-        Indexing::Wrappers::Category::Location.install_on category, range, precision
-        Indexed::Wrappers::Category::Location.install_on category, range, precision
+        Wrappers::Category::Location.wrap category, range, precision
       end
     end
     alias define_ranged_category ranged_category

data/lib/picky/index_indexing.rb

@@ -27,7 +27,7 @@ module Picky
         index_in_parallel
       else
         with_data_snapshot do
-          categories.each &:index
+          categories.index
         end
       end
     end
@@ -73,9 +73,10 @@ module Picky
     # Only use where the category does have a #each source defined.
     #
     def index_in_parallel
+      categories.empty
       indexer = Indexers::Parallel.new self
       indexer.index categories
-      categories.each &:cache
+      categories.cache
     end
 
     # Returns the installed tokenizer or the default.

data/lib/picky/indexers/base.rb

@@ -20,18 +20,10 @@ module Picky
       #
       def index categories
         start_indexing_message
-        prepare categories
         process categories
         finish_indexing_message
       end
 
-      # By default, an indexer
-      #  * prepares the index directories.
-      #
-      def prepare categories
-        categories.each &:prepare_index_directory
-      end
-
     end
 
   end

data/lib/picky/indexers/parallel.rb

@@ -45,7 +45,7 @@ module Picky
           # Is it a good idea that not the tokenizer has control over when he gets the next text?
           #
           combined.each do |category, cache, _, tokenizer|
-            tokens, _ = tokenizer.tokenize object.send(category.from).to_s # Note: Originals not needed.
+            tokens, _ = tokenizer.tokenize object.send(category.from) # Note: Originals not needed.
             tokens.each do |token_text|
               next unless token_text
               cache << id << comma << token_text << newline

data/lib/picky/loader.rb

@@ -83,6 +83,7 @@ module Picky
       load_relative 'generators/partial/strategy'
       load_relative 'generators/partial/none'
       load_relative 'generators/partial/substring'
+      load_relative 'generators/partial/postfix'
       load_relative 'generators/partial/infix'
       load_relative 'generators/partial/default'
 
@@ -133,21 +134,23 @@ module Picky
       # Indexing and Indexed things.
       #
       load_relative 'bundle'
+      load_relative 'bundle_indexing'
+      load_relative 'bundle_indexed'
+      load_relative 'bundle_realtime'
 
-      load_relative 'indexing/bundle'
-      load_relative 'indexing/wrappers/category/location'
-
-      load_relative 'indexed/bundle'
-      load_relative 'indexed/bundle_realtime'
-      load_relative 'indexed/wrappers/exact_first'
-
-      # Bundle Wrapper
+      # Wrappers.
       #
-      load_relative 'indexed/wrappers/bundle/wrapper'
-      load_relative 'indexed/wrappers/bundle/calculation'
-      load_relative 'indexed/wrappers/bundle/location'
+      load_relative 'wrappers/category/location'
+      load_relative 'wrappers/category/exact_first'
 
-      load_relative 'indexed/wrappers/category/location'
+      load_relative 'wrappers/bundle/delegators'
+      load_relative 'wrappers/bundle/wrapper'
+      load_relative 'wrappers/bundle/calculation'
+      load_relative 'wrappers/bundle/location'
+      load_relative 'wrappers/bundle/exact_partial'
+
+      load_relative 'wrappers/sources/base'
+      load_relative 'wrappers/sources/location'
 
       # Tokens.
       #
@@ -245,9 +248,6 @@ module Picky
       load_relative 'sources/couch'
       load_relative 'sources/mongo'
 
-      load_relative 'sources/wrappers/base'
-      load_relative 'sources/wrappers/location'
-
       # Interfaces
       #
       load_relative 'interfaces/live_parameters'

data/lib/picky/query/qualifier_category_mapper.rb

@@ -31,7 +31,7 @@ module Picky
       # Returns nil if it is not allowed, the normalized qualifier if it is.
       #
       def map qualifier
-        return nil if qualifier.blank?
+        return nil if qualifier.empty?
 
         @mapping[qualifier.to_sym]
       end

data/lib/picky/rack/harakiri.rb

@@ -17,7 +17,9 @@ module Picky
 
       # Set the amount of requests before the Unicorn commits Harakiri.
       #
-      cattr_accessor :after
+      class << self
+        attr_accessor :after
+      end
 
       def initialize app
         @app = app

data/lib/picky/sources/db.rb

@@ -33,11 +33,22 @@ module Picky
       @@traversal_id = :__picky_id
 
       def initialize select_statement, options = { file: 'app/db.yml' }
+        check_gem
+
         @select_statement = select_statement
         @database         = create_database_adapter
         @options          = options
       end
 
+      # Tries to require the active_record gem.
+      #
+      def check_gem # :nodoc:
+        require 'active_record'
+      rescue LoadError
+        warn_gem_missing 'active_record', 'the (ActiveRecord) DB source'
+        exit 1
+      end
+
       def to_s
         parameters = [select_statement.inspect]
         parameters << options unless options.empty?

data/lib/picky/statistics.rb

@@ -34,8 +34,8 @@ module Picky
         @indexes << <<-ANALYSIS
   #{"#{category.index_name}".indented_to_s}\n
   #{"#{category.name}".indented_to_s(4)}\n
-  #{"exact\n#{Analyzer.new.analyze(category.indexed_exact).indented_to_s}".indented_to_s(6)}\n
-  #{"partial\n#{Analyzer.new.analyze(category.indexed_partial).indented_to_s}".indented_to_s(6)}
+  #{"exact\n#{Analyzer.new.analyze(category.exact).indented_to_s}".indented_to_s(6)}\n
+  #{"partial\n#{Analyzer.new.analyze(category.partial).indented_to_s}".indented_to_s(6)}
   ANALYSIS
       end
     end

data/lib/picky/tokenizer.rb

@@ -172,7 +172,7 @@ Case sensitive?     #{@case_sensitive ? "Yes." : "-"}
     #  [[:token1, :token2], ["Original1", "Original2"]]
     #
     def tokenize text
-      text   = preprocess text  # processing the text
+      text   = preprocess text.to_s # processing the text
       return empty_tokens if text.blank?
       words  = pretokenize text # splitting and preparations for tokenizing
       return empty_tokens if words.empty?

data/lib/picky/tokenizers/location.rb

@@ -43,7 +43,7 @@ module Picky
 
         # TODO Move to the right place.
         #
-        category.indexing_exact[:location_minimum] = minimum
+        category.exact[:location_minimum] = minimum
       end
 
     end

data/lib/picky/wrappers/bundle/calculation.rb

@@ -0,0 +1,45 @@
+module Picky
+
+  module Wrappers
+
+    module Bundle
+
+      # A calculation rewrites the symbol into a float.
+      #
+      # Note: A calculation will try to find a float in the index,
+      #       not a sym.
+      #
+      # TODO I really need to allow integers as keys.
+      #      The code below is just not up to the needed quality.
+      #      Use key_format :to_i?
+      #
+      class Calculation < Wrapper
+
+        # API.
+        #
+        # By default, a calculation does not
+        # recalculate anything.
+        #
+        def recalculate float
+          float
+        end
+
+        #
+        #
+        def ids float_as_sym
+          @bundle.ids recalculate(float_as_sym.to_s.to_f).to_s.to_sym
+        end
+
+        #
+        #
+        def weight float_as_sym
+          @bundle.weight recalculate(float_as_sym.to_s.to_f).to_s.to_sym
+        end
+
+      end
+
+    end
+
+  end
+
+end