picky 3.0.1 → 3.1.0

data/lib/picky/indexing/bundle.rb

@@ -0,0 +1,188 @@
+module Picky
+
+  module Indexing # :nodoc:all
+
+    # A Bundle is a number of indexes
+    # per [index, category] combination.
+    #
+    # At most, there are three indexes:
+    # * *core* index (always used)
+    # * *weights* index (always used)
+    # * *similarity* index (used with similarity)
+    #
+    # In Picky, indexing is separated from the index
+    # handling itself through a parallel structure.
+    #
+    # Both use methods provided by this base class, but
+    # have very different goals:
+    #
+    # * *Indexing*::*Bundle* is just concerned with creating index files
+    #   and providing helper functions to e.g. check the indexes.
+    #
+    # * *Index*::*Bundle* is concerned with loading these index files into
+    #   memory and looking up search data as fast as possible.
+    #
+    # This is the indexing bundle.
+    #
+    # It does all menial tasks that have nothing to do
+    # with the actual index running etc.
+    # (Find these in Indexed::Bundle)
+    #
+    class Bundle < Picky::Bundle
+
+      attr_reader :backend,
+                  :prepared
+
+      attr_accessor :partial_strategy,
+                    :weights_strategy
+
+      # When indexing, clear only clears the inverted index.
+      #
+      delegate :clear, :to => :inverted
+
+      def initialize name, category, backend, weights_strategy, partial_strategy, similarity_strategy, options = {}
+        super name, category, backend, similarity_strategy, options
+
+        @weights_strategy = weights_strategy
+        @partial_strategy = partial_strategy
+        @key_format       = options[:key_format]
+        @prepared         = Backends::File::Text.new category.prepared_index_path
+
+        @inverted      = {}
+        @weights       = {}
+        @similarity    = {}
+        @configuration = {}
+      end
+
+      # Sets up a piece of the index for the given token.
+      #
+      def initialize_inverted_index_for token
+        self.inverted[token] ||= []
+      end
+
+      # Generation
+      #
+
+      # This method
+      # * Loads the base index from the "prepared..." file.
+      # * Generates derived indexes.
+      # * Dumps all the indexes into files.
+      #
+      def generate_caches_from_source
+        load_from_prepared_index_file
+        generate_caches_from_memory
+      end
+      # Generates derived indexes from the index and dumps.
+      #
+      # Note: assumes that there is something in the index
+      #
+      def generate_caches_from_memory
+        cache_from_memory_generation_message
+        generate_derived
+      end
+      def cache_from_memory_generation_message
+        timed_exclaim %Q{"#{identifier}": Caching from intermediate in-memory index.}
+      end
+
+      # Generates the weights and similarity from the main index.
+      #
+      def generate_derived
+        generate_weights
+        generate_similarity
+      end
+
+      # Load the data from the db.
+      #
+      def load_from_prepared_index_file
+        load_from_prepared_index_generation_message
+        clear
+        retrieve
+      end
+      def load_from_prepared_index_generation_message
+        timed_exclaim %Q{"#{identifier}": Loading prepared data into memory.}
+      end
+      # Retrieves the prepared index data into the index.
+      #
+      # This is in preparation for generating
+      # derived indexes (like weights, similarity)
+      # and later dumping the optimized index.
+      #
+      # TODO Move this out to the category?
+      #
+      def retrieve
+        format = category.key_format || :to_i # Optimization.
+        prepared.retrieve do |id, token|
+          initialize_inverted_index_for token
+          self.inverted[token] << id.send(format)
+        end
+      end
+
+      # Generates a new index (writes its index) using the
+      # partial caching strategy of this bundle.
+      #
+      def generate_partial
+        generator = Generators::PartialGenerator.new self.inverted
+        self.inverted = generator.generate self.partial_strategy
+      end
+      # Generate a partial index from the given exact inverted index.
+      #
+      def generate_partial_from exact_inverted_index
+        timed_exclaim %Q{"#{identifier}": Generating partial index for index.}
+        self.inverted = exact_inverted_index
+        self.generate_partial
+        self
+      end
+      # Generates a new weights index (writes its index) using the
+      # given weight caching strategy.
+      #
+      def generate_weights
+        generator = Generators::WeightsGenerator.new self.inverted
+        self.weights = generator.generate self.weights_strategy
+      end
+      # Generates a new similarity index (writes its index) using the
+      # given similarity caching strategy.
+      #
+      def generate_similarity
+        generator = Generators::SimilarityGenerator.new self.inverted
+        self.similarity = generator.generate self.similarity_strategy
+      end
+
+      # Saves the indexes in a dump file.
+      #
+      def dump
+        timed_exclaim %Q{"#{identifier}": Dumping data.}
+        dump_inverted
+        dump_similarity
+        dump_weights
+        dump_configuration
+      end
+      # Dumps the core index.
+      #
+      def dump_inverted
+        # timed_exclaim %Q{"#{identifier}": Dumping inverted index.}
+        @backend_inverted.dump self.inverted
+      end
+      # Dumps the weights index.
+      #
+      def dump_weights
+        # timed_exclaim %Q{"#{identifier}": Dumping index weights.}
+        @backend_weights.dump self.weights
+      end
+      # Dumps the similarity index.
+      #
+      def dump_similarity
+        # timed_exclaim %Q{"#{identifier}": Dumping similarity index.}
+        @backend_similarity.dump self.similarity
+      end
+      # Dumps the similarity index.
+      #
+      def dump_configuration
+        # timed_exclaim %Q{"#{identifier}": Dumping configuration.}
+        @backend_configuration.dump self.configuration
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/indexing/wrappers/category/location.rb

@@ -11,7 +11,7 @@ module Picky
 
             category.class_eval do
               def tokenizer
-                @tokenizer ||= Tokenizers::Index.new
+                @tokenizer ||= Tokenizer.new
               end
               define_method :source do
                 new_source

data/lib/picky/interfaces/live_parameters.rb

@@ -144,29 +144,29 @@ module Picky
       # TODO Move to Interface object.
       #
       def querying_removes_characters
-        regexp = Tokenizers::Query.default.instance_variable_get :@removes_characters_regexp
+        regexp = Tokenizer.query_default.instance_variable_get :@removes_characters_regexp
         regexp && regexp.source
       end
       def querying_removes_characters= new_value
-        Tokenizers::Query.default.instance_variable_set(:@removes_characters_regexp, %r{#{new_value}})
+        Tokenizer.query_default.instance_variable_set(:@removes_characters_regexp, %r{#{new_value}})
       end
       def querying_stopwords
-        regexp = Tokenizers::Query.default.instance_variable_get :@remove_stopwords_regexp
+        regexp = Tokenizer.query_default.instance_variable_get :@remove_stopwords_regexp
         regexp && regexp.source
       end
       def querying_stopwords= new_value
-        Tokenizers::Query.default.instance_variable_set(:@remove_stopwords_regexp, %r{#{new_value}})
+        Tokenizer.query_default.instance_variable_set(:@remove_stopwords_regexp, %r{#{new_value}})
       end
       def querying_splits_text_on
-        splits = Tokenizers::Query.default.instance_variable_get :@splits_text_on
+        splits = Tokenizer.query_default.instance_variable_get :@splits_text_on
         splits && splits.respond_to?(:source) ? splits.source : splits
       end
       def querying_splits_text_on= new_value
-        splits = Tokenizers::Query.default.instance_variable_get :@splits_text_on
+        splits = Tokenizer.query_default.instance_variable_get :@splits_text_on
         if splits.respond_to?(:source)
-          Tokenizers::Query.default.instance_variable_set(:@splits_text_on, %r{#{new_value}})
+          Tokenizer.query_default.instance_variable_set(:@splits_text_on, %r{#{new_value}})
         else
-          Tokenizers::Query.default.instance_variable_set(:@splits_text_on, new_value)
+          Tokenizer.query_default.instance_variable_set(:@splits_text_on, new_value)
         end
       end
 

data/lib/picky/loader.rb

@@ -110,34 +110,28 @@ module Picky
 
       # Index store handling.
       #
-      load_relative 'backend/base'
+      load_relative 'backends/backend'
 
-      load_relative 'backend/redis'
-      load_relative 'backend/redis/basic'
-      load_relative 'backend/redis/list_hash'
-      load_relative 'backend/redis/string_hash'
+      load_relative 'backends/redis'
+      load_relative 'backends/redis/basic'
+      load_relative 'backends/redis/list_hash'
+      load_relative 'backends/redis/string_hash'
+      load_relative 'backends/redis/float_hash'
 
-      load_relative 'backend/file/basic'
-      load_relative 'backend/file/text'
-      load_relative 'backend/file/marshal'
-      load_relative 'backend/file/json'
-
-      load_relative 'backend/files'
+      load_relative 'backends/file/basic'
+      load_relative 'backends/file/text'
+      load_relative 'backends/file/marshal'
+      load_relative 'backends/file/json'
+      load_relative 'backends/memory'
 
       # Indexing and Indexed things.
       #
       load_relative 'bundle'
 
-      load_relative 'indexing/bundle/base'
-      load_relative 'indexing/bundle/memory'
-      load_relative 'indexing/bundle/redis'
-
+      load_relative 'indexing/bundle'
       load_relative 'indexing/wrappers/category/location'
 
-      load_relative 'indexed/bundle/base'
-      load_relative 'indexed/bundle/memory'
-      load_relative 'indexed/bundle/redis'
-
+      load_relative 'indexed/bundle'
       load_relative 'indexed/wrappers/exact_first'
 
       # Bundle Wrapper
@@ -153,18 +147,14 @@ module Picky
       load_relative 'query/token'
       load_relative 'query/tokens'
 
-      # Tokenizers types.
+      # Tokenizer.
       #
-      load_relative 'tokenizers/base'
-      load_relative 'tokenizers/index'
-      load_relative 'tokenizers/query'
+      load_relative 'tokenizer'
 
       # Query combinations, qualifiers, weigher.
       #
       load_relative 'query/combination'
-      load_relative 'query/combinations/base'
-      load_relative 'query/combinations/memory'
-      load_relative 'query/combinations/redis'
+      load_relative 'query/combinations'
 
       load_relative 'query/allocation'
       load_relative 'query/allocations'
@@ -174,10 +164,7 @@ module Picky
       load_relative 'query/weights'
 
       load_relative 'query/indexes'
-
-      # Configuration.
-      #
-      # load_internals 'configuration/index'
+      load_relative 'query/indexes_check'
 
       # Adapters.
       #
@@ -227,11 +214,9 @@ module Picky
       load_relative 'indexes_indexed'
       load_relative 'indexes_indexing'
 
-      load_relative 'indexes/index'
-      load_relative 'indexes/index_indexed'
-      load_relative 'indexes/index_indexing'
-      load_relative 'indexes/memory'
-      load_relative 'indexes/redis'
+      load_relative 'index'
+      load_relative 'index_indexed'
+      load_relative 'index_indexing'
 
       # Results.
       #
@@ -240,9 +225,6 @@ module Picky
       # Search.
       #
       load_relative 'search'
-      load_relative 'query'
-      #
-      # load_relative 'query/solr'
 
       # Sources.
       #
@@ -267,6 +249,10 @@ module Picky
       # Load tools. Load in specific case?
       #
       load_relative 'cores'
+
+      # Load migration notices.
+      #
+      load_relative 'migrations/from_30_to_31'
     end
 
     # Loads the framework.

data/lib/picky/migrations/from_30_to_31.rb

@@ -0,0 +1,61 @@
+module Picky
+
+  class Indexes
+
+    class Memory
+
+      def initialize(*)
+        raise <<-MESSAGE
+
+The Picky::Indexes::Memory is not available anymore and has been replaced by Picky::Index.
+
+So instead of
+
+  index = Picky::Indexes::Memory.new :name do
+    # your config
+  end
+
+use
+
+  index = Picky::Index.new :name do
+    # your config
+  end
+
+Thanks and sorry for the inconvenience!
+
+MESSAGE
+      end
+
+    end
+
+    class Redis
+
+      def initialize(*)
+        raise <<-RAISE
+
+The Picky::Indexes::Redis is not available anymore and has been replaced by Picky::Index.
+(with the addition of a "backend" option)
+
+So instead of
+
+  index = Picky::Indexes::Redis.new :name do
+    # your config
+  end
+
+use
+
+  index = Picky::Index.new :name do
+    backend Picky::Backends::Redis.new
+    # your config
+  end
+
+Thanks and sorry for the inconvenience!
+
+RAISE
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/query/allocation.rb

@@ -7,13 +7,18 @@ module Picky
     #
     class Allocation # :nodoc:all
 
-      attr_reader :count, :ids, :score, :combinations, :result_identifier
+      attr_reader :count,
+                  :ids,
+                  :score,
+                  :combinations,
+                  :result_identifier
 
       #
       #
-      def initialize combinations, result_identifier
+      def initialize index, combinations
         @combinations      = combinations
-        @result_identifier = result_identifier
+        @result_identifier = index.result_identifier # TODO Make cleverer.
+        @backend           = index.backend           # TODO Make cleverer.
       end
 
       def hash
@@ -30,10 +35,10 @@ module Picky
         @score ||= @combinations.calculate_score(weights)
       end
 
-      # Asks the combinations for the (intersected) ids.
+      # Asks the backend for the (intersected) ids.
       #
       def calculate_ids amount, offset
-        @combinations.ids amount, offset # Calculate as many ids as are necessary.
+        @backend.ids combinations, amount, offset
       end
 
       # This starts the searching process.

data/lib/picky/query/combinations.rb

@@ -0,0 +1,70 @@
+module Picky
+
+  module Query
+
+    # Combinations are a number of Combination-s.
+    #
+    # They are the core of an allocation.
+    # An allocation consists of a number of combinations.
+    #
+    # Base Combinations contain methods for calculating score and ids.
+    #
+    class Combinations # :nodoc:all
+
+      attr_reader :combinations
+
+      delegate :empty?, :inject, :to => :@combinations
+
+      def initialize combinations = []
+        @combinations = combinations
+      end
+
+      def hash
+        @combinations.hash
+      end
+
+      # Uses user specific weights to calculate a score for the combinations.
+      #
+      def calculate_score weights
+        total_score + weighted_score(weights)
+      end
+      def total_score
+        @combinations.sum &:weight
+      end
+      def weighted_score weights
+        weights.score_for @combinations
+      end
+
+      # Filters the tokens and identifiers such that only identifiers
+      # that are passed in, remain, including their tokens.
+      #
+      # Note: This method is not totally independent of the calculate_ids one.
+      #       Since identifiers are only nullified, we need to not include the
+      #       ids that have an associated identifier that is nil.
+      #
+      def keep identifiers = []
+        @combinations.reject! { |combination| !combination.in?(identifiers) }
+      end
+
+      # Filters the tokens and identifiers such that identifiers
+      # that are passed in, are removed, including their tokens.
+      #
+      # Note: This method is not totally independent of the calculate_ids one.
+      #       Since identifiers are only nullified, we need to not include the
+      #       ids that have an associated identifier that is nil.
+      #
+      def remove identifiers = []
+        @combinations.reject! { |combination| combination.in?(identifiers) }
+      end
+
+      #
+      #
+      def to_result
+        @combinations.map &:to_result
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/query/indexes.rb

@@ -20,10 +20,11 @@ module Picky
       # Its job is to generate all possible combinations.
       # Note: We cannot mix memory and redis indexes just yet.
       #
-      def initialize *indexes, combinations_type
-        @indexes           = indexes
-        @combinations_type = combinations_type
-        @mapper            = Query::QualifierCategoryMapper.new
+      def initialize *indexes
+        IndexesCheck.check_backend_types indexes
+
+        @indexes = indexes
+        @mapper  = Query::QualifierCategoryMapper.new
         map_categories
       end
       def map_categories
@@ -81,12 +82,12 @@ module Picky
 
         # Generate all possible combinations.
         #
-        expanded_combinations = expand_combinations_from possible_combinations
+        all_possible_combinations = expand_combinations_from possible_combinations
 
         # Add the wrapped possible allocations to the ones we already have.
         #
-        expanded_combinations.map! do |expanded_combination|
-          Allocation.new @combinations_type.new(expanded_combination), index.result_identifier # TODO Do not extract result_identifier.
+        all_possible_combinations.map! do |expanded_combinations|
+          Allocation.new index, Query::Combinations.new(expanded_combinations)
         end
       end
 

data/lib/picky/query/indexes_check.rb

@@ -0,0 +1,47 @@
+module Picky
+
+  module Query
+
+    # TODO Remove.
+    #
+    class IndexesCheck
+
+      class << self
+
+        # Returns the right combinations strategy for
+        # a number of query indexes.
+        #
+        # Currently it isn't possible using Memory and Redis etc.
+        # indexes in the same query index group.
+        #
+        # Picky will raise a Query::Indexes::DifferentTypesError.
+        #
+        def check_backend_types index_definitions_ary # :nodoc:
+          backend_types = index_definitions_ary.map(&:backend).map(&:class)
+          backend_types.uniq!
+          raise_different backend_types if backend_types.size > 1
+          backend_types
+        end
+        def raise_different backend_types # :nodoc:
+          raise DifferentTypesError.new(backend_types)
+        end
+
+      end
+
+    end
+
+    # Currently it isn't possible using Memory and Redis etc.
+    # indexes in the same query index group.
+    #
+    class DifferentTypesError < StandardError # :nodoc:all
+      def initialize types
+        @types = types
+      end
+      def to_s
+        "Currently it isn't possible to mix Indexes with backends #{@types.join(" and ")} in the same Search instance."
+      end
+    end
+
+  end
+
+end

data/lib/picky/query/token.rb

@@ -19,8 +19,9 @@ module Picky
       #
       # Note: Use this if you do not want a normalized token.
       #
-      def initialize text
-        @text = text
+      def initialize text, original = nil
+        @text     = text
+        @original = original
       end
 
       # Returns a qualified and normalized token.
@@ -28,20 +29,24 @@ module Picky
       # Note: Use this in the search engine if you need a qualified
       #       and normalized token. I.e. one prepared for a search.
       #
-      def self.processed text, downcase = true
-        new(text).process downcase
+      def self.processed text, original = nil
+        new(text, original).process
       end
-      def process downcased = true
-        qualify
-        extract_original
-        downcase if downcased
-        partialize
-        similarize
-        remove_illegals
+      def process # TODO Move this into the processed method and let the token have more params?
+        qualify # TODO Should this operate on the original?
+        partialize # TODO Should this operate on the original?
+        similarize # TODO Should this operate on the original?
+        remove_illegals # TODO Remove?
         symbolize
         self
       end
 
+      #
+      #
+      def symbolize
+        @text = @text.to_sym
+      end
+
       # Translates this token's qualifiers into actual categories.
       #
       # Note: If this is not done, there is no mapping.
@@ -52,18 +57,6 @@ module Picky
         end.compact
       end
 
-      # Dups the original text.
-      #
-      def extract_original
-        @original = @text.dup
-      end
-
-      # Downcases the text.
-      #
-      def downcase
-        @text.downcase!
-      end
-
       # Partial is a conditional setter.
       #
       # It is only settable if it hasn't been set yet.
@@ -108,12 +101,6 @@ module Picky
         @text.gsub! @@illegals, '' unless @text.blank?
       end
 
-      #
-      #
-      def symbolize
-        @text = @text.to_sym
-      end
-
       # Returns an array of possible combinations.
       #
       def possible_combinations_in index