picky 1.4.1 → 1.4.2

data/lib/picky/internals/indexed/bundle/base.rb

@@ -0,0 +1,72 @@
+module Internals
+
+  module Indexed # :nodoc:all
+    
+    # TODO Rewrite.
+    #
+    # A Bundle is a number of indexes
+    # per [index, category] combination.
+    #
+    # At most, there are three indexes:
+    # * *core* index (always used)
+    # * *weights* index (always used)
+    # * *similarity* index (used with similarity)
+    # 
+    # In Picky, indexing is separated from the index
+    # handling itself through a parallel structure.
+    #
+    # Both use methods provided by this base class, but
+    # have very different goals:
+    #
+    # * *Indexing*::*Bundle* is just concerned with creating index files
+    #   and providing helper functions to e.g. check the indexes.
+    #
+    # * *Index*::*Bundle* is concerned with loading these index files into
+    #   memory and looking up search data as fast as possible.
+    #
+    module Bundle
+    
+      class Base
+    
+        attr_reader   :identifier, :configuration
+        attr_accessor :similarity_strategy
+      
+        delegate :[], :to => :configuration
+    
+        def initialize name, configuration, similarity_strategy
+          @identifier = "#{configuration.identifier}:#{name}"
+      
+          @index         = {}
+          @weights       = {}
+          @similarity    = {}
+      
+          @similarity_strategy = similarity_strategy
+        end
+    
+        # Get a list of similar texts.
+        #
+        # Note: Does not return itself.
+        #
+        def similar text
+          code = similarity_strategy.encoded text
+          similar_codes = code && @similarity[code]
+          similar_codes.delete text if similar_codes
+          similar_codes || []
+        end
+      
+        # Loads all indexes.
+        #
+        def load
+          load_index
+          load_weights
+          load_similarity
+          load_configuration
+        end
+      
+      end
+    
+    end
+    
+  end
+  
+end

data/lib/picky/internals/indexed/bundle/memory.rb

@@ -0,0 +1,69 @@
+module Internals
+
+  # encoding: utf-8
+  #
+  module Indexed # :nodoc:all
+  
+    #
+    #
+    module Bundle
+    
+      # This is the _actual_ index (based on memory).
+      #
+      # Handles exact/partial index, weights index, and similarity index.
+      #
+      # Delegates file handling and checking to an *Indexed*::*Files* object.
+      #
+      class Memory < Base
+      
+        attr_accessor :index, :weights, :similarity, :configuration
+      
+        delegate :[], :to => :configuration
+      
+        def initialize name, configuration, *args
+          super name, configuration, *args
+        
+          @configuration = {} # A hash with config options.
+        
+          @backend = Internals::Index::Files.new name, configuration
+        end
+      
+        # Get the ids for the given symbol.
+        #
+        def ids sym
+          @index[sym] || []
+        end
+        # Get a weight for the given symbol.
+        #
+        def weight sym
+          @weights[sym]
+        end
+      
+        # Loads the core index.
+        #
+        def load_index
+          self.index = @backend.load_index
+        end
+        # Loads the weights index.
+        #
+        def load_weights
+          self.weights = @backend.load_weights
+        end
+        # Loads the similarity index.
+        #
+        def load_similarity
+          self.similarity = @backend.load_similarity
+        end
+        # Loads the configuration.
+        #
+        def load_configuration
+          self.configuration = @backend.load_configuration
+        end
+      
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/internals/indexed/bundle/redis.rb

@@ -0,0 +1,70 @@
+module Internals
+
+  # encoding: utf-8
+  #
+  module Indexed # :nodoc:all
+  
+    #
+    #
+    module Bundle
+    
+      # This is the _actual_ index (based on Redis).
+      #
+      # Handles exact/partial index, weights index, and similarity index.
+      #
+      class Redis < Base
+      
+        def initialize name, configuration, *args
+          super name, configuration, *args
+        
+          @backend = Internals::Index::Redis.new name, configuration
+        end
+      
+        # Get the ids for the given symbol.
+        #
+        # Ids are an array of string values in Redis.
+        #
+        def ids sym
+          @backend.ids sym
+        end
+        # Get a weight for the given symbol.
+        #
+        # A weight is a string value in Redis. TODO Convert?
+        #
+        def weight sym
+          @backend.weight sym
+        end
+        # TODO Spec. Doc.
+        #
+        def [] sym
+          @backend.setting sym
+        end
+      
+        # Loads the core index.
+        #
+        def load_index
+          # TODO check if it is there.
+        end
+        # Loads the weights index.
+        #
+        def load_weights
+          # TODO check if it is there.
+        end
+        # Loads the similarity index.
+        #
+        def load_similarity
+          # TODO check if it is there.
+        end
+        # Loads the configuration.
+        #
+        def load_configuration
+          # TODO check if it is there.
+        end
+      
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/internals/indexed/categories.rb

@@ -0,0 +1,135 @@
+module Internals
+
+  module Indexed
+  
+    class Categories
+    
+      attr_reader :categories, :category_hash, :ignore_unassigned_tokens
+    
+      each_delegate :load_from_cache,
+                    :to => :categories
+    
+      # A list of indexed categories.
+      #
+      # Options:
+      #  * ignore_unassigned_tokens: Ignore the given token if it cannot be matched to a category.
+      #                              The default behaviour is that if a token does not match to
+      #                              any category, the query will not return anything (since a
+      #                              single token cannot be matched). If you set this option to
+      #                              true, any token that cannot be matched to a category will be
+      #                              simply ignored.
+      #                              Use this if only a few matched words are important, like for
+      #                              example of the query "Jonathan Myers 86455 Las Cucarachas"
+      #                              you only want to match the zipcode, to have the search engine
+      #                              display advertisements on the side for the zipcode.
+      #                              Nifty! :)
+      #
+      def initialize options = {}
+        clear
+      
+        @ignore_unassigned_tokens = options[:ignore_unassigned_tokens] || false
+      end
+    
+      # Clears both the array of categories and the hash of categories.
+      #
+      def clear
+        @categories    = []
+        @category_hash = {}
+      end
+    
+      # Add the given category to the list of categories.
+      #
+      def << category
+        categories << category
+        # Note: [category] is an optimization, since I need an array
+        #       of categories.
+        #       It's faster to just package it in an array on loading
+        #       Picky than doing it over and over with each query.
+        #
+        category_hash[category.name] = [category]
+      end
+    
+      # Return all possible combinations for the given token.
+      #
+      # This checks if it needs to also search through similar
+      # tokens, if for example, the token is one with ~.
+      # If yes, it puts together all solutions.
+      #
+      def possible_combinations_for token
+        token.similar? ? similar_possible_for(token) : possible_for(token)
+      end
+      # Gets all similar tokens and puts together the possible combinations
+      # for each found similar token.
+      # 
+      def similar_possible_for token
+        # Get as many similar tokens as necessary
+        #
+        tokens = similar_tokens_for token
+        # possible combinations
+        #
+        inject_possible_for tokens
+      end
+      def similar_tokens_for token
+        text = token.text
+        categories.inject([]) do |result, category|
+          next_token = token
+          # Note: We could also break off here if not all the available
+          #       similars are needed.
+          #       Wait for a concrete case that needs this before taking
+          #       action.
+          #
+          while next_token = next_token.next_similar_token(category)
+            result << next_token if next_token && next_token.text != text
+          end
+          result
+        end
+      end
+      def inject_possible_for tokens
+        tokens.inject([]) do |result, token|
+          possible = possible_categories token
+          result + possible_for(token, possible)
+        end
+      end
+    
+      # Returns possible Combinations for the token.
+      #
+      # Note: The preselected_categories param is an optimization.
+      #
+      # Note: Returns [] if no categories matched (will produce no result).
+      #       Returns nil if this token needs to be removed from the query.
+      #       (Also none of the categories matched, but the ignore unassigned
+      #       tokens option is true)
+      #
+      def possible_for token, preselected_categories = nil
+        possible = (preselected_categories || possible_categories(token)).map { |category| category.combination_for(token) }
+        possible.compact!
+        # This is an optimization to mark tokens that are ignored.
+        #
+        return if ignore_unassigned_tokens && possible.empty?
+        possible # wrap in combinations
+      end
+      # This returns the possible categories for this token.
+      # If the user has already preselected a category for this token,
+      # like "artist:moby", if not just return all for the given token,
+      # since all are possible.
+      #
+      # Note: Once I thought this was called too often. But it is not (18.01.2011).
+      #
+      def possible_categories token
+        user_defined_categories(token) || categories
+      end
+      # This returns the array of categories if the user has defined
+      # an existing category.
+      #
+      # Note: Returns nil if the user did not define one
+      #       or if he/she has defined a non-existing one. 
+      #
+      def user_defined_categories token
+        category_hash[token.user_defined_category_name]
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/internals/indexed/category.rb

@@ -0,0 +1,90 @@
+module Internals
+
+  module Indexed
+  
+    # An index category holds a exact and a partial index for a given category.
+    #
+    # For example an index category for names holds a exact and
+    # a partial index bundle for names.
+    #
+    class Category
+    
+      attr_accessor :exact
+      attr_reader   :identifier, :name
+      attr_writer   :partial
+    
+      #
+      #
+      def initialize name, index, options = {}
+        @name = name
+      
+        configuration = Configuration::Index.new index, self
+      
+        @identifier    = configuration.identifier
+      
+        # TODO Push the defaults out into the index.
+        #
+        @partial_strategy = options[:partial] || Internals::Generators::Partial::Default
+        similarity = options[:similarity] || Internals::Generators::Similarity::Default
+      
+        bundle_class = options[:indexed_bundle_class] || Bundle::Memory
+        @exact   = bundle_class.new :exact,   configuration, similarity
+        @partial = bundle_class.new :partial, configuration, similarity
+      
+        # @exact   = exact_lambda.call(@exact, @partial)   if exact_lambda   = options[:exact_lambda]
+        # @partial = partial_lambda.call(@exact, @partial) if partial_lambda = options[:partial_lambda]
+      
+        # TODO Extract?
+        #
+        Query::Qualifiers.add(configuration.category_name, generate_qualifiers_from(options) || [name])
+      end
+    
+      # TODO Move to Index.
+      #
+      def generate_qualifiers_from options
+        options[:qualifiers] || options[:qualifier] && [options[:qualifier]]
+      end
+    
+      # Loads the index from cache.
+      #
+      def load_from_cache
+        timed_exclaim "Loading index #{identifier}."
+        exact.load
+        partial.load
+      end
+    
+      # Gets the weight for this token's text.
+      #
+      def weight token
+        bundle_for(token).weight token.text
+      end
+    
+      # Gets the ids for this token's text.
+      #
+      def ids token
+        bundle_for(token).ids token.text
+      end
+    
+      # Returns the right index bundle for this token.
+      #
+      def bundle_for token
+        token.partial?? partial : exact
+      end
+    
+      # The partial strategy defines whether to really use the partial index.
+      #
+      def partial
+        @partial_strategy.use_exact_for_partial?? @exact : @partial
+      end
+    
+      #
+      #
+      def combination_for token
+        weight(token) && Internals::Query::Combination.new(token, self)
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/internals/indexed/index.rb

@@ -0,0 +1,57 @@
+module Internals
+
+  module Indexed
+  
+    #
+    #
+    class Index
+    
+      attr_reader :name, :result_identifier, :combinator, :categories
+    
+      delegate :load_from_cache,
+               :to => :categories
+    
+      # TODO Externalize?
+      #
+      def initialize name, options = {}
+        @name                     = name
+      
+        @result_identifier        = options[:result_identifier] || name
+        @bundle_class             = options[:indexed_bundle_class] # TODO This should actually be a fixed parameter.
+        ignore_unassigned_tokens  = options[:ignore_unassigned_tokens] || false # TODO Move to query, somehow.
+      
+        @categories = Categories.new ignore_unassigned_tokens: ignore_unassigned_tokens
+      end
+    
+      # TODO Doc. Externalize?
+      #
+      def define_category category_name, options = {}
+        options = default_category_options.merge options
+      
+        new_category = Category.new category_name, self, options
+        categories << new_category
+        new_category
+      end
+    
+      # By default, the category uses
+      # * the index's bundle type.
+      #
+      def default_category_options
+        {
+          :indexed_bundle_class => @bundle_class
+        }
+      end
+    
+      # Return the possible combinations for this token.
+      #
+      # A combination is a tuple <token, index_bundle>.
+      #
+      def possible_combinations token
+        categories.possible_combinations_for token
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/{indexed → internals/indexed}/wrappers/bundle/location.rb

@@ -13,7 +13,7 @@ module Indexed
           precision  = options[:precision] || 1
           user_grid  = options[:grid] || raise("Gridsize needs to be given for location #{bundle.identifier}.")
           
-          @calculation         = Calculations::Location.new user_grid, precision
+          @calculation = Calculations::Location.new user_grid, precision
         end
         
         #
@@ -28,7 +28,9 @@ module Indexed
           # Load first the bundle, then extract the config.
           #
           bundle.load
-          minimum = bundle[:location_minimum] || raise("Configuration :location_minimum for #{bundle.identifier} missing. Did you run rake index already?")
+          # TODO Move the to_f to the backend.
+          #
+          minimum = bundle[:location_minimum] && bundle[:location_minimum].to_f || raise("Configuration :location_minimum for #{bundle.identifier} missing. Did you run rake index already?")
           @calculation.minimum = minimum
         end
         

data/lib/picky/{indexed → internals/indexed}/wrappers/bundle/wrapper.rb

@@ -15,7 +15,7 @@ module Indexed
           @bundle = bundle
         end
         
-        delegate :load, :ids, :weight, :to => :@bundle
+        delegate :load, :ids, :weight, :identifier, :to => :@bundle
         
       end
       

data/lib/picky/internals/indexed/wrappers/exact_first.rb

@@ -0,0 +1,65 @@
+module Internals
+
+  # encoding: utf-8
+  #
+  module Indexed
+  
+    # TODO Spec
+    #
+    module Wrappers
+    
+      # This index combines an exact and partial index.
+      # It serves to order the results such that exact hits are found first.
+      #
+      # TODO Need to use the right subtokens. Bake in?
+      #
+      class ExactFirst < Indexed::Bundle::Memory
+      
+        delegate :similar,
+                 :identifier,
+                 :name,
+                 :to => :@exact
+        delegate :index,
+                 :category,
+                 :weight,
+                 :generate_partial_from,
+                 :generate_caches_from_memory,
+                 :generate_derived,
+                 :dump,
+                 :load,
+                 :to => :@partial
+      
+        def initialize category
+          @exact   = category.exact
+          @partial = category.partial
+        end
+      
+        def self.wrap index_or_category
+          if index_or_category.respond_to? :categories
+            wrap_each_of index_or_category.categories
+            index_or_category
+          else
+            new index_or_category
+          end
+        end
+        # TODO Do not extract categories!
+        #
+        def self.wrap_each_of categories
+          categories.categories.collect! { |category| new(category) }
+        end
+      
+        def ids text
+          @exact.ids(text) + @partial.ids(text)
+        end
+      
+        def weight text
+          [@exact.weight(text) || 0, @partial.weight(text) || 0].max
+        end
+      
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/{indexers → internals/indexers}/serial.rb

@@ -17,7 +17,7 @@ module Indexers
     # Raise a no source exception.
     #
     def raise_no_source
-      raise NoSourceSpecifiedException.new("No source given for #{@configuration.identifier}.")
+      raise NoSourceSpecifiedException.new("No source given for #{@configuration}.")
     end
     
     # Delegates the key format to the source.
@@ -65,7 +65,7 @@ module Indexers
       end
     end
     def indexing_message
-      timed_exclaim "INDEX #{@configuration.identifier}" # TODO from ...
+      timed_exclaim "INDEX #{@configuration}" # TODO from ...
     end
     
   end