picky 2.5.2 → 2.6.0

data/lib/picky/backend/redis/string_hash.rb

@@ -0,0 +1,40 @@
+module Backend
+
+  class Redis
+
+    class StringHash < Basic
+
+      # Writes the hash into Redis.
+      #
+      # Note: We could use multi, but it did not help.
+      #
+      def dump hash
+        clear
+        hash.each_pair do |key, value|
+          backend.hset namespace, key, value
+        end
+      end
+      
+      # Clears the hash.
+      #
+      def clear
+        backend.del namespace
+      end
+
+      # Get a collection.
+      #
+      def collection sym
+        raise "Can't retrieve a collection from a StringHash. Use Index::Redis::ListHash."
+      end
+
+      # Get a single value.
+      #
+      def member sym
+        backend.hget namespace, sym
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/backend/redis.rb

@@ -0,0 +1,40 @@
+module Backend
+
+  # TODO Needs a reconnect to be run after forking.
+  #
+  class Redis < Backend
+
+    def initialize bundle_name, category
+      super bundle_name, category
+
+      # Refine a few Redis "types".
+      #
+      @index         = Redis::ListHash.new "#{category.identifier}:#{bundle_name}:index"
+      @weights       = Redis::StringHash.new "#{category.identifier}:#{bundle_name}:weights"
+      @similarity    = Redis::ListHash.new "#{category.identifier}:#{bundle_name}:similarity"
+      @configuration = Redis::StringHash.new "#{category.identifier}:#{bundle_name}:configuration"
+    end
+
+    # Delegate to the right collection.
+    #
+    def ids sym
+      index.collection sym
+    end
+
+    # Delegate to the right member value.
+    #
+    # Note: Converts to float.
+    #
+    def weight sym
+      weights.member(sym).to_f
+    end
+
+    # Delegate to a member value.
+    #
+    def setting sym
+      configuration.member sym
+    end
+
+  end
+
+end

data/lib/picky/calculations/location.rb

@@ -0,0 +1,57 @@
+module Calculations # :nodoc:all
+
+  # A location calculation recalculates a 1-d location
+  # to the Picky internal 1-d "grid".
+  #
+  # For example, if you have a location x == 12.3456,
+  # it will be recalculated into 3, if the minimum is 9
+  # and the gridlength is 1.
+  #
+  class Location
+
+    attr_reader :minimum, :precision, :grid
+
+    def initialize user_grid, precision = nil
+      @user_grid = user_grid
+      @precision = precision || 1
+      @grid      = @user_grid / (@precision + 0.5)
+    end
+
+    def minimum= minimum
+      # Add a margin of 1 user grid.
+      #
+      minimum -= @user_grid
+
+      # Add plus 1 grid so that the index key never falls on 0.
+      # Why? to_i maps by default to 0.
+      #
+      minimum -= @grid
+
+      @minimum = minimum
+    end
+
+    #
+    #
+    def add_margin length
+      @minimum -= length
+    end
+
+    #
+    #
+    def recalculated_range location
+      range recalculate(location)
+    end
+    #
+    #
+    def range around_location
+      (around_location - @precision)..(around_location + @precision)
+    end
+    #
+    #
+    def recalculate location
+      ((location - @minimum) / @grid).floor
+    end
+
+  end
+
+end

data/lib/picky/categories.rb

@@ -0,0 +1,62 @@
+class Categories
+
+  attr_reader :categories, :category_hash
+
+  delegate :each,
+           :first,
+           :map,
+           :to => :categories
+
+  each_delegate :reindex,
+                :to => :categories
+
+  # A list of indexed categories.
+  #
+  # Options:
+  #  * ignore_unassigned_tokens: Ignore the given token if it cannot be matched to a category.
+  #                              The default behaviour is that if a token does not match to
+  #                              any category, the query will not return anything (since a
+  #                              single token cannot be matched). If you set this option to
+  #                              true, any token that cannot be matched to a category will be
+  #                              simply ignored.
+  #                              Use this if only a few matched words are important, like for
+  #                              example of the query "Jonathan Myers 86455 Las Cucarachas"
+  #                              you only want to match the zipcode, to have the search engine
+  #                              display advertisements on the side for the zipcode.
+  #                              Nifty! :)
+  #
+  def initialize options = {}
+    clear
+
+    @ignore_unassigned_tokens = options[:ignore_unassigned_tokens] || false
+  end
+
+  # Clears both the array of categories and the hash of categories.
+  #
+  def clear
+    @categories    = []
+    @category_hash = {}
+  end
+
+  # Find a given category in the categories.
+  #
+  def [] category_name
+    category_name = category_name.to_sym
+    category_hash[category_name] || raise_not_found(category_name)
+  end
+  def raise_not_found category_name
+    raise %Q{Index category "#{category_name}" not found. Possible categories: "#{categories.map(&:name).join('", "')}".}
+  end
+
+  # Add the given category to the list of categories.
+  #
+  def << category
+    categories << category
+    category_hash[category.name] = category
+  end
+
+  def to_s
+    categories.join(', ')
+  end
+
+end

data/lib/picky/categories_indexed.rb

@@ -0,0 +1,93 @@
+class Categories
+ 
+  attr_reader :ignore_unassigned_tokens
+
+  each_delegate :load_from_cache,
+                :analyze,
+                :to => :categories
+
+  # Return all possible combinations for the given token.
+  #
+  # This checks if it needs to also search through similar
+  # tokens, if for example, the token is one with ~.
+  # If yes, it puts together all solutions.
+  #
+  def possible_combinations_for token
+    token.similar? ? similar_possible_for(token) : possible_for(token)
+  end
+  # Gets all similar tokens and puts together the possible combinations
+  # for each found similar token.
+  #
+  def similar_possible_for token
+    # Get as many tokens as necessary
+    #
+    tokens = similar_tokens_for token
+    # possible combinations
+    #
+    inject_possible_for tokens
+  end
+  def similar_tokens_for token
+    text = token.text
+    categories.inject([]) do |result, category|
+      next_token = token
+      # Note: We could also break off here if not all the available
+      #       similars are needed.
+      #       Wait for a concrete case that needs this before taking
+      #       action.
+      #
+      while next_token = next_token.next_similar_token(category)
+        result << next_token if next_token && next_token.text != text
+      end
+      result
+    end
+  end
+  def inject_possible_for tokens
+    tokens.inject([]) do |result, token|
+      possible = possible_categories token
+      result + possible_for(token, possible)
+    end
+  end
+
+  # Returns possible Combinations for the token.
+  #
+  # Note: The preselected_categories param is an optimization.
+  #
+  # Note: Returns [] if no categories matched (will produce no result).
+  #       Returns nil if this token needs to be removed from the query.
+  #       (Also none of the categories matched, but the ignore unassigned
+  #       tokens option is true)
+  #
+  def possible_for token, preselected_categories = nil
+    possible = (preselected_categories || possible_categories(token)).inject([]) do |combinations, category|
+      combination = category.combination_for token
+      combination ? combinations << combination : combinations
+    end
+    # This is an optimization to mark tokens that are ignored.
+    #
+    return if ignore_unassigned_tokens && possible.empty?
+    possible # wrap in combinations
+  end
+  # This returns the possible categories for this token.
+  # If the user has already preselected a category for this token,
+  # like "artist:moby", if not just return all for the given token,
+  # since all are possible.
+  #
+  # Note: Once I thought this was called too often. But it is not (18.01.2011).
+  #
+  def possible_categories token
+    user_defined_categories(token) || categories
+  end
+  # This returns the array of categories if the user has defined
+  # an existing category.
+  #
+  # Note: Returns nil if the user did not define one
+  #       or [] if he/she has defined a non-existing one.
+  #
+  def user_defined_categories token
+    names = token.user_defined_category_names
+    names && names.map do |name|
+      category_hash[name]
+    end.compact
+  end
+  
+end

data/lib/picky/categories_indexing.rb

@@ -0,0 +1,12 @@
+class Categories
+  
+  each_delegate :backup_caches,
+                :cache,
+                :check_caches,
+                :clear_caches,
+                :create_directory_structure,
+                :generate_caches,
+                :restore_caches,
+                :to => :categories
+  
+end

data/lib/picky/category.rb

@@ -0,0 +1,127 @@
+class Category
+
+  attr_reader :name,
+              :index
+
+  # Mandatory params:
+  #  * name: Category name to use as identifier and file names.
+  #  * index: Index to which this category is attached to.
+  #
+  # Options:
+  #  * partial: Partial::None.new, Partial::Substring.new(from:start_char, to:up_to_char) (defaults from:-3, to:-1)
+  #  * similarity: Similarity::None.new (default), Similarity::DoubleMetaphone.new(amount_of_similarly_linked_words)
+  #  * from: The source category identifier to take the data from.
+  #
+  # Advanced Options:
+  #  * source: Use if the category should use a different source.
+  #  * weights: Query::Weights.new( [:category1, :category2] => +2, ... )
+  #  * tokenizer: Use a subclass of Tokenizers::Base that implements #tokens_for and #empty_tokens.
+  #  * key_format: What this category's keys are formatted with (default is :to_i)
+  #
+  def initialize name, index, options = {}
+    @name  = name
+    @index = index
+
+    # Indexing.
+    #
+    @source     = options[:source]
+    @from       = options[:from]
+    @tokenizer  = options[:tokenizer]
+    @key_format = options[:key_format]
+
+    # TODO Push into Bundle. At least the weights.
+    #
+    partial    = options[:partial]    || Generators::Partial::Default
+    weights    = options[:weights]    || Generators::Weights::Default
+    similarity = options[:similarity] || Generators::Similarity::Default
+
+    @indexing_exact   = index.indexing_bundle_class.new(:exact,   self, similarity, Generators::Partial::None.new, weights)
+    @indexing_partial = index.indexing_bundle_class.new(:partial, self, Generators::Similarity::None.new, partial, weights)
+
+    # Indexed.
+    #
+    # TODO Push the defaults out into the index.
+    #
+    @partial_strategy = partial # TODO Duplicate work.
+
+    @indexed_exact   = index.indexed_bundle_class.new :exact,   self, similarity
+    @indexed_partial = index.indexed_bundle_class.new :partial, self, similarity
+
+    # @exact   = exact_lambda.call(@exact, @partial)   if exact_lambda   = options[:exact_lambda]
+    # @partial = partial_lambda.call(@exact, @partial) if partial_lambda = options[:partial_lambda]
+
+    # TODO Extract? Yes.
+    #
+    Query::Qualifiers.add(name, generate_qualifiers_from(options) || [name])
+  end
+
+  # Indexes and reloads the category.
+  #
+  def reindex
+    index
+    reload
+  end
+
+  # Category name.
+  #
+  def category_name
+    name
+  end
+
+  # Index name.
+  #
+  def index_name
+    @index.name
+  end
+
+  # Path and partial filename of a specific index on this category.
+  #
+  def index_path bundle_name, type
+    "#{index_directory}/#{name}_#{bundle_name}_#{type}"
+  end
+
+  # Path and partial filename of the prepared index on this category.
+  #
+  def prepared_index_path
+    @prepared_index_path ||= "#{index_directory}/prepared_#{name}_index"
+  end
+  def prepared_index_file &block
+    @prepared_index_file ||= Backend::File::Text.new prepared_index_path
+    @prepared_index_file.open_for_indexing &block
+  end
+
+  # The index directory for this category.
+  #
+  def index_directory
+    @index_directory ||= "#{PICKY_ROOT}/index/#{PICKY_ENVIRONMENT}/#{@index.name}"
+  end
+
+  # Creates the index directory including all necessary paths above it.
+  #
+  def prepare_index_directory
+    FileUtils.mkdir_p index_directory
+  end
+
+  # Identifier for internal use.
+  #
+  # TODO What internal use?
+  #
+  def identifier
+    @identifier ||= "#{@index.name}:#{name}"
+  end
+
+  def to_info
+<<-CATEGORY
+Category(#{name}):
+Exact:
+#{exact.indented_to_s(4)}
+Partial:
+#{partial.indented_to_s(4)}
+CATEGORY
+  end
+
+  def to_s
+    "Category(#{name})"
+  end
+
+end

data/lib/picky/category_indexed.rb

@@ -0,0 +1,64 @@
+#
+#
+class Category
+
+  attr_reader :indexed_exact
+
+  # TODO Move to Index.
+  #
+  def generate_qualifiers_from options
+    options[:qualifiers] || options[:qualifier] && [options[:qualifier]]
+  end
+
+  # Loads the index from cache.
+  #
+  def load_from_cache
+    timed_exclaim %Q{"#{identifier}": Loading index from cache.}
+    indexed_exact.load
+    indexed_partial.load
+  end
+  alias reload load_from_cache
+
+  # Loads, analyzes, and clears the index.
+  #
+  # Note: The idea is not to run this while the search engine is running.
+  #
+  def analyze collector
+    collector[identifier] = {
+      :exact   => Analyzer.new.analyze(indexed_exact),
+      :partial => Analyzer.new.analyze(indexed_partial)
+    }
+    collector
+  end
+
+  # Gets the weight for this token's text.
+  #
+  def weight token
+    bundle_for(token).weight token.text
+  end
+
+  # Gets the ids for this token's text.
+  #
+  def ids token
+    bundle_for(token).ids token.text
+  end
+
+  # Returns the right index bundle for this token.
+  #
+  def bundle_for token
+    token.partial? ? indexed_partial : indexed_exact
+  end
+
+  # The partial strategy defines whether to really use the partial index.
+  #
+  def indexed_partial
+    @partial_strategy.use_exact_for_partial? ? @indexed_exact : @indexed_partial
+  end
+
+  #
+  #
+  def combination_for token
+    weight(token) && Query::Combination.new(token, self)
+  end
+
+end

data/lib/picky/category_indexing.rb

@@ -0,0 +1,145 @@
+#
+#
+class Category
+
+  attr_reader :indexing_exact,
+              :indexing_partial
+
+  # Prepares and caches this category.
+  #
+  # This one should be used by users.
+  #
+  def index
+    prepare
+    cache
+  end
+
+  # Return an appropriate source.
+  #
+  def source
+    @source || @index.source
+  end
+
+  # Return the key format.
+  #
+  # If the source has no key format, then
+  # check for an explicit key format, and
+  # if none is defined, ask the index for
+  # one.
+  #
+  def key_format
+    source.respond_to?(:key_format) && source.key_format || @key_format || @index.key_format
+  end
+
+  # Where the data is taken from.
+  #
+  def from
+    @from || name
+  end
+
+  # The indexer is lazily generated and cached.
+  #
+  def indexer
+    @indexer ||= source.respond_to?(:each) ? Indexers::Parallel.new(self) : Indexers::Serial.new(self)
+  end
+
+  # TODO This is a hack to get the parallel indexer working.
+  #
+  def categories
+    [self]
+  end
+
+  # Returns an appropriate tokenizer.
+  # If one isn't set on this category, will try the index,
+  # and finally the default index tokenizer.
+  #
+  def tokenizer
+    @tokenizer || @index.tokenizer || Tokenizers::Index.default
+  end
+
+  # Backup the caches.
+  # (Revert with restore_caches)
+  #
+  def backup_caches
+    timed_exclaim "Backing up #{identifier}."
+    indexing_exact.backup
+    indexing_partial.backup
+  end
+
+  # Restore the caches.
+  # (Revert with backup_caches)
+  #
+  def restore_caches
+    timed_exclaim "Restoring #{identifier}."
+    indexing_exact.restore
+    indexing_partial.restore
+  end
+
+  # Checks the caches for existence.
+  #
+  def check_caches
+    timed_exclaim "Checking #{identifier}."
+    indexing_exact.raise_unless_cache_exists
+    indexing_partial.raise_unless_cache_exists
+  end
+
+  # Deletes the caches.
+  #
+  def clear_caches
+    timed_exclaim "Deleting #{identifier}."
+    indexing_exact.delete
+    indexing_partial.delete
+  end
+
+  # We need to set what formatting method should be used.
+  # Uses the one defined in the indexer.
+  #
+  # TODO Make this more dynamic.
+  #
+  def configure
+    indexing_exact[:key_format] = self.key_format
+    indexing_partial[:key_format] = self.key_format
+  end
+
+  # Indexes, creates the "prepared_..." file.
+  #
+  # TODO This step could already prepare the id (if a
+  #      per category key_format is not really needed).
+  #
+  def prepare
+    prepare_index_directory
+    indexer.index
+  end
+
+  # Generates all caches for this category.
+  #
+  def cache
+    prepare_index_directory
+    generate_caches
+  end
+
+  # Generate the cache data.
+  #
+  def generate_caches
+    configure
+    generate_caches_from_source
+    generate_partial
+    generate_caches_from_memory
+    dump_caches
+    timed_exclaim %Q{"#{identifier}": Caching finished.}
+  end
+  def generate_caches_from_source
+    indexing_exact.generate_caches_from_source
+  end
+  def generate_partial
+    indexing_partial.generate_partial_from indexing_exact.index
+  end
+  def generate_caches_from_memory
+    indexing_partial.generate_caches_from_memory
+  end
+  def dump_caches
+    indexing_exact.dump
+    indexing_partial.dump
+  end
+
+end

data/lib/picky/extensions/class.rb

@@ -0,0 +1,11 @@
+# The original Class class.
+#
+class Class # :nodoc:all
+
+  def instance_delegate *methods
+    methods.each do |method|
+      module_eval("def self.#{method}(*args, &block)\nself.instance.__send__(#{method.inspect}, *args, &block)\nend\n", "(__DELEGATION__)", 1)
+    end
+  end
+
+end