picky 2.5.2 → 2.6.0

data/lib/picky/query/combinations/base.rb

@@ -0,0 +1,70 @@
+module Query
+
+  # Combinations are a number of Combination-s.
+  #
+  # They are the core of an allocation.
+  # An allocation consists of a number of combinations.
+  #
+  module Combinations # :nodoc:all
+
+    # Base Combinations contain methods for calculating score and ids.
+    #
+    class Base
+
+      attr_reader :combinations
+
+      delegate :empty?, :to => :@combinations
+
+      def initialize combinations = []
+        @combinations = combinations
+      end
+
+      def hash
+        @combinations.hash
+      end
+
+      # Uses user specific weights to calculate a score for the combinations.
+      #
+      def calculate_score weights
+        total_score + weighted_score(weights)
+      end
+      def total_score
+        @combinations.sum &:weight
+      end
+      def weighted_score weights
+        weights.score @combinations
+      end
+
+      # Filters the tokens and identifiers such that only identifiers
+      # that are passed in, remain, including their tokens.
+      #
+      # Note: This method is not totally independent of the calculate_ids one.
+      #       Since identifiers are only nullified, we need to not include the
+      #       ids that have an associated identifier that is nil.
+      #
+      def keep identifiers = []
+        @combinations.reject! { |combination| !combination.in?(identifiers) }
+      end
+
+      # Filters the tokens and identifiers such that identifiers
+      # that are passed in, are removed, including their tokens.
+      #
+      # Note: This method is not totally independent of the calculate_ids one.
+      #       Since identifiers are only nullified, we need to not include the
+      #       ids that have an associated identifier that is nil.
+      #
+      def remove identifiers = []
+        @combinations.reject! { |combination| combination.in?(identifiers) }
+      end
+
+      #
+      #
+      def to_result
+        @combinations.map &:to_result
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/query/combinations/memory.rb

@@ -0,0 +1,48 @@
+module Query
+
+  # Combinations are a number of Combination-s.
+  #
+  # They are the core of an allocation.
+  # An allocation consists of a number of combinations.
+  #
+  module Combinations # :nodoc:all
+
+    # Memory Combinations contain specific methods for
+    # calculating score and ids in memory.
+    #
+    class Memory < Base
+
+      # Returns the result ids for the allocation.
+      #
+      # Sorts the ids by size and & through them in the following order (sizes):
+      # 0. [100_000, 400, 30, 2]
+      # 1. [2, 30, 400, 100_000]
+      # 2. (100_000 & (400 & (30 & 2))) # => result
+      #
+      # Note: Uses a C-optimized intersection routine (in performant.c)
+      #       for speed and memory efficiency.
+      #
+      # Note: In the memory based version we ignore the (amount) needed hint.
+      #       We cannot use the information to speed up the algorithm, unfortunately.
+      #
+      def ids _, _
+        return [] if @combinations.empty?
+
+        # Get the ids for each combination.
+        #
+        id_arrays = @combinations.inject([]) do |total, combination|
+          total << combination.ids
+        end
+
+        # Call the optimized C algorithm.
+        #
+        # Note: It orders the passed arrays by size.
+        #
+        Performant::Array.memory_efficient_intersect id_arrays
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/query/combinations/redis.rb

@@ -0,0 +1,86 @@
+module Query
+
+  # Combinations are a number of Combination-s.
+  #
+  # They are the core of an allocation.
+  # An allocation consists of a number of combinations.
+  #
+  module Combinations # :nodoc:all
+
+    # Redis Combinations contain specific methods for
+    # calculating score and ids in memory.
+    #
+    class Redis < Base
+
+      # Connect to the backend.
+      #
+      # TODO Use specific Picky Redis wrapper.
+      #
+      def self.redis
+        @redis ||= ::Redis.new :db => 15
+      end
+
+      attr_reader :redis
+
+      #
+      #
+      def initialize combinations
+        super combinations
+
+        @redis = self.class.redis
+      end
+
+      # Returns the result ids for the allocation.
+      #
+      def ids amount, offset
+        return [] if @combinations.empty?
+
+        identifiers = @combinations.inject([]) do |identifiers, combination|
+          identifiers << "#{combination.identifier}"
+        end
+
+        result_id = generate_intermediate_result_id
+
+        # Intersect and store.
+        #
+        redis.zinterstore result_id, identifiers
+
+        # Get the stored result.
+        #
+        results = redis.zrange result_id, offset, (offset + amount)
+
+        # Delete the stored result as it was only for temporary purposes.
+        #
+        # Note: I could also not delete it, but that would not be clean at all.
+        #
+        redis.del result_id
+
+        results
+      end
+
+      # Generate a multiple host/process safe result id.
+      #
+      # Note: Generated when this class loads.
+      #
+      require 'socket'
+      def self.extract_host
+        @host ||= Socket.gethostname
+      end
+      def host
+        self.class.extract_host
+      end
+      extract_host
+      def pid
+        @pid ||= Process.pid
+      end
+      # Use the host and pid (generated lazily in child processes) for the result.
+      #
+      def generate_intermediate_result_id
+        :"#{host}:#{pid}:picky:result"
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/query/indexes.rb

@@ -0,0 +1,195 @@
+module Query
+
+  # The query indexes class bundles indexes given to a query.
+  #
+  # Example:
+  #   # If you call
+  #   Search.new dvd_index, mp3_index, video_index
+  #
+  #   # What it does is take the three given (API-) indexes and
+  #   # bundle them in an index bundle.
+  #
+  class Indexes
+
+    attr_reader :indexes
+
+    # Creates a new Query::Indexes.
+    #
+    # Its job is to generate all possible combinations.
+    # Note: We cannot mix memory and redis indexes just yet.
+    #
+    def initialize *indexes, combinations_type
+      @indexes           = indexes
+      @combinations_type = combinations_type
+    end
+
+    # Returns a number of prepared (sorted, reduced etc.) allocations for the given tokens.
+    #
+    def prepared_allocations_for tokens, weights = {}
+      allocations = allocations_for tokens
+
+      # Remove double allocations.
+      #
+      allocations.uniq
+
+      # Score the allocations using weights as bias.
+      #
+      allocations.calculate_score weights
+
+      # Sort the allocations.
+      # (allocations are sorted according to score, highest to lowest)
+      #
+      allocations.sort!
+
+      # Reduce the amount of allocations.
+      #
+      # allocations.reduce_to some_amount
+
+      # Remove identifiers from allocations.
+      #
+      # allocations.remove some_array_of_identifiers_to_remove
+
+      allocations
+    end
+    # Returns a number of possible allocations for the given tokens.
+    #
+    def allocations_for tokens
+      Allocations.new allocations_ary_for(tokens)
+    end
+    def allocations_ary_for tokens
+      indexes.inject([]) do |allocations, index|
+        allocations + allocation_for(tokens, index)
+      end
+    end
+    def allocation_for tokens, index
+      # Expand the combinations.
+      #
+      possible_combinations = tokens.possible_combinations_in index
+
+      # Generate all possible combinations.
+      #
+      expanded_combinations = expand_combinations_from possible_combinations
+
+      # Add the wrapped possible allocations to the ones we already have.
+      #
+      expanded_combinations.map! do |expanded_combination|
+        Allocation.new @combinations_type.new(expanded_combination), index.result_identifier # TODO Do not extract result_identifier.
+      end
+    end
+
+    # This is the core of the search engine. No kidding.
+    #
+    # Gets an array of
+    # [
+    #  [<combinations for token1>],
+    #  [<combinations for token2>],
+    #  [<combinations for token3>]
+    # ]
+    #
+    # Generates all possible allocations of combinations.
+    # [
+    #  [first combination of token1, first c of t2, first c of t3],
+    #  [first combination of token1, first c of t2, second c of t3]
+    #  ...
+    # ]
+    #
+    # Generates all possible combinations of array elements:
+    # [1,2,3] x [a,b,c] x [k,l,m] => [[1,a,k], [1,a,l], [1,a,m], [1,b,k], [1,b,l], [1,b,m], [1,c,k], ..., [3,c,m]]
+    # Note: Also calculates the weights and sorts them accordingly.
+    #
+    # Note: This is a heavily optimized ruby version.
+    #
+    # Works like this:
+    # [1,2,3], [a,b,c], [k,l,m] are expanded to
+    #  group mult: 1
+    #  <- single mult ->
+    # [1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3] = 27 elements
+    #  group mult: 3
+    #  <- -> s/m
+    # [a,a,a,b,b,b,c,c,c,a,a,a,b,b,b,c,c,c,a,a,a,b,b,b,c,c,c] = 27 elements
+    #  group mult: 9
+    #  <> s/m
+    # [k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m] = 27 elements
+    #
+    # It is then recombined, where
+    # [
+    #  [a,a,b,b,c,c]
+    #  [d,e,d,e,d,e]
+    # ]
+    # becomes
+    # [
+    #  [a,d],
+    #  [a,e],
+    #  [b,d],
+    #  [b,e],
+    #  [c,d],
+    #  [c,e]
+    # ]
+    #
+    # Note: Not using transpose as it is slower.
+    #
+    # Returns nil if there are no combinations.
+    #
+    # Note: Of course I could split this method up into smaller
+    #       ones, but I guess I am a bit sentimental.
+    #
+    def expand_combinations_from possible_combinations
+      # If an element has size 0, this means one of the
+      # tokens could not be allocated.
+      #
+      return [] if possible_combinations.any?(&:empty?)
+
+      # Generate the first multiplicator "with which" (well, not quite) to multiply the smallest amount of combinations.
+      #
+      single_mult = possible_combinations.inject(1) { |total, combinations| total * combinations.size }
+
+      # Initialize a group multiplicator.
+      #
+      group_mult = 1
+
+      # The expanding part to line up the combinations
+      # for later combination in allocations.
+      #
+      possible_combinations.collect! do |combinations|
+
+        # Get the size of the combinations of the first token.
+        #
+        combinations_size = combinations.size
+
+        # Special case: If there is no combination for one of the tokens.
+        #               In that case, we just use the same single mult for
+        #               the next iteration.
+        #               If there are combinations, we divide the single mult
+        #               by the number of combinations.
+        #
+        single_mult /= combinations_size unless combinations_size.zero?
+
+        # Expand each combination by the single mult:
+        # [a,b,c]
+        # [a,a,a,  b,b,b,  c,c,c]
+        # Then, expand the result by the group mult:
+        # [a,a,a,b,b,b,c,c,c,  a,a,a,b,b,b,c,c,c,  a,a,a,b,b,b,c,c,c]
+        #
+        combinations = combinations.inject([]) do |total, combination|
+          total + Array.new(single_mult, combination)
+        end * group_mult
+
+        # Multiply the group mult by the combinations size,
+        # since the next combinations' single mult is smaller
+        # and we need to adjust for that.
+        #
+        group_mult = group_mult * combinations_size
+
+        # Return the combinations.
+        #
+        combinations
+      end
+
+      return [] if possible_combinations.empty?
+
+      possible_combinations.shift.zip *possible_combinations
+    end
+
+  end
+
+end

data/lib/picky/query/qualifiers.rb

@@ -0,0 +1,76 @@
+# coding: utf-8
+#
+module Query
+
+  # A single qualifier.
+  #
+  class Qualifier # :nodoc:all
+
+    attr_reader :normalized_qualifier, :codes
+
+    #
+    #
+    # codes is an array.
+    #
+    def initialize normalized_qualifier, codes
+      @normalized_qualifier = normalized_qualifier
+      @codes                = codes.map &:to_sym
+    end
+
+    # Will overwrite if the key is present in the hash.
+    #
+    def inject_into hash
+      codes.each do |code|
+        hash[code] = normalized_qualifier
+      end
+    end
+
+  end
+
+  # Collection class for qualifiers.
+  #
+  class Qualifiers # :nodoc:all
+
+    attr_reader :qualifiers, :normalization_mapping
+
+    delegate :<<, :to => :qualifiers
+
+    #
+    #
+    def initialize
+      @qualifiers = []
+      @normalization_mapping = {}
+    end
+    def self.instance
+      @instanec ||= new
+    end
+
+    # TODO Spec.
+    #
+    def self.add name, qualifiers
+      instance << Qualifier.new(name, qualifiers)
+    end
+
+    # Uses the qualifiers to prepare (optimize) the qualifier handling.
+    #
+    def prepare
+      qualifiers.each do |qualifier|
+        qualifier.inject_into normalization_mapping
+      end
+    end
+
+    # Normalizes the given qualifier.
+    #
+    # Returns nil if it is not allowed, the normalized qualifier if it is.
+    #
+    # Note: Normalizes.
+    #
+    def normalize qualifier
+      return nil if qualifier.blank?
+
+      normalization_mapping[qualifier.to_sym]
+    end
+
+  end
+
+end

data/lib/picky/query/token.rb

@@ -0,0 +1,198 @@
+module Query
+
+  # This is a query token. Together with other tokens it makes up a query.
+  #
+  # It remembers the original form, and and a normalized form.
+  #
+  # It also knows whether it needs to look for similarity (bla~), or whether it is a partial (bla*).
+  #
+  class Token # :nodoc:all
+
+    attr_reader :text, :original
+    attr_writer :similar
+
+    delegate :blank?, :to => :text
+
+    # Normal initializer.
+    #
+    # Note: Use this if you do not want a qualified and normalized token.
+    #
+    def initialize text
+      @text = text
+    end
+
+    # Returns a qualified and normalized token.
+    #
+    # Note: Use this in the search engine if you need a qualified
+    #       and normalized token. I.e. one prepared for a search.
+    #
+    def self.processed text, downcase = true
+      new(text).process downcase
+    end
+    def process downcases = true
+      qualify
+      extract_original
+      downcase if downcases
+      partialize
+      similarize
+      remove_illegals
+      symbolize
+      self
+    end
+
+    # This returns an array of predefined category names if the user has given any.
+    #
+    def user_defined_category_names
+      @qualifiers
+    end
+
+    # Extracts a qualifier for this token and pre-assigns an allocation.
+    #
+    # Note: Removes the qualifier if it is not allowed.
+    #
+    def qualify
+      @qualifiers, @text = split @text
+      @qualifiers && @qualifiers.collect! { |qualifier| Query::Qualifiers.instance.normalize qualifier }.compact!
+      @qualifiers
+    end
+    def extract_original
+      @original = @text.dup
+    end
+
+    # Downcases the text.
+    #
+    def downcase
+      @text.downcase!
+    end
+
+    # Partial is a conditional setter.
+    #
+    # It is only settable if it hasn't been set yet.
+    #
+    def partial= partial
+      @partial = partial if @partial.nil?
+    end
+    def partial?
+      !@similar && @partial
+    end
+
+    # If the text ends with *, partialize it. If with ", don't.
+    #
+    # The latter wins. So "hello*" will not be partially searched.
+    #
+    @@no_partial = /\"\Z/
+    @@partial    = /\*\Z/
+    def partialize
+      self.partial = false and return unless @text !~ @@no_partial
+      self.partial = true unless @text !~ @@partial
+    end
+
+    # If the text ends with ~ similarize it. If with ", don't.
+    #
+    # The latter wins.
+    #
+    @@no_similar = /\"\Z/
+    @@similar    = /\~\Z/
+    def similarize
+      self.similar = false and return if @text =~ @@no_similar
+      self.similar = true if @text =~ @@similar
+    end
+
+    def similar?
+      @similar
+    end
+
+    # Normalizes this token's text.
+    #
+    @@illegals = /["*~]/
+    def remove_illegals
+      @text.gsub! @@illegals, '' unless @text.blank?
+    end
+
+    #
+    #
+    def symbolize
+      @text = @text.to_sym
+    end
+
+    # Returns an array of possible combinations.
+    #
+    def possible_combinations_in index
+      index.possible_combinations self
+    end
+
+    # Returns a token with the next similar text.
+    #
+    # TODO Rewrite this. It is hard to understand. Also spec performance.
+    #
+    def next_similar_token category
+      token = self.dup
+      token if token.next_similar category.bundle_for(token)
+    end
+    # Sets and returns the next similar word.
+    #
+    # Note: Also overrides the original.
+    #
+    def next_similar bundle
+      @text = @original = (similarity(bundle).shift || return) if similar?
+    end
+    # Lazy similar reader.
+    #
+    def similarity bundle = nil
+      @similarity || @similarity = generate_similarity_for(bundle)
+    end
+    # Returns an enumerator that traverses over the similar.
+    #
+    # Note: The dup isn't too nice – since it is needed on account of the shift, above.
+    #       (We avoid a StopIteration exception. Which of both is less evil?)
+    #
+    def generate_similarity_for bundle
+      bundle.similar(@text).dup || []
+    end
+
+    #
+    #
+    def to_result
+      [@original, @text]
+    end
+
+    # Internal identifier.
+    #
+    def identifier
+      "#{similar?? :similarity : :index}:#{@text}"
+    end
+
+    # If the originals & the text are the same, they are the same.
+    #
+    def == other
+      self.original == other.original && self.text == other.text
+    end
+
+    # Displays the qualifier text and the text, joined.
+    #
+    # e.g. name:meier
+    #
+    @@split_qualifier_text = ':'
+    @@split_qualifiers     = ','
+    def to_s
+      [@qualifiers && @qualifiers.join(@@split_qualifiers), @text].compact.join @@split_qualifier_text
+    end
+
+    private
+
+      # Splits text into a qualifier and text.
+      #
+      # Returns [qualifier, text].
+      #
+      def split unqualified_text
+        qualifiers, text = (unqualified_text || '').split(@@split_qualifier_text, 2)
+        if text.blank?
+          [nil, (qualifiers || '')]
+        else
+          [qualifiers.split(@@split_qualifiers), text]
+        end
+      end
+
+  end
+
+end