picky 1.4.1 → 1.4.2

data/lib/picky/internals/query/allocations.rb

@@ -0,0 +1,137 @@
+module Internals
+
+  module Query
+    # Container class for allocations.
+    #
+    class Allocations # :nodoc:all
+
+      # TODO Remove size
+      #
+      delegate :each, :inject, :empty?, :size, :to => :@allocations
+      attr_reader :total
+
+      def initialize allocations = []
+        @allocations = allocations
+      end
+
+      # Score each allocation.
+      #
+      def calculate_score weights
+        @allocations.each do |allocation|
+          allocation.calculate_score weights
+        end
+      end
+      # Sort the allocations.
+      #
+      def sort
+        @allocations.sort!
+      end
+
+      # Reduces the amount of allocations to x.
+      #
+      def reduce_to amount
+        @allocations = @allocations.shift amount
+      end
+
+      # Keeps combinations.
+      #
+      # Only those passed in remain.
+      #
+      def keep identifiers = []
+        @allocations.each { |allocation| allocation.keep identifiers } unless identifiers.empty?
+      end
+      # Removes combinations.
+      #
+      # Only those passed in are removed.
+      #
+      # TODO Rewrite.
+      #
+      def remove identifiers = []
+        @allocations.each { |allocation| allocation.remove identifiers } unless identifiers.empty?
+      end
+
+      # Returns the top amount ids.
+      #
+      def ids amount = 20
+        @allocations.inject([]) do |total, allocation|
+          total.size >= amount ? (return total.shift(amount)) : total + allocation.ids
+        end
+      end
+
+      # Returns a random id from the allocations.
+      #
+      # Note: This is an ok algorithm for small id sets.
+      #
+      # But still TODO try for a faster one.
+      #
+      # TODO For the 1 amount random case this needs to be improved.
+      #
+      def random_ids amount = 1
+        return [] if @allocations.empty?
+        ids = @allocations.first.ids
+        indexes = Array.new(ids.size) { |id| id }.sort_by { rand }
+        indexes.first(amount).map { |id| ids[id] }
+      end
+
+      # This is the main method of this class that will replace ids and count.
+      #
+      # What it does is calculate the ids and counts of its allocations
+      # for being used in the results. It also calculates the total
+      #
+      # Parameters:
+      #  * amount: the amount of ids to calculate
+      #  * offset: the offset from where in the result set to take the ids
+      #
+      # Note: With an amount of 0, an offset > 0 doesn't make much
+      #       sense, as seen in the live search.
+      #
+      # Note: Each allocation caches its count, but not its ids (thrown away).
+      #       The ids are cached in this class.
+      #
+      # Note: It's possible that no ids are returned by an allocation, but a count. (In case of an offset)
+      #
+      def process! amount, offset = 0
+        @total = 0
+        current_offset = 0
+        @allocations.each do |allocation|
+          ids = allocation.process! amount, offset
+          @total = @total + allocation.count # the total mixed in
+          if ids.empty?
+            offset = offset - allocation.count unless offset.zero?
+          else
+            amount = amount - ids.size # we need less results from the following allocation
+            offset = 0                 # we have already passed the offset
+          end
+        end
+      end
+
+      def uniq
+        @allocations.uniq!
+      end
+
+      def to_a
+        @allocations
+      end
+
+      # Simply inspects the internal allocations.
+      #
+      def to_s
+        @allocations.inspect
+      end
+
+      # Allocations for results are in the form:
+      # [
+      #   allocation1.to_result,
+      #   allocation2.to_result
+      #   ...
+      # ]
+      #
+      def to_result
+        @allocations.map(&:to_result).compact
+      end
+
+    end
+    
+  end
+  
+end

data/lib/picky/internals/query/combination.rb

@@ -0,0 +1,80 @@
+module Internals
+
+  module Query
+
+    # Describes the combination of a token (the text) and
+    # the index (the bundle): [text, index_bundle]
+    #
+    # A combination is a single part of an allocation:
+    # [..., [text2, index_bundle2], ...]
+    #
+    # An allocation consists of a number of combinations:
+    # [[text1, index_bundle1], [text2, index_bundle2], [text3, index_bundle1]]
+    #
+    class Combination # :nodoc:all
+
+      attr_reader :token, :bundle, :category_name
+
+      def initialize token, category
+        @token         = token
+        @category_name = category.name
+        @bundle        = category.bundle_for token
+        @text          = @token.text # don't want to use reset_similar already
+      end
+    
+      # Note: Required for uniq!
+      #
+      def hash
+        [@token.to_s, @bundle].hash
+      end
+    
+      # Returns the weight of this combination.
+      #
+      # Note: Caching is most of the time useful.
+      #
+      def weight
+        @weight ||= @bundle.weight(@text)
+      end
+    
+      # Returns an array of ids for the given text.
+      #
+      # Note: Caching is most of the time useful.
+      #
+      def ids
+        @ids ||= @bundle.ids(@text)
+      end
+    
+      # The identifier for this combination.
+      #
+      def identifier
+        "#{bundle.identifier}:#{@token.identifier}"
+      end
+    
+      # Is the identifier in the given identifiers?
+      #
+      def in? identifiers
+        identifiers.include? identifier
+      end
+
+      # Combines the category names with the original names.
+      # [
+      #  [:title,    'Flarbl', :flarbl],
+      #  [:category, 'Gnorf',  :gnorf]
+      # ]
+      #
+      def to_result
+        [@category_name, *@token.to_result]
+      end
+    
+      # Example:
+      #  "exact title:Peter*:peter"
+      #
+      def to_s
+        "#{bundle.identifier} #{to_result.join(':')}"
+      end
+    
+    end
+
+  end
+  
+end

data/lib/picky/internals/query/combinations/base.rb

@@ -0,0 +1,84 @@
+module Internals
+
+  module Query
+
+    # Combinations are a number of Combination-s.
+    #
+    # They are the core of an allocation.
+    # An allocation consists of a number of combinations.
+    #
+    module Combinations # :nodoc:all
+    
+      # Base Combinations contain methods for calculating score and ids.
+      #
+      class Base
+      
+        attr_reader :combinations
+
+        delegate :empty?, :to => :@combinations
+
+        def initialize combinations = []
+          @combinations = combinations
+        end
+
+        def hash
+          @combinations.hash
+        end
+
+        # Uses user specific weights to calculate a score for the combinations.
+        #
+        def calculate_score weights
+          total_score + weighted_score(weights)
+        end
+        def total_score
+          @combinations.sum &:weight
+        end
+        def weighted_score weights
+          weights.score @combinations
+        end
+    
+        # Wrap the combinations into an allocation with the result_identifier.
+        #
+        def pack_into_allocation result_identifier
+          Allocation.new self, result_identifier
+        end
+    
+        # Filters the tokens and identifiers such that only identifiers
+        # that are passed in, remain, including their tokens.
+        #
+        # Note: This method is not totally independent of the calculate_ids one.
+        #       Since identifiers are only nullified, we need to not include the
+        #       ids that have an associated identifier that is nil.
+        #
+        def keep identifiers = []
+          # TODO Rewrite to use the category!!!
+          #
+          @combinations.reject! { |combination| !combination.in?(identifiers) }
+        end
+
+        # Filters the tokens and identifiers such that identifiers
+        # that are passed in, are removed, including their tokens.
+        #
+        # Note: This method is not totally independent of the calculate_ids one.
+        #       Since identifiers are only nullified, we need to not include the
+        #       ids that have an associated identifier that is nil.
+        #
+        def remove identifiers = []
+          # TODO Rewrite to use the category!!!
+          #
+          @combinations.reject! { |combination| combination.in?(identifiers) }
+        end
+
+        #
+        #
+        def to_result
+          @combinations.map &:to_result
+        end
+    
+      end
+
+    end
+
+  end
+  
+end

data/lib/picky/internals/query/combinations/memory.rb

@@ -0,0 +1,58 @@
+module Internals
+
+  module Query
+
+    # Combinations are a number of Combination-s.
+    #
+    # They are the core of an allocation.
+    # An allocation consists of a number of combinations.
+    #
+    module Combinations # :nodoc:all
+    
+      # Memory Combinations contain specific methods for
+      # calculating score and ids in memory.
+      #
+      class Memory < Base
+      
+        # Returns the result ids for the allocation.
+        #
+        # Sorts the ids by size and & through them in the following order (sizes):
+        # 0. [100_000, 400, 30, 2]
+        # 1. [2, 30, 400, 100_000]
+        # 2. (100_000 & (400 & (30 & 2))) # => result
+        #
+        # Note: Uses a C-optimized intersection routine for speed and memory efficiency.
+        #
+        # Note: In the memory based version we ignore the (amount) needed hint.
+        # TODO Not ignore it?
+        #
+        def ids _, _
+          return [] if @combinations.empty?
+
+          # Get the ids for each combination.
+          #
+          # TODO For combinations with Redis
+          #
+          id_arrays = @combinations.inject([]) do |total, combination|
+            total << combination.ids
+          end
+
+          # Order by smallest size first such that the intersect can be performed faster.
+          #
+          # TODO Move into the memory_efficient_intersect such that
+          #      this precondition for a fast algorithm is always given.
+          #
+          id_arrays.sort! { |this_array, that_array| this_array.size <=> that_array.size }
+      
+          # Call the optimized C algorithm.
+          #
+          Performant::Array.memory_efficient_intersect id_arrays
+        end
+      
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/internals/query/combinations/redis.rb

@@ -0,0 +1,59 @@
+module Internals
+
+  module Query
+
+    # Combinations are a number of Combination-s.
+    #
+    # They are the core of an allocation.
+    # An allocation consists of a number of combinations.
+    #
+    module Combinations # :nodoc:all
+    
+      # Redis Combinations contain specific methods for
+      # calculating score and ids in memory.
+      #
+      class Redis < Base
+      
+        # TODO Err… yeah. Wrap in Picky specific wrapper.
+        #
+        def initialize combinations
+          super combinations
+        
+          @@redis ||= ::Redis.new
+        end
+      
+        # Returns the result ids for the allocation.
+        #
+        def ids amount, offset
+          return [] if @combinations.empty?
+        
+          identifiers = @combinations.inject([]) do |identifiers, combination|
+            identifiers << "#{combination.identifier}"
+          end
+        
+          result_id = generate_intermediate_result_id
+        
+          # TODO multi?
+          #
+        
+          @@redis.zinterstore result_id, identifiers
+        
+          @@redis.zrange result_id, offset, (offset + amount)
+        end
+      
+        # Generate a multiple host/process safe result id.
+        #
+        # TODO How expensive is Process.pid? If it changes once, remember forever?
+        #
+        def generate_intermediate_result_id
+          # TODO host -> extract host.
+          :"host:#{Process.pid}:picky:result"
+        end
+      
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/internals/query/indexes.rb

@@ -0,0 +1,180 @@
+module Internals
+
+  module Query
+  
+    # The query indexes class bundles indexes given to a query.
+    #
+    # Example:
+    #   # If you call
+    #   Query::Full.new dvd_index, mp3_index, video_index
+    #   
+    #   # What it does is take the three given (API-) indexes and
+    #   # bundle them in an index bundle.
+    #
+    class Indexes
+    
+      attr_reader :indexes
+    
+      # Creates a new Query::Indexes.
+      #
+      # Its job is to generate all possible combinations, but also
+      # checking whether the query indexes are all of the same type.
+      # Note: We cannot mix memory and redis indexes just yet.
+      #
+      def initialize *index_definitions, combinations_type
+        @combinations_type = combinations_type
+        @indexes           = index_definitions.map &:indexed
+      end
+
+      # Returns a number of possible allocations for the given tokens.
+      #
+      def allocations_for tokens
+        Allocations.new(indexes.inject([]) do |previous_allocations, index|
+          # Expand the combinations.
+          #
+          possible_combinations = tokens.possible_combinations_in index
+        
+          # Optimization for ignoring tokens that allocate to nothing and
+          # can be ignored.
+          # For example in a special search, where "florian" is not
+          # mapped to any category.
+          #
+          possible_combinations.compact!
+        
+          # Generate all possible combinations.
+          #
+          expanded_combinations = expand_combinations_from possible_combinations
+        
+          # If there are none, try the next allocation.
+          #
+          next previous_allocations unless expanded_combinations
+        
+          # Add the wrapped possible allocations to the ones we already have.
+          #
+          previous_allocations + expanded_combinations.map! do |expanded_combination|
+            # TODO Insert Redis here?
+            #
+            @combinations_type.new(expanded_combination).pack_into_allocation(index.result_identifier) # TODO Do not extract result_identifier. Remove pack_into_allocation.
+          end
+        end)
+      end
+    
+      # This is the core of the search engine.
+      #
+      # Gets an array of
+      # [
+      #  [<combinations for token1>],
+      #  [<combinations for token2>],
+      #  [<combinations for token3>]
+      # ]
+      #
+      # Generates all possible allocations of combinations.
+      # [
+      #  [first combination of token1, first c of t2, first c of t3],
+      #  [first combination of token1, first c of t2, second c of t3]
+      #  ...
+      # ]
+      #
+      # Generates all possible combinations of array elements:
+      # [1,2,3] x [a,b,c] x [k,l,m] => [[1,a,k], [1,a,l], [1,a,m], [1,b,k], [1,b,l], [1,b,m], [1,c,k], ..., [3,c,m]]
+      # Note: Also calculates the weights and sorts them accordingly.
+      #
+      # Note: This is a heavily optimized ruby version.
+      #
+      # Works like this:
+      # [1,2,3], [a,b,c], [k,l,m] are expanded to
+      #  group mult: 1
+      #  <- single mult ->
+      # [1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3] = 27 elements
+      #  group mult: 3
+      #  <- -> s/m
+      # [a,a,a,b,b,b,c,c,c,a,a,a,b,b,b,c,c,c,a,a,a,b,b,b,c,c,c] = 27 elements
+      #  group mult: 9
+      #  <> s/m
+      # [k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m] = 27 elements
+      #
+      # It is then recombined, where
+      # [
+      #  [a,a,b,b,c,c]
+      #  [d,e,d,e,d,e]
+      # ]
+      # becomes
+      # [
+      #  [a,d],
+      #  [a,e],
+      #  [b,d],
+      #  [b,e],
+      #  [c,d],
+      #  [c,e]
+      # ]
+      #
+      # Note: Not using transpose as it is slower.
+      #
+      # Returns nil if there are no combinations.
+      #
+      # Note: Of course I could split this method up into smaller
+      #       ones, but I guess I am a bit sentimental.
+      #
+      def expand_combinations_from possible_combinations
+        return if possible_combinations.any?(&:empty?)
+      
+        # Generate the first multiplicator "with which" (well, not quite) to multiply the smallest amount of combinations.
+        #
+        # TODO How does this work if an element has size 0? Since below we account for size 0.
+        #      Should we even continue if an element has size 0?
+        #      This means one of the tokens cannot be allocated.
+        #
+        single_mult = possible_combinations.inject(1) { |total, combinations| total * combinations.size }
+
+        # Initialize a group multiplicator.
+        #
+        group_mult = 1
+
+        # The expanding part to line up the combinations
+        # for later combination in allocations.
+        #
+        possible_combinations.collect! do |combinations|
+        
+          # Get the size of the combinations of the first token.
+          #
+          combinations_size = combinations.size
+
+          # Special case: If there is no combination for one of the tokens.
+          #               In that case, we just use the same single mult for
+          #               the next iteration.
+          #               If there are combinations, we divide the single mult
+          #               by the number of combinations.
+          #
+          single_mult /= combinations_size unless combinations_size.zero?
+        
+          # Expand each combination by the single mult:
+          # [a,b,c]
+          # [a,a,a,  b,b,b,  c,c,c]
+          # Then, expand the result by the group mult:
+          # [a,a,a,b,b,b,c,c,c,  a,a,a,b,b,b,c,c,c,  a,a,a,b,b,b,c,c,c]
+          #
+          combinations = combinations.inject([]) do |total, combination|
+            total + Array.new(single_mult, combination)
+          end * group_mult
+        
+          # Multiply the group mult by the combinations size,
+          # since the next combinations' single mult is smaller
+          # and we need to adjust for that.
+          #
+          group_mult = group_mult * combinations_size
+        
+          # Return the combinations.
+          #
+          combinations
+        end
+      
+        return if possible_combinations.empty?
+      
+        possible_combinations.shift.zip *possible_combinations
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/internals/query/qualifiers.rb

@@ -0,0 +1,81 @@
+# coding: utf-8
+#
+module Internals
+
+  #
+  #
+  module Query
+
+    # A single qualifier.
+    #
+    class Qualifier # :nodoc:all
+
+      attr_reader :normalized_qualifier, :codes
+
+      #
+      #
+      # codes is an array.
+      #
+      def initialize normalized_qualifier, codes
+        @normalized_qualifier = normalized_qualifier
+        @codes                = codes.map &:to_sym
+      end
+
+      # Will overwrite if the key is present in the hash.
+      #
+      def inject_into hash
+        codes.each do |code|
+          hash[code] = normalized_qualifier
+        end
+      end
+
+    end
+
+    # Collection class for qualifiers.
+    #
+    class Qualifiers # :nodoc:all
+
+      include Singleton
+
+      attr_reader :qualifiers, :normalization_mapping
+
+      delegate :<<, :to => :qualifiers
+
+      #
+      #
+      def initialize
+        @qualifiers = []
+        @normalization_mapping = {}
+      end
+    
+      # TODO Spec.
+      #
+      def self.add name, qualifiers
+        instance << Qualifier.new(name, qualifiers)
+      end
+    
+      # Uses the qualifiers to prepare (optimize) the qualifier handling.
+      #
+      def prepare
+        qualifiers.each do |qualifier|
+          qualifier.inject_into normalization_mapping
+        end
+      end
+
+      # Normalizes the given qualifier.
+      #
+      # Returns nil if it is not allowed, the normalized qualifier if it is.
+      #
+      # Note: Normalizes.
+      #
+      def normalize qualifier
+        return nil if qualifier.blank?
+
+        normalization_mapping[qualifier.to_sym]
+      end
+
+    end
+    
+  end
+  
+end