picky 2.7.0 → 3.0.0.pre1

data/lib/picky/query/combinations/memory.rb

@@ -1,44 +1,48 @@
-module Query
+module Picky
 
-  # Combinations are a number of Combination-s.
-  #
-  # They are the core of an allocation.
-  # An allocation consists of a number of combinations.
-  #
-  module Combinations # :nodoc:all
+  module Query
 
-    # Memory Combinations contain specific methods for
-    # calculating score and ids in memory.
+    # Combinations are a number of Combination-s.
     #
-    class Memory < Base
+    # They are the core of an allocation.
+    # An allocation consists of a number of combinations.
+    #
+    module Combinations # :nodoc:all
 
-      # Returns the result ids for the allocation.
-      #
-      # Sorts the ids by size and & through them in the following order (sizes):
-      # 0. [100_000, 400, 30, 2]
-      # 1. [2, 30, 400, 100_000]
-      # 2. (100_000 & (400 & (30 & 2))) # => result
-      #
-      # Note: Uses a C-optimized intersection routine (in performant.c)
-      #       for speed and memory efficiency.
+      # Memory Combinations contain specific methods for
+      # calculating score and ids in memory.
       #
-      # Note: In the memory based version we ignore the (amount) needed hint.
-      #       We cannot use the information to speed up the algorithm, unfortunately.
-      #
-      def ids _, _
-        return [] if @combinations.empty?
+      class Memory < Base
 
-        # Get the ids for each combination.
+        # Returns the result ids for the allocation.
         #
-        id_arrays = @combinations.inject([]) do |total, combination|
-          total << combination.ids
-        end
-
-        # Call the optimized C algorithm.
+        # Sorts the ids by size and & through them in the following order (sizes):
+        # 0. [100_000, 400, 30, 2]
+        # 1. [2, 30, 400, 100_000]
+        # 2. (100_000 & (400 & (30 & 2))) # => result
+        #
+        # Note: Uses a C-optimized intersection routine (in performant.c)
+        #       for speed and memory efficiency.
         #
-        # Note: It orders the passed arrays by size.
+        # Note: In the memory based version we ignore the (amount) needed hint.
+        #       We cannot use the information to speed up the algorithm, unfortunately.
         #
-        Performant::Array.memory_efficient_intersect id_arrays
+        def ids _, _
+          return [] if @combinations.empty?
+
+          # Get the ids for each combination.
+          #
+          id_arrays = @combinations.inject([]) do |total, combination|
+            total << combination.ids
+          end
+
+          # Call the optimized C algorithm.
+          #
+          # Note: It orders the passed arrays by size.
+          #
+          Performant::Array.memory_efficient_intersect id_arrays
+        end
+
       end
 
     end

data/lib/picky/query/combinations/redis.rb

@@ -1,82 +1,86 @@
-module Query
+module Picky
 
-  # Combinations are a number of Combination-s.
-  #
-  # They are the core of an allocation.
-  # An allocation consists of a number of combinations.
-  #
-  module Combinations # :nodoc:all
+  module Query
 
-    # Redis Combinations contain specific methods for
-    # calculating score and ids in memory.
+    # Combinations are a number of Combination-s.
     #
-    class Redis < Base
+    # They are the core of an allocation.
+    # An allocation consists of a number of combinations.
+    #
+    module Combinations # :nodoc:all
 
-      # Connect to the backend.
-      #
-      # TODO Use specific Picky Redis wrapper.
+      # Redis Combinations contain specific methods for
+      # calculating score and ids in memory.
       #
-      def self.redis
-        @redis ||= ::Redis.new :db => 15
-      end
+      class Redis < Base
 
-      attr_reader :redis
+        # Connect to the backend.
+        #
+        # TODO Use specific Picky Redis wrapper.
+        #
+        def self.redis
+          @redis ||= ::Redis.new :db => 15
+        end
 
-      #
-      #
-      def initialize combinations
-        super combinations
+        attr_reader :redis
 
-        @redis = self.class.redis
-      end
-
-      # Returns the result ids for the allocation.
-      #
-      def ids amount, offset
-        return [] if @combinations.empty?
+        #
+        #
+        def initialize combinations
+          super combinations
 
-        identifiers = @combinations.inject([]) do |identifiers, combination|
-          identifiers << "#{combination.identifier}"
+          @redis = self.class.redis
         end
 
-        result_id = generate_intermediate_result_id
-
-        # Intersect and store.
+        # Returns the result ids for the allocation.
         #
-        redis.zinterstore result_id, identifiers
+        def ids amount, offset
+          return [] if @combinations.empty?
 
-        # Get the stored result.
-        #
-        results = redis.zrange result_id, offset, (offset + amount)
+          identifiers = @combinations.inject([]) do |identifiers, combination|
+            identifiers << "#{combination.identifier}"
+          end
+
+          result_id = generate_intermediate_result_id
+
+          # Intersect and store.
+          #
+          redis.zinterstore result_id, identifiers
 
-        # Delete the stored result as it was only for temporary purposes.
+          # Get the stored result.
+          #
+          results = redis.zrange result_id, offset, (offset + amount)
+
+          # Delete the stored result as it was only for temporary purposes.
+          #
+          # Note: I could also not delete it, but that would not be clean at all.
+          #
+          redis.del result_id
+
+          results
+        end
+
+        # Generate a multiple host/process safe result id.
         #
-        # Note: I could also not delete it, but that would not be clean at all.
+        # Note: Generated when this class loads.
         #
-        redis.del result_id
-
-        results
-      end
+        require 'socket'
+        def self.extract_host
+          @host ||= Socket.gethostname
+        end
+        def host
+          self.class.extract_host
+        end
+        extract_host
+        def pid
+          @pid ||= Process.pid
+        end
+        # Use the host and pid (generated lazily in child processes) for the result.
+        #
+        def generate_intermediate_result_id
+          :"#{host}:#{pid}:picky:result"
+        end
 
-      # Generate a multiple host/process safe result id.
-      #
-      # Note: Generated when this class loads.
-      #
-      require 'socket'
-      def self.extract_host
-        @host ||= Socket.gethostname
-      end
-      def host
-        self.class.extract_host
-      end
-      extract_host
-      def pid
-        @pid ||= Process.pid
-      end
-      # Use the host and pid (generated lazily in child processes) for the result.
-      #
-      def generate_intermediate_result_id
-        :"#{host}:#{pid}:picky:result"
       end
 
     end

data/lib/picky/query/indexes.rb

@@ -1,193 +1,208 @@
-module Query
-
-  # The query indexes class bundles indexes given to a query.
-  #
-  # Example:
-  #   # If you call
-  #   Search.new dvd_index, mp3_index, video_index
-  #
-  #   # What it does is take the three given (API-) indexes and
-  #   # bundle them in an index bundle.
-  #
-  class Indexes
-
-    attr_reader :indexes
-
-    # Creates a new Query::Indexes.
+module Picky
+
+  module Query
+
+    # The query indexes class bundles indexes given to a query.
     #
-    # Its job is to generate all possible combinations.
-    # Note: We cannot mix memory and redis indexes just yet.
+    # Example:
+    #   # If you call
+    #   Search.new dvd_index, mp3_index, video_index
     #
-    def initialize *indexes, combinations_type
-      @indexes           = indexes
-      @combinations_type = combinations_type
-    end
-
-    # Returns a number of prepared (sorted, reduced etc.) allocations for the given tokens.
+    #   # What it does is take the three given (API-) indexes and
+    #   # bundle them in an index bundle.
     #
-    def prepared_allocations_for tokens, weights = {}
-      allocations = allocations_for tokens
+    class Indexes
 
-      # Remove double allocations.
-      #
-      allocations.uniq
+      attr_reader :indexes
 
-      # Score the allocations using weights as bias.
+      # Creates a new Query::Indexes.
       #
-      allocations.calculate_score weights
-
-      # Sort the allocations.
-      # (allocations are sorted according to score, highest to lowest)
+      # Its job is to generate all possible combinations.
+      # Note: We cannot mix memory and redis indexes just yet.
       #
-      allocations.sort!
+      def initialize *indexes, combinations_type
+        @indexes           = indexes
+        @combinations_type = combinations_type
+        @mapper            = Query::QualifierCategoryMapper.new
+        map_categories
+      end
+      def map_categories
+        @indexes.each do |index|
+          index.each_category do |category|
+            @mapper.add category
+          end
+        end
+      end
 
-      # Reduce the amount of allocations.
+      # Returns a number of prepared (sorted, reduced etc.) allocations for the given tokens.
       #
-      # allocations.reduce_to some_amount
+      def prepared_allocations_for tokens, weights = {}
+        allocations = allocations_for tokens
 
-      # Remove identifiers from allocations.
-      #
-      # allocations.remove some_array_of_identifiers_to_remove
+        # Remove double allocations.
+        #
+        allocations.uniq
 
-      allocations
-    end
-    # Returns a number of possible allocations for the given tokens.
-    #
-    def allocations_for tokens
-      Allocations.new allocations_ary_for(tokens)
-    end
-    def allocations_ary_for tokens
-      indexes.inject([]) do |allocations, index|
-        allocations + allocation_for(tokens, index)
-      end
-    end
-    def allocation_for tokens, index
-      # Expand the combinations.
-      #
-      possible_combinations = tokens.possible_combinations_in index
+        # Score the allocations using weights as bias.
+        #
+        allocations.calculate_score weights
 
-      # Generate all possible combinations.
-      #
-      expanded_combinations = expand_combinations_from possible_combinations
+        # Sort the allocations.
+        # (allocations are sorted according to score, highest to lowest)
+        #
+        allocations.sort!
 
-      # Add the wrapped possible allocations to the ones we already have.
-      #
-      expanded_combinations.map! do |expanded_combination|
-        Allocation.new @combinations_type.new(expanded_combination), index.result_identifier # TODO Do not extract result_identifier.
-      end
-    end
+        # Reduce the amount of allocations.
+        #
+        # allocations.reduce_to some_amount
 
-    # This is the core of the search engine. No kidding.
-    #
-    # Gets an array of
-    # [
-    #  [<combinations for token1>],
-    #  [<combinations for token2>],
-    #  [<combinations for token3>]
-    # ]
-    #
-    # Generates all possible allocations of combinations.
-    # [
-    #  [first combination of token1, first c of t2, first c of t3],
-    #  [first combination of token1, first c of t2, second c of t3]
-    #  ...
-    # ]
-    #
-    # Generates all possible combinations of array elements:
-    # [1,2,3] x [a,b,c] x [k,l,m] => [[1,a,k], [1,a,l], [1,a,m], [1,b,k], [1,b,l], [1,b,m], [1,c,k], ..., [3,c,m]]
-    # Note: Also calculates the weights and sorts them accordingly.
-    #
-    # Note: This is a heavily optimized ruby version.
-    #
-    # Works like this:
-    # [1,2,3], [a,b,c], [k,l,m] are expanded to
-    #  group mult: 1
-    #  <- single mult ->
-    # [1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3] = 27 elements
-    #  group mult: 3
-    #  <- -> s/m
-    # [a,a,a,b,b,b,c,c,c,a,a,a,b,b,b,c,c,c,a,a,a,b,b,b,c,c,c] = 27 elements
-    #  group mult: 9
-    #  <> s/m
-    # [k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m] = 27 elements
-    #
-    # It is then recombined, where
-    # [
-    #  [a,a,b,b,c,c]
-    #  [d,e,d,e,d,e]
-    # ]
-    # becomes
-    # [
-    #  [a,d],
-    #  [a,e],
-    #  [b,d],
-    #  [b,e],
-    #  [c,d],
-    #  [c,e]
-    # ]
-    #
-    # Note: Not using transpose as it is slower.
-    #
-    # Returns nil if there are no combinations.
-    #
-    # Note: Of course I could split this method up into smaller
-    #       ones, but I guess I am a bit sentimental.
-    #
-    def expand_combinations_from possible_combinations
-      # If an element has size 0, this means one of the
-      # tokens could not be allocated.
-      #
-      return [] if possible_combinations.any?(&:empty?)
+        # Remove identifiers from allocations.
+        #
+        # allocations.remove some_array_of_identifiers_to_remove
 
-      # Generate the first multiplicator "with which" (well, not quite) to multiply the smallest amount of combinations.
+        allocations
+      end
+      # Returns a number of possible allocations for the given tokens.
       #
-      single_mult = possible_combinations.inject(1) { |total, combinations| total * combinations.size }
+      def allocations_for tokens
+        tokens.categorize @mapper
 
-      # Initialize a group multiplicator.
-      #
-      group_mult = 1
+        Allocations.new allocations_ary_for(tokens)
+      end
+      def allocations_ary_for tokens
+        indexes.inject([]) do |allocations, index|
+          allocations + allocation_for(tokens, index)
+        end
+      end
+      def allocation_for tokens, index
+        # Expand the combinations.
+        #
+        possible_combinations = tokens.possible_combinations_in index
 
-      # The expanding part to line up the combinations
-      # for later combination in allocations.
-      #
-      possible_combinations.collect! do |combinations|
+        # Generate all possible combinations.
+        #
+        expanded_combinations = expand_combinations_from possible_combinations
 
-        # Get the size of the combinations of the first token.
+        # Add the wrapped possible allocations to the ones we already have.
         #
-        combinations_size = combinations.size
+        expanded_combinations.map! do |expanded_combination|
+          Allocation.new @combinations_type.new(expanded_combination), index.result_identifier # TODO Do not extract result_identifier.
+        end
+      end
 
-        # Special case: If there is no combination for one of the tokens.
-        #               In that case, we just use the same single mult for
-        #               the next iteration.
-        #               If there are combinations, we divide the single mult
-        #               by the number of combinations.
+      # This is the core of the search engine. No kidding.
+      #
+      # Gets an array of
+      # [
+      #  [<combinations for token1>],
+      #  [<combinations for token2>],
+      #  [<combinations for token3>]
+      # ]
+      #
+      # Generates all possible allocations of combinations.
+      # [
+      #  [first combination of token1, first c of t2, first c of t3],
+      #  [first combination of token1, first c of t2, second c of t3]
+      #  ...
+      # ]
+      #
+      # Generates all possible combinations of array elements:
+      # [1,2,3] x [a,b,c] x [k,l,m] => [[1,a,k], [1,a,l], [1,a,m], [1,b,k], [1,b,l], [1,b,m], [1,c,k], ..., [3,c,m]]
+      # Note: Also calculates the weights and sorts them accordingly.
+      #
+      # Note: This is a heavily optimized ruby version.
+      #
+      # Works like this:
+      # [1,2,3], [a,b,c], [k,l,m] are expanded to
+      #  group mult: 1
+      #  <- single mult ->
+      # [1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3] = 27 elements
+      #  group mult: 3
+      #  <- -> s/m
+      # [a,a,a,b,b,b,c,c,c,a,a,a,b,b,b,c,c,c,a,a,a,b,b,b,c,c,c] = 27 elements
+      #  group mult: 9
+      #  <> s/m
+      # [k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m] = 27 elements
+      #
+      # It is then recombined, where
+      # [
+      #  [a,a,b,b,c,c]
+      #  [d,e,d,e,d,e]
+      # ]
+      # becomes
+      # [
+      #  [a,d],
+      #  [a,e],
+      #  [b,d],
+      #  [b,e],
+      #  [c,d],
+      #  [c,e]
+      # ]
+      #
+      # Note: Not using transpose as it is slower.
+      #
+      # Returns nil if there are no combinations.
+      #
+      # Note: Of course I could split this method up into smaller
+      #       ones, but I guess I am a bit sentimental.
+      #
+      def expand_combinations_from possible_combinations
+        # If an element has size 0, this means one of the
+        # tokens could not be allocated.
         #
-        single_mult /= combinations_size unless combinations_size.zero?
+        return [] if possible_combinations.any?(&:empty?)
 
-        # Expand each combination by the single mult:
-        # [a,b,c]
-        # [a,a,a,  b,b,b,  c,c,c]
-        # Then, expand the result by the group mult:
-        # [a,a,a,b,b,b,c,c,c,  a,a,a,b,b,b,c,c,c,  a,a,a,b,b,b,c,c,c]
+        # Generate the first multiplicator "with which" (well, not quite) to multiply the smallest amount of combinations.
         #
-        combinations = combinations.inject([]) do |total, combination|
-          total + Array.new(single_mult, combination)
-        end * group_mult
+        single_mult = possible_combinations.inject(1) { |total, combinations| total * combinations.size }
 
-        # Multiply the group mult by the combinations size,
-        # since the next combinations' single mult is smaller
-        # and we need to adjust for that.
+        # Initialize a group multiplicator.
         #
-        group_mult = group_mult * combinations_size
+        group_mult = 1
 
-        # Return the combinations.
+        # The expanding part to line up the combinations
+        # for later combination in allocations.
         #
-        combinations
+        possible_combinations.collect! do |combinations|
+
+          # Get the size of the combinations of the first token.
+          #
+          combinations_size = combinations.size
+
+          # Special case: If there is no combination for one of the tokens.
+          #               In that case, we just use the same single mult for
+          #               the next iteration.
+          #               If there are combinations, we divide the single mult
+          #               by the number of combinations.
+          #
+          single_mult /= combinations_size unless combinations_size.zero?
+
+          # Expand each combination by the single mult:
+          # [a,b,c]
+          # [a,a,a,  b,b,b,  c,c,c]
+          # Then, expand the result by the group mult:
+          # [a,a,a,b,b,b,c,c,c,  a,a,a,b,b,b,c,c,c,  a,a,a,b,b,b,c,c,c]
+          #
+          combinations = combinations.inject([]) do |total, combination|
+            total + Array.new(single_mult, combination)
+          end * group_mult
+
+          # Multiply the group mult by the combinations size,
+          # since the next combinations' single mult is smaller
+          # and we need to adjust for that.
+          #
+          group_mult = group_mult * combinations_size
+
+          # Return the combinations.
+          #
+          combinations
+        end
+
+        return [] if possible_combinations.empty?
+
+        possible_combinations.shift.zip *possible_combinations
       end
 
-      return [] if possible_combinations.empty?
-
-      possible_combinations.shift.zip *possible_combinations
     end
 
   end

data/lib/picky/query/qualifier_category_mapper.rb

@@ -0,0 +1,43 @@
+module Picky
+
+  # coding: utf-8
+  #
+  module Query
+
+    # Collection class for qualifiers.
+    #
+    class QualifierCategoryMapper # :nodoc:all
+
+      attr_reader :mapping
+
+      #
+      #
+      def initialize
+        @mapping = {}
+      end
+
+      #
+      #
+      def add category
+        category.qualifiers.each do |qualifier|
+          sym_qualifier = qualifier.to_sym
+          warn %Q{Warning: Qualifier "#{qualifier}" already mapped to category #{mapping[sym_qualifier].identifier} (ambiguous qualifier mapping).} if mapping.has_key? sym_qualifier
+          mapping[sym_qualifier] = category
+        end
+      end
+
+      # Normalizes the given qualifier.
+      #
+      # Returns nil if it is not allowed, the normalized qualifier if it is.
+      #
+      def map qualifier
+        return nil if qualifier.blank?
+
+        @mapping[qualifier.to_sym]
+      end
+
+    end
+
+  end
+
+end