picky 2.5.2 → 2.6.0

data/lib/picky/internals/query/combinations/base.rb

@@ -1,74 +0,0 @@
-module Internals
-
-  module Query
-
-    # Combinations are a number of Combination-s.
-    #
-    # They are the core of an allocation.
-    # An allocation consists of a number of combinations.
-    #
-    module Combinations # :nodoc:all
-
-      # Base Combinations contain methods for calculating score and ids.
-      #
-      class Base
-
-        attr_reader :combinations
-
-        delegate :empty?, :to => :@combinations
-
-        def initialize combinations = []
-          @combinations = combinations
-        end
-
-        def hash
-          @combinations.hash
-        end
-
-        # Uses user specific weights to calculate a score for the combinations.
-        #
-        def calculate_score weights
-          total_score + weighted_score(weights)
-        end
-        def total_score
-          @combinations.sum &:weight
-        end
-        def weighted_score weights
-          weights.score @combinations
-        end
-
-        # Filters the tokens and identifiers such that only identifiers
-        # that are passed in, remain, including their tokens.
-        #
-        # Note: This method is not totally independent of the calculate_ids one.
-        #       Since identifiers are only nullified, we need to not include the
-        #       ids that have an associated identifier that is nil.
-        #
-        def keep identifiers = []
-          @combinations.reject! { |combination| !combination.in?(identifiers) }
-        end
-
-        # Filters the tokens and identifiers such that identifiers
-        # that are passed in, are removed, including their tokens.
-        #
-        # Note: This method is not totally independent of the calculate_ids one.
-        #       Since identifiers are only nullified, we need to not include the
-        #       ids that have an associated identifier that is nil.
-        #
-        def remove identifiers = []
-          @combinations.reject! { |combination| combination.in?(identifiers) }
-        end
-
-        #
-        #
-        def to_result
-          @combinations.map &:to_result
-        end
-
-      end
-
-    end
-
-  end
-
-end

data/lib/picky/internals/query/combinations/memory.rb

@@ -1,52 +0,0 @@
-module Internals
-
-  module Query
-
-    # Combinations are a number of Combination-s.
-    #
-    # They are the core of an allocation.
-    # An allocation consists of a number of combinations.
-    #
-    module Combinations # :nodoc:all
-
-      # Memory Combinations contain specific methods for
-      # calculating score and ids in memory.
-      #
-      class Memory < Base
-
-        # Returns the result ids for the allocation.
-        #
-        # Sorts the ids by size and & through them in the following order (sizes):
-        # 0. [100_000, 400, 30, 2]
-        # 1. [2, 30, 400, 100_000]
-        # 2. (100_000 & (400 & (30 & 2))) # => result
-        #
-        # Note: Uses a C-optimized intersection routine (in performant.c)
-        #       for speed and memory efficiency.
-        #
-        # Note: In the memory based version we ignore the (amount) needed hint.
-        #       We cannot use the information to speed up the algorithm, unfortunately.
-        #
-        def ids _, _
-          return [] if @combinations.empty?
-
-          # Get the ids for each combination.
-          #
-          id_arrays = @combinations.inject([]) do |total, combination|
-            total << combination.ids
-          end
-
-          # Call the optimized C algorithm.
-          #
-          # Note: It orders the passed arrays by size.
-          #
-          Performant::Array.memory_efficient_intersect id_arrays
-        end
-
-      end
-
-    end
-
-  end
-
-end

data/lib/picky/internals/query/combinations/redis.rb

@@ -1,90 +0,0 @@
-module Internals
-
-  module Query
-
-    # Combinations are a number of Combination-s.
-    #
-    # They are the core of an allocation.
-    # An allocation consists of a number of combinations.
-    #
-    module Combinations # :nodoc:all
-
-      # Redis Combinations contain specific methods for
-      # calculating score and ids in memory.
-      #
-      class Redis < Base
-
-        # Connect to the backend.
-        #
-        # TODO Use specific Picky Redis wrapper.
-        #
-        def self.redis
-          @redis ||= ::Redis.new :db => 15
-        end
-
-        attr_reader :redis
-
-        #
-        #
-        def initialize combinations
-          super combinations
-
-          @redis = self.class.redis
-        end
-
-        # Returns the result ids for the allocation.
-        #
-        def ids amount, offset
-          return [] if @combinations.empty?
-
-          identifiers = @combinations.inject([]) do |identifiers, combination|
-            identifiers << "#{combination.identifier}"
-          end
-
-          result_id = generate_intermediate_result_id
-
-          # Intersect and store.
-          #
-          redis.zinterstore result_id, identifiers
-
-          # Get the stored result.
-          #
-          results = redis.zrange result_id, offset, (offset + amount)
-
-          # Delete the stored result as it was only for temporary purposes.
-          #
-          # Note: I could also not delete it, but that would not be clean at all.
-          #
-          redis.del result_id
-
-          results
-        end
-
-        # Generate a multiple host/process safe result id.
-        #
-        # Note: Generated when this class loads.
-        #
-        require 'socket'
-        def self.extract_host
-          @host ||= Socket.gethostname
-        end
-        def host
-          self.class.extract_host
-        end
-        extract_host
-        def pid
-          @pid ||= Process.pid
-        end
-        # Use the host and pid (generated lazily in child processes) for the result.
-        #
-        def generate_intermediate_result_id
-          :"#{host}:#{pid}:picky:result"
-        end
-
-      end
-
-    end
-
-  end
-
-end

data/lib/picky/internals/query/indexes.rb

@@ -1,199 +0,0 @@
-module Internals
-
-  module Query
-
-    # The query indexes class bundles indexes given to a query.
-    #
-    # Example:
-    #   # If you call
-    #   Search.new dvd_index, mp3_index, video_index
-    #
-    #   # What it does is take the three given (API-) indexes and
-    #   # bundle them in an index bundle.
-    #
-    class Indexes
-
-      attr_reader :indexes
-
-      # Creates a new Query::Indexes.
-      #
-      # Its job is to generate all possible combinations.
-      # Note: We cannot mix memory and redis indexes just yet.
-      #
-      def initialize *index_definitions, combinations_type
-        @combinations_type = combinations_type
-        @indexes           = index_definitions.map(&:internal_indexed)
-      end
-
-      # Returns a number of prepared (sorted, reduced etc.) allocations for the given tokens.
-      #
-      def prepared_allocations_for tokens, weights = {}
-        allocations = allocations_for tokens
-
-        # Remove double allocations.
-        #
-        allocations.uniq
-
-        # Score the allocations using weights as bias.
-        #
-        allocations.calculate_score weights
-
-        # Sort the allocations.
-        # (allocations are sorted according to score, highest to lowest)
-        #
-        allocations.sort!
-
-        # Reduce the amount of allocations.
-        #
-        # allocations.reduce_to some_amount
-
-        # Remove identifiers from allocations.
-        #
-        # allocations.remove some_array_of_identifiers_to_remove
-
-        allocations
-      end
-      # Returns a number of possible allocations for the given tokens.
-      #
-      def allocations_for tokens
-        Allocations.new allocations_ary_for(tokens)
-      end
-      def allocations_ary_for tokens
-        indexes.inject([]) do |allocations, index|
-          allocations + allocation_for(tokens, index)
-        end
-      end
-      def allocation_for tokens, index
-        # Expand the combinations.
-        #
-        possible_combinations = tokens.possible_combinations_in index
-
-        # Generate all possible combinations.
-        #
-        expanded_combinations = expand_combinations_from possible_combinations
-
-        # Add the wrapped possible allocations to the ones we already have.
-        #
-        expanded_combinations.map! do |expanded_combination|
-          Allocation.new @combinations_type.new(expanded_combination), index.result_identifier # TODO Do not extract result_identifier.
-        end
-      end
-
-      # This is the core of the search engine. No kidding.
-      #
-      # Gets an array of
-      # [
-      #  [<combinations for token1>],
-      #  [<combinations for token2>],
-      #  [<combinations for token3>]
-      # ]
-      #
-      # Generates all possible allocations of combinations.
-      # [
-      #  [first combination of token1, first c of t2, first c of t3],
-      #  [first combination of token1, first c of t2, second c of t3]
-      #  ...
-      # ]
-      #
-      # Generates all possible combinations of array elements:
-      # [1,2,3] x [a,b,c] x [k,l,m] => [[1,a,k], [1,a,l], [1,a,m], [1,b,k], [1,b,l], [1,b,m], [1,c,k], ..., [3,c,m]]
-      # Note: Also calculates the weights and sorts them accordingly.
-      #
-      # Note: This is a heavily optimized ruby version.
-      #
-      # Works like this:
-      # [1,2,3], [a,b,c], [k,l,m] are expanded to
-      #  group mult: 1
-      #  <- single mult ->
-      # [1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3] = 27 elements
-      #  group mult: 3
-      #  <- -> s/m
-      # [a,a,a,b,b,b,c,c,c,a,a,a,b,b,b,c,c,c,a,a,a,b,b,b,c,c,c] = 27 elements
-      #  group mult: 9
-      #  <> s/m
-      # [k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m] = 27 elements
-      #
-      # It is then recombined, where
-      # [
-      #  [a,a,b,b,c,c]
-      #  [d,e,d,e,d,e]
-      # ]
-      # becomes
-      # [
-      #  [a,d],
-      #  [a,e],
-      #  [b,d],
-      #  [b,e],
-      #  [c,d],
-      #  [c,e]
-      # ]
-      #
-      # Note: Not using transpose as it is slower.
-      #
-      # Returns nil if there are no combinations.
-      #
-      # Note: Of course I could split this method up into smaller
-      #       ones, but I guess I am a bit sentimental.
-      #
-      def expand_combinations_from possible_combinations
-        # If an element has size 0, this means one of the
-        # tokens could not be allocated.
-        #
-        return [] if possible_combinations.any?(&:empty?)
-
-        # Generate the first multiplicator "with which" (well, not quite) to multiply the smallest amount of combinations.
-        #
-        single_mult = possible_combinations.inject(1) { |total, combinations| total * combinations.size }
-
-        # Initialize a group multiplicator.
-        #
-        group_mult = 1
-
-        # The expanding part to line up the combinations
-        # for later combination in allocations.
-        #
-        possible_combinations.collect! do |combinations|
-
-          # Get the size of the combinations of the first token.
-          #
-          combinations_size = combinations.size
-
-          # Special case: If there is no combination for one of the tokens.
-          #               In that case, we just use the same single mult for
-          #               the next iteration.
-          #               If there are combinations, we divide the single mult
-          #               by the number of combinations.
-          #
-          single_mult /= combinations_size unless combinations_size.zero?
-
-          # Expand each combination by the single mult:
-          # [a,b,c]
-          # [a,a,a,  b,b,b,  c,c,c]
-          # Then, expand the result by the group mult:
-          # [a,a,a,b,b,b,c,c,c,  a,a,a,b,b,b,c,c,c,  a,a,a,b,b,b,c,c,c]
-          #
-          combinations = combinations.inject([]) do |total, combination|
-            total + Array.new(single_mult, combination)
-          end * group_mult
-
-          # Multiply the group mult by the combinations size,
-          # since the next combinations' single mult is smaller
-          # and we need to adjust for that.
-          #
-          group_mult = group_mult * combinations_size
-
-          # Return the combinations.
-          #
-          combinations
-        end
-
-        return [] if possible_combinations.empty?
-
-        possible_combinations.shift.zip *possible_combinations
-      end
-
-    end
-
-  end
-
-end

data/lib/picky/internals/query/qualifiers.rb

@@ -1,82 +0,0 @@
-# coding: utf-8
-#
-module Internals
-
-  #
-  #
-  module Query
-
-    # A single qualifier.
-    #
-    class Qualifier # :nodoc:all
-
-      attr_reader :normalized_qualifier, :codes
-
-      #
-      #
-      # codes is an array.
-      #
-      def initialize normalized_qualifier, codes
-        @normalized_qualifier = normalized_qualifier
-        @codes                = codes.map &:to_sym
-      end
-
-      # Will overwrite if the key is present in the hash.
-      #
-      def inject_into hash
-        codes.each do |code|
-          hash[code] = normalized_qualifier
-        end
-      end
-
-    end
-
-    # Collection class for qualifiers.
-    #
-    class Qualifiers # :nodoc:all
-
-      attr_reader :qualifiers, :normalization_mapping
-
-      delegate :<<, :to => :qualifiers
-
-      #
-      #
-      def initialize
-        @qualifiers = []
-        @normalization_mapping = {}
-      end
-      def self.instance
-        @instanec ||= new
-      end
-
-      # TODO Spec.
-      #
-      def self.add name, qualifiers
-        instance << Qualifier.new(name, qualifiers)
-      end
-
-      # Uses the qualifiers to prepare (optimize) the qualifier handling.
-      #
-      def prepare
-        qualifiers.each do |qualifier|
-          qualifier.inject_into normalization_mapping
-        end
-      end
-
-      # Normalizes the given qualifier.
-      #
-      # Returns nil if it is not allowed, the normalized qualifier if it is.
-      #
-      # Note: Normalizes.
-      #
-      def normalize qualifier
-        return nil if qualifier.blank?
-
-        normalization_mapping[qualifier.to_sym]
-      end
-
-    end
-
-  end
-
-end

data/lib/picky/internals/query/token.rb

@@ -1,202 +0,0 @@
-module Internals
-
-  module Query
-
-    # This is a query token. Together with other tokens it makes up a query.
-    #
-    # It remembers the original form, and and a normalized form.
-    #
-    # It also knows whether it needs to look for similarity (bla~), or whether it is a partial (bla*).
-    #
-    class Token # :nodoc:all
-
-      attr_reader :text, :original
-      attr_writer :similar
-
-      delegate :blank?, :to => :text
-
-      # Normal initializer.
-      #
-      # Note: Use this if you do not want a qualified and normalized token.
-      #
-      def initialize text
-        @text = text
-      end
-
-      # Returns a qualified and normalized token.
-      #
-      # Note: Use this in the search engine if you need a qualified
-      #       and normalized token. I.e. one prepared for a search.
-      #
-      def self.processed text, downcase = true
-        new(text).process downcase
-      end
-      def process downcases = true
-        qualify
-        extract_original
-        downcase if downcases
-        partialize
-        similarize
-        remove_illegals
-        symbolize
-        self
-      end
-
-      # This returns an array of predefined category names if the user has given any.
-      #
-      def user_defined_category_names
-        @qualifiers
-      end
-
-      # Extracts a qualifier for this token and pre-assigns an allocation.
-      #
-      # Note: Removes the qualifier if it is not allowed.
-      #
-      def qualify
-        @qualifiers, @text = split @text
-        @qualifiers && @qualifiers.collect! { |qualifier| Query::Qualifiers.instance.normalize qualifier }.compact!
-        @qualifiers
-      end
-      def extract_original
-        @original = @text.dup
-      end
-
-      # Downcases the text.
-      #
-      def downcase
-        @text.downcase!
-      end
-
-      # Partial is a conditional setter.
-      #
-      # It is only settable if it hasn't been set yet.
-      #
-      def partial= partial
-        @partial = partial if @partial.nil?
-      end
-      def partial?
-        !@similar && @partial
-      end
-
-      # If the text ends with *, partialize it. If with ", don't.
-      #
-      # The latter wins. So "hello*" will not be partially searched.
-      #
-      @@no_partial = /\"\Z/
-      @@partial    = /\*\Z/
-      def partialize
-        self.partial = false and return unless @text !~ @@no_partial
-        self.partial = true unless @text !~ @@partial
-      end
-
-      # If the text ends with ~ similarize it. If with ", don't.
-      #
-      # The latter wins.
-      #
-      @@no_similar = /\"\Z/
-      @@similar    = /\~\Z/
-      def similarize
-        self.similar = false and return if @text =~ @@no_similar
-        self.similar = true if @text =~ @@similar
-      end
-
-      def similar?
-        @similar
-      end
-
-      # Normalizes this token's text.
-      #
-      @@illegals = /["*~]/
-      def remove_illegals
-        @text.gsub! @@illegals, '' unless @text.blank?
-      end
-
-      #
-      #
-      def symbolize
-        @text = @text.to_sym
-      end
-
-      # Returns an array of possible combinations.
-      #
-      def possible_combinations_in index
-        index.possible_combinations self
-      end
-
-      # Returns a token with the next similar text.
-      #
-      # TODO Rewrite this. It is hard to understand. Also spec performance.
-      #
-      def next_similar_token category
-        token = self.dup
-        token if token.next_similar category.bundle_for(token)
-      end
-      # Sets and returns the next similar word.
-      #
-      # Note: Also overrides the original.
-      #
-      def next_similar bundle
-        @text = @original = (similarity(bundle).shift || return) if similar?
-      end
-      # Lazy similar reader.
-      #
-      def similarity bundle = nil
-        @similarity || @similarity = generate_similarity_for(bundle)
-      end
-      # Returns an enumerator that traverses over the similar.
-      #
-      # Note: The dup isn't too nice – since it is needed on account of the shift, above.
-      #       (We avoid a StopIteration exception. Which of both is less evil?)
-      #
-      def generate_similarity_for bundle
-        bundle.similar(@text).dup || []
-      end
-
-      #
-      #
-      def to_result
-        [@original, @text]
-      end
-
-      # Internal identifier.
-      #
-      def identifier
-        "#{similar?? :similarity : :index}:#{@text}"
-      end
-
-      # If the originals & the text are the same, they are the same.
-      #
-      def == other
-        self.original == other.original && self.text == other.text
-      end
-
-      # Displays the qualifier text and the text, joined.
-      #
-      # e.g. name:meier
-      #
-      @@split_qualifier_text = ':'
-      @@split_qualifiers     = ','
-      def to_s
-        [@qualifiers && @qualifiers.join(@@split_qualifiers), @text].compact.join @@split_qualifier_text
-      end
-
-      private
-
-        # Splits text into a qualifier and text.
-        #
-        # Returns [qualifier, text].
-        #
-        def split unqualified_text
-          qualifiers, text = (unqualified_text || '').split(@@split_qualifier_text, 2)
-          if text.blank?
-            [nil, (qualifiers || '')]
-          else
-            [qualifiers.split(@@split_qualifiers), text]
-          end
-        end
-
-    end
-
-  end
-
-end