picky 3.0.1 → 3.1.0

data/lib/picky/indexing/bundle/base.rb

@@ -1,242 +0,0 @@
-module Picky
-
-  module Indexing # :nodoc:all
-
-    # A Bundle is a number of indexes
-    # per [index, category] combination.
-    #
-    # At most, there are three indexes:
-    # * *core* index (always used)
-    # * *weights* index (always used)
-    # * *similarity* index (used with similarity)
-    #
-    # In Picky, indexing is separated from the index
-    # handling itself through a parallel structure.
-    #
-    # Both use methods provided by this base class, but
-    # have very different goals:
-    #
-    # * *Indexing*::*Bundle* is just concerned with creating index files
-    #   and providing helper functions to e.g. check the indexes.
-    #
-    # * *Index*::*Bundle* is concerned with loading these index files into
-    #   memory and looking up search data as fast as possible.
-    #
-    module Bundle
-
-      # This is the indexing bundle.
-      #
-      # It does all menial tasks that have nothing to do
-      # with the actual index running etc.
-      #
-      class Base < Picky::Bundle
-
-        attr_reader :backend,
-                    :prepared
-
-        attr_accessor :partial_strategy,
-                      :weights_strategy
-
-        def initialize name, category, weights_strategy, partial_strategy, similarity_strategy, options = {}
-          super name, category, similarity_strategy, options
-
-          @weights_strategy = weights_strategy
-          @partial_strategy = partial_strategy
-          @key_format       = options[:key_format]
-          @prepared         = Backend::File::Text.new category.prepared_index_path
-        end
-
-        # Sets up a piece of the index for the given token.
-        #
-        def initialize_inverted_index_for token
-          self.inverted[token] ||= []
-        end
-
-        # Generation
-        #
-
-        # This method
-        # * Loads the base index from the "prepared..." file.
-        # * Generates derived indexes.
-        # * Dumps all the indexes into files.
-        #
-        def generate_caches_from_source
-          load_from_prepared_index_file
-          generate_caches_from_memory
-        end
-        # Generates derived indexes from the index and dumps.
-        #
-        # Note: assumes that there is something in the index
-        #
-        def generate_caches_from_memory
-          cache_from_memory_generation_message
-          generate_derived
-        end
-        def cache_from_memory_generation_message
-          timed_exclaim %Q{"#{identifier}": Caching from intermediate in-memory index.}
-        end
-
-        # Generates the weights and similarity from the main index.
-        #
-        def generate_derived
-          generate_weights
-          generate_similarity
-        end
-
-        # Load the data from the db.
-        #
-        def load_from_prepared_index_file
-          load_from_prepared_index_generation_message
-          clear
-          retrieve
-        end
-        def load_from_prepared_index_generation_message
-          timed_exclaim %Q{"#{identifier}": Loading prepared data into memory.}
-        end
-        # Retrieves the prepared index data into the index.
-        #
-        # This is in preparation for generating
-        # derived indexes (like weights, similarity)
-        # and later dumping the optimized index.
-        #
-        # TODO Move this out to the category?
-        #
-        def retrieve
-          format = category.key_format || :to_i # Optimization.
-          prepared.retrieve do |id, token|
-            initialize_inverted_index_for token
-            self.inverted[token] << id.send(format)
-          end
-        end
-
-        # Generates a new index (writes its index) using the
-        # partial caching strategy of this bundle.
-        #
-        def generate_partial
-          generator = Generators::PartialGenerator.new self.inverted
-          self.inverted = generator.generate self.partial_strategy
-        end
-        # Generate a partial index from the given exact inverted index.
-        #
-        def generate_partial_from exact_inverted_index
-          timed_exclaim %Q{"#{identifier}": Generating partial index for index.}
-          self.inverted = exact_inverted_index
-          self.generate_partial
-          self
-        end
-        # Generates a new weights index (writes its index) using the
-        # given weight caching strategy.
-        #
-        def generate_weights
-          generator = Generators::WeightsGenerator.new self.inverted
-          self.weights = generator.generate self.weights_strategy
-        end
-        # Generates a new similarity index (writes its index) using the
-        # given similarity caching strategy.
-        #
-        def generate_similarity
-          generator = Generators::SimilarityGenerator.new self.inverted
-          self.similarity = generator.generate self.similarity_strategy
-        end
-
-        # Saves the indexes in a dump file.
-        #
-        def dump
-          timed_exclaim %Q{"#{identifier}": Dumping data.}
-          dump_inverted
-          dump_similarity
-          dump_weights
-          dump_configuration
-        end
-        # Dumps the core index.
-        #
-        def dump_inverted
-          # timed_exclaim %Q{"#{identifier}": Dumping inverted index.}
-          backend.dump_inverted self.inverted
-        end
-        # Dumps the weights index.
-        #
-        def dump_weights
-          # timed_exclaim %Q{"#{identifier}": Dumping index weights.}
-          backend.dump_weights self.weights
-        end
-        # Dumps the similarity index.
-        #
-        def dump_similarity
-          # timed_exclaim %Q{"#{identifier}": Dumping similarity index.}
-          backend.dump_similarity self.similarity
-        end
-        # Dumps the similarity index.
-        #
-        def dump_configuration
-          # timed_exclaim %Q{"#{identifier}": Dumping configuration.}
-          backend.dump_configuration self.configuration
-        end
-
-        # Alerts the user if an index is missing.
-        #
-        def raise_unless_cache_exists
-          raise_unless_index_exists
-          raise_unless_similarity_exists
-        end
-        # Alerts the user if one of the necessary indexes
-        # (core, weights) is missing.
-        #
-        def raise_unless_index_exists
-          if partial_strategy.saved?
-            warn_if_index_small
-            raise_unless_index_ok
-          end
-        end
-        # Alerts the user if the similarity
-        # index is missing (given that it's used).
-        #
-        def raise_unless_similarity_exists
-          if similarity_strategy.saved?
-            warn_if_similarity_small
-            raise_unless_similarity_ok
-          end
-        end
-
-        # Outputs a warning for the given cache.
-        #
-        def warn_cache_small what
-          warn "Warning: #{what} cache for #{identifier} smaller than 16 bytes."
-        end
-        # Raises an appropriate error message for the given cache.
-        #
-        def raise_cache_missing what
-          raise "Error: The #{what} cache for #{identifier} is missing."
-        end
-
-        # Warns the user if the similarity index is small.
-        #
-        def warn_if_similarity_small
-          warn_cache_small :similarity if backend.similarity_cache_small?
-        end
-        # Alerts the user if the similarity index is not there.
-        #
-        def raise_unless_similarity_ok
-          raise_cache_missing :similarity unless backend.similarity_cache_ok?
-        end
-
-        # Warns the user if the core or weights indexes are small.
-        #
-        def warn_if_index_small
-          warn_cache_small :inverted if backend.inverted_cache_small?
-          warn_cache_small :weights  if backend.weights_cache_small?
-        end
-        # Alerts the user if the core or weights indexes are not there.
-        #
-        def raise_unless_index_ok
-          raise_cache_missing :inverted unless backend.inverted_cache_ok?
-          raise_cache_missing :weights  unless backend.weights_cache_ok?
-        end
-
-      end
-
-    end
-
-  end
-
-end

data/lib/picky/indexing/bundle/memory.rb

@@ -1,26 +0,0 @@
-module Picky
-
-  # encoding: utf-8
-  #
-  module Indexing # :nodoc:all
-
-    module Bundle
-
-      # The memory version dumps its generated indexes to disk
-      # (mostly JSON) to load them into memory on startup.
-      #
-      class Memory < Base
-
-        def initialize name, category, *args
-          super name, category, *args
-
-          @backend = Backend::Files.new self
-        end
-
-      end
-
-    end
-
-  end
-
-end

data/lib/picky/indexing/bundle/redis.rb

@@ -1,26 +0,0 @@
-module Picky
-
-  # encoding: utf-8
-  #
-  module Indexing # :nodoc:all
-
-    module Bundle
-
-      # The Redis version dumps its generated indexes to
-      # the Redis backend.
-      #
-      class Redis < Base
-
-        def initialize name, category, *args
-          super name, category, *args
-
-          @backend = Backend::Redis.new self
-        end
-
-      end
-
-    end
-
-  end
-
-end

data/lib/picky/query/combinations/base.rb

@@ -1,74 +0,0 @@
-module Picky
-
-  module Query
-
-    # Combinations are a number of Combination-s.
-    #
-    # They are the core of an allocation.
-    # An allocation consists of a number of combinations.
-    #
-    module Combinations # :nodoc:all
-
-      # Base Combinations contain methods for calculating score and ids.
-      #
-      class Base
-
-        attr_reader :combinations
-
-        delegate :empty?, :to => :@combinations
-
-        def initialize combinations = []
-          @combinations = combinations
-        end
-
-        def hash
-          @combinations.hash
-        end
-
-        # Uses user specific weights to calculate a score for the combinations.
-        #
-        def calculate_score weights
-          total_score + weighted_score(weights)
-        end
-        def total_score
-          @combinations.sum &:weight
-        end
-        def weighted_score weights
-          weights.score_for @combinations
-        end
-
-        # Filters the tokens and identifiers such that only identifiers
-        # that are passed in, remain, including their tokens.
-        #
-        # Note: This method is not totally independent of the calculate_ids one.
-        #       Since identifiers are only nullified, we need to not include the
-        #       ids that have an associated identifier that is nil.
-        #
-        def keep identifiers = []
-          @combinations.reject! { |combination| !combination.in?(identifiers) }
-        end
-
-        # Filters the tokens and identifiers such that identifiers
-        # that are passed in, are removed, including their tokens.
-        #
-        # Note: This method is not totally independent of the calculate_ids one.
-        #       Since identifiers are only nullified, we need to not include the
-        #       ids that have an associated identifier that is nil.
-        #
-        def remove identifiers = []
-          @combinations.reject! { |combination| combination.in?(identifiers) }
-        end
-
-        #
-        #
-        def to_result
-          @combinations.map &:to_result
-        end
-
-      end
-
-    end
-
-  end
-
-end

data/lib/picky/query/combinations/memory.rb

@@ -1,52 +0,0 @@
-module Picky
-
-  module Query
-
-    # Combinations are a number of Combination-s.
-    #
-    # They are the core of an allocation.
-    # An allocation consists of a number of combinations.
-    #
-    module Combinations # :nodoc:all
-
-      # Memory Combinations contain specific methods for
-      # calculating score and ids in memory.
-      #
-      class Memory < Base
-
-        # Returns the result ids for the allocation.
-        #
-        # Sorts the ids by size and & through them in the following order (sizes):
-        # 0. [100_000, 400, 30, 2]
-        # 1. [2, 30, 400, 100_000]
-        # 2. (100_000 & (400 & (30 & 2))) # => result
-        #
-        # Note: Uses a C-optimized intersection routine (in performant.c)
-        #       for speed and memory efficiency.
-        #
-        # Note: In the memory based version we ignore the (amount) needed hint.
-        #       We cannot use the information to speed up the algorithm, unfortunately.
-        #
-        def ids _, _
-          return [] if @combinations.empty?
-
-          # Get the ids for each combination.
-          #
-          id_arrays = @combinations.inject([]) do |total, combination|
-            total << combination.ids
-          end
-
-          # Call the optimized C algorithm.
-          #
-          # Note: It orders the passed arrays by size.
-          #
-          Performant::Array.memory_efficient_intersect id_arrays
-        end
-
-      end
-
-    end
-
-  end
-
-end

data/lib/picky/query/combinations/redis.rb

@@ -1,90 +0,0 @@
-module Picky
-
-  module Query
-
-    # Combinations are a number of Combination-s.
-    #
-    # They are the core of an allocation.
-    # An allocation consists of a number of combinations.
-    #
-    module Combinations # :nodoc:all
-
-      # Redis Combinations contain specific methods for
-      # calculating score and ids in memory.
-      #
-      class Redis < Base
-
-        # Connect to the backend.
-        #
-        # TODO Use specific Picky Redis wrapper.
-        #
-        def self.redis
-          @redis ||= ::Redis.new :db => 15
-        end
-
-        attr_reader :redis
-
-        #
-        #
-        def initialize combinations
-          super combinations
-
-          @redis = self.class.redis
-        end
-
-        # Returns the result ids for the allocation.
-        #
-        def ids amount, offset
-          return [] if @combinations.empty?
-
-          identifiers = @combinations.inject([]) do |identifiers, combination|
-            identifiers << "#{combination.identifier}"
-          end
-
-          result_id = generate_intermediate_result_id
-
-          # Intersect and store.
-          #
-          redis.zinterstore result_id, identifiers
-
-          # Get the stored result.
-          #
-          results = redis.zrange result_id, offset, (offset + amount)
-
-          # Delete the stored result as it was only for temporary purposes.
-          #
-          # Note: I could also not delete it, but that would not be clean at all.
-          #
-          redis.del result_id
-
-          results
-        end
-
-        # Generate a multiple host/process safe result id.
-        #
-        # Note: Generated when this class loads.
-        #
-        require 'socket'
-        def self.extract_host
-          @host ||= Socket.gethostname
-        end
-        def host
-          self.class.extract_host
-        end
-        extract_host
-        def pid
-          @pid ||= Process.pid
-        end
-        # Use the host and pid (generated lazily in child processes) for the result.
-        #
-        def generate_intermediate_result_id
-          :"#{host}:#{pid}:picky:result"
-        end
-
-      end
-
-    end
-
-  end
-
-end

data/lib/picky/query.rb

@@ -1,6 +0,0 @@
-module Picky
-
-  module Query # :nodoc:all
-  end
-
-end