picky 2.5.2 → 2.6.0

data/lib/picky/indexed/bundle/redis.rb

@@ -0,0 +1,45 @@
+# encoding: utf-8
+#
+module Indexed # :nodoc:all
+
+  #
+  #
+  module Bundle
+
+    # This is the _actual_ index (based on Redis).
+    #
+    # Handles exact/partial index, weights index, and similarity index.
+    #
+    class Redis < Base
+
+      def initialize name, category, *args
+        super name, category, *args
+
+        @backend = Backend::Redis.new name, category
+      end
+
+      # Get the ids for the given symbol.
+      #
+      # Ids are an array of string values in Redis.
+      #
+      def ids sym
+        @backend.ids sym
+      end
+      # Get a weight for the given symbol.
+      #
+      # A weight is a string value in Redis. TODO Convert?
+      #
+      def weight sym
+        @backend.weight sym
+      end
+      # Settings of this bundle can be accessed via [].
+      #
+      def [] sym
+        @backend.setting sym
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/indexed/wrappers/bundle/calculation.rb

@@ -0,0 +1,35 @@
+module Indexed
+  module Wrappers
+
+    module Bundle
+
+      # A calculation rewrites the symbol into a float.
+      #
+      # TODO I really need to allow integers as keys. The code below is just not up to the needed quality.
+      #
+      class Calculation < Wrapper
+
+        #
+        #
+        def recalculate float
+          float
+        end
+
+        #
+        #
+        def ids sym
+          @bundle.ids recalculate(sym.to_s.to_f).to_s.to_sym
+        end
+
+        #
+        #
+        def weight sym
+          @bundle.weight recalculate(sym.to_s.to_f).to_s.to_sym
+        end
+
+      end
+
+    end
+
+  end
+end

data/lib/picky/indexed/wrappers/bundle/location.rb

@@ -0,0 +1,42 @@
+module Indexed
+  module Wrappers
+
+    module Bundle
+
+      # A location calculation recalculates a location to the Picky internal location.
+      #
+      class Location < Calculation
+
+        def initialize bundle, options = {}
+          super bundle
+
+          precision  = options[:precision] || 1
+          user_grid  = options[:grid] || raise("Gridsize needs to be given for location #{bundle.identifier}.")
+
+          @calculation = Calculations::Location.new user_grid, precision
+        end
+
+        #
+        #
+        def recalculate float
+          @calculation.recalculate float
+        end
+
+        #
+        #
+        def load
+          # Load first the bundle, then extract the config.
+          #
+          bundle.load
+          # TODO Move the to_f to the backend.
+          #
+          minimum = bundle[:location_minimum] && bundle[:location_minimum].to_f || raise("Configuration :location_minimum for #{bundle.identifier} missing. Did you run rake index already?")
+          @calculation.minimum = minimum
+        end
+
+      end
+
+    end
+
+  end
+end

data/lib/picky/indexed/wrappers/bundle/wrapper.rb

@@ -0,0 +1,43 @@
+module Indexed
+  module Wrappers
+
+    # Per Bundle wrappers.
+    #
+    module Bundle
+
+      # Base wrapper. Just delegates all methods to the bundle.
+      #
+      class Wrapper
+
+        attr_reader :bundle
+
+        def initialize bundle
+          @bundle = bundle
+        end
+
+        delegate :load,
+                 :load_index,
+                 :load_weights,
+                 :load_similarity,
+                 :load_configuration,
+                 :clear_index,
+                 :clear_weights,
+                 :clear_similarity,
+                 :clear_configuration,
+                 :ids,
+                 :weight,
+                 :identifier,
+                 :analyze,
+                 :size,
+                 :index,
+                 :weights,
+                 :similarity,
+                 :configuration,
+                 :to => :@bundle
+
+      end
+
+    end
+
+  end
+end

data/lib/picky/indexed/wrappers/category/location.rb

@@ -0,0 +1,25 @@
+module Indexed
+  module Wrappers
+    module Category
+
+      module Location
+
+        def self.install_on category, grid, precision = 1
+          wrapped_exact = Indexed::Wrappers::Bundle::Location.new category.indexed_exact, grid: grid, precision: precision
+
+          category.class_eval do
+            define_method :indexed_exact do
+              wrapped_exact
+            end
+            define_method :indexed_partial do
+              wrapped_exact
+            end
+          end
+
+        end
+
+      end
+
+    end
+  end
+end

data/lib/picky/indexed/wrappers/exact_first.rb

@@ -0,0 +1,55 @@
+# encoding: utf-8
+#
+module Indexed
+
+  module Wrappers
+
+    # This index combines an exact and partial index.
+    # It serves to order the results such that exact hits are found first.
+    #
+    class ExactFirst < Indexed::Bundle::Base
+
+      delegate :similar,
+               :identifier,
+               :name,
+               :to => :@exact
+      delegate :index,
+               :category,
+               :weight,
+               :generate_partial_from,
+               :generate_caches_from_memory,
+               :generate_derived,
+               :dump,
+               :load,
+               :to => :@partial
+
+      def initialize category
+        @exact   = category.indexed_exact
+        @partial = category.indexed_partial
+      end
+
+      def self.wrap index_or_category
+        if index_or_category.respond_to? :categories
+          wrap_each_of index_or_category.categories
+          index_or_category
+        else
+          new index_or_category
+        end
+      end
+      def self.wrap_each_of categories
+        categories.categories.collect! { |category| new(category) }
+      end
+
+      def ids text
+        @exact.ids(text) + @partial.ids(text)
+      end
+
+      def weight text
+        [@exact.weight(text) || 0, @partial.weight(text) || 0].max
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/indexes.rb

@@ -0,0 +1,73 @@
+# Holds all indexes and provides operations
+# for extracting and working on them.
+#
+# Delegates a number of operations to the
+# indexes.
+#
+class Indexes
+
+  attr_reader :indexes,
+              :index_mapping
+
+  delegate :size,
+           :each,
+           :to => :indexes
+
+  each_delegate :reindex,
+                :to => :indexes
+
+  def initialize
+    clear
+  end
+
+  # Return the Indexes instance.
+  #
+  def self.instance
+    @instance ||= new
+  end
+
+  instance_delegate :clear,
+                    :register,
+                    :reindex,
+                    :[],
+                    :to_s,
+                    :size,
+                    :each
+
+  # Clears the indexes and the mapping.
+  #
+  def clear
+    @indexes       = []
+    @index_mapping = {}
+  end
+
+  # Registers an index with the indexes.
+  #
+  def register index
+    self.indexes << index
+    self.index_mapping[index.name] = index
+  end
+  def self.register index
+    self.instance.register index
+  end
+
+  # Extracts an index, given its identifier.
+  #
+  def [] identifier
+    index_name = identifier.to_sym
+    index_mapping[index_name] || raise_not_found(index_name)
+  end
+
+  # Raises a not found for the index.
+  #
+  def raise_not_found index_name
+    raise %Q{Index "#{index_name}" not found. Possible indexes: "#{indexes.map(&:name).join('", "')}".}
+  end
+
+  #
+  #
+  def to_s
+    indexes.indented_to_s
+  end
+
+end

data/lib/picky/indexes_indexed.rb

@@ -0,0 +1,29 @@
+# Registers the indexes held at runtime, for queries.
+#
+class Indexes
+  
+  instance_delegate :load_from_cache,
+                    :reload,
+                    :analyze
+  
+  each_delegate :load_from_cache,
+                :to => :indexes
+  
+  # Reloads all indexes, one after another,
+  # in the order they were added.
+  #
+  alias reload load_from_cache
+
+  # Load each index, and analyze it.
+  #
+  # Returns a hash with the findings.
+  #
+  def analyze
+    result = {}
+    indexes.each do |index|
+      index.analyze result
+    end
+    result
+  end
+
+end

data/lib/picky/indexes_indexing.rb

@@ -0,0 +1,49 @@
+# Indexes indexing.
+#
+class Indexes
+
+  instance_delegate :take_snapshot,
+                    :generate_caches,
+                    :backup_caches,
+                    :restore_caches,
+                    :check_caches,
+                    :clear_caches,
+                    :create_directory_structure,
+                    :index,
+                    :index_for_tests
+
+  each_delegate :take_snapshot,
+                :generate_caches,
+                :backup_caches,
+                :restore_caches,
+                :check_caches,
+                :clear_caches,
+                :create_directory_structure,
+                :to => :indexes
+
+  # Runs the indexers in parallel (prepare + cache).
+  #
+  def index randomly = true
+    take_snapshot
+
+    # Run in parallel.
+    #
+    timed_exclaim "Indexing using #{Cores.max_processors} processors, in #{randomly ? 'random' : 'given'} order."
+
+    # Run indexing/caching forked.
+    #
+    Cores.forked self.indexes, { randomly: randomly }, &:index
+
+    timed_exclaim "Indexing finished."
+  end
+
+  # For integration testing – indexes for the tests
+  # without forking and shouting ;)
+  #
+  def index_for_tests
+    take_snapshot
+
+    indexes.each(&:index)
+  end
+
+end

data/lib/picky/indexing/bundle/base.rb

@@ -0,0 +1,212 @@
+# encoding: utf-8
+#
+module Indexing # :nodoc:all
+
+  module Bundle
+
+    # This is the indexing bundle.
+    # It does all menial tasks that have nothing to do
+    # with the actual index running etc.
+    #
+    class Base < SuperBase
+
+      attr_accessor :partial_strategy, :weights_strategy
+
+      # Path is in which directory the cache is located.
+      #
+      def initialize name, category, similarity_strategy, partial_strategy, weights_strategy
+        super name, category, similarity_strategy
+
+        @partial_strategy    = partial_strategy
+        @weights_strategy    = weights_strategy
+      end
+
+      # Sets up a piece of the index for the given token.
+      #
+      def initialize_index_for token
+        index[token] ||= []
+      end
+
+      # Generation
+      #
+
+      # This method
+      # * Loads the base index from the "prepared..." file.
+      # * Generates derived indexes.
+      # * Dumps all the indexes into files.
+      #
+      def generate_caches_from_source
+        load_from_index_file
+        generate_caches_from_memory
+      end
+      # Generates derived indexes from the index and dumps.
+      #
+      # Note: assumes that there is something in the index
+      #
+      def generate_caches_from_memory
+        cache_from_memory_generation_message
+        generate_derived
+      end
+      def cache_from_memory_generation_message
+        timed_exclaim %Q{"#{identifier}": Caching from intermediate in-memory index.}
+      end
+
+      # Generates the weights and similarity from the main index.
+      #
+      def generate_derived
+        generate_weights
+        generate_similarity
+      end
+
+      # Load the data from the db.
+      #
+      def load_from_index_file
+        load_from_index_generation_message
+        clear
+        retrieve
+      end
+      def load_from_index_generation_message
+        timed_exclaim %Q{"#{identifier}": Loading index.}
+      end
+      # Retrieves the prepared index data into the index.
+      #
+      # This is in preparation for generating
+      # derived indexes (like weights, similarity)
+      # and later dumping the optimized index.
+      #
+      def retrieve
+        key_format = self[:key_format] || :to_i
+        files.retrieve do |id, token|
+          initialize_index_for token
+          index[token] << id.send(key_format)
+        end
+      end
+
+      # Generates a new index (writes its index) using the
+      # partial caching strategy of this bundle.
+      #
+      def generate_partial
+        generator = Generators::PartialGenerator.new self.index
+        self.index = generator.generate self.partial_strategy
+      end
+      # Generate a partial index from the given exact index.
+      #
+      def generate_partial_from exact_index
+        timed_exclaim %Q{"#{identifier}": Generating partial index for index.}
+        self.index = exact_index
+        self.generate_partial
+        self
+      end
+      # Generates a new similarity index (writes its index) using the
+      # given similarity caching strategy.
+      #
+      def generate_similarity
+        generator = Generators::SimilarityGenerator.new self.index
+        self.similarity = generator.generate self.similarity_strategy
+      end
+      # Generates a new weights index (writes its index) using the
+      # given weight caching strategy.
+      #
+      def generate_weights
+        generator = Generators::WeightsGenerator.new self.index
+        self.weights = generator.generate self.weights_strategy
+      end
+
+      # Saves the indexes in a dump file.
+      #
+      def dump
+        dump_index
+        dump_similarity
+        dump_weights
+        dump_configuration
+      end
+      # Dumps the core index.
+      #
+      def dump_index
+        timed_exclaim %Q{"#{identifier}": Dumping index.}
+        backend.dump_index index
+      end
+      # Dumps the weights index.
+      #
+      def dump_weights
+        timed_exclaim %Q{"#{identifier}": Dumping weights of index.}
+        backend.dump_weights weights
+      end
+      # Dumps the similarity index.
+      #
+      def dump_similarity
+        timed_exclaim %Q{"#{identifier}": Dumping similarity of index.}
+        backend.dump_similarity similarity
+      end
+      # Dumps the similarity index.
+      #
+      def dump_configuration
+        timed_exclaim %Q{"#{identifier}": Dumping configuration for index.}
+        backend.dump_configuration configuration
+      end
+
+      # Alerts the user if an index is missing.
+      #
+      def raise_unless_cache_exists
+        raise_unless_index_exists
+        raise_unless_similarity_exists
+      end
+      # Alerts the user if one of the necessary indexes
+      # (core, weights) is missing.
+      #
+      def raise_unless_index_exists
+        if partial_strategy.saved?
+          warn_if_index_small
+          raise_unless_index_ok
+        end
+      end
+      # Alerts the user if the similarity
+      # index is missing (given that it's used).
+      #
+      def raise_unless_similarity_exists
+        if similarity_strategy.saved?
+          warn_if_similarity_small
+          raise_unless_similarity_ok
+        end
+      end
+
+      # Outputs a warning for the given cache.
+      #
+      def warn_cache_small what
+        warn "Warning: #{what} cache for #{identifier} smaller than 16 bytes."
+      end
+      # Raises an appropriate error message for the given cache.
+      #
+      def raise_cache_missing what
+        raise "Error: The #{what} cache for #{identifier} is missing."
+      end
+
+      # Warns the user if the similarity index is small.
+      #
+      def warn_if_similarity_small
+        warn_cache_small :similarity if backend.similarity_cache_small?
+      end
+      # Alerts the user if the similarity index is not there.
+      #
+      def raise_unless_similarity_ok
+        raise_cache_missing :similarity unless backend.similarity_cache_ok?
+      end
+
+      # Warns the user if the core or weights indexes are small.
+      #
+      def warn_if_index_small
+        warn_cache_small :index   if backend.index_cache_small?
+        warn_cache_small :weights if backend.weights_cache_small?
+      end
+      # Alerts the user if the core or weights indexes are not there.
+      #
+      def raise_unless_index_ok
+        raise_cache_missing :index   unless backend.index_cache_ok?
+        raise_cache_missing :weights unless backend.weights_cache_ok?
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/indexing/bundle/memory.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+#
+module Indexing # :nodoc:all
+
+  module Bundle
+
+    # The memory version dumps its generated indexes to disk
+    # (mostly JSON) to load them into memory on startup.
+    #
+    class Memory < Base
+
+      # We're using files for the memory backend.
+      # E.g. dump writes files.
+      #
+      alias backend files
+
+      def to_s
+        "Memory\n#{@backend.indented_to_s}"
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/indexing/bundle/redis.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+#
+module Indexing # :nodoc:all
+
+  module Bundle
+
+    # The Redis version dumps its generated indexes to
+    # the Redis backend.
+    #
+    class Redis < Base
+
+      attr_reader :backend
+
+      def initialize name, category, *args
+        super name, category, *args
+
+        @backend = Backend::Redis.new name, category
+      end
+
+    end
+
+  end
+
+end