picky 1.4.1 → 1.4.2

data/lib/picky/internals/indexing/bundle/base.rb

@@ -0,0 +1,219 @@
+# encoding: utf-8
+#
+module Internals
+
+  module Indexing # :nodoc:all
+  
+    module Bundle
+    
+      # This is the indexing bundle.
+      # It does all menial tasks that have nothing to do
+      # with the actual index running etc.
+      #
+      class Base < SuperBase
+      
+        attr_accessor :partial_strategy, :weights_strategy
+      
+        # Path is in which directory the cache is located.
+        #
+        def initialize name, configuration, similarity_strategy, partial_strategy, weights_strategy
+          super name, configuration, similarity_strategy
+        
+          @partial_strategy    = partial_strategy
+          @weights_strategy    = weights_strategy
+        end
+      
+        # Sets up a piece of the index for the given token.
+        #
+        def initialize_index_for token
+          index[token] ||= []
+        end
+      
+        # Generation
+        #
+      
+        # This method
+        # * loads the base index from the db
+        # * generates derived indexes
+        # * dumps all the indexes into files
+        #
+        def generate_caches_from_source
+          load_from_index_file
+          generate_caches_from_memory
+        end
+        # Generates derived indexes from the index and dumps.
+        #
+        # Note: assumes that there is something in the index
+        #
+        def generate_caches_from_memory
+          cache_from_memory_generation_message
+          generate_derived
+        end
+        def cache_from_memory_generation_message
+          timed_exclaim "CACHE FROM MEMORY #{identifier}."
+        end
+      
+        # Generates the weights and similarity from the main index.
+        #
+        def generate_derived
+          generate_weights
+          generate_similarity
+        end
+      
+        # Load the data from the db.
+        #
+        def load_from_index_file
+          load_from_index_generation_message
+          clear
+          retrieve
+        end
+        def load_from_index_generation_message
+          timed_exclaim "LOAD INDEX #{identifier}."
+        end
+        # Retrieves the prepared index data into the index.
+        #
+        # This is in preparation for generating
+        # derived indexes (like weights, similarity)
+        # and later dumping the optimized index.
+        #
+        def retrieve
+          key_format = self[:key_format] || :to_i
+          files.retrieve do |id, token|
+            initialize_index_for token
+            index[token] << id.send(key_format) # TODO Rewrite. Move this into the specific indexing.
+          end
+        end
+      
+        # Generates a new index (writes its index) using the
+        # partial caching strategy of this bundle.
+        #
+        def generate_partial
+          generator = Generators::PartialGenerator.new self.index
+          self.index = generator.generate self.partial_strategy
+        end
+        # Generate a partial index from the given exact index.
+        #
+        def generate_partial_from exact_index
+          timed_exclaim "PARTIAL GENERATE #{identifier}."
+          self.index = exact_index
+          self.generate_partial
+          self
+        end
+        # Generates a new similarity index (writes its index) using the
+        # given similarity caching strategy.
+        #
+        def generate_similarity
+          generator = Generators::SimilarityGenerator.new self.index
+          self.similarity = generator.generate self.similarity_strategy
+        end
+        # Generates a new weights index (writes its index) using the
+        # given weight caching strategy.
+        #
+        def generate_weights
+          generator = Generators::WeightsGenerator.new self.index
+          self.weights = generator.generate self.weights_strategy
+        end
+
+        # Saves the indexes in a dump file.
+        #
+        def dump
+          dump_index
+          dump_similarity
+          dump_weights
+          dump_configuration
+        end
+        # Dumps the core index.
+        #
+        def dump_index
+          timed_exclaim "DUMP INDEX #{identifier}."
+          backend.dump_index index
+        end
+        # Dumps the weights index.
+        #
+        def dump_weights
+          timed_exclaim "DUMP WEIGHTS #{identifier}."
+          backend.dump_weights weights
+        end
+        # Dumps the similarity index.
+        #
+        def dump_similarity
+          timed_exclaim "DUMP SIMILARITY #{identifier}."
+          backend.dump_similarity similarity
+        end
+        # Dumps the similarity index.
+        #
+        def dump_configuration
+          timed_exclaim "DUMP CONFIGURATION #{identifier}."
+          backend.dump_configuration configuration
+        end
+    
+        # Alerts the user if an index is missing.
+        #
+        def raise_unless_cache_exists
+          raise_unless_index_exists
+          raise_unless_similarity_exists
+        end
+        # Alerts the user if one of the necessary indexes
+        # (core, weights) is missing.
+        #
+        def raise_unless_index_exists
+          if partial_strategy.saved?
+            warn_if_index_small
+            raise_unless_index_ok
+          end
+        end
+        # Alerts the user if the similarity
+        # index is missing (given that it's used).
+        #
+        def raise_unless_similarity_exists
+          if similarity_strategy.saved?
+            warn_if_similarity_small
+            raise_unless_similarity_ok
+          end
+        end
+      
+        # Outputs a warning for the given cache.
+        #
+        def warn_cache_small what
+          puts "Warning: #{what} cache for #{identifier} smaller than 16 bytes."
+        end
+        # Raises an appropriate error message for the given cache.
+        #
+        def raise_cache_missing what
+          raise "#{what} cache for #{identifier} missing."
+        end
+      
+        # Warns the user if the similarity index is small.
+        #
+        def warn_if_similarity_small
+          warn_cache_small :similarity if backend.similarity_cache_small?
+        end
+        # Alerts the user if the similarity index is not there.
+        #
+        def raise_unless_similarity_ok
+          raise_cache_missing :similarity unless backend.similarity_cache_ok?
+        end
+    
+        # TODO Spec on down.
+        #
+    
+        # Warns the user if the core or weights indexes are small.
+        #
+        def warn_if_index_small
+          warn_cache_small :index   if backend.index_cache_small?
+          warn_cache_small :weights if backend.weights_cache_small?
+        end
+        # Alerts the user if the core or weights indexes are not there.
+        #
+        def raise_unless_index_ok
+          raise_cache_missing :index   unless backend.index_cache_ok?
+          raise_cache_missing :weights unless backend.weights_cache_ok?
+        end
+    
+      end
+    
+    end
+    
+  end
+  
+end

data/lib/picky/internals/indexing/bundle/memory.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+#
+module Internals
+
+  module Indexing # :nodoc:all
+  
+    module Bundle
+    
+      # The memory version dumps its generated indexes to disk
+      # (mostly JSON) to load them into memory on startup.
+      #
+      class Memory < Base
+      
+        # We're using files for the memory backend.
+        # E.g. dump writes files.
+        #
+        alias backend files
+    
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/internals/indexing/bundle/redis.rb

@@ -0,0 +1,28 @@
+# encoding: utf-8
+#
+module Internals
+
+  module Indexing # :nodoc:all
+  
+    module Bundle
+    
+      # The memory version dumps its generated indexes to disk
+      # (mostly JSON) to load them into memory on startup.
+      #
+      class Redis < Base
+      
+        attr_reader :backend
+      
+        def initialize name, configuration, *args
+          super name, configuration, *args
+        
+          @backend = Internals::Index::Redis.new name, configuration # TODO Needed?
+        end
+    
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/internals/indexing/bundle/super_base.rb

@@ -0,0 +1,65 @@
+module Internals
+
+  # FIXME Merge into Base, extract common with Indexed::Base.
+  #
+  module Indexing # :nodoc:all
+    # A Bundle is a number of indexes
+    # per [index, category] combination.
+    #
+    # At most, there are three indexes:
+    # * *core* index (always used)
+    # * *weights* index (always used)
+    # * *similarity* index (used with similarity)
+    # 
+    # In Picky, indexing is separated from the index
+    # handling itself through a parallel structure.
+    #
+    # Both use methods provided by this base class, but
+    # have very different goals:
+    #
+    # * *Indexing*::*Bundle* is just concerned with creating index files
+    #   and providing helper functions to e.g. check the indexes.
+    #
+    # * *Index*::*Bundle* is concerned with loading these index files into
+    #   memory and looking up search data as fast as possible.
+    #
+    module Bundle
+    
+      class SuperBase
+    
+        attr_reader   :identifier, :files
+        attr_accessor :index, :weights, :similarity, :configuration, :similarity_strategy
+    
+        delegate :clear,    :to => :index
+        delegate :[], :[]=, :to => :configuration
+    
+        def initialize name, configuration, similarity_strategy
+          @identifier = "#{configuration.identifier}:#{name}"
+          @files      = Internals::Index::Files.new name, configuration
+      
+          @index         = {}
+          @weights       = {}
+          @similarity    = {}
+          @configuration = {} # A hash with config options.
+      
+          @similarity_strategy = similarity_strategy
+        end
+    
+        # Get a list of similar texts.
+        #
+        # Note: Does not return itself.
+        #
+        def similar text
+          code = similarity_strategy.encoded text
+          similar_codes = code && @similarity[code]
+          similar_codes.delete text if similar_codes
+          similar_codes || []
+        end
+    
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/internals/indexing/categories.rb

@@ -0,0 +1,42 @@
+module Internals
+
+  module Indexing
+  
+    class Categories
+    
+      attr_reader :categories
+    
+      each_delegate :index,
+                    :cache,
+                    :generate_caches,
+                    :backup_caches,
+                    :restore_caches,
+                    :check_caches,
+                    :clear_caches,
+                    :create_directory_structure,
+                    :to => :categories
+    
+      def initialize
+        @categories = []
+      end
+    
+      def << category
+        categories << category
+      end
+    
+      def find category_name
+        category_name = category_name.to_sym
+      
+        categories.each do |category|
+          next unless category.name == category_name
+          return category
+        end
+      
+        raise %Q{Index category "#{category_name}" not found. Possible categories: "#{categories.map(&:name).join('", "')}".}
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/internals/indexing/category.rb

@@ -0,0 +1,120 @@
+module Internals
+
+  module Indexing
+  
+    class Category
+    
+      attr_reader :exact, :partial, :name, :configuration, :indexer
+    
+      # Mandatory params:
+      #  * name: Category name to use as identifier and file names.
+      #  * index: Index to which this category is attached to.
+      # Options:
+      #  * partial: Partial::None.new, Partial::Substring.new(from:start_char, to:up_to_char) (defaults from:-3, to:-1)
+      #  * similarity: Similarity::None.new (default), Similarity::Phonetic.new(amount_of_similarly_linked_words)
+      #  * source: Use if the category should use a different source.
+      #  * from: The source category identifier to take the data from.
+      #
+      # Advanced Options (TODO):
+      #
+      #  * weights: 
+      #  * tokenizer: 
+      #
+      def initialize name, index, options = {}
+        @name = name
+        @from = options[:from]
+      
+        # Now we have enough info to combine the index and the category.
+        #
+        @configuration = Configuration::Index.new index, self
+      
+        @tokenizer = options[:tokenizer] || Tokenizers::Index.default
+        @indexer = Indexers::Serial.new configuration, options[:source], @tokenizer
+      
+        # TODO Push into Bundle. At least the weights.
+        #
+        partial    = options[:partial]    || Generators::Partial::Default
+        weights    = options[:weights]    || Generators::Weights::Default
+        similarity = options[:similarity] || Generators::Similarity::Default
+      
+        bundle_class = options[:indexing_bundle_class] || Bundle::Memory
+        @exact   = bundle_class.new(:exact,   configuration, similarity, Generators::Partial::None.new, weights)
+        @partial = bundle_class.new(:partial, configuration, Generators::Similarity::None.new, partial, weights)
+      end
+    
+      delegate :identifier, :prepare_index_directory, :to => :configuration
+      delegate :source, :source=, :tokenizer, :tokenizer=, :to => :indexer
+    
+      def from
+        @from || name
+      end
+    
+      # TODO Spec.
+      #
+      def backup_caches
+        timed_exclaim "Backing up #{identifier}."
+        exact.backup
+        partial.backup
+      end
+      def restore_caches
+        timed_exclaim "Restoring #{identifier}."
+        exact.restore
+        partial.restore
+      end
+      def check_caches
+        timed_exclaim "Checking #{identifier}."
+        exact.raise_unless_cache_exists
+        partial.raise_unless_cache_exists
+      end
+      def clear_caches
+        timed_exclaim "Deleting #{identifier}."
+        exact.delete
+        partial.delete
+      end
+    
+      def index
+        prepare_index_directory
+        indexer.index
+      end
+    
+      # Generates all caches for this category.
+      #
+      def cache
+        prepare_index_directory
+        configure
+        generate_caches
+      end
+      # We need to set what formatting method should be used.
+      # Uses the one defined in the indexer.
+      #
+      def configure
+        key_format = indexer.key_format
+        exact[:key_format] = key_format
+        partial[:key_format] = key_format
+      end
+      def generate_caches
+        generate_caches_from_source
+        generate_partial
+        generate_caches_from_memory
+        dump_caches
+        timed_exclaim "CACHE FINISHED #{identifier}."
+      end
+      def generate_caches_from_source
+        exact.generate_caches_from_source
+      end
+      def generate_partial
+        partial.generate_partial_from exact.index
+      end
+      def generate_caches_from_memory
+        partial.generate_caches_from_memory
+      end
+      def dump_caches
+        exact.dump
+        partial.dump
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/internals/indexing/index.rb

@@ -0,0 +1,67 @@
+# TODO Move to the API.
+#
+module Internals
+
+  module Indexing
+  
+    class Index
+    
+      attr_reader :name, :source, :categories, :after_indexing
+    
+      # Delegators for indexing.
+      #
+      delegate :connect_backend,
+               :to => :source
+             
+      delegate :index,
+               :cache,
+               :generate_caches,
+               :backup_caches,
+               :restore_caches,
+               :check_caches,
+               :clear_caches,
+               :create_directory_structure,
+               :to => :categories
+    
+      def initialize name, source, options = {}
+        @name   = name
+        @source = source
+              
+        @after_indexing = options[:after_indexing]
+        @bundle_class   = options[:indexing_bundle_class] # TODO This should actually be a fixed parameter.
+      
+        @categories = Categories.new
+      end
+    
+      # TODO Spec. Doc.
+      #
+      def define_category category_name, options = {}
+        options = default_category_options.merge options
+      
+        new_category = Category.new category_name, self, options
+        categories << new_category
+        new_category
+      end
+    
+      # By default, the category uses
+      # * the index's source.
+      # * the index's bundle type.
+      #
+      def default_category_options
+        {
+          :source => @source,
+          :indexing_bundle_class => @bundle_class
+        }
+      end
+    
+      # Indexing.
+      #
+      def take_snapshot
+        source.take_snapshot self
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/internals/query/allocation.rb

@@ -0,0 +1,88 @@
+module Internals
+
+  module Query
+
+    # An allocation has a number of combinations:
+    # [token, index] [other_token, other_index], ...
+    #
+    class Allocation # :nodoc:all
+
+      attr_reader :count, :ids, :score, :combinations, :result_identifier
+    
+      #
+      #
+      def initialize combinations, result_identifier
+        @combinations      = combinations
+        @result_identifier = result_identifier
+      end
+
+      def hash
+        @combinations.hash
+      end
+      def eql? other_allocation
+        true # FIXME
+        # @combinations.eql? other_allocation.combinations
+      end
+
+      # Scores its combinations and caches the result.
+      #
+      def calculate_score weights
+        @score ||= @combinations.calculate_score(weights)
+      end
+
+      # Asks the combinations for the (intersected) ids.
+      #
+      def calculate_ids amount, offset
+        @combinations.ids amount, offset # Calculate as many ids as are necessary.
+      end
+
+      # This starts the searching process.
+      #
+      def process! amount, offset
+        ids    = calculate_ids amount, offset
+        @count = ids.size                         # cache the count before throwing away the ids
+        @ids   = ids.slice!(offset, amount) || [] # slice out the relevant part
+      end
+
+      #
+      #
+      def keep identifiers = [] # categories
+        @combinations.keep identifiers
+      end
+      #
+      #
+      def remove identifiers = [] # categories
+        @combinations.remove identifiers
+      end
+
+      # Sort highest score first.
+      #
+      def <=> other_allocation
+         other_allocation.score <=> self.score
+      end
+
+      # Transform the allocation into result form.
+      #
+      def to_result
+        [self.result_identifier, self.score, self.count, @combinations.to_result, self.ids] if self.count > 0
+      end
+
+      # Json representation of this allocation.
+      #
+      # Note: Delegates to to_result.
+      #
+      def to_json
+        to_result.to_json
+      end
+
+      #
+      #
+      def to_s
+        "Allocation: #{to_result.join(', ')}"
+      end
+
+    end
+    
+  end
+  
+end