picky 1.5.2 → 1.5.3

data/lib/picky/internals/indexing/bundle/base.rb

@@ -3,35 +3,35 @@
 module Internals
 
   module Indexing # :nodoc:all
-  
+
     module Bundle
-    
+
       # This is the indexing bundle.
       # It does all menial tasks that have nothing to do
       # with the actual index running etc.
       #
       class Base < SuperBase
-      
+
         attr_accessor :partial_strategy, :weights_strategy
-      
+
         # Path is in which directory the cache is located.
         #
         def initialize name, configuration, similarity_strategy, partial_strategy, weights_strategy
           super name, configuration, similarity_strategy
-        
+
           @partial_strategy    = partial_strategy
           @weights_strategy    = weights_strategy
         end
-      
+
         # Sets up a piece of the index for the given token.
         #
         def initialize_index_for token
           index[token] ||= []
         end
-      
+
         # Generation
         #
-      
+
         # This method
         # * loads the base index from the db
         # * generates derived indexes
@@ -50,16 +50,16 @@ module Internals
           generate_derived
         end
         def cache_from_memory_generation_message
-          timed_exclaim "CACHE FROM MEMORY #{identifier}."
+          timed_exclaim %Q{"#{identifier}": Caching from intermediate in-memory index.}
         end
-      
+
         # Generates the weights and similarity from the main index.
         #
         def generate_derived
           generate_weights
           generate_similarity
         end
-      
+
         # Load the data from the db.
         #
         def load_from_index_file
@@ -68,7 +68,7 @@ module Internals
           retrieve
         end
         def load_from_index_generation_message
-          timed_exclaim "LOAD INDEX #{identifier}."
+          timed_exclaim %Q{"#{identifier}": Loading index.}
         end
         # Retrieves the prepared index data into the index.
         #
@@ -83,7 +83,7 @@ module Internals
             index[token] << id.send(key_format) # TODO Rewrite. Move this into the specific indexing.
           end
         end
-      
+
         # Generates a new index (writes its index) using the
         # partial caching strategy of this bundle.
         #
@@ -94,7 +94,7 @@ module Internals
         # Generate a partial index from the given exact index.
         #
         def generate_partial_from exact_index
-          timed_exclaim "PARTIAL GENERATE #{identifier}."
+          timed_exclaim %Q{"#{identifier}": Generating partial index for index.}
           self.index = exact_index
           self.generate_partial
           self
@@ -125,28 +125,28 @@ module Internals
         # Dumps the core index.
         #
         def dump_index
-          timed_exclaim "DUMP INDEX #{identifier}."
+          timed_exclaim %Q{"#{identifier}": Dumping index.}
           backend.dump_index index
         end
         # Dumps the weights index.
         #
         def dump_weights
-          timed_exclaim "DUMP WEIGHTS #{identifier}."
+          timed_exclaim %Q{"#{identifier}": Dumping weights of index.}
           backend.dump_weights weights
         end
         # Dumps the similarity index.
         #
         def dump_similarity
-          timed_exclaim "DUMP SIMILARITY #{identifier}."
+          timed_exclaim %Q{"#{identifier}": Dumping similarity of index.}
           backend.dump_similarity similarity
         end
         # Dumps the similarity index.
         #
         def dump_configuration
-          timed_exclaim "DUMP CONFIGURATION #{identifier}."
+          timed_exclaim %Q{"#{identifier}": Dumping configuration for index.}
           backend.dump_configuration configuration
         end
-    
+
         # Alerts the user if an index is missing.
         #
         def raise_unless_cache_exists
@@ -171,18 +171,18 @@ module Internals
             raise_unless_similarity_ok
           end
         end
-      
+
         # Outputs a warning for the given cache.
         #
         def warn_cache_small what
-          puts "Warning: #{what} cache for #{identifier} smaller than 16 bytes."
+          warn "Warning: #{what} cache for #{identifier} smaller than 16 bytes."
         end
         # Raises an appropriate error message for the given cache.
         #
         def raise_cache_missing what
           raise "#{what} cache for #{identifier} missing."
         end
-      
+
         # Warns the user if the similarity index is small.
         #
         def warn_if_similarity_small
@@ -193,10 +193,10 @@ module Internals
         def raise_unless_similarity_ok
           raise_cache_missing :similarity unless backend.similarity_cache_ok?
         end
-    
+
         # TODO Spec on down.
         #
-    
+
         # Warns the user if the core or weights indexes are small.
         #
         def warn_if_index_small
@@ -209,11 +209,11 @@ module Internals
           raise_cache_missing :index   unless backend.index_cache_ok?
           raise_cache_missing :weights unless backend.weights_cache_ok?
         end
-    
+
       end
-    
+
     end
-    
+
   end
-  
+
 end

data/lib/picky/internals/indexing/bundle/memory.rb

@@ -3,23 +3,30 @@
 module Internals
 
   module Indexing # :nodoc:all
-  
+
     module Bundle
-    
+
       # The memory version dumps its generated indexes to disk
       # (mostly JSON) to load them into memory on startup.
       #
       class Memory < Base
-      
+
         # We're using files for the memory backend.
         # E.g. dump writes files.
         #
         alias backend files
-    
+
+        def to_s
+          <<-MEMORY
+Memory
+#{@backend.indented_to_s}
+MEMORY
+        end
+
       end
-    
+
     end
-  
+
   end
-  
+
 end

data/lib/picky/internals/indexing/categories.rb

@@ -1,11 +1,11 @@
 module Internals
 
   module Indexing
-  
+
     class Categories
-    
+
       attr_reader :categories
-    
+
       each_delegate :index,
                     :cache,
                     :generate_caches,
@@ -15,28 +15,32 @@ module Internals
                     :clear_caches,
                     :create_directory_structure,
                     :to => :categories
-    
+
       def initialize
         @categories = []
       end
-    
+
+      def to_s
+        categories.indented_to_s
+      end
+
       def << category
         categories << category
       end
-    
+
       def find category_name
         category_name = category_name.to_sym
-      
+
         categories.each do |category|
           next unless category.name == category_name
           return category
         end
-      
+
         raise %Q{Index category "#{category_name}" not found. Possible categories: "#{categories.map(&:name).join('", "')}".}
       end
-    
+
     end
-  
+
   end
-  
+
 end

data/lib/picky/internals/indexing/category.rb

@@ -1,11 +1,14 @@
 module Internals
 
   module Indexing
-  
+
     class Category
-    
+
       attr_reader :exact, :partial, :name, :configuration, :indexer
-    
+
+      delegate :identifier, :prepare_index_directory, :to => :configuration
+      delegate :source, :source=, :tokenizer, :tokenizer=, :to => :indexer
+
       # Mandatory params:
       #  * name: Category name to use as identifier and file names.
       #  * index: Index to which this category is attached to.
@@ -17,38 +20,45 @@ module Internals
       #
       # Advanced Options (TODO):
       #
-      #  * weights: 
-      #  * tokenizer: 
+      #  * weights:
+      #  * tokenizer:
       #
       def initialize name, index, options = {}
         @name = name
         @from = options[:from]
-      
+
         # Now we have enough info to combine the index and the category.
         #
         @configuration = Configuration::Index.new index, self
-      
+
         @tokenizer = options[:tokenizer] || Tokenizers::Index.default
         @indexer = Indexers::Serial.new configuration, options[:source], @tokenizer
-      
+
         # TODO Push into Bundle. At least the weights.
         #
         partial    = options[:partial]    || Generators::Partial::Default
         weights    = options[:weights]    || Generators::Weights::Default
         similarity = options[:similarity] || Generators::Similarity::Default
-      
+
         bundle_class = options[:indexing_bundle_class] || Bundle::Memory
         @exact   = bundle_class.new(:exact,   configuration, similarity, Generators::Partial::None.new, weights)
         @partial = bundle_class.new(:partial, configuration, Generators::Similarity::None.new, partial, weights)
       end
-    
-      delegate :identifier, :prepare_index_directory, :to => :configuration
-      delegate :source, :source=, :tokenizer, :tokenizer=, :to => :indexer
-    
+
+      def to_s
+        <<-CATEGORY
+Category(#{name} from #{from}):
+  Exact:
+#{exact.indented_to_s(4)}
+  Partial:
+#{partial.indented_to_s(4)}
+        CATEGORY
+      end
+
       def from
         @from || name
       end
-    
+
       # TODO Spec.
       #
       def backup_caches
@@ -71,12 +81,12 @@ module Internals
         exact.delete
         partial.delete
       end
-    
+
       def index
         prepare_index_directory
         indexer.index
       end
-    
+
       # Generates all caches for this category.
       #
       def cache
@@ -97,7 +107,7 @@ module Internals
         generate_partial
         generate_caches_from_memory
         dump_caches
-        timed_exclaim "CACHE FINISHED #{identifier}."
+        timed_exclaim %Q{"#{identifier}": Caching finished.}
       end
       def generate_caches_from_source
         exact.generate_caches_from_source
@@ -112,9 +122,9 @@ module Internals
         exact.dump
         partial.dump
       end
-    
+
     end
-  
+
   end
-  
+
 end

data/lib/picky/internals/indexing/index.rb

@@ -3,16 +3,16 @@
 module Internals
 
   module Indexing
-  
+
     class Index
-    
+
       attr_reader :name, :source, :categories, :after_indexing
-    
+
       # Delegators for indexing.
       #
       delegate :connect_backend,
                :to => :source
-             
+
       delegate :index,
                :cache,
                :generate_caches,
@@ -22,27 +22,27 @@ module Internals
                :clear_caches,
                :create_directory_structure,
                :to => :categories
-    
+
       def initialize name, source, options = {}
         @name   = name
         @source = source
-              
+
         @after_indexing = options[:after_indexing]
         @bundle_class   = options[:indexing_bundle_class] # TODO This should actually be a fixed parameter.
-      
+
         @categories = Categories.new
       end
-    
+
       # TODO Spec. Doc.
       #
       def define_category category_name, options = {}
         options = default_category_options.merge options
-      
+
         new_category = Category.new category_name, self, options
         categories << new_category
         new_category
       end
-    
+
       # By default, the category uses
       # * the index's source.
       # * the index's bundle type.
@@ -53,15 +53,23 @@ module Internals
           :indexing_bundle_class => @bundle_class
         }
       end
-    
+
       # Indexing.
       #
       def take_snapshot
         source.take_snapshot self
       end
-    
+
+      def to_s
+        <<-INDEX
+Indexing(#{name}):
+#{"source: #{source}".indented_to_s}
+#{"Categories:\n#{categories.indented_to_s}".indented_to_s}
+INDEX
+      end
+
     end
-  
+
   end
-  
+
 end

data/lib/picky/internals/query/allocations.rb

@@ -58,21 +58,6 @@ module Internals
         end
       end
 
-      # Returns a random id from the allocations.
-      #
-      # Note: This is an ok algorithm for small id sets.
-      #
-      # But still TODO try for a faster one.
-      #
-      # TODO For the 1 amount random case this needs to be improved.
-      #
-      def random_ids amount = 1
-        return [] if @allocations.empty?
-        ids = @allocations.first.ids
-        indexes = Array.new(ids.size) { |id| id }.sort_by { rand }
-        indexes.first(amount).map { |id| ids[id] }
-      end
-
       # This is the main method of this class that will replace ids and count.
       #
       # What it does is calculate the ids and counts of its allocations

data/lib/picky/internals/query/combinations/base.rb

@@ -51,8 +51,6 @@ module Internals
         #       ids that have an associated identifier that is nil.
         #
         def keep identifiers = []
-          # TODO Rewrite to use the category!!!
-          #
           @combinations.reject! { |combination| !combination.in?(identifiers) }
         end
 
@@ -64,8 +62,6 @@ module Internals
         #       ids that have an associated identifier that is nil.
         #
         def remove identifiers = []
-          # TODO Rewrite to use the category!!!
-          #
           @combinations.reject! { |combination| combination.in?(identifiers) }
         end
 

data/lib/picky/internals/query/combinations/redis.rb

@@ -19,7 +19,7 @@ module Internals
         def initialize combinations
           super combinations
         
-          @@redis ||= ::Redis.new
+          @@redis ||= ::Redis.new :db => 15
         end
       
         # Returns the result ids for the allocation.
@@ -32,22 +32,33 @@ module Internals
           end
         
           result_id = generate_intermediate_result_id
-        
-          # TODO multi?
+          
+          # Intersect and store.
           #
-        
           @@redis.zinterstore result_id, identifiers
         
-          @@redis.zrange result_id, offset, (offset + amount)
+          # Get the stored result.
+          #
+          results = @@redis.zrange result_id, offset, (offset + amount)
+          
+          # Delete the stored result as it was only for temporary purposes.
+          #
+          @@redis.del result_id
+          
+          results
         end
       
         # Generate a multiple host/process safe result id.
         #
-        # TODO How expensive is Process.pid? If it changes once, remember forever?
+        require 'socket'
+        @@host = Socket.gethostname
+        define_method :host do
+          @@host
+        end
+        # Use the host and pid (generated lazily in child processes) for the result.
         #
         def generate_intermediate_result_id
-          # TODO host -> extract host.
-          :"host:#{Process.pid}:picky:result"
+          :"#{host}:#{@pid ||= Process.pid}:picky:result"
         end
       
       end

data/lib/picky/internals/query/indexes.rb

@@ -52,8 +52,6 @@ module Internals
           # Add the wrapped possible allocations to the ones we already have.
           #
           previous_allocations + expanded_combinations.map! do |expanded_combination|
-            # TODO Insert Redis here?
-            #
             @combinations_type.new(expanded_combination).pack_into_allocation(index.result_identifier) # TODO Do not extract result_identifier. Remove pack_into_allocation.
           end
         end)
@@ -116,14 +114,13 @@ module Internals
       #       ones, but I guess I am a bit sentimental.
       #
       def expand_combinations_from possible_combinations
+        # If an element has size 0, this means one of the
+        # tokens could not be allocated.
+        #
         return if possible_combinations.any?(&:empty?)
       
         # Generate the first multiplicator "with which" (well, not quite) to multiply the smallest amount of combinations.
         #
-        # TODO How does this work if an element has size 0? Since below we account for size 0.
-        #      Should we even continue if an element has size 0?
-        #      This means one of the tokens cannot be allocated.
-        #
         single_mult = possible_combinations.inject(1) { |total, combinations| total * combinations.size }
 
         # Initialize a group multiplicator.

data/lib/picky/internals/query/token.rb

@@ -8,8 +8,6 @@ module Internals
     #
     # It also knows whether it needs to look for similarity (bla~), or whether it is a partial (bla*).
     #
-    # TODO Make partial / similarity char configurable.
-    #
     class Token # :nodoc:all
 
       attr_reader :text, :original
@@ -179,8 +177,6 @@ module Internals
     
       # Internal identifier.
       #
-      # TODO Uh.
-      #
       def identifier
         "#{similar?? :similarity : :index}:#{@text}"
       end

data/lib/picky/internals/query/weights.rb

@@ -7,14 +7,7 @@ module Query
     #
     #
     def initialize weights = {}
-      # @weights_cache = {} # TODO
-      @weights = prepare weights
-    end
-    
-    # Get the category indexes for the given bonuses.
-    #
-    def prepare weights
-      weights
+      @weights = weights
     end
     
     # Get the weight of an allocation.
@@ -38,10 +31,8 @@ module Query
     # Note: Cache this if more complicated weighings become necessary.
     #
     def score combinations
-      # TODO Beautify? Use categories for weights?
+      # TODO Or hide: combinations#to_weights_key
       #
-      # weight_for combinations.map(&:category).clustered_uniq_fast.map!(&:name)
-      
       # TODO combinations could cluster uniq as combinations are added (since combinations don't change).
       #
       weight_for combinations.map(&:category_name).clustered_uniq_fast

data/lib/picky/internals/results/base.rb

@@ -15,8 +15,8 @@ module Internals
       # Takes instances of Query::Allocations as param.
       #
       def initialize offset = 0, allocations = Query::Allocations.new
-        @offset = offset
-        @allocations = allocations # || Query::Allocations.new
+        @offset      = offset
+        @allocations = allocations
       end
       # Create new results and calculate the ids.
       #
@@ -26,7 +26,7 @@ module Internals
         results
       end
 
-      #
+      # Returns a hash with the allocations, offset, duration and total.
       #
       def serialize
         { allocations: allocations.to_result,
@@ -85,13 +85,6 @@ module Internals
       def ids amount = 20
         allocations.ids amount
       end
-      # Gets an amout of random ids from the allocations.
-      #
-      # Note: Basically delegates to the allocations.
-      #
-      def random_ids amount = 1
-        allocations.random_ids amount
-      end
 
       # Human readable log.
       #