picky 2.6.0 → 2.7.0

data/lib/picky/index/base_indexing.rb

@@ -3,33 +3,67 @@ module Index
   #
   #
   class Base
-    
+
     attr_reader :after_indexing,
-                :bundle_class,
-                :tokenizer
-    
+                :bundle_class
+
     # Delegators for indexing.
     #
-    delegate :backup_caches,
-             :cache,
-             :check_caches,
-             :clear_caches,
-             :create_directory_structure,
-             :generate_caches,
-             :restore_caches,
+    delegate :cache,
+             :check,
+             :clear,
+             :backup,
+             :restore,
              :to => :categories
-    
-    delegate :connect_backend,
-             :to => :source
-    
-    # Calling index on an index will
-    #  * prepare (the data)
-    #  * cache (the data)
+
+    # Calling index on an index will call index
     # on every category.
     #
+    # Decides whether to use a parallel indexer or whether to
+    # delegate to each category to index themselves.
+    #
     def index
-      prepare
-      cache
+      if source.respond_to?(:each)
+        check_source_empty
+        index_in_parallel
+      else
+        with_data_snapshot do
+          categories.each &:index
+        end
+      end
+    end
+
+    # Check if the given enumerable source is empty.
+    #
+    # Note: Checking as early as possible to tell the
+    #       user as early as possible.
+    #
+    def check_source_empty
+      warn %Q{\n\033[1mWarning\033[m, source for index "#{name}" is empty: #{source} (responds true to empty?).\n} if source.respond_to?(:empty?) && source.empty?
+    end
+
+    # Note: Duplicated in category_indexing.rb.
+    #
+    # Take a data snapshot if the source offers it.
+    #
+    def with_data_snapshot
+      if source.respond_to? :with_snapshot
+        source.with_snapshot(self) do
+          yield
+        end
+      else
+        yield
+      end
+    end
+
+    # Indexes the categories in parallel.
+    #
+    # Only use where the category does have a #each source defined.
+    #
+    def index_in_parallel
+      indexer = Indexers::Parallel.new self
+      indexer.index categories
+      categories.each &:cache
     end
 
     # Define an index tokenizer on the index.
@@ -40,7 +74,15 @@ module Index
       @tokenizer = Tokenizers::Index.new options
     end
     alias define_indexing indexing
-    
+
+    # Returns the installed tokenizer or the default.
+    #
+    # TODO Spec.
+    #
+    def tokenizer
+      @tokenizer || Indexes.tokenizer
+    end
+
     # Define a source on the index.
     #
     # Parameter is a source, either one of the standard sources or
@@ -68,7 +110,7 @@ end
       NO_SOURCE
 )
     end
-    
+
     # Define a key_format on the index.
     #
     # Parameter is a method name to use on the key (e.g. :to_i, :to_s, :strip).
@@ -79,41 +121,7 @@ end
     def define_key_format key_format
       @key_format = key_format
     end
-    
-    # Decides whether to use a parallel indexer or whether to
-    # delegate to each category to index themselves.
-    #
-    # TODO Rename to prepare.
-    #
-    def prepare
-      # TODO Duplicated in category.rb def indexer.
-      #
-      if source.respond_to?(:each)
-        warn %Q{\n\033[1mWarning\033[m, source for index "#{name}" is empty: #{source} (responds true to empty?).\n} if source.respond_to?(:empty?) && source.empty?
-        index_parallel
-      else
-        categories.each &:prepare
-      end
-    end
-
-    # Indexes the categories in parallel.
-    #
-    # Only use where the category does not have a non-#each source defined.
-    #
-    def index_parallel
-      indexer = Indexers::Parallel.new self
-      categories.first.prepare_index_directory # TODO Unnice.
-      indexer.index
-    end
 
-    # Indexing.
-    #
-    # Note: If it is an each source we do not take a snapshot.
-    #
-    def take_snapshot
-      source.take_snapshot self unless source.respond_to? :each
-    end
-    
   end
-  
+
 end

data/lib/picky/indexed/bundle/base.rb

@@ -1,106 +1,41 @@
+# TODO Extract duplicate code from base bundle classes.
+#
 module Indexed # :nodoc:all
 
-  # A Bundle is a number of indexes
-  # per [index, category] combination.
+  # An indexed bundle is a number of memory/redis
+  # indexes that compose the indexes for a single category:
+  #  * core (inverted) index
+  #  * weights index
+  #  * similarity index
+  #  * index configuration
   #
-  # At most, there are three indexes:
-  # * *core* index (always used)
-  # * *weights* index (always used)
-  # * *similarity* index (used with similarity)
+  # Indexed refers to them being indexed.
+  # This class notably offers the methods:
+  #  * load
+  #  * clear
   #
-  # In Picky, indexing is separated from the index
-  # handling itself through a parallel structure.
-  #
-  # Both use methods provided by this base class, but
-  # have very different goals:
-  #
-  # * *Indexing*::*Bundle* is just concerned with creating index files
-  #   and providing helper functions to e.g. check the indexes.
-  #
-  # * *Index*::*Bundle* is concerned with loading these index files into
-  #   memory and looking up search data as fast as possible.
+  # To (re)load or clear the current indexes.
   #
   module Bundle
 
-    class Base
-
-      attr_reader   :identifier, :configuration
-      attr_accessor :similarity_strategy
-      attr_accessor :index, :weights, :similarity, :configuration
-
-      delegate :[], :to => :configuration
-      delegate :size, :to => :index
-
-      def initialize name, category, similarity_strategy
-        @identifier = "#{category.identifier}:#{name}"
-
-        @index         = {}
-        @weights       = {}
-        @similarity    = {}
-
-        @similarity_strategy = similarity_strategy
-      end
-
-      # Get a list of similar texts.
-      #
-      # Note: Does not return itself.
-      #
-      def similar text
-        code = similarity_strategy.encoded text
-        similar_codes = code && @similarity[code]
-        similar_codes.delete text if similar_codes
-        similar_codes || []
-      end
+    class Base < ::Bundle
 
       # Loads all indexes.
       #
       def load
-        load_index
+        load_inverted
         load_weights
         load_similarity
         load_configuration
       end
 
-      # Loads the core index.
-      #
-      def load_index
-        # No loading needed.
-      end
-      # Loads the weights index.
-      #
-      def load_weights
-        # No loading needed.
-      end
-      # Loads the similarity index.
-      #
-      def load_similarity
-        # No loading needed.
-      end
-      # Loads the configuration.
-      #
-      def load_configuration
-        # No loading needed.
-      end
-
-      # Loads the core index.
-      #
-      def clear_index
-        # No loading needed.
-      end
-      # Loads the weights index.
-      #
-      def clear_weights
-        # No loading needed.
-      end
-      # Loads the similarity index.
-      #
-      def clear_similarity
-        # No loading needed.
-      end
-      # Loads the configuration.
+      # Clears all indexes.
       #
-      def clear_configuration
-        # No loading needed.
+      def clear
+        clear_inverted
+        clear_weights
+        clear_similarity
+        clear_configuration
       end
 
     end

data/lib/picky/indexed/bundle/memory.rb

@@ -24,17 +24,10 @@ module Indexed # :nodoc:all
         @backend = Backend::Files.new name, configuration
       end
 
-      def to_s
-        <<-MEMORY
-Memory
-#{@backend.indented_to_s}
-MEMORY
-      end
-
       # Get the ids for the given symbol.
       #
       def ids sym
-        @index[sym] || []
+        @inverted[sym] || []
       end
       # Get a weight for the given symbol.
       #
@@ -44,8 +37,8 @@ MEMORY
 
       # Loads the core index.
       #
-      def load_index
-        self.index = @backend.load_index
+      def load_inverted
+        self.inverted = @backend.load_inverted
       end
       # Loads the weights index.
       #
@@ -65,8 +58,8 @@ MEMORY
 
       # Loads the core index.
       #
-      def clear_index
-        self.index = {}
+      def clear_inverted
+        self.inverted = {}
       end
       # Loads the weights index.
       #

data/lib/picky/indexed/bundle/redis.rb

@@ -38,6 +38,48 @@ module Indexed # :nodoc:all
         @backend.setting sym
       end
 
+      # Loads the inverted index.
+      #
+      def load_inverted
+        # No loading needed.
+      end
+      # Loads the weights index.
+      #
+      def load_weights
+        # No loading needed.
+      end
+      # Loads the similarity index.
+      #
+      def load_similarity
+        # No loading needed.
+      end
+      # Loads the configuration.
+      #
+      def load_configuration
+        # No loading needed.
+      end
+
+      # Loads the inverted index.
+      #
+      def clear_inverted
+        # No clearing possible, currently.
+      end
+      # Loads the weights index.
+      #
+      def clear_weights
+        # No clearing possible, currently.
+      end
+      # Loads the similarity index.
+      #
+      def clear_similarity
+        # No clearing possible, currently.
+      end
+      # Loads the configuration.
+      #
+      def clear_configuration
+        # No clearing possible, currently.
+      end
+
     end
 
   end

data/lib/picky/indexed/wrappers/bundle/wrapper.rb

@@ -16,11 +16,11 @@ module Indexed
         end
 
         delegate :load,
-                 :load_index,
+                 :load_inverted,
                  :load_weights,
                  :load_similarity,
                  :load_configuration,
-                 :clear_index,
+                 :clear_inverted,
                  :clear_weights,
                  :clear_similarity,
                  :clear_configuration,
@@ -29,7 +29,7 @@ module Indexed
                  :identifier,
                  :analyze,
                  :size,
-                 :index,
+                 :inverted,
                  :weights,
                  :similarity,
                  :configuration,

data/lib/picky/indexers/base.rb

@@ -6,11 +6,28 @@ module Indexers
   #
   class Base
 
+    attr_reader :index_or_category
+
+    delegate :source, :to => :index_or_category
+
+    def initialize index_or_category
+      @index_or_category = index_or_category
+    end
+
     # Starts the indexing process.
     #
-    def index
-      indexing_message
-      process
+    def index categories
+      start_indexing_message
+      prepare categories
+      process categories
+      finish_indexing_message
+    end
+
+    # By default, an indexer
+    #  * prepares the index directories.
+    #
+    def prepare categories
+      categories.each &:prepare_index_directory
     end
 
   end

data/lib/picky/indexers/parallel.rb

@@ -1,4 +1,4 @@
-# encoding: utf-8
+  # encoding: utf-8
 #
 module Indexers
 
@@ -6,27 +6,35 @@ module Indexers
   #
   # The tokenizer is taken from each category if specified, from the index, if not.
   #
-  # TODO Think about this one more. It should work on an index, but also a single category.
-  #
   class Parallel < Base
 
-    delegate :categories, :source, :to => :@index
-
-    def initialize index
-      @index = index
-    end
-
-    def process
+    # Process does the actual indexing.
+    #
+    # Parameters:
+    #  * categories: An Enumerable of Category-s.
+    #
+    def process categories
       comma   = ?,
       newline = ?\n
 
       # Prepare a combined object - array.
       #
-      combined = categories.map { |category| [category, [], category.prepared_index_file, (category.tokenizer || tokenizer)] }
+      combined = categories.map do |category|
+        [category, [], category.prepared_index_file, (category.tokenizer || tokenizer)]
+      end
 
       # Index.
       #
+      # TODO Extract into flush_every(100_000) do
+      #
       i = 0
+
+      # Explicitly reset the source to avoid caching trouble.
+      #
+      source.reset if source.respond_to?(:reset)
+
+      # Go through each object in the source.
+      #
       source.each do |object|
         id = object.id
 
@@ -48,17 +56,27 @@ module Indexers
         i += 1
       end
       flush combined
-      combined.each { |_, _, file, _| file.close }
+      combined.each do |_, _, file, _|
+        timed_exclaim %Q{"#{@index_or_category.identifier}":   => #{file.path}.}
+        file.close
+      end
     end
+
+    # Flush the combined array into the file.
+    #
     def flush combined # :nodoc:
       combined.each do |_, cache, file, _|
         file.write(cache.join) && cache.clear
       end
     end
+
     #
     #
-    def indexing_message # :nodoc:
-      timed_exclaim %Q{"#{@index.name}": Starting parallel indexing.}
+    def start_indexing_message # :nodoc:
+      timed_exclaim %Q{"#{@index_or_category.identifier}": Starting parallel data preparation.}
+    end
+    def finish_indexing_message # :nodoc:
+      timed_exclaim %Q{"#{@index_or_category.identifier}": Finished parallel data preparation.}
     end
 
   end

data/lib/picky/indexers/serial.rb

@@ -8,44 +8,47 @@ module Indexers
   #
   class Serial < Base
 
-    attr_reader :category
-
-    delegate :source, :to => :category
-
-    def initialize category
-      @category = category
-    end
-
-    # The tokenizer used is a cached tokenizer from the category.
-    #
-    def tokenizer
-      @tokenizer ||= category.tokenizer
-    end
-
     # Harvest the data from the source, tokenize,
     # and write to an intermediate "prepared index" file.
     #
-    def process
+    # Parameters:
+    #  * categories: An enumerable of Category-s.
+    #
+    def process categories
       comma   = ?,
       newline = ?\n
 
-      local_tokenizer = tokenizer
-      category.prepared_index_file do |file|
-        result = []
-        source.harvest(category) do |indexed_id, text|
-          local_tokenizer.tokenize(text).each do |token_text|
-            next unless token_text
-            result << indexed_id << comma << token_text << newline
+      categories.each do |category|
+
+        tokenizer = category.tokenizer
+
+        category.prepared_index_file do |file|
+          result = []
+
+          source.harvest(category) do |indexed_id, text|
+            tokenizer.tokenize(text).each do |token_text|
+              next unless token_text
+              result << indexed_id << comma << token_text << newline
+            end
+            file.write(result.join) && result.clear if result.size > 100_000
           end
-          file.write(result.join) && result.clear if result.size > 100_000
+
+          timed_exclaim %Q{"#{@index_or_category.identifier}":   => #{file.path}.}
+
+          file.write result.join
         end
-        file.write result.join
+
       end
+
     end
+
     #
     #
-    def indexing_message # :nodoc:
-      timed_exclaim %Q{"#{@category.identifier}": Starting serial indexing.}
+    def start_indexing_message # :nodoc:
+      timed_exclaim %Q{"#{@index_or_category.identifier}": Starting serial data preparation.}
+    end
+    def finish_indexing_message # :nodoc:
+      timed_exclaim %Q{"#{@index_or_category.identifier}": Finished serial data preparation.}
     end
 
   end

data/lib/picky/indexes.rb

@@ -14,10 +14,11 @@ class Indexes
            :to => :indexes
 
   each_delegate :reindex,
+                :each_category,
                 :to => :indexes
 
   def initialize
-    clear
+    clear_indexes
   end
 
   # Return the Indexes instance.
@@ -32,11 +33,12 @@ class Indexes
                     :[],
                     :to_s,
                     :size,
-                    :each
+                    :each,
+                    :each_category
 
   # Clears the indexes and the mapping.
   #
-  def clear
+  def clear_indexes
     @indexes       = []
     @index_mapping = {}
   end

data/lib/picky/indexes_indexed.rb

@@ -1,29 +1,17 @@
 # Registers the indexes held at runtime, for queries.
 #
 class Indexes
-  
+
   instance_delegate :load_from_cache,
                     :reload,
                     :analyze
-  
+
   each_delegate :load_from_cache,
                 :to => :indexes
-  
+
   # Reloads all indexes, one after another,
   # in the order they were added.
   #
   alias reload load_from_cache
 
-  # Load each index, and analyze it.
-  #
-  # Returns a hash with the findings.
-  #
-  def analyze
-    result = {}
-    indexes.each do |index|
-      index.analyze result
-    end
-    result
-  end
-
 end