picky 2.6.0 → 2.7.0

data/lib/picky/category.rb

@@ -1,7 +1,6 @@
 class Category
 
-  attr_reader :name,
-              :index
+  attr_reader :name
 
   # Mandatory params:
   #  * name: Category name to use as identifier and file names.
@@ -31,12 +30,12 @@ class Category
 
     # TODO Push into Bundle. At least the weights.
     #
-    partial    = options[:partial]    || Generators::Partial::Default
     weights    = options[:weights]    || Generators::Weights::Default
+    partial    = options[:partial]    || Generators::Partial::Default
     similarity = options[:similarity] || Generators::Similarity::Default
 
-    @indexing_exact   = index.indexing_bundle_class.new(:exact,   self, similarity, Generators::Partial::None.new, weights)
-    @indexing_partial = index.indexing_bundle_class.new(:partial, self, Generators::Similarity::None.new, partial, weights)
+    @indexing_exact   = index.indexing_bundle_class.new(:exact,   self, weights, Generators::Partial::None.new, similarity)
+    @indexing_partial = index.indexing_bundle_class.new(:partial, self, weights, partial, Generators::Similarity::None.new)
 
     # Indexed.
     #
@@ -55,6 +54,12 @@ class Category
     Query::Qualifiers.add(name, generate_qualifiers_from(options) || [name])
   end
 
+  # TODO Move to Index.
+  #
+  def generate_qualifiers_from options
+    options[:qualifiers] || options[:qualifier] && [options[:qualifier]]
+  end
+
   # Indexes and reloads the category.
   #
   def reindex
@@ -74,10 +79,10 @@ class Category
     @index.name
   end
 
-  # Path and partial filename of a specific index on this category.
+  # The category itself just yields itself.
   #
-  def index_path bundle_name, type
-    "#{index_directory}/#{name}_#{bundle_name}_#{type}"
+  def each_category
+    yield self
   end
 
   # Path and partial filename of the prepared index on this category.
@@ -85,9 +90,20 @@ class Category
   def prepared_index_path
     @prepared_index_path ||= "#{index_directory}/prepared_#{name}_index"
   end
+  # Get an opened index file.
+  #
+  # Note: If you don't use it with the block, do not forget to close it.
+  #
   def prepared_index_file &block
     @prepared_index_file ||= Backend::File::Text.new prepared_index_path
-    @prepared_index_file.open_for_indexing &block
+    @prepared_index_file.open &block
+  end
+  # Creates the index directory including all necessary paths above it.
+  #
+  # Note: Interface method called by any indexers.
+  #
+  def prepare_index_directory
+    FileUtils.mkdir_p index_directory
   end
 
   # The index directory for this category.
@@ -96,30 +112,26 @@ class Category
     @index_directory ||= "#{PICKY_ROOT}/index/#{PICKY_ENVIRONMENT}/#{@index.name}"
   end
 
-  # Creates the index directory including all necessary paths above it.
+  # Path and partial filename of a specific subindex on this category.
   #
-  def prepare_index_directory
-    FileUtils.mkdir_p index_directory
+  # Subindexes are:
+  #  * inverted index
+  #  * weights index
+  #  * partial index
+  #  * similarity index
+  #
+  def index_path bundle_name, type
+    "#{index_directory}/#{name}_#{bundle_name}_#{type}"
   end
 
-  # Identifier for internal use.
-  #
-  # TODO What internal use?
+  # Identifier for technical output.
   #
   def identifier
-    @identifier ||= "#{@index.name}:#{name}"
-  end
-
-  def to_info
-<<-CATEGORY
-Category(#{name}):
-Exact:
-#{exact.indented_to_s(4)}
-Partial:
-#{partial.indented_to_s(4)}
-CATEGORY
+    @identifier ||= "#{PICKY_ENVIRONMENT}:#{index_name}:#{name}"
   end
 
+  #
+  #
   def to_s
     "Category(#{name})"
   end

data/lib/picky/category_indexed.rb

@@ -4,12 +4,6 @@ class Category
 
   attr_reader :indexed_exact
 
-  # TODO Move to Index.
-  #
-  def generate_qualifiers_from options
-    options[:qualifiers] || options[:qualifier] && [options[:qualifier]]
-  end
-
   # Loads the index from cache.
   #
   def load_from_cache
@@ -19,18 +13,6 @@ class Category
   end
   alias reload load_from_cache
 
-  # Loads, analyzes, and clears the index.
-  #
-  # Note: The idea is not to run this while the search engine is running.
-  #
-  def analyze collector
-    collector[identifier] = {
-      :exact   => Analyzer.new.analyze(indexed_exact),
-      :partial => Analyzer.new.analyze(indexed_partial)
-    }
-    collector
-  end
-
   # Gets the weight for this token's text.
   #
   def weight token
@@ -49,13 +31,15 @@ class Category
     token.partial? ? indexed_partial : indexed_exact
   end
 
-  # The partial strategy defines whether to really use the partial index.
+  # The partial strategy defines whether to
+  # really use the partial index.
   #
   def indexed_partial
     @partial_strategy.use_exact_for_partial? ? @indexed_exact : @indexed_partial
   end
 
-  #
+  # Returns a combination for the token,
+  # or nil, if there is none.
   #
   def combination_for token
     weight(token) && Query::Combination.new(token, self)

data/lib/picky/category_indexing.rb

@@ -14,8 +14,56 @@ class Category
     cache
   end
 
+  # Indexes, creates the "prepared_..." file.
+  #
+  def prepare
+    with_data_snapshot do
+      indexer.index [self]
+    end
+  end
+
+  # Take a data snapshot if the source offers it.
+  #
+  def with_data_snapshot
+    if source.respond_to? :with_snapshot
+      source.with_snapshot(@index) do
+        yield
+      end
+    else
+      yield
+    end
+  end
+
+  # Generates all caches for this category.
+  #
+  def cache
+    configure
+    generate_caches_from_source
+    generate_partial
+    generate_caches_from_memory
+    dump_caches
+    timed_exclaim %Q{"#{identifier}": Caching finished.}
+  end
+  # Generate the cache data.
+  #
+  def generate_caches_from_source
+    indexing_exact.generate_caches_from_source
+  end
+  def generate_partial
+    indexing_partial.generate_partial_from indexing_exact.inverted
+  end
+  def generate_caches_from_memory
+    indexing_partial.generate_caches_from_memory
+  end
+  def dump_caches
+    indexing_exact.dump
+    indexing_partial.dump
+  end
+
   # Return an appropriate source.
   #
+  # If we have no explicit source, we'll check the index for one.
+  #
   def source
     @source || @index.source
   end
@@ -39,45 +87,33 @@ class Category
 
   # The indexer is lazily generated and cached.
   #
+  # TODO Really cache?
+  #
   def indexer
     @indexer ||= source.respond_to?(:each) ? Indexers::Parallel.new(self) : Indexers::Serial.new(self)
   end
 
-  # TODO This is a hack to get the parallel indexer working.
-  #
-  def categories
-    [self]
-  end
-
   # Returns an appropriate tokenizer.
   # If one isn't set on this category, will try the index,
   # and finally the default index tokenizer.
   #
   def tokenizer
-    @tokenizer || @index.tokenizer || Tokenizers::Index.default
+    @tokenizer || @index.tokenizer
   end
 
-  # Backup the caches.
-  # (Revert with restore_caches)
+  # We need to set what formatting method should be used.
+  # Uses the one defined in the indexer.
   #
-  def backup_caches
-    timed_exclaim "Backing up #{identifier}."
-    indexing_exact.backup
-    indexing_partial.backup
-  end
-
-  # Restore the caches.
-  # (Revert with backup_caches)
+  # TODO Make this more dynamic.
   #
-  def restore_caches
-    timed_exclaim "Restoring #{identifier}."
-    indexing_exact.restore
-    indexing_partial.restore
+  def configure
+    indexing_exact[:key_format] = self.key_format
+    indexing_partial[:key_format] = self.key_format
   end
 
   # Checks the caches for existence.
   #
-  def check_caches
+  def check
     timed_exclaim "Checking #{identifier}."
     indexing_exact.raise_unless_cache_exists
     indexing_partial.raise_unless_cache_exists
@@ -85,61 +121,28 @@ class Category
 
   # Deletes the caches.
   #
-  def clear_caches
+  def clear
     timed_exclaim "Deleting #{identifier}."
     indexing_exact.delete
     indexing_partial.delete
   end
 
-  # We need to set what formatting method should be used.
-  # Uses the one defined in the indexer.
-  #
-  # TODO Make this more dynamic.
-  #
-  def configure
-    indexing_exact[:key_format] = self.key_format
-    indexing_partial[:key_format] = self.key_format
-  end
-
-  # Indexes, creates the "prepared_..." file.
-  #
-  # TODO This step could already prepare the id (if a
-  #      per category key_format is not really needed).
-  #
-  def prepare
-    prepare_index_directory
-    indexer.index
-  end
-
-  # Generates all caches for this category.
+  # Backup the caches.
+  # (Revert with restore_caches)
   #
-  def cache
-    prepare_index_directory
-    generate_caches
+  def backup
+    timed_exclaim "Backing up #{identifier}."
+    indexing_exact.backup
+    indexing_partial.backup
   end
 
-  # Generate the cache data.
+  # Restore the caches.
+  # (Revert with backup_caches)
   #
-  def generate_caches
-    configure
-    generate_caches_from_source
-    generate_partial
-    generate_caches_from_memory
-    dump_caches
-    timed_exclaim %Q{"#{identifier}": Caching finished.}
-  end
-  def generate_caches_from_source
-    indexing_exact.generate_caches_from_source
-  end
-  def generate_partial
-    indexing_partial.generate_partial_from indexing_exact.index
-  end
-  def generate_caches_from_memory
-    indexing_partial.generate_caches_from_memory
-  end
-  def dump_caches
-    indexing_exact.dump
-    indexing_partial.dump
+  def restore
+    timed_exclaim "Restoring #{identifier}."
+    indexing_exact.restore
+    indexing_partial.restore
   end
 
 end

data/lib/picky/generators/base.rb

@@ -3,13 +3,13 @@ module Generators # :nodoc:all
   # A cache generator holds an index.
   #
   class Base
-  
-    attr_reader :index
-  
-    def initialize index
-      @index = index
+
+    attr_reader :inverted
+
+    def initialize inverted
+      @inverted = inverted
     end
-  
+
   end
 
 end

data/lib/picky/generators/partial/substring.rb

@@ -1,16 +1,16 @@
 module Generators
 
   module Partial
-  
+
     # Generates the right substrings for use in the substring strategy.
     #
     class SubstringGenerator
-    
+
       attr_reader :from, :to
-    
+
       def initialize from, to
         @from, @to = from, to
-      
+
         if @to.zero?
           def each_subtoken token, &block
             token.each_subtoken @from, &block
@@ -20,11 +20,11 @@ module Generators
             token[0..@to].intern.each_subtoken @from, &block
           end
         end
-      
+
       end
-    
+
     end
-  
+
     # The subtoken partial strategy.
     #
     # If given "florian"
@@ -32,7 +32,7 @@ module Generators
     # (Depending on what the given from value is, the example is with option from: 1)
     #
     class Substring < Strategy
-    
+
       # The from option signifies where in the symbol it
       # will start in generating the subtokens.
       #
@@ -51,48 +51,50 @@ module Generators
         to = options[:to] || -1
         @generator = SubstringGenerator.new from, to
       end
-    
+
       # Delegator to generator#from.
       #
       def from
         @generator.from
       end
-    
+
       # Delegator to generator#to.
       #
       def to
         @generator.to
       end
-    
-      # Generates a partial index from the given index.
+
+      # Generates a partial index from the given inverted index.
       #
-      def generate_from index
+      def generate_from inverted
         result = {}
-      
+
         # Generate for each key token the subtokens.
         #
         i = 0
-        index.each_key do |token|
+        j = 0
+        inverted.each_key do |token|
           i += 1
           if i == 5000
-            timed_exclaim "Generating partial tokens for token #{token}. This appears every 5000 tokens."
+            j += 1
+            timed_exclaim %Q{#{"%8i" % (i*j)} generated (current token: "#{token}").}
             i = 0
           end
-          generate_for token, index, result
+          generate_for token, inverted, result
         end
-      
+
         # Remove duplicate ids.
         #
         # THINK If it is unique for a subtoken, it is
         #       unique for all derived longer tokens.
         #
         result.each_value &:uniq!
-      
+
         result
       end
-    
+
       private
-      
+
         # To each shortened token of :test
         # :test, :tes, :te, :t
         # add all ids of :test
@@ -101,18 +103,18 @@ module Generators
         #
         # THINK Could be improved by appending the aforegoing ids?
         #
-        def generate_for token, index, result
+        def generate_for token, inverted, result
           @generator.each_subtoken(token) do |subtoken|
             if result[subtoken]
-              result[subtoken] += index[token] # unique
+              result[subtoken] += inverted[token] # unique
             else
-              result[subtoken] = index[token].dup
+              result[subtoken] = inverted[token].dup
             end
           end
         end
-      
+
     end
-  
+
   end
 
 end

data/lib/picky/generators/partial_generator.rb

@@ -3,11 +3,11 @@ module Generators
   # The partial generator uses a subtoken(downto:1) generator as default.
   #
   class PartialGenerator < Base
-  
-    # Generate a partial index based on the given index.
+
+    # Generate a partial index based on the given inverted index.
     #
     def generate strategy = Partial::Substring.new(from: 1)
-      strategy.generate_from self.index
+      strategy.generate_from self.inverted
     end
 
   end

data/lib/picky/generators/similarity/phonetic.rb

@@ -26,8 +26,8 @@ module Generators
       # In the following form:
       # [:meier, :mueller, :peter, :pater] => { MR: [:meier], MLR: [:mueller], PTR: [:peter, :pater] }
       #
-      def generate_from index
-        hash = hashify index.keys
+      def generate_from inverted
+        hash = hashify inverted.keys
         sort hash
       end
 
@@ -35,12 +35,12 @@ module Generators
 
         # Sorts the index values in place.
         #
-        def sort index
-          index.each_pair.each do |code, ary|
+        def sort hash
+          hash.each_pair.each do |code, ary|
             ary.sort_by_levenshtein! code
             ary.slice! amount, ary.size # size is not perfectly correct, but anyway
           end
-          index
+          hash
         end
 
         # Hashifies a list of symbols.

data/lib/picky/generators/similarity_generator.rb

@@ -4,10 +4,10 @@ module Generators
   #
   class SimilarityGenerator < Base
 
-    # Generate a similarity index based on the given index.
+    # Generate a similarity index based on the given inverted index.
     #
     def generate strategy = Similarity::None.new
-      strategy.generate_from self.index
+      strategy.generate_from self.inverted
     end
 
   end

data/lib/picky/generators/weights/logarithmic.rb

@@ -9,10 +9,10 @@ module Generators
     #
     class Logarithmic < Strategy
 
-      # Generates a partial index from the given index.
+      # Generates a partial index from the given inverted index.
       #
-      def generate_from index
-        index.inject({}) do |hash, text_ids|
+      def generate_from inverted
+        inverted.inject({}) do |hash, text_ids|
           text, ids = *text_ids
           weight = weight_for ids.size
           hash[text] ||= weight.round(2) if weight

data/lib/picky/generators/weights_generator.rb

@@ -4,10 +4,10 @@ module Generators
   #
   class WeightsGenerator < Base
 
-    # Generate a weights index based on the given index.
+    # Generate a weights index based on the given inverted index.
     #
     def generate strategy = Weights::Logarithmic.new
-      strategy.generate_from self.index
+      strategy.generate_from self.inverted
     end
 
   end

data/lib/picky/index/base.rb

@@ -89,6 +89,7 @@ module Index
                 :categories
 
     delegate :[],
+             :each_category,
              :to => :categories
 
     # Create a new index with a given source.
@@ -381,16 +382,6 @@ SOURCE
 ) unless source.respond_to?(:each) || source.respond_to?(:harvest)
     end
 
-    def method_name
-
-    end
-
-    #
-    #
-    def to_s
-      "#{self.class}(#{name}, result_id: #{result_identifier}, source: #{source}, categories: #{categories})"
-    end
-
     def to_stats # :nodoc:
       stats = <<-INDEX
 #{name} (#{self.class}):
@@ -401,6 +392,18 @@ INDEX
       stats
     end
 
+    # Identifier used for technical output.
+    #
+    def identifier
+      "#{PICKY_ENVIRONMENT}:#{name}"
+    end
+
+    #
+    #
+    def to_s
+      "#{self.class}(#{name}, result_id: #{result_identifier}, source: #{source}, categories: #{categories})"
+    end
+
   end
 
 end

data/lib/picky/index/base_indexed.rb

@@ -18,6 +18,8 @@ module Index
     #
     # A combination is a tuple <token, index_bundle>.
     #
+    # TODO Rename and delegate.
+    #
     def possible_combinations token
       categories.possible_combinations_for token
     end