picky 2.7.0 → 3.0.0.pre1

data/lib/picky/indexers/base.rb

@@ -1,33 +1,37 @@
 # encoding: utf-8
 #
-module Indexers
+module Picky
 
-  #
-  #
-  class Base
+  module Indexers
 
-    attr_reader :index_or_category
+    #
+    #
+    class Base
 
-    delegate :source, :to => :index_or_category
+      attr_reader :index_or_category
 
-    def initialize index_or_category
-      @index_or_category = index_or_category
-    end
+      delegate :source, :to => :index_or_category
 
-    # Starts the indexing process.
-    #
-    def index categories
-      start_indexing_message
-      prepare categories
-      process categories
-      finish_indexing_message
-    end
+      def initialize index_or_category
+        @index_or_category = index_or_category
+      end
+
+      # Starts the indexing process.
+      #
+      def index categories
+        start_indexing_message
+        prepare categories
+        process categories
+        finish_indexing_message
+      end
+
+      # By default, an indexer
+      #  * prepares the index directories.
+      #
+      def prepare categories
+        categories.each &:prepare_index_directory
+      end
 
-    # By default, an indexer
-    #  * prepares the index directories.
-    #
-    def prepare categories
-      categories.each &:prepare_index_directory
     end
 
   end

data/lib/picky/indexers/parallel.rb

@@ -1,82 +1,86 @@
-  # encoding: utf-8
+# encoding: utf-8
 #
-module Indexers
+module Picky
 
-  # Uses a number of categories, a source, and a tokenizer to index data.
-  #
-  # The tokenizer is taken from each category if specified, from the index, if not.
-  #
-  class Parallel < Base
+  module Indexers
 
-    # Process does the actual indexing.
+    # Uses a number of categories, a source, and a tokenizer to index data.
     #
-    # Parameters:
-    #  * categories: An Enumerable of Category-s.
+    # The tokenizer is taken from each category if specified, from the index, if not.
     #
-    def process categories
-      comma   = ?,
-      newline = ?\n
+    class Parallel < Base
 
-      # Prepare a combined object - array.
+      # Process does the actual indexing.
       #
-      combined = categories.map do |category|
-        [category, [], category.prepared_index_file, (category.tokenizer || tokenizer)]
-      end
-
-      # Index.
+      # Parameters:
+      #  * categories: An Enumerable of Category-s.
       #
-      # TODO Extract into flush_every(100_000) do
-      #
-      i = 0
+      def process categories
+        comma   = ?,
+        newline = ?\n
 
-      # Explicitly reset the source to avoid caching trouble.
-      #
-      source.reset if source.respond_to?(:reset)
+        # Prepare a combined object - array.
+        #
+        combined = categories.map do |category|
+          [category, [], category.prepared_index_file, (category.tokenizer || tokenizer)]
+        end
 
-      # Go through each object in the source.
-      #
-      source.each do |object|
-        id = object.id
+        # Index.
+        #
+        # TODO Extract into flush_every(100_000) do
+        #
+        i = 0
 
-        # This needs to be rewritten.
+        # Explicitly reset the source to avoid caching trouble.
         #
-        # Is it a good idea that not the tokenizer has control over when he gets the next text?
+        source.reset if source.respond_to?(:reset)
+
+        # Go through each object in the source.
         #
-        combined.each do |category, cache, _, tokenizer|
-          tokenizer.tokenize(object.send(category.from).to_s).each do |token_text|
-            next unless token_text
-            cache << id << comma << token_text << newline
+        source.each do |object|
+          id = object.id
+
+          # This needs to be rewritten.
+          #
+          # Is it a good idea that not the tokenizer has control over when he gets the next text?
+          #
+          combined.each do |category, cache, _, tokenizer|
+            tokenizer.tokenize(object.send(category.from).to_s).each do |token_text|
+              next unless token_text
+              cache << id << comma << token_text << newline
+            end
           end
-        end
 
-        if i >= 100_000
-          flush combined
-          i = 0
+          if i >= 100_000
+            flush combined
+            i = 0
+          end
+          i += 1
+        end
+        flush combined
+        combined.each do |_, _, file, _|
+          timed_exclaim %Q{"#{@index_or_category.identifier}":   => #{file.path}.}
+          file.close
         end
-        i += 1
       end
-      flush combined
-      combined.each do |_, _, file, _|
-        timed_exclaim %Q{"#{@index_or_category.identifier}":   => #{file.path}.}
-        file.close
+
+      # Flush the combined array into the file.
+      #
+      def flush combined # :nodoc:
+        combined.each do |_, cache, file, _|
+          file.write(cache.join) && cache.clear
+        end
       end
-    end
 
-    # Flush the combined array into the file.
-    #
-    def flush combined # :nodoc:
-      combined.each do |_, cache, file, _|
-        file.write(cache.join) && cache.clear
+      #
+      #
+      def start_indexing_message # :nodoc:
+        timed_exclaim %Q{"#{@index_or_category.identifier}": Starting parallel data preparation.}
+      end
+      def finish_indexing_message # :nodoc:
+        timed_exclaim %Q{"#{@index_or_category.identifier}": Finished parallel data preparation.}
       end
-    end
 
-    #
-    #
-    def start_indexing_message # :nodoc:
-      timed_exclaim %Q{"#{@index_or_category.identifier}": Starting parallel data preparation.}
-    end
-    def finish_indexing_message # :nodoc:
-      timed_exclaim %Q{"#{@index_or_category.identifier}": Finished parallel data preparation.}
     end
 
   end

data/lib/picky/indexers/serial.rb

@@ -1,55 +1,59 @@
 # encoding: utf-8
 #
-module Indexers
+module Picky
 
-  # Uses a category to index its data.
-  #
-  # Note: It is called serial since it indexes each category separately.
-  #
-  class Serial < Base
+  module Indexers
 
-    # Harvest the data from the source, tokenize,
-    # and write to an intermediate "prepared index" file.
+    # Uses a category to index its data.
     #
-    # Parameters:
-    #  * categories: An enumerable of Category-s.
+    # Note: It is called serial since it indexes each category separately.
     #
-    def process categories
-      comma   = ?,
-      newline = ?\n
-
-      categories.each do |category|
-
-        tokenizer = category.tokenizer
+    class Serial < Base
+
+      # Harvest the data from the source, tokenize,
+      # and write to an intermediate "prepared index" file.
+      #
+      # Parameters:
+      #  * categories: An enumerable of Category-s.
+      #
+      def process categories
+        comma   = ?,
+        newline = ?\n
+
+        categories.each do |category|
+
+          tokenizer = category.tokenizer
+
+          category.prepared_index_file do |file|
+            result = []
+
+            source.harvest(category) do |indexed_id, text|
+              tokenizer.tokenize(text).each do |token_text|
+                next unless token_text
+                result << indexed_id << comma << token_text << newline
+              end
+              file.write(result.join) && result.clear if result.size > 100_000
+            end
 
-        category.prepared_index_file do |file|
-          result = []
+            timed_exclaim %Q{"#{@index_or_category.identifier}":   => #{file.path}.}
 
-          source.harvest(category) do |indexed_id, text|
-            tokenizer.tokenize(text).each do |token_text|
-              next unless token_text
-              result << indexed_id << comma << token_text << newline
-            end
-            file.write(result.join) && result.clear if result.size > 100_000
+            file.write result.join
           end
 
-          timed_exclaim %Q{"#{@index_or_category.identifier}":   => #{file.path}.}
-
-          file.write result.join
         end
 
       end
 
-    end
+      #
+      #
+      def start_indexing_message # :nodoc:
+        timed_exclaim %Q{"#{@index_or_category.identifier}": Starting serial data preparation.}
+      end
+      def finish_indexing_message # :nodoc:
+        timed_exclaim %Q{"#{@index_or_category.identifier}": Finished serial data preparation.}
+      end
 
-    #
-    #
-    def start_indexing_message # :nodoc:
-      timed_exclaim %Q{"#{@index_or_category.identifier}": Starting serial data preparation.}
     end
-    def finish_indexing_message # :nodoc:
-      timed_exclaim %Q{"#{@index_or_category.identifier}": Finished serial data preparation.}
-    end
-
   end
+
 end