picky 2.1.2 → 2.2.0

data/lib/picky/internals/index/file/text.rb

@@ -1,13 +1,13 @@
 module Internals
 
   module Index
-  
+
     module File
-    
+
       # Index data dumped in the text format.
       #
       class Text < Basic
-      
+
         # Uses the extension "txt".
         #
         def extension
@@ -20,12 +20,12 @@ module Internals
           raise "Can't load from text file. Use JSON or Marshal."
         end
         # Text files are used exclusively for
-        # prepared data files. 
+        # prepared data files.
         #
         def dump hash
           raise "Can't dump to text file. Use JSON or Marshal."
         end
-      
+
         # Retrieves prepared index data in the form
         # * id,data\n
         # * id,data\n
@@ -43,18 +43,18 @@ module Internals
             end
           end
         end
-      
+
         #
         #
         def open_for_indexing &block
           ::File.open cache_path, 'w:binary', &block
         end
-      
-      
+
+
       end
-    
+
     end
-  
+
   end
-  
+
 end

data/lib/picky/internals/index/files.rb

@@ -4,17 +4,17 @@ module Internals
 
     class Files < Backend
 
-      def initialize bundle_name, config
-        super bundle_name, config
+      def initialize bundle_name, category
+        super bundle_name, category
 
         # Note: We marshal the similarity, as the
         #       Yajl json lib cannot load symbolized
         #       values, just keys.
         #
-        @index         = File::JSON.new    config.index_path(bundle_name, :index)
-        @weights       = File::JSON.new    config.index_path(bundle_name, :weights)
-        @similarity    = File::Marshal.new config.index_path(bundle_name, :similarity)
-        @configuration = File::JSON.new    config.index_path(bundle_name, :configuration)
+        @index         = File::JSON.new    category.index_path(bundle_name, :index)
+        @weights       = File::JSON.new    category.index_path(bundle_name, :weights)
+        @similarity    = File::Marshal.new category.index_path(bundle_name, :similarity)
+        @configuration = File::JSON.new    category.index_path(bundle_name, :configuration)
       end
 
       def to_s

data/lib/picky/internals/index/redis.rb

@@ -1,28 +1,28 @@
 module Internals
 
   module Index
-  
+
     # TODO Needs a reconnect to be run after forking.
     #
     class Redis < Backend
-    
-      def initialize bundle_name, config
-        super bundle_name, config
+
+      def initialize bundle_name, category
+        super bundle_name, category
 
         # Refine a few Redis "types".
         #
-        @index         = Redis::ListHash.new "#{config.identifier}:#{bundle_name}:index"
-        @weights       = Redis::StringHash.new "#{config.identifier}:#{bundle_name}:weights"
-        @similarity    = Redis::ListHash.new "#{config.identifier}:#{bundle_name}:similarity"
-        @configuration = Redis::StringHash.new "#{config.identifier}:#{bundle_name}:configuration"
+        @index         = Redis::ListHash.new "#{category.identifier}:#{bundle_name}:index"
+        @weights       = Redis::StringHash.new "#{category.identifier}:#{bundle_name}:weights"
+        @similarity    = Redis::ListHash.new "#{category.identifier}:#{bundle_name}:similarity"
+        @configuration = Redis::StringHash.new "#{category.identifier}:#{bundle_name}:configuration"
       end
-    
+
       # Delegate to the right collection.
       #
       def ids sym
         index.collection sym
       end
-    
+
       # Delegate to the right member value.
       #
       # Note: Converts to float.
@@ -30,15 +30,15 @@ module Internals
       def weight sym
         weights.member(sym).to_f
       end
-    
+
       # Delegate to a member value.
       #
       def setting sym
         configuration.member sym
       end
-    
+
     end
-  
+
   end
-  
+
 end

data/lib/picky/internals/indexed/bundle/base.rb

@@ -35,8 +35,8 @@ module Internals
         delegate :[], :to => :configuration
         delegate :size, :to => :index
 
-        def initialize name, configuration, similarity_strategy
-          @identifier = "#{configuration.identifier}:#{name}"
+        def initialize name, category, similarity_strategy
+          @identifier = "#{category.identifier}:#{name}"
 
           @index         = {}
           @weights       = {}

data/lib/picky/internals/indexed/bundle/redis.rb

@@ -14,10 +14,10 @@ module Internals
       #
       class Redis < Base
 
-        def initialize name, configuration, *args
-          super name, configuration, *args
+        def initialize name, category, *args
+          super name, category, *args
 
-          @backend = Internals::Index::Redis.new name, configuration
+          @backend = Internals::Index::Redis.new name, category
         end
 
         # Get the ids for the given symbol.

data/lib/picky/internals/indexed/category.rb

@@ -9,18 +9,17 @@ module Internals
     #
     class Category
 
+      include Internals::Shared::Category
+
       attr_accessor :exact
-      attr_reader   :identifier, :name
+      attr_reader   :name, :index
       attr_writer   :partial
 
       #
       #
       def initialize name, index, options = {}
-        @name = name
-
-        configuration = Configuration::Index.new index, self
-
-        @identifier    = configuration.identifier
+        @name  = name
+        @index = index
 
         # TODO Push the defaults out into the index.
         #
@@ -28,15 +27,15 @@ module Internals
         similarity = options[:similarity] || Internals::Generators::Similarity::Default
 
         bundle_class = options[:indexed_bundle_class] || Bundle::Memory
-        @exact   = bundle_class.new :exact,   configuration, similarity
-        @partial = bundle_class.new :partial, configuration, similarity
+        @exact   = bundle_class.new :exact,   self, similarity
+        @partial = bundle_class.new :partial, self, similarity
 
         # @exact   = exact_lambda.call(@exact, @partial)   if exact_lambda   = options[:exact_lambda]
         # @partial = partial_lambda.call(@exact, @partial) if partial_lambda = options[:partial_lambda]
 
         # TODO Extract?
         #
-        Query::Qualifiers.add(configuration.category_name, generate_qualifiers_from(options) || [name])
+        Query::Qualifiers.add(name, generate_qualifiers_from(options) || [name])
       end
 
       def to_s

data/lib/picky/internals/indexed/wrappers/bundle/calculation.rb

@@ -1,35 +1,37 @@
-module Indexed
-  module Wrappers
+module Internals
+  module Indexed
+    module Wrappers
 
-    module Bundle
-
-      # A calculation rewrites the symbol into a float.
-      #
-      # TODO I really need to allow integers as keys. The code below is just not up to the needed quality.
-      #
-      class Calculation < Wrapper
+      module Bundle
 
+        # A calculation rewrites the symbol into a float.
         #
+        # TODO I really need to allow integers as keys. The code below is just not up to the needed quality.
         #
-        def recalculate float
-          float
-        end
+        class Calculation < Wrapper
+
+          #
+          #
+          def recalculate float
+            float
+          end
+
+          #
+          #
+          def ids sym
+            @bundle.ids recalculate(sym.to_s.to_f).to_s.to_sym
+          end
+
+          #
+          #
+          def weight sym
+            @bundle.weight recalculate(sym.to_s.to_f).to_s.to_sym
+          end
 
-        #
-        #
-        def ids sym
-          @bundle.ids recalculate(sym.to_s.to_f).to_s.to_sym
-        end
-
-        #
-        #
-        def weight sym
-          @bundle.weight recalculate(sym.to_s.to_f).to_s.to_sym
         end
 
       end
 
     end
-
   end
 end

data/lib/picky/internals/indexed/wrappers/bundle/location.rb

@@ -1,42 +1,44 @@
-module Indexed
-  module Wrappers
-    
-    module Bundle
-      
-      # A location calculation recalculates a location to the Picky internal location.
-      #
-      class Location < Calculation
-        
-        def initialize bundle, options = {}
-          super bundle
-          
-          precision  = options[:precision] || 1
-          user_grid  = options[:grid] || raise("Gridsize needs to be given for location #{bundle.identifier}.")
-          
-          @calculation = Calculations::Location.new user_grid, precision
-        end
-        
-        #
-        #
-        def recalculate float
-          @calculation.recalculate float
-        end
-        
+module Internals
+  module Indexed
+    module Wrappers
+
+      module Bundle
+
+        # A location calculation recalculates a location to the Picky internal location.
         #
-        #
-        def load
-          # Load first the bundle, then extract the config.
+        class Location < Calculation
+
+          def initialize bundle, options = {}
+            super bundle
+
+            precision  = options[:precision] || 1
+            user_grid  = options[:grid] || raise("Gridsize needs to be given for location #{bundle.identifier}.")
+
+            @calculation = Internals::Calculations::Location.new user_grid, precision
+          end
+
+          #
+          #
+          def recalculate float
+            @calculation.recalculate float
+          end
+
           #
-          bundle.load
-          # TODO Move the to_f to the backend.
           #
-          minimum = bundle[:location_minimum] && bundle[:location_minimum].to_f || raise("Configuration :location_minimum for #{bundle.identifier} missing. Did you run rake index already?")
-          @calculation.minimum = minimum
+          def load
+            # Load first the bundle, then extract the config.
+            #
+            bundle.load
+            # TODO Move the to_f to the backend.
+            #
+            minimum = bundle[:location_minimum] && bundle[:location_minimum].to_f || raise("Configuration :location_minimum for #{bundle.identifier} missing. Did you run rake index already?")
+            @calculation.minimum = minimum
+          end
+
         end
-        
+
       end
-      
+
     end
-    
   end
 end

data/lib/picky/internals/indexed/wrappers/bundle/wrapper.rb

@@ -1,43 +1,45 @@
-module Indexed
-  module Wrappers
+module Internals
+  module Indexed
+    module Wrappers
 
-    # Per Bundle wrappers.
-    #
-    module Bundle
-
-      # Base wrapper. Just delegates all methods to the bundle.
+      # Per Bundle wrappers.
       #
-      class Wrapper
-
-        attr_reader :bundle
+      module Bundle
+
+        # Base wrapper. Just delegates all methods to the bundle.
+        #
+        class Wrapper
+
+          attr_reader :bundle
+
+          def initialize bundle
+            @bundle = bundle
+          end
+
+          delegate :load,
+                   :load_index,
+                   :load_weights,
+                   :load_similarity,
+                   :load_configuration,
+                   :clear_index,
+                   :clear_weights,
+                   :clear_similarity,
+                   :clear_configuration,
+                   :ids,
+                   :weight,
+                   :identifier,
+                   :analyze,
+                   :size,
+                   :index,
+                   :weights,
+                   :similarity,
+                   :configuration,
+                   :to => :@bundle
 
-        def initialize bundle
-          @bundle = bundle
         end
 
-        delegate :load,
-                 :load_index,
-                 :load_weights,
-                 :load_similarity,
-                 :load_configuration,
-                 :clear_index,
-                 :clear_weights,
-                 :clear_similarity,
-                 :clear_configuration,
-                 :ids,
-                 :weight,
-                 :identifier,
-                 :analyze,
-                 :size,
-                 :index,
-                 :weights,
-                 :similarity,
-                 :configuration,
-                 :to => :@bundle
-
       end
 
     end
-
   end
 end

data/lib/picky/internals/indexed/wrappers/category/location.rb

@@ -0,0 +1,27 @@
+module Internals
+  module Indexed
+    module Wrappers
+      module Category
+
+        module Location
+
+          def self.install_on category, grid, precision = 1
+            wrapped_exact = Internals::Indexed::Wrappers::Bundle::Location.new category.exact, grid: grid, precision: precision
+
+            category.class_eval do
+              define_method :exact do
+                wrapped_exact
+              end
+              define_method :partial do
+                wrapped_exact
+              end
+            end
+
+          end
+
+        end
+
+      end
+    end
+  end
+end

data/lib/picky/internals/indexers/base.rb

@@ -0,0 +1,28 @@
+# encoding: utf-8
+#
+module Indexers
+
+  #
+  #
+  class Base
+
+    # Selects the original id (indexed id) and a column to process. The column data is called "token".
+    #
+    # Note: Puts together the parts first in an array, then releasing the array from time to time by joining.
+    #
+    def index
+      indexing_message
+      process
+    end
+
+    # Delegates the key format to the source.
+    #
+    # Default is to_i.
+    #
+    def key_format
+      source.respond_to?(:key_format) && source.key_format || :to_i
+    end
+
+  end
+
+end

data/lib/picky/internals/indexers/parallel.rb

@@ -0,0 +1,64 @@
+# encoding: utf-8
+#
+module Indexers
+
+  # Uses a number of categories, a source, and a tokenizer to index data.
+  #
+  # The tokenizer is taken from each category if specified, from the index, if not.
+  #
+  class Parallel < Base
+
+    delegate :categories, :source, :to => :@index
+
+    def initialize index
+      @index = index
+    end
+
+    def process
+      comma   = ?,
+      newline = ?\n
+
+      # Prepare a combined object - array.
+      #
+      combined = categories.map { |category| [category, [], category.prepared_index_file, (category.tokenizer || tokenizer)] }
+
+      # Index.
+      #
+      i = 0
+      source.each do |object|
+        id = object.id
+
+        # This needs to be rewritten.
+        #
+        # Is it a good idea that not the tokenizer has control over when he gets the next text?
+        #
+        combined.each do |category, cache, _, tokenizer|
+          tokenizer.tokenize(object.send(category.from).to_s).each do |token_text|
+            next unless token_text
+            cache << id << comma << token_text << newline
+          end
+        end
+
+        if i >= 100_000
+          flush combined
+          i = 0
+        end
+        i += 1
+      end
+      flush combined
+      combined.each { |_, _, file, _| file.close }
+    end
+    def flush combined
+      combined.each do |_, cache, file, _|
+        file.write(cache.join) && cache.clear
+      end
+    end
+    #
+    #
+    def indexing_message
+      timed_exclaim %Q{"#{@index.name}": Starting parallel indexing.}
+    end
+
+  end
+
+end

data/lib/picky/internals/indexers/serial.rb

@@ -2,40 +2,28 @@
 #
 module Indexers
 
-  # The indexer defines the control flow.
+  # Uses a category to index its data.
   #
-  class Serial
+  # Note: It is called serial since it indexes each
+  #
+  # FIXME Giving the serial a category would be enough, since it already contains an index!
+  #
+  class Serial < Base
 
-    attr_accessor :tokenizer, :source
+    attr_reader :category
 
-    def initialize configuration, source, tokenizer
-      @configuration = configuration
-      @source        = source || raise_no_source
-      @tokenizer     = tokenizer
-    end
+    delegate :source, :to => :category
 
-    # Raise a no source exception.
-    #
-    def raise_no_source
-      raise NoSourceSpecifiedException.new("No source given for #{@configuration}.")
+    def initialize category
+      @category = category
     end
 
-    # Delegates the key format to the source.
-    #
-    # Default is to_i.
+    # The tokenizer used is a cached tokenizer from the category.
     #
-    def key_format
-      @source.key_format || :to_i
+    def tokenizer
+      @tokenizer ||= category.tokenizer
     end
 
-    # Selects the original id (indexed id) and a column to process. The column data is called "token".
-    #
-    # Note: Puts together the parts first in an array, then releasing the array from time to time by joining.
-    #
-    def index
-      indexing_message
-      process
-    end
     def process
       comma   = ?,
       newline = ?\n
@@ -52,10 +40,11 @@ module Indexers
       #   end
       # end
       #
-      @configuration.prepared_index_file do |file|
+      local_tokenizer = tokenizer
+      category.prepared_index_file do |file|
         result = []
-        source.harvest(@configuration.index, @configuration.category) do |indexed_id, text|
-          tokenizer.tokenize(text).each do |token_text|
+        source.harvest(category) do |indexed_id, text|
+          local_tokenizer.tokenize(text).each do |token_text|
             next unless token_text
             result << indexed_id << comma << token_text << newline
           end
@@ -64,8 +53,10 @@ module Indexers
         file.write result.join
       end
     end
+    #
+    #
     def indexing_message
-      timed_exclaim %Q{"#{@configuration.identifier}": Starting indexing.}
+      timed_exclaim %Q{"#{@category.identifier}": Starting serial indexing.}
     end
 
   end

data/lib/picky/internals/indexing/bundle/base.rb

@@ -16,8 +16,8 @@ module Internals
 
         # Path is in which directory the cache is located.
         #
-        def initialize name, configuration, similarity_strategy, partial_strategy, weights_strategy
-          super name, configuration, similarity_strategy
+        def initialize name, category, similarity_strategy, partial_strategy, weights_strategy
+          super name, category, similarity_strategy
 
           @partial_strategy    = partial_strategy
           @weights_strategy    = weights_strategy

data/lib/picky/internals/indexing/bundle/super_base.rb

@@ -33,9 +33,9 @@ module Internals
         delegate :clear,    :to => :index
         delegate :[], :[]=, :to => :configuration
 
-        def initialize name, configuration, similarity_strategy
-          @identifier = "#{configuration.identifier}:#{name}"
-          @files      = Internals::Index::Files.new name, configuration
+        def initialize name, category, similarity_strategy
+          @identifier    = "#{category.identifier}:#{name}"
+          @files         = Internals::Index::Files.new name, category
 
           @index         = {}
           @weights       = {}