picky 4.6.3 → 4.6.4

data/lib/performant.c

@@ -2,7 +2,7 @@
 
 // Copying internal ruby methods.
 //
-inline VALUE rb_ary_elt(ary, offset)
+static inline VALUE rb_ary_elt(ary, offset)
     VALUE ary;
     long offset;
 {
@@ -12,7 +12,7 @@ inline VALUE rb_ary_elt(ary, offset)
     }
     return RARRAY_PTR(ary)[offset];
 }
-inline VALUE ary_make_hash(ary1, ary2)
+static inline VALUE ary_make_hash(ary1, ary2)
     VALUE ary1, ary2;
 {
     VALUE hash = rb_hash_new();
@@ -28,7 +28,7 @@ inline VALUE ary_make_hash(ary1, ary2)
     }
     return hash;
 }
-inline VALUE rb_ary_length(VALUE ary) {
+static inline VALUE rb_ary_length(VALUE ary) {
   long length = RARRAY_LEN(ary);
   return LONG2NUM(length);
 }
@@ -37,7 +37,7 @@ inline VALUE rb_ary_length(VALUE ary) {
 //  * orders the arrays by ascending size, small to large.
 //  * calls the & consecutively for all arrays.
 //
-inline VALUE memory_efficient_intersect(VALUE self, VALUE unsorted_array_of_arrays) {
+static inline VALUE memory_efficient_intersect(VALUE self, VALUE unsorted_array_of_arrays) {
   // Counters.
   //
   long i, j;

data/lib/picky/analyzer.rb

@@ -135,17 +135,20 @@ class Analyzer
     return if analysis[:__keys].zero?
     ary = ["index key cardinality:                #{"%10d" % analysis[:__keys]}"]
     return ary.join "\n" unless analysis[:index]
-    ary << "index key length range (avg):         #{"%10s" % analysis[:index][:key_length]} (#{analysis[:index][:key_length_average].round(2)})"
-    ary << "index ids per key length range (avg): #{"%10s" % analysis[:index][:ids_length]} (#{analysis[:index][:ids_length_average].round(2)})"
+    ary << formatted('',        :key_length)
+    ary << formatted('ids per', :ids_length)
     ary.join "\n"
   end
+  def formatted description, key, index = :index
+    "index #{description} key length range (avg): #{"%10s" % analysis[index][key]} (#{analysis[index][:"#{key}_average"].round(2)})"
+  end
   def weights_to_s
     return unless analysis[:weights]
     %Q{weights range (avg):                  #{"%10s" % analysis[:weights][:weight_range]} (#{analysis[:weights][:weight_average].round(2)})}
   end
   def similarity_to_s
     return unless analysis[:similarity]
-    %Q{similarity key length range (avg):    #{"%10s" % analysis[:similarity][:key_length]} (#{analysis[:similarity][:key_length_average].round(2)})}
+    formatted('similarity', :key_length, :similarity)
   end
   def configuration_to_s
     # analysis[:configuration]

data/lib/picky/backends/backend.rb

@@ -5,6 +5,46 @@ module Picky
     #
     #
     class Backend
+      
+      # This is the default behaviour and should be overridden
+      # for different backends.
+      #
+      
+      # Returns an object that on #initial, #load returns
+      # an object that responds to:
+      #   [:token] # => [id, id, id, id, id] (an array of ids)
+      #
+      def create_inverted bundle
+        json bundle.index_path(:inverted)
+      end
+      # Returns an object that on #initial, #load returns
+      # an object that responds to:
+      #   [:token] # => 1.23 (a weight)
+      #
+      def create_weights bundle
+        json bundle.index_path(:weights)
+      end
+      # # Returns an object that on #initial, #load returns
+      # # an object that responds to:
+      # #   [:encoded] # => [:original, :original] (an array of original symbols this similarity encoded thing maps to)
+      # #
+      # def create_similarity bundle
+      #   Marshal.new bundle.index_path(:similarity)
+      # end
+      # Returns an object that on #initial, #load returns
+      # an object that responds to:
+      #   [:key] # => value (a value for this config key)
+      #
+      def create_configuration bundle
+        json bundle.index_path(:configuration)
+      end
+      # Returns an object that on #initial, #load returns
+      # an object that responds to:
+      #   [id] # => [:sym1, :sym2]
+      #
+      def create_realtime bundle
+        json bundle.index_path(:realtime)
+      end
 
       # Returns the total score of the combinations.
       #

data/lib/picky/backends/file/json.rb

@@ -3,6 +3,10 @@ module Picky
   module Backends
 
     class File
+      
+      def json *args
+        JSON.new *args
+      end
 
       # File-based index files dumped in the JSON format.
       #

data/lib/picky/backends/file.rb

@@ -10,35 +10,11 @@ module Picky
     #
     class File < Backend
 
-      # Returns an object that on #initial, #load returns an object that responds to:
-      #   [:token] # => [id, id, id, id, id] (an array of ids)
-      #
-      def create_inverted bundle
-        JSON.new bundle.index_path(:inverted)
-      end
-      # Returns an object that on #initial, #load returns an object that responds to:
-      #   [:token] # => 1.23 (a weight)
-      #
-      def create_weights bundle
-        JSON.new bundle.index_path(:weights)
-      end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:encoded] # => [:original, :original] (an array of original symbols this similarity encoded thing maps to)
       #
       def create_similarity bundle
-        JSON.new bundle.index_path(:similarity)
-      end
-      # Returns an object that on #initial, #load returns an object that responds to:
-      #   [:key] # => value (a value for this config key)
-      #
-      def create_configuration bundle
-        JSON.new bundle.index_path(:configuration)
-      end
-      # Returns an object that on #initial, #load returns an object that responds to:
-      #   [id] # => [:sym1, :sym2]
-      #
-      def create_realtime bundle
-        JSON.new bundle.index_path(:realtime)
+        json bundle.index_path(:similarity)
       end
 
     end

data/lib/picky/backends/memory/json.rb

@@ -3,6 +3,10 @@ module Picky
   module Backends
 
     class Memory
+      
+      def json *args
+        JSON.new *args
+      end
 
       # Memory-based index files dumped in the JSON format.
       #

data/lib/picky/backends/memory.rb

@@ -3,21 +3,7 @@ module Picky
   module Backends
 
     class Memory < Backend
-
-      # Returns an object that on #initial, #load returns
-      # an object that responds to:
-      #   [:token] # => [id, id, id, id, id] (an array of ids)
-      #
-      def create_inverted bundle
-        JSON.new bundle.index_path(:inverted)
-      end
-      # Returns an object that on #initial, #load returns
-      # an object that responds to:
-      #   [:token] # => 1.23 (a weight)
-      #
-      def create_weights bundle
-        JSON.new bundle.index_path(:weights)
-      end
+      
       # Returns an object that on #initial, #load returns
       # an object that responds to:
       #   [:encoded] # => [:original, :original] (an array of original symbols this similarity encoded thing maps to)
@@ -25,20 +11,6 @@ module Picky
       def create_similarity bundle
         Marshal.new bundle.index_path(:similarity)
       end
-      # Returns an object that on #initial, #load returns
-      # an object that responds to:
-      #   [:key] # => value (a value for this config key)
-      #
-      def create_configuration bundle
-        JSON.new bundle.index_path(:configuration)
-      end
-      # Returns an object that on #initial, #load returns
-      # an object that responds to:
-      #   [id] # => [:sym1, :sym2]
-      #
-      def create_realtime bundle
-        JSON.new bundle.index_path(:realtime)
-      end
 
     end
 

data/lib/picky/backends/redis/directly_manipulable.rb

@@ -18,25 +18,33 @@ module Picky
         #
         @@append_index = 0
         def << value
-          super value
-          backend.client.zadd "#{backend.namespace}:#{key}", (@@append_index+=1), value
-          backend[key]
+          super
+          zadd value, @@append_index+=1
         end
 
         # THINK Current implementation is very brittle.
         #
         @@unshift_index = 0
         def unshift value
-          super value
-          backend.client.zadd "#{backend.namespace}:#{key}", (@@unshift_index-=1), value
-          backend[key]
+          super
+          zadd value, @@unshift_index -= 1
         end
-
+        
+        # Deletes the value.
+        #
         def delete value
           result = super value
           backend.client.zrem "#{backend.namespace}:#{key}", value if result
           result
         end
+        
+        # Z-Adds a value with the given index.
+        #
+        def zadd value, index
+          backend.client.zadd "#{backend.namespace}:#{key}", index, value
+          backend[key]
+        end
+        
       end
 
     end

data/lib/picky/backends/redis.rb

@@ -169,104 +169,24 @@ module Picky
             require 'digest/sha1'
             @@ids_sent_once = nil
 
-            # Scripting version of #ids.
+            # Overrides _this_ method.
             #
-            class << self
-              def ids combinations, amount, offset
-                identifiers = combinations.inject([]) do |identifiers, combination|
-                  identifiers << "#{combination.identifier}"
-                end
-
-                # Assume it's using EVALSHA.
-                #
-                begin
-                  if identifiers.size > 1
-                    client.evalsha @@ids_sent_once,
-                                   identifiers.size,
-                                   *identifiers,
-                                   generate_intermediate_result_id,
-                                   offset,
-                                   (offset + amount)
-                  else
-                    client.zrange identifiers.first,
-                                  offset,
-                                  (offset + amount)
-                  end
-                rescue RuntimeError => e
-                  # Make the server have a SHA-1 for the script.
-                  #
-                  @@ids_sent_once = Digest::SHA1.hexdigest @@ids_script
-                  client.eval @@ids_script,
-                              identifiers.size,
-                              *identifiers,
-                              generate_intermediate_result_id,
-                              offset,
-                              (offset + amount)
-                end
-              end
-            end
+            extend Scripting
           else
-            # Non-Scripting version of #ids.
+            # Overrides _this_ method.
             #
-            class << self
-              def ids combinations, amount, offset
-                identifiers = combinations.inject([]) do |identifiers, combination|
-                  identifiers << "#{combination.identifier}"
-                end
-
-                result_id = generate_intermediate_result_id
-
-                # Little optimization.
-                #
-                if identifiers.size > 1
-                  # Intersect and store.
-                  #
-                  intersected = client.zinterstore result_id, identifiers
-
-                  # Return clean and early if there has been no intersection.
-                  #
-                  if intersected.zero?
-                    client.del result_id
-                    return []
-                  end
-
-                  # Get the stored result.
-                  #
-                  results = client.zrange result_id, offset, (offset + amount)
-
-                  # Delete the stored result as it was only for temporary purposes.
-                  #
-                  # Note: I could also not delete it, but that
-                  #       would not be clean at all.
-                  #
-                  client.del result_id
-                else
-                  results = client.zrange identifiers.first, offset, (offset + amount)
-                end
-
-                results
-              end
-            end
+            extend NonScripting
           end
         else
-          class << self
-            def ids combinations, _, _
-              # Get the ids for each combination.
-              #
-              id_arrays = combinations.inject([]) do |total, combination|
-                total << combination.ids
-              end
-
-              # Call the optimized C algorithm.
-              #
-              # Note: It orders the passed arrays by size.
-              #
-              Performant::Array.memory_efficient_intersect id_arrays
-            end
-          end
+          # Remove _this_ method and use the super
+          # class method from now on.
+          #
+          # Note: This fails if there are multiple
+          # Redis backends with different versions.
+          #
+          self.class.send :remove_method, __method__
         end
-
-        # Call the newly installed version.
+        # Call the newly installed / super class version.
         #
         ids combinations, amount, offset
       end
@@ -291,6 +211,85 @@ module Picky
       def generate_intermediate_result_id
         @intermediate_result_id ||= "#{host}:#{pid}:picky:result"
       end
+      
+      # Uses Lua scripting on Redis 2.6.
+      #
+      module Scripting
+        def ids combinations, amount, offset
+          identifiers = combinations.inject([]) do |identifiers, combination|
+            identifiers << "#{combination.identifier}"
+          end
+
+          # Assume it's using EVALSHA.
+          #
+          begin
+            if identifiers.size > 1
+              client.evalsha @@ids_sent_once,
+                             identifiers.size,
+                             *identifiers,
+                             generate_intermediate_result_id,
+                             offset,
+                             (offset + amount)
+            else
+              client.zrange identifiers.first,
+                            offset,
+                            (offset + amount)
+            end
+          rescue RuntimeError => e
+            # Make the server have a SHA-1 for the script.
+            #
+            @@ids_sent_once = Digest::SHA1.hexdigest @@ids_script
+            client.eval @@ids_script,
+                        identifiers.size,
+                        *identifiers,
+                        generate_intermediate_result_id,
+                        offset,
+                        (offset + amount)
+          end
+        end
+      end
+      
+      # Does not use Lua scripting, < Redis 2.6.
+      #
+      module NonScripting
+        def ids combinations, amount, offset
+          identifiers = combinations.inject([]) do |identifiers, combination|
+            identifiers << "#{combination.identifier}"
+          end
+
+          result_id = generate_intermediate_result_id
+
+          # Little optimization.
+          #
+          if identifiers.size > 1
+            # Intersect and store.
+            #
+            intersected = client.zinterstore result_id, identifiers
+
+            # Return clean and early if there has been no intersection.
+            #
+            if intersected.zero?
+              client.del result_id
+              return []
+            end
+
+            # Get the stored result.
+            #
+            results = client.zrange result_id, offset, (offset + amount)
+
+            # Delete the stored result as it was only for temporary purposes.
+            #
+            # Note: I could also not delete it, but that
+            #       would not be clean at all.
+            #
+            client.del result_id
+          else
+            results = client.zrange identifiers.first, offset, (offset + amount)
+          end
+
+          results
+        end
+      end
 
     end
 

data/lib/picky/backends/sqlite/basic.rb

@@ -17,6 +17,12 @@ module Picky
           @realtime   = options[:realtime]
 
           lazily_initialize_client
+        
+        # Note: If on OSX, too many files get opened during
+        #       the specs -> ulimit -n 3000
+        #  
+        # rescue SQLite3::CantOpenException => e
+        #  
         end
 
         def initial

data/lib/picky/bundle.rb

@@ -91,22 +91,24 @@ module Picky
     # the strategy itself pretends to be an index.
     #
     def initialize_backends
-      @inverted      = @backend_inverted.initial
-      @weights       = @weight_strategy.respond_to?(:saved?) && !@weight_strategy.saved? ? @weight_strategy : @backend_weights.initial
-      @similarity    = @backend_similarity.initial
-      @configuration = @backend_configuration.initial
-      @realtime      = @backend_realtime.initial
+      on_all_indexes_call :initial
     end
 
     # "Empties" the index(es) by getting a new empty
     # internal backend instance.
     #
     def empty
-      @inverted      = @backend_inverted.empty
-      @weights       = @weight_strategy.respond_to?(:saved?) && !@weight_strategy.saved? ? @weight_strategy : @backend_weights.empty
-      @similarity    = @backend_similarity.empty
-      @configuration = @backend_configuration.empty
-      @realtime      = @backend_realtime.empty
+      on_all_indexes_call :empty
+    end
+    
+    # Extracted to avoid duplicate code.
+    #
+    def on_all_indexes_call method_name
+      @inverted      = @backend_inverted.send method_name
+      @weights       = @weight_strategy.respond_to?(:saved?) && !@weight_strategy.saved? ? @weight_strategy : @backend_weights.send(method_name)
+      @similarity    = @backend_similarity.send method_name
+      @configuration = @backend_configuration.send method_name
+      @realtime      = @backend_realtime.send method_name
     end
 
     # Delete all index files.

data/lib/picky/categories_indexing.rb

@@ -8,19 +8,6 @@ module Picky
                   :clear,
                   :prepare,
                   :to => :categories
-
-    # First prepares all categories,
-    # then caches all categories.
-    #
-    def index scheduler = Scheduler.new
-      timed_indexing scheduler do
-        categories.prepare scheduler
-        scheduler.finish
-
-        categories.cache scheduler
-        scheduler.finish
-      end
-    end
   end
 
 end

data/lib/picky/category.rb

@@ -3,10 +3,6 @@ module Picky
   class Category
 
     include API::Tokenizer
-    include API::Source
-    include API::Category::Weight
-    include API::Category::Partial
-    include API::Category::Similarity
 
     attr_accessor :exact,
                   :partial
@@ -40,11 +36,17 @@ module Picky
       @name  = name
       @index = index
 
-      # Indexing.
-      #
-      @source    = extract_source options[:source], nil_ok: true
+      configure_from options
+      configure_indexes_from options
+    end
+    
+    def configure_from options
       @from      = options[:from]
-      @tokenizer = extract_tokenizer options[:indexing]
+      
+      # Instantly extracted to raise an error instantly.
+      #
+      @source    = Source.from options[:source], true, @index.name
+      @tokenizer = Tokenizer.from options[:indexing], @index.name, name
 
       @key_format = options.delete :key_format
       @backend    = options.delete :backend
@@ -52,10 +54,16 @@ module Picky
       @qualifiers = extract_qualifiers_from options
 
       # @symbols    = options[:use_symbols] || index.use_symbols? # SYMBOLS.
+    end
 
-      weights    = extract_weight options[:weight]
-      partial    = extract_partial options[:partial]
-      similarity = extract_similarity options[:similarity]
+    # TODO I do a lot of helper method calls here. Refactor?
+    #
+    def configure_indexes_from options
+      index_name = @index.name
+      
+      weights    = Generators::Weights.from    options[:weight],     index_name, name
+      partial    = Generators::Partial.from    options[:partial],    index_name, name
+      similarity = Generators::Similarity.from options[:similarity], index_name, name
 
       no_partial    = Generators::Partial::None.new
       no_similarity = Generators::Similarity::None.new
@@ -99,12 +107,6 @@ module Picky
       partial.reset_backend
     end
 
-    # Index name.
-    #
-    def index_name
-      @index.name
-    end
-
     # Returns the qualifiers if set or
     # just the name if not.
     #
@@ -136,11 +138,12 @@ module Picky
       @prepared_index_file ||= Backends::Prepared::Text.new prepared_index_path
       @prepared_index_file.open &block
     end
-
-    # The index directory for this category.
-    #
+    
     def index_directory
-      @index_directory ||= ::File.join(PICKY_ROOT, 'index', PICKY_ENVIRONMENT, @index.name.to_s)
+      @index.directory
+    end
+    def index_name
+      @index.name
     end
 
     # Identifier for technical output.

data/lib/picky/category_indexing.rb

@@ -9,20 +9,6 @@ module Picky
     attr_reader :exact,
                 :partial
 
-    # Prepares and caches this category.
-    #
-    # This one should be used by users.
-    #
-    def index scheduler = Scheduler.new
-      timed_indexing scheduler do
-        prepare scheduler
-        scheduler.finish
-
-        cache scheduler
-        scheduler.finish
-      end
-    end
-
     # Indexes, creates the "prepared_..." file.
     #
     def prepare scheduler = Scheduler.new
@@ -73,14 +59,6 @@ module Picky
       prepared.retrieve { |id, token| add_tokenized_token id, token, :<< }
     end
 
-    # Return an appropriate source.
-    #
-    # If we have no explicit source, we'll check the index for one.
-    #
-    def source
-      (@source = extract_source(@source, nil_ok: true)) || @index.source
-    end
-
     # Return the key format.
     #
     # If no key_format is defined on the category
@@ -98,6 +76,14 @@ module Picky
     def from
       @from || name
     end
+    
+    # Return an appropriate source.
+    #
+    # If we have no explicit source, we'll check the index for one.
+    #
+    def source
+      @source || @index.source
+    end
 
     # The indexer is lazily generated and cached.
     #

data/lib/picky/constants.rb

@@ -4,7 +4,6 @@
 # Use rack's environment for the search engine.
 #
 ENV['PICKY_ENV'] ||= ENV['RUBY_ENV'] || ENV['RACK_ENV']
-
 PICKY_ENVIRONMENT = ENV['PICKY_ENV'] || 'development' unless defined? PICKY_ENVIRONMENT
 PICKY_ROOT        = Dir.pwd unless defined? PICKY_ROOT
 

data/lib/picky/generators/aliases.rb

@@ -1,3 +1,5 @@
+# Removes and reinstalls convenience generator accessors.
+#
 module Picky
   remove_const :Partial if defined? Partial
   Partial = Generators::Partial