picky 3.6.16 → 4.0.0pre1

data/lib/picky/application.rb

@@ -200,7 +200,7 @@ module Picky
         Loader.load_user 'app'             # Sinatra appfile.
         Loader.load_user 'app/application' # Standard Picky appfile.
         finalize_apps
-        exclaim "Application #{apps.map(&:name).join(', ')} loaded."
+        exclaim "Loaded Picky application(s) with environment '#{PICKY_ENVIRONMENT}' in #{PICKY_ROOT} on Ruby #{RUBY_VERSION}."
       end
 
       # Finalize the subclass as soon as it

data/lib/picky/backends/backend.rb

@@ -2,6 +2,8 @@ module Picky
 
   module Backends
 
+    #
+    #
     class Backend
 
       attr_reader :inverted,

data/lib/picky/backends/memory.rb

@@ -4,35 +4,40 @@ module Picky
 
     class Memory < Backend
 
-      # Returns an object that on #initial, #load returns an object that responds to:
+      # Returns an object that on #initial, #load returns
+      # an object that responds to:
       #   [:token] # => [id, id, id, id, id] (an array of ids)
       #
       def create_inverted bundle
         extract_lambda_or(inverted, bundle) ||
           JSON.new(bundle.index_path(:inverted))
       end
-      # Returns an object that on #initial, #load returns an object that responds to:
+      # Returns an object that on #initial, #load returns
+      # an object that responds to:
       #   [:token] # => 1.23 (a weight)
       #
       def create_weights bundle
         extract_lambda_or(weights, bundle) ||
           JSON.new(bundle.index_path(:weights))
       end
-      # Returns an object that on #initial, #load returns an object that responds to:
+      # Returns an object that on #initial, #load returns
+      # an object that responds to:
       #   [:encoded] # => [:original, :original] (an array of original symbols this similarity encoded thing maps to)
       #
       def create_similarity bundle
         extract_lambda_or(similarity, bundle) ||
           Marshal.new(bundle.index_path(:similarity))
       end
-      # Returns an object that on #initial, #load returns an object that responds to:
+      # Returns an object that on #initial, #load returns
+      # an object that responds to:
       #   [:key] # => value (a value for this config key)
       #
       def create_configuration bundle
         extract_lambda_or(configuration, bundle) ||
           JSON.new(bundle.index_path(:configuration))
       end
-      # Returns an object that on #initial, #load returns an object that responds to:
+      # Returns an object that on #initial, #load returns
+      # an object that responds to:
       #   [id] # => [:sym1, :sym2]
       #
       def create_realtime bundle
@@ -50,8 +55,10 @@ module Picky
       # Note: Uses a C-optimized intersection routine (in performant.c)
       #       for speed and memory efficiency.
       #
-      # Note: In the memory based version we ignore the amount and offset hints.
-      #       We cannot use the information to speed up the algorithm, unfortunately.
+      # Note: In the memory based version we ignore the amount and
+      # offset hints.
+      # We cannot use the information to speed up the algorithm,
+      # unfortunately.
       #
       def ids combinations, _, _
         # Get the ids for each combination.

data/lib/picky/backends/{memory → prepared}/text.rb

@@ -2,13 +2,19 @@ module Picky
 
   module Backends
 
-    class Memory
+    class Prepared
 
       # Index data dumped in the text format.
       #
-      # TODO Should this really be Memory::Text?
-      #
-      class Text < Basic
+      class Text
+
+        include Helpers::File
+
+        attr_reader :cache_path
+
+        def initialize cache_path, options = {}
+          @cache_path = "#{cache_path}.prepared.#{extension}"
+        end
 
         # Uses the extension "txt".
         #

data/lib/picky/backends/redis/directly_manipulable.rb

@@ -14,7 +14,7 @@ module Picky
           list.key     = key
         end
 
-        # TODO Current implementation is very brittle.
+        # THINK Current implementation is very brittle.
         #
         @@append_index = 0
         def << value
@@ -23,7 +23,7 @@ module Picky
           backend[key]
         end
 
-        # TODO Current implementation is very brittle.
+        # THINK Current implementation is very brittle.
         #
         @@unshift_index = 0
         def unshift value
@@ -34,9 +34,7 @@ module Picky
 
         def delete value
           result = super value
-          if result
-            backend.client.zrem "#{backend.namespace}:#{key}", value # TODO if super(value) ?
-          end
+          backend.client.zrem "#{backend.namespace}:#{key}", value if result
           result
         end
       end

data/lib/picky/backends/redis/list.rb

@@ -54,6 +54,8 @@ module Picky
 
         # Set a single list.
         #
+        # TODO We should optimize this by only adding. Maybe only add the difference?
+        #
         def []= key, values
           redis_key = "#{namespace}:#{key}"
           i = 0
@@ -62,7 +64,9 @@ module Picky
             client.zadd redis_key, i, value
           end
 
-          self[key] # TODO Performance?
+          # We need to return the whole list.
+          #
+          self[key]
         end
 
       end

data/lib/picky/backends/sqlite/basic.rb

@@ -67,9 +67,11 @@ module Picky
           self
         end
 
+        # Drops the table and creates it anew.
+        #
+        # THINK Could this be replaced by a truncate (DELETE FROM) statement?
+        #
         def truncate_db
-          # TODO Could this be replaced by a truncate statement?
-          #
           drop_table
           create_table
         end

data/lib/picky/bundle.rb

@@ -47,15 +47,13 @@ module Picky
       @name     = name
       @category = category
 
-      # TODO Tidy up a bit.
-      #
-      @key_format = options.delete :key_format
-      @backend    = options.delete :backend
-
       @weights_strategy    = weights_strategy
       @partial_strategy    = partial_strategy
       @similarity_strategy = similarity_strategy
 
+      @key_format = options.delete :key_format
+      @backend    = options.delete :backend
+
       reset_backend
     end
     def identifier
@@ -106,7 +104,7 @@ module Picky
       @inverted = @backend_inverted.empty
     end
     def empty_weights
-      # TODO THINK about this. Perhaps the strategies should implement the backend methods?
+      # THINK about this. Perhaps the strategies should implement the backend methods?
       #
       @weights = @weights_strategy.saved?? @backend_weights.empty : @weights_strategy
     end
@@ -126,7 +124,8 @@ module Picky
     #
     def similar text
       code = similarity_strategy.encoded text
-      similar_codes = code && @similarity[code]
+      return [] unless code
+      similar_codes = @similarity[code]
       if similar_codes.blank?
         [] # Return a simple array.
       else

data/lib/picky/bundle_indexed.rb

@@ -20,8 +20,8 @@ module Picky
     #
     # Returns a (potentially empty) array of ids.
     #
-    # TODO Empty string ok, or does it need to be from the backend?
-    #      Should the backend always return an empty array? (Probably no)
+    # Note: If the backend wants to return a special
+    # enumerable, the backend should do so.
     #
     def ids sym_or_string
       @inverted[sym_or_string] || []

data/lib/picky/bundle_realtime.rb

@@ -47,8 +47,6 @@ module Picky
         #
         str_or_syms << str_or_sym
 
-        # TODO Introduce a new method?
-        #
         ids = @inverted[str_or_sym] ||= []
         ids.send where, id
       end
@@ -68,8 +66,6 @@ module Picky
 
     # Add string/symbol to similarity index.
     #
-    # TODO Probably where makes no sense here. Should have its own order.
-    #
     def add_similarity str_or_sym, where = :unshift
       if encoded = self.similarity_strategy.encoded(str_or_sym)
         similars = @similarity[encoded] ||= []
@@ -77,7 +73,11 @@ module Picky
         # Not completely correct, as others will also be affected, but meh.
         #
         similars.delete str_or_sym if similars.include? str_or_sym
-        similars.send where, str_or_sym
+        similars << str_or_sym
+
+        # Uses the sort order of the strategy.
+        #
+        self.similarity_strategy.sort similars, str_or_sym
       end
     end
 
@@ -91,9 +91,10 @@ module Picky
 
     # Builds the realtime mapping.
     #
-    # Note: Experimental feature.
+    # Note: Experimental feature. Might be removed in 4.0.
     #
-    # TODO Subset of #add. Rewrite.
+    # TODO Subset of #add. Rewrite, optimize.
+    # THINK Maybe load it and just replace the arrays with the corresponding ones.
     #
     def build_realtime
       clear_realtime

data/lib/picky/categories.rb

@@ -12,7 +12,6 @@ module Picky
     each_delegate :cache,
                   :dump,
                   :empty,
-                  :index,
                   :inject,
                   :reindex,
                   :reset_backend,

data/lib/picky/categories_indexing.rb

@@ -2,11 +2,25 @@ module Picky
 
   class Categories
 
+    include Helpers::Indexing
+
     each_delegate :cache,
                   :clear,
                   :prepare,
                   :to => :categories
 
+    # First prepares all categories,
+    # then caches all categories.
+    #
+    def index scheduler = Scheduler.new
+      timed_indexing scheduler do
+        categories.prepare scheduler
+        scheduler.finish
+
+        categories.cache scheduler
+        scheduler.finish
+      end
+    end
   end
 
 end

data/lib/picky/category.rb

@@ -55,7 +55,7 @@ module Picky
         @partial = Bundle.new :partial, self, weights, partial, no_similarity, options
       end
 
-      @prepared = Backends::Memory::Text.new prepared_index_path
+      @prepared = Backends::Prepared::Text.new prepared_index_path
     end
 
     # Indexes and loads the category.
@@ -70,7 +70,7 @@ module Picky
     def dump
       exact.dump
       partial.dump
-      timed_exclaim %Q{"#{identifier}": Generated -> #{index_directory.gsub("#{PICKY_ROOT}/", '')}.}
+      timed_exclaim %Q{  "#{identifier}": Dumped -> #{index_directory.gsub("#{PICKY_ROOT}/", '')}/#{name}_*.}
     end
 
     # Returns the backend.
@@ -121,14 +121,12 @@ module Picky
     # Note: If you don't use it with the block, do not forget to close it.
     #
     def prepared_index_file &block
-      @prepared_index_file ||= Backends::Memory::Text.new prepared_index_path
+      @prepared_index_file ||= Backends::Prepared::Text.new prepared_index_path
       @prepared_index_file.open &block
     end
 
     # The index directory for this category.
     #
-    # TODO Push down into files? Yes.
-    #
     def index_directory
       @index_directory ||= "#{PICKY_ROOT}/index/#{PICKY_ENVIRONMENT}/#{@index.name}"
     end

data/lib/picky/category_indexed.rb

@@ -7,14 +7,11 @@ module Picky
     # Loads the index from cache.
     #
     def load
-      timed_exclaim %Q{"#{identifier}": Loading index from cache.}
-      clear_realtime # TODO What to do?
+      timed_exclaim %Q{  "#{identifier}": Loading index from cache.}
+      clear_realtime # THINK Should we really explicitly clear the realtime? Or should it just be loaded?
       exact.load
       partial.load
     end
-    # TODO Remove in 4.0.
-    #
-    alias reload load
 
     # Gets the weight for this token's text.
     #

data/lib/picky/category_indexing.rb

@@ -4,6 +4,8 @@ module Picky
   #
   class Category
 
+    include Helpers::Indexing
+
     attr_reader :exact,
                 :partial
 
@@ -11,18 +13,37 @@ module Picky
     #
     # This one should be used by users.
     #
-    def index
-      prepare
-      cache
+    def index scheduler = Scheduler.new
+      timed_indexing scheduler do
+        prepare scheduler
+        scheduler.finish
+
+        cache scheduler
+        scheduler.finish
+      end
     end
 
     # Indexes, creates the "prepared_..." file.
     #
-    def prepare
+    def prepare scheduler = Scheduler.new
       categories = Categories.new
       categories << self
       with_data_snapshot do
-        indexer.index categories
+        scheduler.schedule do
+          indexer.prepare categories, scheduler
+          nil # TODO Needed so procrastinate is happy. Remove in 4.0.
+        end
+      end
+    end
+
+    # Generates all caches for this category.
+    #
+    def cache scheduler = Scheduler.new
+      scheduler.schedule do
+        empty
+        retrieve
+        dump
+        nil # TODO Needed so procrastinate is happy. Remove in 4.0.
       end
     end
 
@@ -45,15 +66,6 @@ module Picky
       end
     end
 
-    # Generates all caches for this category.
-    #
-    def cache
-      empty
-      retrieve
-      dump
-      clear_realtime # TODO To call or not to call, that is the question.
-    end
-
     # Retrieves the prepared index data into the indexes and
     # generates the necessary derived indexes.
     #
@@ -109,8 +121,8 @@ module Picky
 
     # Clears the caches.
     #
-    # TODO Think about the semantics of clear.
-    #      Is a delete even needed or is it clear+dump?
+    # THINK about the semantics of clear.
+    # Is a delete even needed or is it clear+dump?
     #
     def clear
       exact.clear

data/lib/picky/constants.rb

@@ -6,4 +6,6 @@
 ENV['PICKY_ENV'] ||= ENV['RACK_ENV']
 
 PICKY_ENVIRONMENT = ENV['PICKY_ENV'] || 'development' unless defined? PICKY_ENVIRONMENT
-PICKY_ROOT        = Dir.pwd unless defined? PICKY_ROOT
+PICKY_ROOT        = Dir.pwd unless defined? PICKY_ROOT
+
+EMPTY_STRING = ''.freeze

data/lib/picky/frontend_adapters/rack.rb

@@ -90,8 +90,8 @@ module Picky
       # Note: Rack-mount already handles the 404.
       #
       STATUSES = {
-        200 => lambda { |_| [200, { 'Content-Type' => 'text/html', 'Content-Length' => '0' }, ['']] },
-        404 => lambda { |_| [404, { 'Content-Type' => 'text/html', 'Content-Length' => '0' }, ['']] }
+        200 => lambda { |_| [200, { 'Content-Type' => 'text/html', 'Content-Length' => '0' }, [EMPTY_STRING]] },
+        404 => lambda { |_| [404, { 'Content-Type' => 'text/html', 'Content-Length' => '0' }, [EMPTY_STRING]] }
       }
 
       #

data/lib/picky/generators/similarity/phonetic.rb

@@ -23,20 +23,12 @@ module Picky
           @amount = amount
         end
 
-        protected
-
-          # Sorts the index values in place.
-          #
-          # TODO Include this again. Sort at the end.
-          #      Or sort when inserting in realtime.
-          #
-          def sort hash
-            hash.each_pair.each do |code, ary|
-              ary.sort_by_levenshtein! code
-              ary.slice! amount, ary.size # size is not perfectly correct, but anyway
-            end
-            hash
-          end
+        # Sorts the index values in place.
+        #
+        def sort ary, code
+            ary.sort_by_levenshtein! code
+            ary.slice! amount, ary.size # THINK size is not perfectly correct, but anyway
+        end
 
       end
 

data/lib/picky/generators/strategy.rb

@@ -7,7 +7,7 @@ module Picky
       # By default, all caches are saved in a
       # storage (like a file).
       #
-      # TODO Move to the backends?
+      # TODO Move to the backends? Rename to backend?
       #
       def saved?
         true

data/lib/picky/generators/weights/runtime.rb

@@ -9,8 +9,8 @@ module Picky
       #       does nothing at all.
       #
       # To override, implement:
-      #   * weight_for(size)    # During indextime. # Probably never used.
-      #   * [] symbol_or_string # During runtime.
+      #   * [symbol_or_string] # During runtime.
+      #   * weight_for(size)   # During indextime. # Probably never used.
       #
       # TODO Find a better name.
       #

data/lib/picky/helpers/indexing.rb

@@ -0,0 +1,20 @@
+module Picky
+
+  # Helper methods for measuring, benchmarking, logging.
+  #
+  module Helpers
+    module Indexing
+
+      include Measuring
+
+      # Returns a duration in seconds.
+      #
+      def timed_indexing scheduler, &block
+        timed_exclaim "Indexing using #{scheduler.fork? ? 'multiple processes' : 'a single process'}."
+        timed_exclaim "Indexing finished after #{timed(&block).round}s."
+      end
+
+    end
+  end
+
+end

data/lib/picky/index.rb

@@ -167,7 +167,7 @@ module Picky
     # * qualifiers: An array of qualifiers with which you can define which category you’d like to search, for example “title:hobbit” will search for hobbit in just title categories. Example: qualifiers: [:t, :titre, :title] (use it for example with multiple languages). Default is the name of the category.
     # * qualifier: Convenience options if you just need a single qualifier, see above. Example: qualifiers => :title. Default is the name of the category.
     # * source: Use a different source than the index uses. If you think you need that, there might be a better solution to your problem. Please post to the mailing list first with your application.rb :)
-    # * from: Take the data from the data category with this name. Example: You have a source Sources::CSV.new(:title, file:'some_file.csv') but you want the category to be called differently. The you use from: define_category(:similar_title, :from => :title).
+    # * from: Take the data from the data category with this name. Example: You have a source Sources::CSV.new(:title, file:'some_file.csv') but you want the category to be called differently. The you use from: category(:similar_title, :from => :title).
     #
     def category category_name, options = {}
       new_category = Category.new category_name.intern, self, options
@@ -177,7 +177,6 @@ module Picky
 
       new_category
     end
-    alias define_category category
 
     # Make this category range searchable with a fixed range. If you need other
     # ranges, define another category with a different range value.
@@ -233,7 +232,7 @@ module Picky
     # Or go crazy and use 4 ranged categories for a space/time search! ;)
     #
     # === Parameters
-    # * category_name: The category_name as used in #define_category.
+    # * category_name: The category_name as used in #category.
     # * range: The range (in the units of your data values) around the query point where we search for results.
     #
     #  -----|<- range  ->*------------|-----
@@ -241,7 +240,7 @@ module Picky
     # === Options
     # * precision: Default is 1 (20% error margin, very fast), up to 5 (5% error margin, slower) makes sense.
     # * anchor: Where to anchor the geo grid.
-    # * ... all options of #define_category.
+    # * ... all options of #category.
     #
     def ranged_category category_name, range, options = {}
       precision = options.delete(:precision) || 1
@@ -251,19 +250,18 @@ module Picky
       #
       options = { partial: Partial::None.new }.merge options
 
-      define_category category_name, options do |category|
-        Wrappers::Category::Location.wrap category, range, precision, anchor
+      category category_name, options do |cat|
+        Wrappers::Category::Location.wrap cat, range, precision, anchor
       end
     end
-    alias define_ranged_category ranged_category
 
     # HIGHLY EXPERIMENTAL Not correctly working yet. Try it if you feel "beta".
     #
     # Also a range search see #ranged_category, but on the earth's surface.
     #
     # Parameters:
-    # * lat_name: The latitude's name as used in #define_category.
-    # * lng_name: The longitude's name as used in #define_category.
+    # * lat_name: The latitude's name as used in #category.
+    # * lng_name: The longitude's name as used in #category.
     # * radius: The distance (in km) around the query point which we search for results.
     #
     # Note: Picky uses a square, not a circle. That should be ok for most usages.
@@ -317,7 +315,6 @@ module Picky
       ranged_category lng_name, radius*0.01796624, options.merge(from: lng_from)
 
     end
-    alias define_geo_categories geo_categories
 
     def to_stats # :nodoc:
       stats = <<-INDEX

data/lib/picky/index_indexed.rb

@@ -10,20 +10,13 @@ module Picky
              :possible_combinations,
              :to => :categories
 
-    # TODO Remove in 4.0.
-    #
-    alias reload load
-
     # Define how the results of this index are identified.
     # (Shown in the client, for example)
     #
     # Default is the name of the index.
     #
     def result_identifier result_identifier = nil
-      result_identifier ? define_result_identifier(result_identifier) : (@result_identifier || @name)
-    end
-    def define_result_identifier result_identifier
-      @result_identifier = result_identifier
+      result_identifier ? (@result_identifier = result_identifier) : (@result_identifier || @name)
     end
 
   end