picky 4.0.0pre1 → 4.0.0pre2

data/aux/picky/cli.rb

@@ -38,7 +38,11 @@ module Picky
       def execute name, args, params
         commands = Picky::CLI.mapping.map do |command, object_and_params|
           _, *params = object_and_params
-          "  picky #{command} #{params_to_s(params)}"
+          ary = []
+          ary << "  picky"
+          ary << command
+          ary << params_to_s(params) unless params.empty?
+          ary.join ' '
         end.join(?\n)
 
         Kernel.puts "Possible commands:\n#{commands}\n"
@@ -103,7 +107,7 @@ module Picky
     # TODO Try to load the other gems and get the commands dynamically.
     #
     @@mapping = {
-      :generate => [Generate, :'{client,server,sinatra_client,unicorn_server,all_in_one}', :'app_directory_name'],
+      :generate => [Generate, :'{client,server,all_in_one}', :'app_directory_name'],
       :help     => [Help],
       :live     => [Live, 'host:port/path (default: localhost:8080/admin)', 'port (default: 4568)'],
       :search   => [Search, :url_or_path, 'amount of ids (default 20)'],

data/lib/picky.rb

@@ -2,20 +2,22 @@ module Picky
 
   # External libraries.
   #
-  require 'active_support/core_ext'
-  require 'text'
-  require 'yajl' # THINK Maybe replace by multi_json?
-  require 'rack' # TODO Remove.
-  require 'rack_fast_escape' # TODO Remove.
+  require 'active_support/core_ext/module/delegation'
+  require 'active_support/core_ext/logger'
+  require 'active_support/core_ext/object/blank'
+  require 'active_support/core_ext/enumerable'
+  require 'active_support/multibyte'
+  require 'yajl'
   require 'procrastinate'
+  require 'rack_fast_escape' if defined? Rack
 
   # Require the constants.
   #
-  require ::File.expand_path '../picky/constants', __FILE__
+  require_relative 'picky/constants'
 
   # Loader which handles framework and app loading.
   #
-  require ::File.expand_path '../picky/loader', __FILE__
+  require_relative 'picky/loader'
 
   # Load the framework
   #
@@ -23,7 +25,7 @@ module Picky
 
   # Check if delegators need to be installed.
   #
-  require ::File.expand_path '../picky/sinatra', __FILE__
+  require_relative 'picky/sinatra'
 
   # This is only used in the classic project style.
   #

data/lib/picky/backends/backend.rb

@@ -22,6 +22,43 @@ module Picky
         thing && (thing.respond_to?(:call) && thing.call(*args) || thing)
       end
 
+      # Returns the total score of the combinations.
+      #
+      # Default implementation. Override to speed up.
+      #
+      def weight combinations
+        combinations.score
+      end
+
+      # Returns the result ids for the allocation.
+      #
+      # Sorts the ids by size and & through them in the following order (sizes):
+      # 0. [100_000, 400, 30, 2]
+      # 1. [2, 30, 400, 100_000]
+      # 2. (100_000 & (400 & (30 & 2))) # => result
+      #
+      # Note: Uses a C-optimized intersection routine (in performant.c)
+      #       for speed and memory efficiency.
+      #
+      # Note: In the memory based version we ignore the amount and
+      # offset hints.
+      # We cannot use the information to speed up the algorithm,
+      # unfortunately.
+      #
+      def ids combinations, _, _
+        # Get the ids for each combination.
+        #
+        id_arrays = combinations.inject([]) do |total, combination|
+          total << combination.ids
+        end
+
+        # Call the optimized C algorithm.
+        #
+        # Note: It orders the passed arrays by size.
+        #
+        Performant::Array.memory_efficient_intersect id_arrays
+      end
+
       #
       #
       def to_s

data/lib/picky/backends/file.rb

@@ -46,26 +46,6 @@ module Picky
           JSON.new(bundle.index_path(:realtime))
       end
 
-      # Currently, the loaded ids are intersected using
-      # the fast C-based intersection.
-      #
-      # However, if we could come up with a clever way
-      # to do this faster, it would be most welcome.
-      #
-      def ids combinations, _, _
-        # Get the ids for each combination.
-        #
-        id_arrays = combinations.inject([]) do |total, combination|
-          total << combination.ids
-        end
-
-        # Call the optimized C algorithm.
-        #
-        # Note: It orders the passed arrays by size.
-        #
-        Performant::Array.memory_efficient_intersect id_arrays
-      end
-
     end
 
   end

data/lib/picky/backends/memory.rb

@@ -45,35 +45,6 @@ module Picky
           JSON.new(bundle.index_path(:realtime))
       end
 
-      # Returns the result ids for the allocation.
-      #
-      # Sorts the ids by size and & through them in the following order (sizes):
-      # 0. [100_000, 400, 30, 2]
-      # 1. [2, 30, 400, 100_000]
-      # 2. (100_000 & (400 & (30 & 2))) # => result
-      #
-      # Note: Uses a C-optimized intersection routine (in performant.c)
-      #       for speed and memory efficiency.
-      #
-      # Note: In the memory based version we ignore the amount and
-      # offset hints.
-      # We cannot use the information to speed up the algorithm,
-      # unfortunately.
-      #
-      def ids combinations, _, _
-        # Get the ids for each combination.
-        #
-        id_arrays = combinations.inject([]) do |total, combination|
-          total << combination.ids
-        end
-
-        # Call the optimized C algorithm.
-        #
-        # Note: It orders the passed arrays by size.
-        #
-        Performant::Array.memory_efficient_intersect id_arrays
-      end
-
     end
 
   end

data/lib/picky/backends/redis.rb

@@ -12,9 +12,25 @@ module Picky
       def initialize options = {}
         super options
 
-        require 'redis'
+        maybe_load_hiredis
+        check_hiredis_gem
+        check_redis_gem
+
         @client    = options[:client] || ::Redis.new(:db => (options[:db] || 15))
         @immediate = options[:immediate]
+      end
+      def maybe_load_hiredis
+        require 'hiredis'
+      rescue LoadError
+        # It's ok.
+      end
+      def check_hiredis_gem
+        require 'redis/connection/hiredis'
+      rescue LoadError
+        # It's ok, the next check will fail if this one does.
+      end
+      def check_redis_gem
+        require 'redis'
       rescue LoadError => e
         warn_gem_missing 'redis', 'the Redis client'
       end
@@ -24,28 +40,28 @@ module Picky
       #
       def create_inverted bundle
         extract_lambda_or(inverted, bundle, client) ||
-          List.new(client, "#{bundle.identifier}:inverted", immediate: immediate)
+          List.new(client, "#{PICKY_ENVIRONMENT}:#{bundle.identifier}:inverted", immediate: immediate)
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:token] # => 1.23 (a weight)
       #
       def create_weights bundle
         extract_lambda_or(weights, bundle, client) ||
-          Float.new(client, "#{bundle.identifier}:weights", immediate: immediate)
+          Float.new(client, "#{PICKY_ENVIRONMENT}:#{bundle.identifier}:weights", immediate: immediate)
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:encoded] # => [:original, :original] (an array of original symbols this similarity encoded thing maps to)
       #
       def create_similarity bundle
         extract_lambda_or(similarity, bundle, client) ||
-          List.new(client, "#{bundle.identifier}:similarity", immediate: immediate)
+          List.new(client, "#{PICKY_ENVIRONMENT}:#{bundle.identifier}:similarity", immediate: immediate)
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:key] # => value (a value for this config key)
       #
       def create_configuration bundle
         extract_lambda_or(configuration, bundle, client) ||
-          String.new(client, "#{bundle.identifier}:configuration", immediate: immediate)
+          String.new(client, "#{PICKY_ENVIRONMENT}:#{bundle.identifier}:configuration", immediate: immediate)
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [id] # => [:sym1, :sym2]
@@ -89,6 +105,54 @@ module Picky
         version_string.split('.').map &:to_i
       end
 
+      # Returns the total weight for the combinations.
+      #
+      def weight combinations
+        # Note: A nice experiment that generated far too many strings.
+        #
+        # if redis_with_scripting?
+        #   @@weight_script = "local sum = 0; for i=1,#(KEYS),2 do local value = redis.call('hget', KEYS[i], KEYS[i+1]); if value then sum = sum + value end end return sum;"
+        #
+        #   require 'digest/sha1'
+        #   @@weight_sent_once = nil
+        #
+        #   # Scripting version of #ids.
+        #   #
+        #   class << self
+        #     def weight combinations
+        #       namespaces_keys = combinations.inject([]) do |namespaces_keys, combination|
+        #         namespaces_keys << "#{combination.bundle.identifier}:weights"
+        #         namespaces_keys << combination.token.text
+        #       end
+        #
+        #       # Assume it's using EVALSHA.
+        #       #
+        #       begin
+        #         client.evalsha @@weight_sent_once,
+        #                        namespaces_keys.size,
+        #                        *namespaces_keys
+        #       rescue RuntimeError => e
+        #         # Make the server have a SHA-1 for the script.
+        #         #
+        #         @@weight_sent_once = Digest::SHA1.hexdigest @@weight_script
+        #         client.eval @@weight_script,
+        #                     namespaces_keys.size,
+        #                     *namespaces_keys
+        #       end
+        #     end
+        #   end
+        # else
+        #   class << self
+        #     def weight combinations
+              combinations.score
+        #     end
+        #   end
+        # end
+        # # Call the newly installed version.
+        # #
+        # weight combinations
+      end
+
       # Returns the result ids for the allocation.
       #
       # Developers wanting to program fast intersection
@@ -97,9 +161,6 @@ module Picky
       #
       # Note: We use the amount and offset hints to speed Redis up.
       #
-      # TODO What if it hasn't been dumped?
-      #      Move this method to the actual backends?
-      #
       def ids combinations, amount, offset
         # TODO FIXME This is actually not correct:
         #            A dumped/loaded Redis backend should use
@@ -110,10 +171,10 @@ module Picky
           # Just checked once on the first call.
           #
           if redis_with_scripting?
-            @@script = "local intersected = redis.call('zinterstore', ARGV[1], #(KEYS), unpack(KEYS)); if intersected == 0 then redis.call('del', ARGV[1]); return {}; end local results = redis.call('zrange', ARGV[1], tonumber(ARGV[2]), tonumber(ARGV[3])); redis.call('del', ARGV[1]); return results;"
+            @@ids_script = "local intersected = redis.call('zinterstore', ARGV[1], #(KEYS), unpack(KEYS)); if intersected == 0 then redis.call('del', ARGV[1]); return {}; end local results = redis.call('zrange', ARGV[1], tonumber(ARGV[2]), tonumber(ARGV[3])); redis.call('del', ARGV[1]); return results;"
 
             require 'digest/sha1'
-            @@sent_once = nil
+            @@ids_sent_once = nil
 
             # Scripting version of #ids.
             #
@@ -126,7 +187,7 @@ module Picky
                 # Assume it's using EVALSHA.
                 #
                 begin
-                  client.evalsha @@sent_once,
+                  client.evalsha @@ids_sent_once,
                                  identifiers.size,
                                  *identifiers,
                                  generate_intermediate_result_id,
@@ -135,8 +196,8 @@ module Picky
                 rescue RuntimeError => e
                   # Make the server have a SHA-1 for the script.
                   #
-                  @@sent_once = Digest::SHA1.hexdigest @@script
-                  client.eval @@script,
+                  @@ids_sent_once = Digest::SHA1.hexdigest @@ids_script
+                  client.eval @@ids_script,
                               identifiers.size,
                               *identifiers,
                               generate_intermediate_result_id,
@@ -191,8 +252,6 @@ module Picky
             end
           end
         else
-          # TODO Refactor!
-          #
           class << self
             def ids combinations, _, _
               # Get the ids for each combination.

data/lib/picky/backends/redis/list.rb

@@ -54,7 +54,7 @@ module Picky
 
         # Set a single list.
         #
-        # TODO We should optimize this by only adding. Maybe only add the difference?
+        # TODO Clear? Maybe only add the difference?
         #
         def []= key, values
           redis_key = "#{namespace}:#{key}"

data/lib/picky/backends/sqlite.rb

@@ -51,33 +51,6 @@ module Picky
           IntegerKeyArray.new(bundle.index_path(:realtime), self_indexed: self_indexed)
       end
 
-      # Returns the result ids for the allocation.
-      #
-      # Sorts the ids by size and & through them in the following order (sizes):
-      # 0. [100_000, 400, 30, 2]
-      # 1. [2, 30, 400, 100_000]
-      # 2. (100_000 & (400 & (30 & 2))) # => result
-      #
-      # Note: Uses a C-optimized intersection routine (in performant.c)
-      #       for speed and memory efficiency.
-      #
-      # Note: In the memory based version we ignore the amount and offset hints.
-      #       We cannot use the information to speed up the algorithm, unfortunately.
-      #
-      def ids combinations, _, _
-        # Get the ids for each combination.
-        #
-        id_arrays = combinations.inject([]) do |total, combination|
-          total << combination.ids
-        end
-
-        # Call the optimized C algorithm.
-        #
-        # Note: It orders the passed arrays by size.
-        #
-        Performant::Array.memory_efficient_intersect id_arrays
-      end
-
     end
 
   end

data/lib/picky/bundle.rb

@@ -57,7 +57,7 @@ module Picky
       reset_backend
     end
     def identifier
-      :"#{category.identifier}:#{name}"
+      @identifier ||= :"#{category.identifier}:#{name}"
     end
 
     # If no specific backend has been set,
@@ -160,7 +160,7 @@ module Picky
     #
     def delete
       @backend_inverted.delete       if @backend_inverted.respond_to? :delete
-      # TODO THINK about this. Perhaps the strategies should implement the backend methods?
+      # THINK about this. Perhaps the strategies should implement the backend methods?
       #
       @backend_weights.delete        if @backend_weights.respond_to?(:delete) && @weights_strategy.saved?
       @backend_similarity.delete     if @backend_similarity.respond_to? :delete

data/lib/picky/bundle_indexed.rb

@@ -64,7 +64,7 @@ module Picky
     # Loads the weights index.
     #
     def load_weights
-      # TODO THINK about this. Perhaps the strategies should implement the backend methods?
+      # THINK about this. Perhaps the strategies should implement the backend methods?
       #
       self.weights = @backend_weights.load if @weights_strategy.saved?
     end

data/lib/picky/bundle_indexing.rb

@@ -45,7 +45,7 @@ module Picky
     # Dumps the weights index.
     #
     def dump_weights
-      # TODO THINK about this. Perhaps the strategies should implement the backend methods? Or only the internal index ones?
+      # THINK about this. Perhaps the strategies should implement the backend methods? Or only the internal index ones?
       #
       @backend_weights.dump @weights if @weights_strategy.saved?
     end

data/lib/picky/categories_indexed.rb

@@ -27,18 +27,8 @@ module Picky
     # Returns all possible similar tokens for the given token.
     #
     def similar_tokens_for token
-      text = token.text
       categories.inject([]) do |result, category|
-        next_token = token
-        # Note: We could also break off here if not all the available
-        #       similars are needed.
-        #       Wait for a concrete case that needs this before taking
-        #       action.
-        #
-        while next_token = next_token.next_similar_token(category)
-          result << next_token if next_token && next_token.text != text
-        end
-        result
+        result + token.similar_tokens_for(category)
       end
     end
 

data/lib/picky/category.rb

@@ -2,9 +2,9 @@ module Picky
 
   class Category
 
+    attr_accessor :exact,
+                  :partial
     attr_reader :name,
-                :exact,
-                :partial,
                 :prepared,
                 :backend
 
@@ -41,7 +41,7 @@ module Picky
 
       # @symbols    = options[:use_symbols] || index.use_symbols? # TODO Symbols.
 
-      weights    = options[:weights]    || Generators::Weights::Default
+      weights    = options[:weight]     || Generators::Weights::Default
       partial    = options[:partial]    || Generators::Partial::Default
       similarity = options[:similarity] || Generators::Similarity::Default
 
@@ -49,7 +49,7 @@ module Picky
       no_similarity = Generators::Similarity::None.new
 
       @exact = Bundle.new :exact, self, weights, no_partial, similarity, options
-      if partial.use_exact_for_partial?
+      if partial.respond_to?(:use_exact_for_partial?) && partial.use_exact_for_partial?
         @partial = Wrappers::Bundle::ExactPartial.new @exact
       else
         @partial = Bundle.new :partial, self, weights, partial, no_similarity, options