picky 3.0.1 → 3.1.0

data/lib/picky/application.rb

@@ -15,14 +15,14 @@ module Picky
   # will generate an example <tt>project_name/app/application.rb</tt> file for you
   # with some example code inside.
   #
-  # == Indexes::Memory.new(name)
+  # == Index.new(name)
   #
-  # Next, define where your data comes from, creating an <tt>Index</tt>. You use the <tt>Indexes::Memory.new</tt> method for that:
-  #   my_index = Indexes::Memory.new :some_index_name
+  # Next, define where your data comes from, creating an <tt>Index</tt>. You use the <tt>Index.new</tt> method for that:
+  #   my_index = Index.new :some_index_name
   # You give the index a name (or identifier), and a source (see Sources), where its data comes from. Let's do that:
   #   class MyGreatSearch < Application
   #
-  #     books = Indexes::Memory.new :books do
+  #     books = Index.new :books do
   #       source Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
   #     end
   #
@@ -31,7 +31,7 @@ module Picky
   #
   # That on itself won't do much good.
   #
-  # Note that a Redis index is also available: Indexes::Redis.new.
+  # Note that a Redis index is also available: Index.new.
   #
   # == category(identifier, options = {})
   #
@@ -43,7 +43,7 @@ module Picky
   # Let's go ahead and define a category:
   #   class MyGreatSearch < Application
   #
-  #     books = Indexes::Memory.new :books do
+  #     books = Index.new :books do
   #       source   Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
   #       category :title
   #     end
@@ -70,7 +70,7 @@ module Picky
   # In full glory:
   #   class MyGreatSearch < Application
   #
-  #     books = Indexes::Memory.new :books do
+  #     books = Index.new :books do
   #       source   Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
   #       category :title
   #     end
@@ -134,7 +134,7 @@ module Picky
   #               substitutes_characters_with: CharacterSubstituters::WestEuropean.new,
   #               maximum_tokens: 4
   #
-  #     books = Indexes::Memory.new :books do
+  #     books = Index.new :books do
   #       source   Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
   #       category :title,
   #                qualifiers: [:t, :title, :titre],
@@ -163,14 +163,14 @@ module Picky
       # is used for indexing by default.
       #
       def indexing options = {}
-        Tokenizers::Index.default = Tokenizers::Index.new(options)
+        Tokenizer.index_default = Tokenizer.new(options)
       end
 
       # Returns a configured tokenizer that
       # is used for querying by default.
       #
       def searching options = {}
-        Tokenizers::Query.default = Tokenizers::Query.new(options)
+        Tokenizer.query_default = Tokenizer.new(options)
       end
 
       # Routes.
@@ -240,10 +240,10 @@ module Picky
       def to_stats
         <<-APP
   \033[1mIndexing (default)\033[m:
-  #{Tokenizers::Index.default.indented_to_s}
+  #{Tokenizer.index_default.indented_to_s}
 
   \033[1mQuerying (default)\033[m:
-  #{Tokenizers::Query.default.indented_to_s}
+  #{Tokenizer.query_default.indented_to_s}
 
   \033[1mIndexes\033[m:
   #{Indexes.to_s.indented_to_s}

data/lib/picky/backends/backend.rb

@@ -0,0 +1,17 @@
+module Picky
+
+  module Backends
+
+    class Backend
+
+      #
+      #
+      def to_s
+        "#{self.class}(#{[inverted, weights, similarity, configuration].join(', ')})"
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/{backend → backends}/file/basic.rb

@@ -1,6 +1,6 @@
 module Picky
 
-  module Backend
+  module Backends
 
     # Handles all aspects of index files, such as dumping/loading.
     #

data/lib/picky/{backend → backends}/file/json.rb

@@ -1,6 +1,6 @@
 module Picky
 
-  module Backend
+  module Backends
 
     module File
 

data/lib/picky/{backend → backends}/file/marshal.rb

@@ -1,6 +1,6 @@
 module Picky
 
-  module Backend
+  module Backends
 
     module File
 

data/lib/picky/{backend → backends}/file/text.rb

@@ -1,6 +1,6 @@
 module Picky
 
-  module Backend
+  module Backends
 
     module File
 

data/lib/picky/backends/memory.rb

@@ -0,0 +1,53 @@
+module Picky
+
+  module Backends
+
+    class Memory < Backend
+
+      def create_inverted bundle
+        File::JSON.new bundle.index_path(:inverted)
+      end
+      def create_weights bundle
+        File::JSON.new bundle.index_path(:weights)
+      end
+      def create_similarity bundle
+        File::Marshal.new bundle.index_path(:similarity)
+      end
+      def create_configuration bundle
+        File::JSON.new bundle.index_path(:configuration)
+      end
+
+      # Returns the result ids for the allocation.
+      #
+      # Sorts the ids by size and & through them in the following order (sizes):
+      # 0. [100_000, 400, 30, 2]
+      # 1. [2, 30, 400, 100_000]
+      # 2. (100_000 & (400 & (30 & 2))) # => result
+      #
+      # Note: Uses a C-optimized intersection routine (in performant.c)
+      #       for speed and memory efficiency.
+      #
+      # Note: In the memory based version we ignore the amount and offset hints.
+      #       We cannot use the information to speed up the algorithm, unfortunately.
+      #
+      def ids combinations, _, _
+        return [] if combinations.empty?
+
+        # Get the ids for each combination.
+        #
+        id_arrays = combinations.inject([]) do |total, combination|
+          total << combination.ids
+        end
+
+        # Call the optimized C algorithm.
+        #
+        # Note: It orders the passed arrays by size.
+        #
+        Performant::Array.memory_efficient_intersect id_arrays
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/{backend → backends}/redis/basic.rb

@@ -1,6 +1,6 @@
 module Picky
 
-  module Backend
+  module Backends
 
     class Redis
 
@@ -13,34 +13,29 @@ module Picky
       #
       class Basic
 
-        attr_reader :namespace, :backend
+        attr_reader :client, :namespace
 
         # An index cache takes a path, without file extension,
         # which will be provided by the subclasses.
         #
-        def initialize namespace
+        def initialize client, namespace
+          @client    = client
           @namespace = namespace
-
-          # TODO Turn this inside out such that people can pass in
-          # their own Redis instance.
-          #
-          # TODO Make the :db a real option.
-          #
-          @backend = ::Redis.new :db => 15
         end
 
-        # Does nothing.
+        # Returns itself.
         #
         def load
-          # Nothing.
+          self
         end
+
         # We do not use Redis to retrieve data.
         #
         def retrieve
           # Nothing.
         end
 
-        # Redis does not backup.
+        # Redis doesn't do backup.
         #
         def backup
           # Nothing.
@@ -79,7 +74,7 @@ module Picky
         #       DBSIZE KEYPATTERN, we are stuck with this.
         #
         def size
-          backend.dbsize
+          client.dbsize
         end
 
         #

data/lib/picky/backends/redis/float_hash.rb

@@ -0,0 +1,26 @@
+module Picky
+
+  module Backends
+
+    class Redis
+
+      class FloatHash < StringHash
+
+        # Get a single value.
+        #
+        # Internal API method for the index.
+        #
+        # Note: Works like the StringHash method, but
+        # returns a float corresponding to that string.
+        #
+        def [] key
+          super.to_f
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/{backend → backends}/redis/list_hash.rb

@@ -1,6 +1,6 @@
 module Picky
 
-  module Backend
+  module Backends
 
     class Redis
 
@@ -15,7 +15,7 @@ module Picky
             i = 0
             values.each do |value|
               i += 1
-              backend.zadd redis_key, i, value
+              client.zadd redis_key, i, value
             end
           end
         end
@@ -27,21 +27,17 @@ module Picky
         #
         def clear
           redis_key = "#{namespace}:*"
-          backend.keys(redis_key).each do |key|
-            backend.del key
+          client.keys(redis_key).each do |key|
+            client.del key
           end
         end
 
         # Get a collection.
         #
-        def collection key
-          backend.zrange "#{namespace}:#{key}", 0, -1
-        end
-
-        # Get a single value.
+        # Internal API method for the index.
         #
-        def member key
-          raise "Can't retrieve single value :#{key} from a Redis ListHash. Use Indexes::Redis::StringHash."
+        def [] key
+          client.zrange "#{namespace}:#{key}", 0, -1
         end
 
       end

data/lib/picky/{backend → backends}/redis/string_hash.rb

@@ -1,6 +1,6 @@
 module Picky
 
-  module Backend
+  module Backends
 
     class Redis
 
@@ -13,26 +13,22 @@ module Picky
         def dump hash
           clear
           hash.each_pair do |key, value|
-            backend.hset namespace, key, value
+            client.hset namespace, key, value
           end
         end
 
         # Clears the hash.
         #
         def clear
-          backend.del namespace
-        end
-
-        # Get a collection.
-        #
-        def collection key
-          raise "Can't retrieve collection for :#{key} from a StringHash. Use Indexes::Redis::ListHash."
+          client.del namespace
         end
 
         # Get a single value.
         #
-        def member key
-          backend.hget namespace, key
+        # Internal API method for the index.
+        #
+        def [] key
+          client.hget namespace, key
         end
 
       end

data/lib/picky/backends/redis.rb

@@ -0,0 +1,87 @@
+module Picky
+
+  module Backends
+
+    #
+    #
+    class Redis < Backend
+
+      attr_reader :client
+
+      def initialize options = {}
+        @client = options[:client] || ::Redis.new(:db => (options[:db] || 15))
+      end
+
+      def create_inverted bundle
+        Redis::ListHash.new client, "#{bundle.identifier}:inverted"
+      end
+      def create_weights bundle
+        Redis::FloatHash.new client, "#{bundle.identifier}:weights"
+      end
+      def create_similarity bundle
+        Redis::ListHash.new client, "#{bundle.identifier}:similarity"
+      end
+      def create_configuration bundle
+        Redis::StringHash.new client, "#{bundle.identifier}:configuration"
+      end
+
+      # Returns the result ids for the allocation.
+      #
+      # Developers wanting to program fast intersection
+      # routines, can do so analogue to this in their own
+      # backend implementations.
+      #
+      # Note: We use the amount and offset hints to speed Redis up.
+      #
+      def ids combinations, amount, offset
+        return [] if combinations.empty?
+
+        identifiers = combinations.inject([]) do |identifiers, combination|
+          identifiers << "#{combination.identifier}"
+        end
+
+        result_id = generate_intermediate_result_id
+
+        # Intersect and store.
+        #
+        client.zinterstore result_id, identifiers
+
+        # Get the stored result.
+        #
+        results = client.zrange result_id, offset, (offset + amount)
+
+        # Delete the stored result as it was only for temporary purposes.
+        #
+        # Note: I could also not delete it, but that would not be clean at all.
+        #
+        client.del result_id
+
+        results
+      end
+
+      # Generate a multiple host/process safe result id.
+      #
+      # Note: Generated when this class loads.
+      #
+      require 'socket'
+      def self.extract_host
+        @host ||= Socket.gethostname
+      end
+      def host
+        self.class.extract_host
+      end
+      extract_host
+      def pid
+        @pid ||= Process.pid
+      end
+      # Use the host and pid (generated lazily in child processes) for the result.
+      #
+      def generate_intermediate_result_id
+        :"#{host}:#{pid}:picky:result"
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/bundle.rb

@@ -23,28 +23,37 @@ module Picky
   class Bundle
 
     attr_reader :name,
-                :category
+                :category,
+                :backend
 
     attr_accessor :inverted,
                   :weights,
                   :similarity,
                   :configuration,
+
+                  :backend_inverted,
+                  :backend_weights,
+                  :backend_similarity,
+                  :backend_configuration,
+
                   :similarity_strategy
 
-    delegate :clear,           :to => :inverted
     delegate :[], :[]=,        :to => :configuration
     delegate :index_directory, :to => :category
 
-    def initialize name, category, similarity_strategy, options = {}
-      @name          = name
-      @category      = category
-
-      @inverted      = {}
-      @weights       = {}
-      @similarity    = {}
-      @configuration = {} # A hash with config options.
-
+    def initialize name, category, backend, similarity_strategy, options = {}
+      @name                = name
+      @category            = category
       @similarity_strategy = similarity_strategy
+
+      # Extract specific indexes from backend.
+      #
+      # TODO Clean up all related.
+      #
+      @backend_inverted      = backend.create_inverted      self
+      @backend_weights       = backend.create_weights       self
+      @backend_similarity    = backend.create_similarity    self
+      @backend_configuration = backend.create_configuration self
     end
     def identifier
       "#{category.identifier}:#{name}"
@@ -79,6 +88,93 @@ module Picky
       ::File.join index_directory, "#{category.name}_#{name}_#{type}"
     end
 
+    # Copies the indexes to the "backup" directory.
+    #
+    def backup
+      @backend_inverted.backup      if @backend_inverted.respond_to? :backup
+      @backend_weights.backup       if @backend_weights.respond_to? :backup
+      @backend_similarity.backup    if @backend_similarity.respond_to? :backup
+      @backend_configuration.backup if @backend_configuration.respond_to? :backup
+    end
+
+    # Restores the indexes from the "backup" directory.
+    #
+    def restore
+      @backend_inverted.restore       if @backend_inverted.respond_to? :restore
+      @backend_weights.restore        if @backend_weights.respond_to? :restore
+      @backend_similarity.restore     if @backend_similarity.respond_to? :restore
+      @backend_configuration.restore  if @backend_configuration.respond_to? :restore
+    end
+
+    # Delete all index files.
+    #
+    def delete
+      @backend_inverted.delete       if @backend_inverted.respond_to? :delete
+      @backend_weights.delete        if @backend_weights.respond_to? :delete
+      @backend_similarity.delete     if @backend_similarity.respond_to? :delete
+      @backend_configuration.delete  if @backend_configuration.respond_to? :delete
+    end
+
+    # Alerts the user if an index is missing.
+    #
+    def raise_unless_cache_exists
+      raise_unless_index_exists
+      raise_unless_similarity_exists
+    end
+    # Alerts the user if one of the necessary indexes
+    # (core, weights) is missing.
+    #
+    def raise_unless_index_exists
+      if partial_strategy.saved?
+        warn_if_index_small
+        raise_unless_index_ok
+      end
+    end
+    # Alerts the user if the similarity
+    # index is missing (given that it's used).
+    #
+    def raise_unless_similarity_exists
+      if similarity_strategy.saved?
+        warn_if_similarity_small
+        raise_unless_similarity_ok
+      end
+    end
+
+    # Outputs a warning for the given cache.
+    #
+    def warn_cache_small what
+      warn "Warning: #{what} cache for #{identifier} smaller than 16 bytes."
+    end
+    # Raises an appropriate error message for the given cache.
+    #
+    def raise_cache_missing what
+      raise "Error: The #{what} cache for #{identifier} is missing."
+    end
+
+    # Warns the user if the similarity index is small.
+    #
+    def warn_if_similarity_small
+      warn_cache_small :similarity if backend_similarity.respond_to?(:cache_small?) && backend_similarity.cache_small?
+    end
+    # Alerts the user if the similarity index is not there.
+    #
+    def raise_unless_similarity_ok
+      raise_cache_missing :similarity if backend_similarity.respond_to?(:cache_ok?) && !backend_similarity.cache_ok?
+    end
+
+    # Warns the user if the core or weights indexes are small.
+    #
+    def warn_if_index_small
+      warn_cache_small :inverted if backend_inverted.respond_to?(:cache_small?) && backend_inverted.cache_small?
+      warn_cache_small :weights  if backend_weights.respond_to?(:cache_small?)  && backend_weights.cache_small?
+    end
+    # Alerts the user if the core or weights indexes are not there.
+    #
+    def raise_unless_index_ok
+      raise_cache_missing :inverted if backend_inverted.respond_to?(:cache_ok?) && !backend_inverted.cache_ok?
+      raise_cache_missing :weights  if backend_weights.respond_to?(:cache_ok?)  && !backend_weights.cache_ok?
+    end
+
     def to_s
       "#{self.class}(#{identifier})"
     end

data/lib/picky/category.rb

@@ -35,16 +35,16 @@ module Picky
       partial    = options[:partial]    || Generators::Partial::Default
       similarity = options[:similarity] || Generators::Similarity::Default
 
-      @indexing_exact   = index.indexing_bundle_class.new :exact,   self, weights, Generators::Partial::None.new, similarity, options
-      @indexing_partial = index.indexing_bundle_class.new :partial, self, weights, partial, Generators::Similarity::None.new, options
+      @indexing_exact   = Indexing::Bundle.new  :exact,  self, index.backend, weights, Generators::Partial::None.new, similarity, options
+      @indexing_partial = Indexing::Bundle.new :partial, self, index.backend, weights, partial, Generators::Similarity::None.new, options
 
       # Indexed.
       #
-      @indexed_exact  = index.indexed_bundle_class.new  :exact, self, similarity
+      @indexed_exact  = Indexed::Bundle.new :exact, self, index.backend, similarity
       if partial.use_exact_for_partial?
         @indexed_partial  = @indexed_exact
       else
-        @indexed_partial  = index.indexed_bundle_class.new  :partial, self, similarity
+        @indexed_partial  = Indexed::Bundle.new :partial, self, index.backend, similarity
       end
 
       # @exact   = exact_lambda.call(@exact, @partial)   if exact_lambda   = options[:exact_lambda]
@@ -92,7 +92,7 @@ module Picky
     # Note: If you don't use it with the block, do not forget to close it.
     #
     def prepared_index_file &block
-      @prepared_index_file ||= Backend::File::Text.new prepared_index_path
+      @prepared_index_file ||= Backends::File::Text.new prepared_index_path
       @prepared_index_file.open &block
     end
     # Creates the index directory including all necessary paths above it.