picky 3.5.4 → 3.6.0

data/lib/picky/backends/backend.rb

@@ -23,7 +23,7 @@ module Picky
       #
       #
       def to_s
-        "#{self.class}(#{[inverted, weights, similarity, configuration].join(', ')})"
+        self.class.name
       end
 
     end

data/lib/picky/backends/file/basic.rb

@@ -55,26 +55,6 @@ module Picky
           @initial && @initial.clone || nil
         end
 
-        # Will copy the index file to a location that
-        # is in a directory named "backup" right under
-        # the directory the index file is in.
-        #
-        def backup
-          mapping_file.backup
-
-          prepare_backup backup_directory(cache_path)
-          FileUtils.cp cache_path, target, verbose: true
-        end
-
-        # Copies the file from its backup location back
-        # to the original location.
-        #
-        def restore
-          mapping_file.restore
-
-          FileUtils.cp backup_file_path_of(cache_path), cache_path, verbose: true
-        end
-
         # Deletes the file.
         #
         def delete
@@ -83,22 +63,6 @@ module Picky
           `rm -Rf #{cache_path}`
         end
 
-        # Is this cache file suspiciously small?
-        # (less than 8 Bytes of size)
-        #
-        def cache_small?
-          size_of(cache_path) < 8
-        end
-
-        # Is the cache ok? (existing and larger than
-        # zero Bytes in size)
-        #
-        # A small cache is still ok.
-        #
-        def cache_ok?
-          size_of(cache_path) > 0
-        end
-
         #
         #
         def to_s

data/lib/picky/backends/file/json.rb

@@ -84,4 +84,4 @@ module Picky
 
   end
 
-end
+end

data/lib/picky/backends/file.rb

@@ -15,28 +15,28 @@ module Picky
       #
       def create_inverted bundle
         extract_lambda_or(inverted, bundle) ||
-        JSON.new(bundle.index_path(:inverted))
+          JSON.new(bundle.index_path(:inverted))
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:token] # => 1.23 (a weight)
       #
       def create_weights bundle
         extract_lambda_or(weights, bundle) ||
-        JSON.new(bundle.index_path(:weights))
+          JSON.new(bundle.index_path(:weights))
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:encoded] # => [:original, :original] (an array of original symbols this similarity encoded thing maps to)
       #
       def create_similarity bundle
         extract_lambda_or(similarity, bundle) ||
-        JSON.new(bundle.index_path(:similarity))
+          JSON.new(bundle.index_path(:similarity))
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:key] # => value (a value for this config key)
       #
       def create_configuration bundle
         extract_lambda_or(configuration, bundle) ||
-        JSON.new(bundle.index_path(:configuration))
+          JSON.new(bundle.index_path(:configuration))
       end
 
       # Currently, the loaded ids are intersected using

data/lib/picky/backends/helpers/file.rb

@@ -14,32 +14,6 @@ module Picky
           FileUtils.mkdir_p ::File.dirname(path)
         end
 
-        # The backup directory of this file.
-        # Equal to the file's dirname plus /backup
-        #
-        def backup_directory path
-          ::File.join ::File.dirname(path), 'backup'
-        end
-
-        # Prepares the backup directory for the file.
-        #
-        def prepare_backup target
-          FileUtils.mkdir target unless Dir.exists?(target)
-        end
-
-        # The backup filename.
-        #
-        def backup_file_path_of path
-          dir, name = ::File.split path
-          ::File.join dir, 'backup', name
-        end
-
-        # Extracts the size of the file in Bytes.
-        #
-        def size_of path
-          `ls -l #{path} | awk '{print $5}'`.to_i
-        end
-
       end
 
     end

data/lib/picky/backends/memory/basic.rb

@@ -47,47 +47,12 @@ module Picky
           @initial && @initial.clone || {}
         end
 
-        # Will copy the index file to a location that
-        # is in a directory named "backup" right under
-        # the directory the index file is in.
-        #
-        def backup
-          prepare_backup backup_directory
-          FileUtils.cp cache_path, target, verbose: true
-        end
-
-        # Copies the file from its backup location back
-        # to the original location.
-        #
-        def restore
-          FileUtils.cp backup_file_path_of(cache_path), cache_path, verbose: true
-        end
-
         # Deletes the file.
         #
         def delete
           `rm -Rf #{cache_path}`
         end
 
-        # Checks.
-        #
-
-        # Is this cache file suspiciously small?
-        # (less than 8 Bytes of size)
-        #
-        def cache_small?
-          size_of(cache_path) < 8
-        end
-
-        # Is the cache ok? (existing and larger than
-        # zero Bytes in size)
-        #
-        # A small cache is still ok.
-        #
-        def cache_ok?
-          size_of(cache_path) > 0
-        end
-
         #
         #
         def to_s

data/lib/picky/backends/memory.rb

@@ -9,28 +9,28 @@ module Picky
       #
       def create_inverted bundle
         extract_lambda_or(inverted, bundle) ||
-        JSON.new(bundle.index_path(:inverted))
+          JSON.new(bundle.index_path(:inverted))
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:token] # => 1.23 (a weight)
       #
       def create_weights bundle
         extract_lambda_or(weights, bundle) ||
-        JSON.new(bundle.index_path(:weights))
+          JSON.new(bundle.index_path(:weights))
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:encoded] # => [:original, :original] (an array of original symbols this similarity encoded thing maps to)
       #
       def create_similarity bundle
         extract_lambda_or(similarity, bundle) ||
-        Marshal.new(bundle.index_path(:similarity))
+          Marshal.new(bundle.index_path(:similarity))
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:key] # => value (a value for this config key)
       #
       def create_configuration bundle
         extract_lambda_or(configuration, bundle) ||
-        JSON.new(bundle.index_path(:configuration))
+          JSON.new(bundle.index_path(:configuration))
       end
 
       # Returns the result ids for the allocation.

data/lib/picky/backends/redis/basic.rb

@@ -54,12 +54,6 @@ module Picky
           # Nothing.
         end
 
-        # Redis doesn't do backup.
-        #
-        def backup
-          # Nothing.
-        end
-
         # Deletes the Redis index namespace.
         #
         def delete
@@ -69,33 +63,6 @@ module Picky
           #       we don't do anything.
         end
 
-        # Checks.
-        #
-
-        # Is this cache suspiciously small?
-        #
-        def cache_small?
-          size < 1
-        end
-
-        # Is the cache ok?
-        #
-        # A small cache is still ok.
-        #
-        def cache_ok?
-          size > 0
-        end
-
-        # Extracts the size of the file in Bytes.
-        #
-        # Note: This is a very forgiving implementation.
-        #       But as long as Redis does not implement
-        #       DBSIZE KEYPATTERN, we are stuck with this.
-        #
-        def size
-          client.dbsize
-        end
-
         #
         #
         def to_s

data/lib/picky/backends/redis.rb

@@ -22,28 +22,28 @@ module Picky
       #
       def create_inverted bundle
         extract_lambda_or(inverted, bundle, client) ||
-        List.new(client, "#{bundle.identifier}:inverted")
+          List.new(client, "#{bundle.identifier}:inverted")
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:token] # => 1.23 (a weight)
       #
       def create_weights bundle
         extract_lambda_or(weights, bundle, client) ||
-        Float.new(client, "#{bundle.identifier}:weights")
+          Float.new(client, "#{bundle.identifier}:weights")
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:encoded] # => [:original, :original] (an array of original symbols this similarity encoded thing maps to)
       #
       def create_similarity bundle
         extract_lambda_or(similarity, bundle, client) ||
-        List.new(client, "#{bundle.identifier}:similarity")
+          List.new(client, "#{bundle.identifier}:similarity")
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:key] # => value (a value for this config key)
       #
       def create_configuration bundle
         extract_lambda_or(configuration, bundle, client) ||
-        String.new(client, "#{bundle.identifier}:configuration")
+          String.new(client, "#{bundle.identifier}:configuration")
       end
 
       # Does the Redis version already include

data/lib/picky/backends/sqlite/db.rb

@@ -0,0 +1,84 @@
+module Picky
+
+  module Backends
+
+    class Sqlite
+
+      class DB
+
+        include Helpers::File
+
+        attr_reader :cache_path, :db
+
+        def initialize cache_path, options = {}
+          @cache_path = "#{cache_path}.sqlite3"
+          @empty      = options[:empty]
+          @initial    = options[:initial]
+        end
+
+        def initial
+          @initial && @initial.clone || {}
+        end
+
+        def empty
+          @empty && @empty.clone || {}
+        end
+
+        def lazily_initialize_client
+          @db ||= SQLite3::Database.new cache_path
+        end
+
+        def dump_sqlite internal
+          lazily_initialize_client
+
+          # TODO Does it make a difference if these
+          #      statements are given as one to the
+          #      @db.execute method?
+          #
+
+          db.execute 'drop table if exists key_value;'
+          db.execute 'create table key_value (key varchar(255), value text);'
+          db.execute 'create index key_idx on key_value (key);'
+          db.execute 'BEGIN;'
+
+          # Note: Internal structures need to
+          #       implement each.
+          #
+          internal.each do |key, value|
+            encoded_value = Yajl::Encoder.encode value
+            db.execute 'insert into key_value values (?,?)', key.to_s, encoded_value
+          end
+
+          db.execute 'COMMIT;'
+        end
+
+        def dump internal
+          create_directory cache_path
+          dump_sqlite internal
+        end
+
+        def load
+          lazily_initialize_client
+          self
+        end
+
+        def [] key
+          res = db.execute "select value from key_value where key = ? limit 1;", key.to_s
+          return nil if res.empty?
+
+          # TODO Slightly speed up by not calling Yajl for the || case?
+          #
+          Yajl::Parser.parse res.first.first || ""
+        end
+
+        def to_s
+          "#{self.class}(#{cache_path})"
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/backends/sqlite.rb

@@ -0,0 +1,75 @@
+module Picky
+
+  module Backends
+
+    class Sqlite < Backend
+
+      def initialize options = {}
+        super options
+
+        require 'sqlite3'
+      rescue LoadError => e
+        warn_gem_missing 'sqlite3', 'SQLite bindings'
+      end
+
+      # Returns an object that on #initial, #load returns an object that responds to:
+      #   [:token] # => [id, id, id, id, id] (an array of ids)
+      #
+      def create_inverted bundle
+        extract_lambda_or(inverted, bundle) ||
+        DB.new(bundle.index_path(:inverted))
+      end
+      # Returns an object that on #initial, #load returns an object that responds to:
+      #   [:token] # => 1.23 (a weight)
+      #
+      def create_weights bundle
+        extract_lambda_or(weights, bundle) ||
+        DB.new(bundle.index_path(:weights))
+      end
+      # Returns an object that on #initial, #load returns an object that responds to:
+      #   [:encoded] # => [:original, :original] (an array of original symbols this similarity encoded thing maps to)
+      #
+      def create_similarity bundle
+        extract_lambda_or(similarity, bundle) ||
+        DB.new(bundle.index_path(:similarity))
+      end
+      # Returns an object that on #initial, #load returns an object that responds to:
+      #   [:key] # => value (a value for this config key)
+      #
+      def create_configuration bundle
+        extract_lambda_or(configuration, bundle) ||
+        DB.new(bundle.index_path(:configuration))
+      end
+
+      # Returns the result ids for the allocation.
+      #
+      # Sorts the ids by size and & through them in the following order (sizes):
+      # 0. [100_000, 400, 30, 2]
+      # 1. [2, 30, 400, 100_000]
+      # 2. (100_000 & (400 & (30 & 2))) # => result
+      #
+      # Note: Uses a C-optimized intersection routine (in performant.c)
+      #       for speed and memory efficiency.
+      #
+      # Note: In the memory based version we ignore the amount and offset hints.
+      #       We cannot use the information to speed up the algorithm, unfortunately.
+      #
+      def ids combinations, _, _
+        # Get the ids for each combination.
+        #
+        id_arrays = combinations.inject([]) do |total, combination|
+          total << combination.ids
+        end
+
+        # Call the optimized C algorithm.
+        #
+        # Note: It orders the passed arrays by size.
+        #
+        Performant::Array.memory_efficient_intersect id_arrays
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/bundle.rb

@@ -55,6 +55,19 @@ module Picky
       @partial_strategy    = partial_strategy
       @similarity_strategy = similarity_strategy
 
+      reset_backend backend
+    end
+    def identifier
+      "#{category.identifier}:#{name}"
+    end
+
+    # Resets the backend with the given one instantly.
+    #
+    # TODO Redesign such that the backend is only
+    #      generated lazily.
+    #      And reset using backend = nil.
+    #
+    def reset_backend backend
       # Extract specific indexes from backend.
       #
       # TODO Clean up all related.
@@ -74,10 +87,7 @@ module Picky
       @similarity    = @backend_similarity.initial
       @configuration = @backend_configuration.initial
 
-      @realtime_mapping = {} # id -> ary of syms.  TODO Always instantiate?
-    end
-    def identifier
-      "#{category.identifier}:#{name}"
+      @realtime_mapping = {} # id -> ary of syms.  TODO Always instantiate? d
     end
 
     # "Empties" the index(es) by getting a new empty
@@ -136,99 +146,17 @@ module Picky
       ::File.join index_directory, "#{category.name}_#{name}#{ "_#{type}" if type }"
     end
 
-    # Copies the indexes to the "backup" directory.
-    #
-    def backup
-      @backend_inverted.backup      if @backend_inverted.respond_to? :backup
-      # TODO THINK about this. Perhaps the strategies should implement the backend methods?
-      #
-      @backend_weights.backup       if @backend_weights.respond_to? :backup && @weights_strategy.saved?
-      @backend_similarity.backup    if @backend_similarity.respond_to? :backup
-      @backend_configuration.backup if @backend_configuration.respond_to? :backup
-    end
-
-    # Restores the indexes from the "backup" directory.
-    #
-    def restore
-      @backend_inverted.restore       if @backend_inverted.respond_to? :restore
-      # TODO THINK about this. Perhaps the strategies should implement the backend methods?
-      #
-      @backend_weights.restore        if @backend_weights.respond_to? :restore && @weights_strategy.saved?
-      @backend_similarity.restore     if @backend_similarity.respond_to? :restore
-      @backend_configuration.restore  if @backend_configuration.respond_to? :restore
-    end
-
     # Delete all index files.
     #
     def delete
       @backend_inverted.delete       if @backend_inverted.respond_to? :delete
       # TODO THINK about this. Perhaps the strategies should implement the backend methods?
       #
-      @backend_weights.delete        if @backend_weights.respond_to? :delete && @weights_strategy.saved?
+      @backend_weights.delete        if @backend_weights.respond_to?(:delete) && @weights_strategy.saved?
       @backend_similarity.delete     if @backend_similarity.respond_to? :delete
       @backend_configuration.delete  if @backend_configuration.respond_to? :delete
     end
 
-    # Alerts the user if an index is missing.
-    #
-    def raise_unless_cache_exists
-      raise_unless_index_exists
-      raise_unless_similarity_exists
-    end
-    # Alerts the user if one of the necessary indexes
-    # (core, weights) is missing.
-    #
-    def raise_unless_index_exists
-      if partial_strategy.saved?
-        warn_if_index_small
-        raise_unless_index_ok
-      end
-    end
-    # Alerts the user if the similarity
-    # index is missing (given that it's used).
-    #
-    def raise_unless_similarity_exists
-      if similarity_strategy.saved?
-        warn_if_similarity_small
-        raise_unless_similarity_ok
-      end
-    end
-
-    # Outputs a warning for the given cache.
-    #
-    def warn_cache_small what
-      warn "Warning: #{what} cache for #{identifier} smaller than 16 bytes."
-    end
-    # Raises an appropriate error message for the given cache.
-    #
-    def raise_cache_missing what
-      raise "Error: The #{what} cache for #{identifier} is missing."
-    end
-
-    # Warns the user if the similarity index is small.
-    #
-    def warn_if_similarity_small
-      warn_cache_small :similarity if backend_similarity.respond_to?(:cache_small?) && backend_similarity.cache_small?
-    end
-    # Alerts the user if the similarity index is not there.
-    #
-    def raise_unless_similarity_ok
-      raise_cache_missing :similarity if backend_similarity.respond_to?(:cache_ok?) && !backend_similarity.cache_ok?
-    end
-
-    # Warns the user if the core or weights indexes are small.
-    #
-    def warn_if_index_small
-      warn_cache_small :inverted if backend_inverted.respond_to?(:cache_small?) && backend_inverted.cache_small?
-      warn_cache_small :weights  if backend_weights.respond_to?(:cache_small?)  && backend_weights.cache_small?
-    end
-    # Alerts the user if the core or weights indexes are not there.
-    #
-    def raise_unless_index_ok
-      raise_cache_missing :inverted if backend_inverted.respond_to?(:cache_ok?) && !backend_inverted.cache_ok?
-      raise_cache_missing :weights  if backend_weights.respond_to?(:cache_ok?)  && !backend_weights.cache_ok?
-    end
-
     def to_s
       "#{self.class}(#{identifier})"
     end

data/lib/picky/bundle_realtime.rb

@@ -42,7 +42,7 @@ module Picky
       #
       ids = if str_or_syms.include? str_or_sym
         ids = @inverted[str_or_sym]
-        ids.delete  id
+        ids.delete id
         ids.send where, id
       else
         str_or_syms << str_or_sym

data/lib/picky/categories.rb

@@ -15,6 +15,7 @@ module Picky
                   :empty,
                   :index,
                   :reindex,
+                  :reset_backend,
                   :to => :categories
 
     # A list of indexed categories.

data/lib/picky/categories_indexed.rb

@@ -2,7 +2,7 @@ module Picky
 
   class Categories
 
-    each_delegate :load_from_cache,
+    each_delegate :load,
                   :analyze,
                   :to => :categories
 

data/lib/picky/categories_indexing.rb

@@ -3,10 +3,8 @@ module Picky
   class Categories
 
     each_delegate :cache,
-                  :check,
                   :clear,
-                  :backup,
-                  :restore,
+                  :prepare,
                   :to => :categories
 
   end

data/lib/picky/category.rb

@@ -35,6 +35,7 @@ module Picky
       @key_format = options[:key_format]
       # @symbols    = options[:use_symbols] || index.use_symbols? # TODO Symbols.
       @qualifiers = extract_qualifiers_from options
+      # TODO backend option!
 
       weights    = options[:weights]    || Generators::Weights::Default
       partial    = options[:partial]    || Generators::Partial::Default
@@ -53,19 +54,32 @@ module Picky
       @prepared = Backends::Memory::Text.new prepared_index_path
     end
 
-    # Indexes and reloads the category.
+    # Indexes and loads the category.
     #
     def reindex
       index
-      reload
+      load
     end
 
+    # Dumps both bundles.
+    #
     def dump
       exact.dump
       partial.dump
       timed_exclaim %Q{"#{identifier}": Generated -> #{index_directory.gsub("#{PICKY_ROOT}/", '')}.}
     end
 
+    # Resets backends in both bundles.
+    #
+    # This will only set the backend
+    #
+    def reset_backend backend
+      unless @backend
+        exact.reset_backend backend
+        partial.reset_backend backend
+      end
+    end
+
     # Index name.
     #
     def index_name