picky 3.6.7 → 3.6.8

data/lib/picky/backends/file/basic.rb

@@ -52,7 +52,7 @@ module Picky
         #       as in #load.
         #
         def initial
-          @initial && @initial.clone || nil
+          @initial && @initial.clone || {}
         end
 
         # Deletes the file.

data/lib/picky/backends/file/json.rb

@@ -8,11 +8,15 @@ module Picky
       #
       class JSON < Basic
 
+        attr_writer :mapping
+
         # The in-memory mapping hash, mapping
         # a Symbol key to [length, offset] of
         # the JSON data in the file.
         #
-        attr_accessor :mapping
+        def mapping
+          @mapping || raise("The File index/category needs to be loaded first.")
+        end
 
         # See lib/picky/backends/file.rb for what this should return.
         #

data/lib/picky/backends/file.rb

@@ -38,6 +38,13 @@ module Picky
         extract_lambda_or(configuration, bundle) ||
           JSON.new(bundle.index_path(:configuration))
       end
+      # Returns an object that on #initial, #load returns an object that responds to:
+      #   [id] # => [:sym1, :sym2]
+      #
+      def create_realtime bundle
+        extract_lambda_or(similarity, bundle) ||
+          JSON.new(bundle.index_path(:realtime))
+      end
 
       # Currently, the loaded ids are intersected using
       # the fast C-based intersection.

data/lib/picky/backends/memory.rb

@@ -32,6 +32,13 @@ module Picky
         extract_lambda_or(configuration, bundle) ||
           JSON.new(bundle.index_path(:configuration))
       end
+      # Returns an object that on #initial, #load returns an object that responds to:
+      #   [id] # => [:sym1, :sym2]
+      #
+      def create_realtime bundle
+        extract_lambda_or(similarity, bundle) ||
+          JSON.new(bundle.index_path(:realtime))
+      end
 
       # Returns the result ids for the allocation.
       #

data/lib/picky/backends/redis/basic.rb

@@ -24,13 +24,14 @@ module Picky
 
           @empty     = options[:empty]
           @initial   = options[:initial]
+          @immediate = options[:immediate]
         end
 
         # The empty index that is used for putting the index
         # together.
         #
         def empty
-          @empty && @empty.clone || {}
+          @empty && @empty.clone || (@immediate ? self : {})
         end
 
         # The initial content before loading.
@@ -39,7 +40,7 @@ module Picky
         #       this just returns the same thing as #load.
         #
         def initial
-          @initial && @initial.clone || self
+          @initial && @initial.clone || (@immediate ? self : {})
         end
 
         # Returns itself.
@@ -54,15 +55,6 @@ module Picky
           # Nothing.
         end
 
-        # Deletes the Redis index namespace.
-        #
-        def delete
-          # Not implemented here.
-          # Note: backend.flushdb might be the way to go,
-          #       but since we cannot delete by key pattern,
-          #       we don't do anything.
-        end
-
         #
         #
         def to_s

data/lib/picky/backends/redis/directly_manipulable.rb

@@ -0,0 +1,48 @@
+module Picky
+
+  module Backends
+
+    class Redis
+
+      module DirectlyManipulable
+
+        attr_accessor :backend, :key
+
+        def self.make backend, list, key
+          list.extend DirectlyManipulable
+          list.backend = backend
+          list.key     = key
+        end
+
+        # TODO Current implementation is very brittle.
+        #
+        @@append_index = 0
+        def << value
+          super value
+          backend.client.zadd "#{backend.namespace}:#{key}", (@@append_index+=1), value
+          backend[key]
+        end
+
+        # TODO Current implementation is very brittle.
+        #
+        @@unshift_index = 0
+        def unshift value
+          super value
+          backend.client.zadd "#{backend.namespace}:#{key}", (@@unshift_index-=1), value
+          backend[key]
+        end
+
+        def delete value
+          result = super value
+          if result
+            backend.client.zrem "#{backend.namespace}:#{key}", value # TODO if super(value) ?
+          end
+          result
+        end
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/backends/redis/list.rb

@@ -6,20 +6,6 @@ module Picky
 
       class List < Basic
 
-        # Writes the hash into Redis.
-        #
-        def dump hash
-          clear
-          hash.each_pair do |key, values|
-            redis_key = "#{namespace}:#{key}"
-            i = 0
-            values.each do |value|
-              i += 1
-              client.zadd redis_key, i, value
-            end
-          end
-        end
-
         # Clear the index for this list.
         #
         # Note: Perhaps we can use a server only command.
@@ -32,12 +18,50 @@ module Picky
           end
         end
 
+        # Deletes the list for the key.
+        #
+        def delete key
+          client.del key
+        end
+
+        # Writes the hash into Redis.
+        #
+        def dump hash
+          unless @immediate
+            clear
+            # client.pipelined do
+              hash.each_pair do |key, values|
+                redis_key = "#{namespace}:#{key}"
+                i = 0
+                values.each do |value|
+                  i += 1
+                  client.zadd redis_key, i, value
+                end
+              end
+            # end
+          end
+        end
+
         # Get a collection.
         #
         # Internal API method for the index.
         #
         def [] key
-          client.zrange "#{namespace}:#{key}", 0, -1
+          list = client.zrange "#{namespace}:#{key}", :'0', :'-1'
+          DirectlyManipulable.make self, list, key
+          list
+        end
+
+        # Set a single list.
+        #
+        def []= key, values
+          redis_key = "#{namespace}:#{key}"
+          i = 0
+          values.each do |value|
+            i += 1
+            client.zadd redis_key, i, value
+          end
+          self[key] # TODO Performance?
         end
 
       end

data/lib/picky/backends/redis/string.rb

@@ -6,23 +6,31 @@ module Picky
 
       class String < Basic
 
+        # Clears the hash.
+        #
+        def clear
+          client.del namespace
+        end
+
+        # Deletes the single value.
+        #
+        def delete key
+          client.hdel namespace, key
+        end
+
         # Writes the hash into Redis.
         #
         # Note: We could use multi, but it did not help.
         #
         def dump hash
-          clear
-          hash.each_pair do |key, value|
-            client.hset namespace, key, value
+          unless @immediate
+            clear
+            hash.each_pair do |key, value|
+              client.hset namespace, key, value
+            end
           end
         end
 
-        # Clears the hash.
-        #
-        def clear
-          client.del namespace
-        end
-
         # Get a single value.
         #
         # Internal API method for the index.

data/lib/picky/backends/redis.rb

@@ -6,13 +6,15 @@ module Picky
     #
     class Redis < Backend
 
-      attr_reader :client
+      attr_reader :client,
+                  :immediate
 
       def initialize options = {}
         super options
 
         require 'redis'
-        @client = options[:client] || ::Redis.new(:db => (options[:db] || 15))
+        @client    = options[:client] || ::Redis.new(:db => (options[:db] || 15))
+        @immediate = options[:immediate]
       rescue LoadError => e
         warn_gem_missing 'redis', 'the Redis client'
       end
@@ -22,28 +24,35 @@ module Picky
       #
       def create_inverted bundle
         extract_lambda_or(inverted, bundle, client) ||
-          List.new(client, "#{bundle.identifier}:inverted")
+          List.new(client, "#{bundle.identifier}:inverted", immediate: immediate)
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:token] # => 1.23 (a weight)
       #
       def create_weights bundle
         extract_lambda_or(weights, bundle, client) ||
-          Float.new(client, "#{bundle.identifier}:weights")
+          Float.new(client, "#{bundle.identifier}:weights", immediate: immediate)
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:encoded] # => [:original, :original] (an array of original symbols this similarity encoded thing maps to)
       #
       def create_similarity bundle
         extract_lambda_or(similarity, bundle, client) ||
-          List.new(client, "#{bundle.identifier}:similarity")
+          List.new(client, "#{bundle.identifier}:similarity", immediate: immediate)
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:key] # => value (a value for this config key)
       #
       def create_configuration bundle
         extract_lambda_or(configuration, bundle, client) ||
-          String.new(client, "#{bundle.identifier}:configuration")
+          String.new(client, "#{bundle.identifier}:configuration", immediate: immediate)
+      end
+      # Returns an object that on #initial, #load returns an object that responds to:
+      #   [id] # => [:sym1, :sym2]
+      #
+      def create_realtime bundle
+        extract_lambda_or(similarity, bundle) ||
+          List.new(client, "#{bundle.identifier}:realtime", immediate: immediate)
       end
 
       # Does the Redis version already include
@@ -88,76 +97,103 @@ module Picky
       #
       # Note: We use the amount and offset hints to speed Redis up.
       #
+      # TODO What if it hasn't been dumped?
+      #      Move this method to the actual backends?
+      #
       def ids combinations, amount, offset
-        # Just checked once on the first call.
-        #
-        if redis_with_scripting?
-          @@script = "local intersected = redis.call('zinterstore', ARGV[1], #(KEYS), unpack(KEYS)); if intersected == 0 then redis.call('del', ARGV[1]); return {}; end local results = redis.call('zrange', ARGV[1], tonumber(ARGV[2]), tonumber(ARGV[3])); redis.call('del', ARGV[1]); return results;"
+        if immediate
+          # Just checked once on the first call.
+          #
+          if redis_with_scripting?
+            @@script = "local intersected = redis.call('zinterstore', ARGV[1], #(KEYS), unpack(KEYS)); if intersected == 0 then redis.call('del', ARGV[1]); return {}; end local results = redis.call('zrange', ARGV[1], tonumber(ARGV[2]), tonumber(ARGV[3])); redis.call('del', ARGV[1]); return results;"
 
-          require 'digest/sha1'
-          @@sent_once = nil
+            require 'digest/sha1'
+            @@sent_once = nil
 
-          # Scripting version of #ids.
-          #
-          def ids combinations, amount, offset
-            identifiers = combinations.inject([]) do |identifiers, combination|
-              identifiers << "#{combination.identifier}"
+            # Scripting version of #ids.
+            #
+            class << self
+              def ids combinations, amount, offset
+                identifiers = combinations.inject([]) do |identifiers, combination|
+                  identifiers << "#{combination.identifier}"
+                end
+
+                # Assume it's using EVALSHA.
+                #
+                begin
+                  client.evalsha @@sent_once,
+                                 identifiers.size,
+                                 *identifiers,
+                                 generate_intermediate_result_id,
+                                 offset,
+                                 (offset + amount)
+                rescue RuntimeError => e
+                  # Make the server have a SHA-1 for the script.
+                  #
+                  @@sent_once = Digest::SHA1.hexdigest @@script
+                  client.eval @@script,
+                              identifiers.size,
+                              *identifiers,
+                              generate_intermediate_result_id,
+                              offset,
+                              (offset + amount)
+                end
+              end
             end
-
-            # Assume it's using EVALSHA.
+          else
+            # Non-Scripting version of #ids.
             #
-            begin
-              client.evalsha @@sent_once,
-                             identifiers.size,
-                             *identifiers,
-                             generate_intermediate_result_id,
-                             offset,
-                             (offset + amount)
-            rescue RuntimeError => e
-              # Make the server have a SHA-1 for the script.
-              #
-              @@sent_once = Digest::SHA1.hexdigest @@script
-              client.eval @@script,
-                          identifiers.size,
-                          *identifiers,
-                          generate_intermediate_result_id,
-                          offset,
-                          (offset + amount)
+            class << self
+              def ids combinations, amount, offset
+                identifiers = combinations.inject([]) do |identifiers, combination|
+                  identifiers << "#{combination.identifier}"
+                end
+
+                result_id = generate_intermediate_result_id
+
+                # Intersect and store.
+                #
+                intersected = client.zinterstore result_id, identifiers
+
+                # Return clean and early if there has been no intersection.
+                #
+                if intersected.zero?
+                  client.del result_id
+                  return []
+                end
+
+                # Get the stored result.
+                #
+                results = client.zrange result_id, offset, (offset + amount)
+
+                # Delete the stored result as it was only for temporary purposes.
+                #
+                # Note: I could also not delete it, but that
+                #       would not be clean at all.
+                #
+                client.del result_id
+
+                results
+              end
             end
           end
         else
-          # Non-Scripting version of #ids.
+          # TODO Refactor!
           #
-          def ids combinations, amount, offset
-            identifiers = combinations.inject([]) do |identifiers, combination|
-              identifiers << "#{combination.identifier}"
-            end
-
-            result_id = generate_intermediate_result_id
-
-            # Intersect and store.
-            #
-            intersected = client.zinterstore result_id, identifiers
+          class << self
+            def ids combinations, _, _
+              # Get the ids for each combination.
+              #
+              id_arrays = combinations.inject([]) do |total, combination|
+                total << combination.ids
+              end
 
-            # Return clean and early if there has been no intersection.
-            #
-            if intersected.zero?
-              client.del result_id
-              return []
+              # Call the optimized C algorithm.
+              #
+              # Note: It orders the passed arrays by size.
+              #
+              Performant::Array.memory_efficient_intersect id_arrays
             end
-
-            # Get the stored result.
-            #
-            results = client.zrange result_id, offset, (offset + amount)
-
-            # Delete the stored result as it was only for temporary purposes.
-            #
-            # Note: I could also not delete it, but that
-            #       would not be clean at all.
-            #
-            client.del result_id
-
-            results
           end
         end
 
@@ -184,7 +220,7 @@ module Picky
       # Use the host and pid (generated lazily in child processes) for the result.
       #
       def generate_intermediate_result_id
-        :"#{host}:#{pid}:picky:result"
+        @intermediate_result_id ||= "#{host}:#{pid}:picky:result"
       end
 
     end

data/lib/picky/backends/sqlite/array.rb

@@ -0,0 +1,38 @@
+module Picky
+
+  module Backends
+
+    class SQLite
+
+      class Array < Basic
+
+        def []= key, array
+          if array.empty?
+            db.execute 'insert or replace into key_value (key, value) values (?,?)', key.to_s, Yajl::Encoder.encode(array)
+          end
+
+          DirectlyManipulable.make self, array, key
+          array
+        end
+
+        def [] key
+          res = db.execute "select value from key_value where key = ? limit 1;", key.to_s
+
+          return res unless res
+
+          array = res.empty? ? [] : Yajl::Parser.parse(res.first.first)
+          DirectlyManipulable.make self, array, key
+          array
+        end
+
+        def delete key
+          db.execute "delete from key_value where key = (?)", key.to_s
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/backends/sqlite/basic.rb

@@ -0,0 +1,100 @@
+module Picky
+
+  module Backends
+
+    class SQLite
+
+      class Basic
+
+        include Helpers::File
+
+        attr_reader :cache_path, :db
+
+        def initialize cache_path, options = {}
+          @cache_path   = "#{cache_path}.sqlite3"
+          @empty        = options[:empty]
+          @initial      = options[:initial]
+          @self_indexed = options[:self_indexed]
+
+          lazily_initialize_client
+        end
+
+        def initial
+          @initial && @initial.clone || (@self_indexed ? self : {})
+        end
+
+        def empty
+          @empty && @empty.clone || (@self_indexed ? self.reset_db.asynchronous : {})
+        end
+
+        def dump internal
+          dump_sqlite internal unless @self_indexed
+          self
+        end
+
+        def load
+          self
+        end
+
+        def clear
+          db.execute 'delete from key_value'
+        end
+
+        def lazily_initialize_client
+          @db ||= SQLite3::Database.new cache_path
+        end
+
+        def dump_sqlite internal
+          reset_db
+
+          transaction do
+            # Note: Internal structures need to
+            #       implement each.
+            #
+            internal.each do |key, value|
+              encoded_value = Yajl::Encoder.encode value
+              db.execute 'insert into key_value values (?,?)', key.to_s, encoded_value
+            end
+          end
+        end
+
+        def reset_db
+          create_directory cache_path
+          lazily_initialize_client
+
+          # TODO Could this be replaced by a truncate statement?
+          #
+          db.execute 'drop table if exists key_value;'
+          db.execute 'create table key_value (key varchar(255), value text);'
+          db.execute 'create index key_idx on key_value (key);'
+
+          self
+        end
+
+        def asynchronous
+          db.execute 'PRAGMA synchronous = OFF;'
+          self
+        end
+
+        def synchronous
+          db.execute 'PRAGMA synchronous = ON;'
+          self
+        end
+
+        def transaction
+          db.execute 'BEGIN;'
+          yield
+          db.execute 'COMMIT;'
+        end
+
+        def to_s
+          "#{self.class}(#{cache_path})"
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/backends/sqlite/directly_manipulable.rb

@@ -0,0 +1,42 @@
+module Picky
+
+  module Backends
+
+    class SQLite
+
+      module DirectlyManipulable
+
+        attr_accessor :backend, :key
+
+        def self.make backend, array, key
+          array.extend DirectlyManipulable
+          array.backend = backend
+          array.key     = key
+        end
+
+        def << value
+          super value
+          backend[key] = self
+          self
+        end
+
+        def unshift value
+          super value
+          backend[key] = self
+          self
+        end
+
+        def delete value
+          value = super value
+          if value
+            backend[key] = self
+          end
+          value
+        end
+      end
+
+    end
+
+  end
+
+end