picky 1.4.1 → 1.4.2

data/lib/picky/internals/generators/weights/logarithmic.rb

@@ -0,0 +1,43 @@
+module Internals
+
+  module Generators
+
+    module Weights
+
+      # Uses a logarithmic weight.
+      # If for a key k we have x ids, the weight is:
+      # w(x): log(x)
+      # Special case: If x < 1, then we use 0.
+      #
+      class Logarithmic < Strategy
+
+        # Generates a partial index from the given index.
+        #
+        def generate_from index
+          index.inject({}) do |hash, text_ids|
+            text, ids = *text_ids
+            weight = weight_for ids.size
+            hash[text] ||= weight.round(2) if weight
+            hash
+          end
+        end
+
+        # Sets the weight value.
+        #
+        # If the size is 0 or one, we would get -Infinity or 0.0.
+        # Thus we do not set a value if there is just one. The default, dynamically, is 0.
+        #
+        # BUT: We need the value, even if 0. To designate that there is a weight!
+        #
+        def weight_for amount
+          return 0 if amount < 1
+          Math.log amount
+        end
+
+      end
+
+    end
+
+  end
+  
+end

data/lib/picky/internals/generators/weights/strategy.rb

@@ -0,0 +1,11 @@
+module Internals
+  module Generators
+    module Weights
+      # Superclass for weighing strategies.
+      #
+      class Strategy < Generators::Strategy
+      
+      end
+    end
+  end
+end

data/lib/picky/internals/generators/weights_generator.rb

@@ -0,0 +1,19 @@
+module Internals
+
+  module Generators
+
+    # Uses a logarithmic algorithm as default.
+    #
+    class WeightsGenerator < Base
+
+      # Generate a weights index based on the given index.
+      #
+      def generate strategy = Weights::Logarithmic.new
+        strategy.generate_from self.index
+      end
+
+    end
+
+  end
+  
+end

data/lib/picky/internals/index/backend.rb

@@ -0,0 +1,113 @@
+module Internals
+  
+  module Index
+  
+    class Backend
+    
+      attr_reader :bundle_name
+      attr_reader :prepared, :index, :weights, :similarity, :configuration
+    
+      delegate :index_name, :category_name, :to => :@config
+    
+      def initialize bundle_name, config
+        @bundle_name = bundle_name
+        @config      = config
+        @prepared    = File::Text.new config.prepared_index_path
+      end
+    
+      # Delegators.
+      #
+    
+      # Retrieving data.
+      #
+      def retrieve &block
+        prepared.retrieve &block
+      end
+    
+      # Dumping.
+      #
+      def dump_index index_hash
+        index.dump index_hash
+      end
+      def dump_weights weights_hash
+        weights.dump weights_hash
+      end
+      def dump_similarity similarity_hash
+        similarity.dump similarity_hash
+      end
+      def dump_configuration configuration_hash
+        configuration.dump configuration_hash
+      end
+    
+      # Loading.
+      #
+      def load_index
+        index.load
+      end
+      def load_similarity
+        similarity.load
+      end
+      def load_weights
+        weights.load
+      end
+      def load_configuration
+        configuration.load
+      end
+    
+      # Cache ok?
+      #
+      def index_cache_ok?
+        index.cache_ok?
+      end
+      def similarity_cache_ok?
+        similarity.cache_ok?
+      end
+      def weights_cache_ok?
+        weights.cache_ok?
+      end
+    
+      # Cache small?
+      #
+      def index_cache_small?
+        index.cache_small?
+      end
+      def similarity_cache_small?
+        similarity.cache_small?
+      end
+      def weights_cache_small?
+        weights.cache_small?
+      end
+    
+      # Copies the indexes to the "backup" directory.
+      #
+      def backup
+        index.backup
+        weights.backup
+        similarity.backup
+        configuration.backup
+      end
+    
+      # Restores the indexes from the "backup" directory.
+      #
+      def restore
+        index.restore
+        weights.restore
+        similarity.restore
+        configuration.restore
+      end
+    
+    
+      # Delete all index files.
+      #
+      def delete
+        index.delete
+        weights.delete
+        similarity.delete
+        configuration.delete
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/internals/index/file/basic.rb

@@ -0,0 +1,101 @@
+module Internals
+
+  module Index
+  
+    # Handles all aspects of index files, such as dumping/loading.
+    #
+    module File
+    
+      # Base class for all index files.
+      #
+      # Provides necessary helper methods for its
+      # subclasses.
+      # Not directly useable, as it does not provide
+      # dump/load methods.
+      #
+      class Basic
+      
+        attr_reader :cache_path
+      
+        # An index cache takes a path, without file extension,
+        # which will be provided by the subclasses.
+        #
+        def initialize cache_path
+          @cache_path = "#{cache_path}.#{extension}"
+        end
+      
+        # The default extension for index files is "index".
+        #
+        def extension
+          :index
+        end
+      
+        # Will copy the index file to a location that
+        # is in a directory named "backup" right under
+        # the directory the index file is in.
+        #
+        def backup
+          prepare_backup backup_directory
+          FileUtils.cp cache_path, target, verbose: true
+        end
+        # The backup directory of this file.
+        # Equal to the file's dirname plus /backup
+        #
+        def backup_directory
+          ::File.join ::File.dirname(cache_path), 'backup'
+        end
+        # Prepares the backup directory for the file.
+        #
+        def prepare_backup target
+          FileUtils.mkdir target unless Dir.exists?(target)
+        end
+      
+        # Copies the file from its backup location back
+        # to the original location.
+        #
+        def restore
+          FileUtils.cp backup_file_path_of(cache_path), cache_path, verbose: true
+        end
+        # The backup filename.
+        #
+        def backup_file_path_of path
+          dir, name = ::File.split path
+          ::File.join dir, 'backup', name
+        end
+      
+        # Deletes the file.
+        #
+        def delete
+          `rm -Rf #{cache_path}`
+        end
+      
+        # Checks.
+        #
+      
+        # Is this cache file suspiciously small?
+        # (less than 8 Bytes of size)
+        #
+        def cache_small?
+          size_of(cache_path) < 8
+        end
+        # Is the cache ok? (existing and larger than
+        # zero Bytes in size)
+        #
+        # A small cache is still ok.
+        #
+        def cache_ok?
+          size_of(cache_path) > 0
+        end
+        # Extracts the size of the file in Bytes.
+        #
+        def size_of path
+          `ls -l #{path} | awk '{print $5}'`.to_i
+        end
+      
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/internals/index/file/json.rb

@@ -0,0 +1,38 @@
+module Internals
+
+  module Index
+  
+    module File
+    
+      # Index files dumped in the JSON format.
+      #
+      class JSON < Basic
+      
+        # Uses the extension "json".
+        #
+        def extension
+          :json
+        end
+        # Loads the index hash from json format.
+        #
+        def load
+          Yajl::Parser.parse ::File.open(cache_path, 'r'), symbolize_keys: true
+        end
+        # Dumps the index hash in json format.
+        #
+        def dump hash
+          hash.dump_json cache_path
+        end
+        # A json file does not provide retrieve functionality.
+        #
+        def retrieve
+          raise "Can't retrieve from JSON file. Use text file."
+        end
+      
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/internals/index/file/marshal.rb

@@ -0,0 +1,38 @@
+module Internals
+
+  module Index
+  
+    module File
+    
+      # Index data in the Ruby Marshal format.
+      #
+      class Marshal < Basic
+      
+        # Uses the extension "dump".
+        #
+        def extension
+          :dump
+        end
+        # Loads the index hash from marshal format.
+        #
+        def load
+          ::Marshal.load ::File.open(cache_path, 'r:binary')
+        end
+        # Dumps the index hash in marshal format.
+        #
+        def dump hash
+          hash.dump_marshalled cache_path
+        end
+        # A marshal file does not provide retrieve functionality.
+        #
+        def retrieve
+          raise "Can't retrieve from marshalled file. Use text file."
+        end
+      
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/internals/index/file/text.rb

@@ -0,0 +1,60 @@
+module Internals
+
+  module Index
+  
+    module File
+    
+      # Index data dumped in the text format.
+      #
+      class Text < Basic
+      
+        # Uses the extension "txt".
+        #
+        def extension
+          :txt
+        end
+        # Text files are used exclusively for
+        # prepared data files.
+        #
+        def load
+          raise "Can't load from text file. Use JSON or Marshal."
+        end
+        # Text files are used exclusively for
+        # prepared data files. 
+        #
+        def dump hash
+          raise "Can't dump to text file. Use JSON or Marshal."
+        end
+      
+        # Retrieves prepared index data in the form
+        # * id,data\n
+        # * id,data\n
+        # * id,data\n
+        #
+        # Yields an id string and a symbol token.
+        #
+        def retrieve
+          id    = nil
+          token = nil
+          ::File.open(cache_path, 'r:binary') do |file|
+            file.each_line do |line|
+              id, token = line.split ?,, 2
+              yield id, (token.chomp! || token).to_sym
+            end
+          end
+        end
+      
+        #
+        #
+        def open_for_indexing &block
+          ::File.open cache_path, 'w:binary', &block
+        end
+      
+      
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/internals/index/files.rb

@@ -0,0 +1,24 @@
+module Internals
+
+  module Index
+  
+    class Files < Backend
+    
+      def initialize bundle_name, config
+        super bundle_name, config
+      
+        # Note: We marshal the similarity, as the
+        #       Yajl json lib cannot load symbolized
+        #       values, just keys.
+        #
+        @index         = File::JSON.new    config.index_path(bundle_name, :index)
+        @weights       = File::JSON.new    config.index_path(bundle_name, :weights)
+        @similarity    = File::Marshal.new config.index_path(bundle_name, :similarity)
+        @configuration = File::JSON.new    config.index_path(bundle_name, :configuration)
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/internals/index/redis/basic.rb

@@ -0,0 +1,77 @@
+module Internals
+
+  module Index
+  
+    class Redis
+    
+      # Redis Backend Accessor.
+      #
+      # Provides necessary helper methods for its
+      # subclasses.
+      # Not directly useable, as it does not provide
+      # dump/load methods.
+      #
+      class Basic
+      
+        attr_reader :namespace
+      
+        # An index cache takes a path, without file extension,
+        # which will be provided by the subclasses.
+        #
+        def initialize namespace
+          @namespace = namespace
+        
+          @backend   = ::Redis.new
+        end
+      
+        # Does nothing.
+        #
+        def load
+          # Nothing.
+        end
+        # We do not use Redis to retrieve data.
+        #
+        def retrieve
+          # Nothing.
+        end
+      
+        # Redis does not backup.
+        #
+        def backup
+          # Nothing.
+        end
+      
+        # Deletes the Redis index namespace.
+        #
+        def delete
+          # TODO @backend.
+        end
+      
+        # Checks.
+        #
+      
+        # Is this cache suspiciously small?
+        #
+        def cache_small?
+          false # TODO
+        end
+        # Is the cache ok?
+        #
+        # A small cache is still ok.
+        #
+        def cache_ok?
+          false # TODO
+        end
+        # Extracts the size of the file in Bytes.
+        #
+        def size_of path
+          # TODO
+        end
+      
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/internals/index/redis/list_hash.rb

@@ -0,0 +1,46 @@
+module Internals
+
+  module Index
+  
+    class Redis
+    
+      class ListHash < Basic
+      
+        # Writes the hash into Redis.
+        #
+        # TODO Performance: rpush as you get the values instead of putting it together in an array first.
+        #
+        def dump hash
+          hash.each_pair do |key, values|
+            redis_key = "#{namespace}:#{key}"
+            i = 0
+            @backend.multi do
+              @backend.del redis_key
+            
+              values.each do |value|
+                i += 1
+                @backend.zadd redis_key, i, value
+              end
+            end
+          end
+        end
+      
+        # Get a collection.
+        #
+        def collection sym
+          @backend.lrange "#{namespace}:#{sym}", 0, -1
+        end
+      
+        # Get a single value.
+        #
+        def member sym
+          raise "Can't retrieve a single value from a Redis ListHash. Use Index::Redis::StringHash."
+        end
+      
+      end
+    
+    end
+  
+  end
+
+end

data/lib/picky/internals/index/redis/string_hash.rb

@@ -0,0 +1,35 @@
+module Internals
+
+  module Index
+  
+    class Redis
+    
+      class StringHash < Basic
+      
+        # Writes the hash into Redis.
+        #
+        def dump hash
+          hash.each_pair do |key, value|
+            @backend.hset namespace, key, value
+          end
+        end
+      
+        # Get a collection.
+        #
+        def collection sym
+          raise "Can't retrieve a collection from a StringHash. Use Index::Redis::ListHash."
+        end
+      
+        # Get a single value.
+        #
+        def member sym
+          @backend.hget namespace, sym
+        end
+      
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/internals/index/redis.rb

@@ -0,0 +1,44 @@
+module Internals
+
+  module Index
+  
+    # TODO Needs a reconnect to be run after forking.
+    #
+    class Redis < Backend
+    
+      def initialize bundle_name, config
+        super bundle_name, config
+      
+        # TODO
+        #
+        @index         = Redis::ListHash.new "#{config.identifier}:#{bundle_name}:index"
+        @weights       = Redis::StringHash.new "#{config.identifier}:#{bundle_name}:weights"
+        @similarity    = Redis::ListHash.new "#{config.identifier}:#{bundle_name}:similarity"
+        @configuration = Redis::StringHash.new "#{config.identifier}:#{bundle_name}:configuration"
+      end
+    
+      # Delegate to the right collection.
+      #
+      def ids sym
+        @index.collection sym
+      end
+    
+      # Delegate to the right member value.
+      #
+      # Note: Converts to float.
+      #
+      def weight sym
+        @weights.member(sym).to_f
+      end
+    
+      # Delegate to a member value.
+      #
+      def setting sym
+        @configuration.member sym
+      end
+    
+    end
+  
+  end
+  
+end