picky 2.5.2 → 2.6.0

data/lib/picky/adapters/rack/base.rb

@@ -0,0 +1,23 @@
+module Adapters
+  # Adapter that is plugged into a Rack outlet.
+  #
+  module Rack
+  
+    # Subclasses of this class should respond to
+    # * to_app(options)
+    #
+    class Base
+    
+      # Puts together an appropriately structured Rack response.
+      #
+      # Note: Bytesize is needed to have special characters not trip up Rack.
+      #
+      def respond_with response, content_type = 'application/json'
+        [200, { 'Content-Type' => content_type, 'Content-Length' => response.bytesize.to_s }, [response]]
+      end
+    
+    end
+  
+  end
+
+end

data/lib/picky/adapters/rack/live_parameters.rb

@@ -0,0 +1,33 @@
+module Adapters
+
+  #
+  #
+  module Rack
+  
+    class LiveParameters < Base
+
+      def initialize live_parameters
+        @live_parameters = live_parameters
+      end
+    
+      #
+      #
+      def to_app options = {}
+        # For capturing by the lambda block.
+        #
+        live_parameters = @live_parameters
+      
+        lambda do |env|
+          params = ::Rack::Request.new(env).params
+        
+          results = live_parameters.parameters params
+        
+          respond_with results.to_json
+        end
+      end
+    
+    end
+  
+  end
+
+end

data/lib/picky/adapters/rack/query.rb

@@ -0,0 +1,65 @@
+module Adapters
+  # This is an adapter that is plugged into a Rack outlet.
+  #
+  # It looks at what is given to it and generate an appropriate
+  # adapter for it.
+  #
+  # For example, if you give it a query, it will extract the query param etc.
+  # and call search_with_text on it if it is called by Rack.
+  #
+  module Rack
+
+    # TODO Rename to Search.
+    #
+    class Query < Base
+
+      @@defaults = {
+        query_key:    'query'.freeze,
+        ids_key:      'ids'.freeze,
+        offset_key:   'offset'.freeze,
+        content_type: 'application/json'.freeze
+      }
+
+      def initialize query
+        @query    = query
+        @defaults = @@defaults.dup
+      end
+
+      def to_app options = {}
+        # For capturing in the lambda.
+        #
+        query        = @query
+        query_key    = options[:query_key]    || @defaults[:query_key]
+        content_type = options[:content_type] || @defaults[:content_type]
+
+        lambda do |env|
+          params  = ::Rack::Request.new(env).params
+
+          results = query.search_with_text *extracted(params)
+
+          PickyLog.log results.to_log(params[query_key])
+
+          respond_with results.to_response, content_type
+        end
+      end
+
+      # Helper method to extract the params
+      #
+      # Defaults are 20 ids, offset 0.
+      #
+      UTF8_STRING = 'UTF-8'.freeze
+      def extracted params
+        [
+          # query is encoded in ASCII
+          #
+          params[@defaults[:query_key]]  && params[@defaults[:query_key]].force_encoding(UTF8_STRING),
+          params[@defaults[:ids_key]]    && params[@defaults[:ids_key]].to_i || 20,
+          params[@defaults[:offset_key]] && params[@defaults[:offset_key]].to_i || 0
+        ]
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/adapters/rack.rb

@@ -0,0 +1,30 @@
+module Adapters
+
+  # This is an adapter that is plugged into a Rack outlet.
+  #
+  # It looks at what is given to it and generate an appropriate
+  # adapter for it.
+  #
+  # For example, if you give it a query, it will extract the query param etc.
+  # and call search_with_text on it if it is called by Rack.
+  #
+  # Usage:
+  #   Adapters::Rack.app_for(thing, options)
+  #
+  module Rack
+  
+    # Generates the appropriate app for Rack.
+    #
+    @@mapping = {
+      :search_with_text => Query,
+      :parameters       => LiveParameters
+    }
+    def self.app_for thing, options = {}
+      @@mapping.each_pair do |method, adapter|
+        return adapter.new(thing).to_app(options) if thing.respond_to?(method)
+      end
+    end
+  
+  end
+
+end

data/lib/picky/application.rb

@@ -161,7 +161,7 @@ class Application
     # is used for indexing by default.
     #
     def indexing options = {}
-      Internals::Tokenizers::Index.default = Internals::Tokenizers::Index.new(options)
+      Tokenizers::Index.default = Tokenizers::Index.new(options)
     end
     alias default_indexing indexing
 
@@ -169,7 +169,7 @@ class Application
     # is used for querying by default.
     #
     def searching options = {}
-      Internals::Tokenizers::Query.default = Internals::Tokenizers::Query.new(options)
+      Tokenizers::Query.default = Tokenizers::Query.new(options)
     end
     alias default_querying searching
     alias querying searching
@@ -190,7 +190,7 @@ class Application
       rack_adapter.call env
     end
     def rack_adapter # :nodoc:
-      @rack_adapter ||= Internals::FrontendAdapters::Rack.new
+      @rack_adapter ||= FrontendAdapters::Rack.new
     end
 
     # Finalize the subclass as soon as it
@@ -236,10 +236,10 @@ APPLICATION
     def to_stats
       <<-APP
 \033[1mIndexing (default)\033[m:
-#{Internals::Tokenizers::Index.default.indented_to_s}
+#{Tokenizers::Index.default.indented_to_s}
 
 \033[1mQuerying (default)\033[m:
-#{Internals::Tokenizers::Query.default.indented_to_s}
+#{Tokenizers::Query.default.indented_to_s}
 
 \033[1mIndexes\033[m:
 #{Indexes.to_s.indented_to_s}

data/lib/picky/backend/backend.rb

@@ -0,0 +1,108 @@
+module Backend
+
+  class Backend
+
+    attr_reader :bundle_name
+    attr_reader :prepared, :index, :weights, :similarity, :configuration
+
+    delegate :index_name, :category_name, :to => :@category
+
+    def initialize bundle_name, category
+      @bundle_name = bundle_name
+      @category    = category
+      @prepared    = File::Text.new category.prepared_index_path
+    end
+
+    # Delegators.
+    #
+
+    # Retrieving data.
+    #
+    def retrieve &block
+      prepared.retrieve &block
+    end
+
+    # Dumping.
+    #
+    def dump_index index_hash
+      index.dump index_hash
+    end
+    def dump_weights weights_hash
+      weights.dump weights_hash
+    end
+    def dump_similarity similarity_hash
+      similarity.dump similarity_hash
+    end
+    def dump_configuration configuration_hash
+      configuration.dump configuration_hash
+    end
+
+    # Loading.
+    #
+    def load_index
+      index.load
+    end
+    def load_similarity
+      similarity.load
+    end
+    def load_weights
+      weights.load
+    end
+    def load_configuration
+      configuration.load
+    end
+
+    # Cache ok?
+    #
+    def index_cache_ok?
+      index.cache_ok?
+    end
+    def similarity_cache_ok?
+      similarity.cache_ok?
+    end
+    def weights_cache_ok?
+      weights.cache_ok?
+    end
+
+    # Cache small?
+    #
+    def index_cache_small?
+      index.cache_small?
+    end
+    def similarity_cache_small?
+      similarity.cache_small?
+    end
+    def weights_cache_small?
+      weights.cache_small?
+    end
+
+    # Copies the indexes to the "backup" directory.
+    #
+    def backup
+      index.backup
+      weights.backup
+      similarity.backup
+      configuration.backup
+    end
+
+    # Restores the indexes from the "backup" directory.
+    #
+    def restore
+      index.restore
+      weights.restore
+      similarity.restore
+      configuration.restore
+    end
+
+    # Delete all index files.
+    #
+    def delete
+      index.delete
+      weights.delete
+      similarity.delete
+      configuration.delete
+    end
+
+  end
+
+end

data/lib/picky/backend/file/basic.rb

@@ -0,0 +1,101 @@
+module Backend
+
+  # Handles all aspects of index files, such as dumping/loading.
+  #
+  module File
+
+    # Base class for all index files.
+    #
+    # Provides necessary helper methods for its
+    # subclasses.
+    # Not directly useable, as it does not provide
+    # dump/load methods.
+    #
+    class Basic
+
+      attr_reader :cache_path
+
+      # An index cache takes a path, without file extension,
+      # which will be provided by the subclasses.
+      #
+      def initialize cache_path
+        @cache_path = "#{cache_path}.#{extension}"
+      end
+
+      def to_s
+        cache_path
+      end
+
+      # The default extension for index files is "index".
+      #
+      def extension
+        :index
+      end
+
+      # Will copy the index file to a location that
+      # is in a directory named "backup" right under
+      # the directory the index file is in.
+      #
+      def backup
+        prepare_backup backup_directory
+        FileUtils.cp cache_path, target, verbose: true
+      end
+      # The backup directory of this file.
+      # Equal to the file's dirname plus /backup
+      #
+      def backup_directory
+        ::File.join ::File.dirname(cache_path), 'backup'
+      end
+      # Prepares the backup directory for the file.
+      #
+      def prepare_backup target
+        FileUtils.mkdir target unless Dir.exists?(target)
+      end
+
+      # Copies the file from its backup location back
+      # to the original location.
+      #
+      def restore
+        FileUtils.cp backup_file_path_of(cache_path), cache_path, verbose: true
+      end
+      # The backup filename.
+      #
+      def backup_file_path_of path
+        dir, name = ::File.split path
+        ::File.join dir, 'backup', name
+      end
+
+      # Deletes the file.
+      #
+      def delete
+        `rm -Rf #{cache_path}`
+      end
+
+      # Checks.
+      #
+
+      # Is this cache file suspiciously small?
+      # (less than 8 Bytes of size)
+      #
+      def cache_small?
+        size_of(cache_path) < 8
+      end
+      # Is the cache ok? (existing and larger than
+      # zero Bytes in size)
+      #
+      # A small cache is still ok.
+      #
+      def cache_ok?
+        size_of(cache_path) > 0
+      end
+      # Extracts the size of the file in Bytes.
+      #
+      def size_of path
+        `ls -l #{path} | awk '{print $5}'`.to_i
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/backend/file/json.rb

@@ -0,0 +1,34 @@
+module Backend
+  
+  module File
+  
+    # Index files dumped in the JSON format.
+    #
+    class JSON < Basic
+    
+      # Uses the extension "json".
+      #
+      def extension
+        :json
+      end
+      # Loads the index hash from json format.
+      #
+      def load
+        Yajl::Parser.parse ::File.open(cache_path, 'r'), symbolize_keys: true
+      end
+      # Dumps the index hash in json format.
+      #
+      def dump hash
+        hash.dump_json cache_path
+      end
+      # A json file does not provide retrieve functionality.
+      #
+      def retrieve
+        raise "Can't retrieve from JSON file. Use text file."
+      end
+    
+    end
+  
+  end
+
+end

data/lib/picky/backend/file/marshal.rb

@@ -0,0 +1,34 @@
+module Backend
+  
+  module File
+  
+    # Index data in the Ruby Marshal format.
+    #
+    class Marshal < Basic
+    
+      # Uses the extension "dump".
+      #
+      def extension
+        :dump
+      end
+      # Loads the index hash from marshal format.
+      #
+      def load
+        ::Marshal.load ::File.open(cache_path, 'r:binary')
+      end
+      # Dumps the index hash in marshal format.
+      #
+      def dump hash
+        hash.dump_marshalled cache_path
+      end
+      # A marshal file does not provide retrieve functionality.
+      #
+      def retrieve
+        raise "Can't retrieve from marshalled file. Use text file."
+      end
+    
+    end
+  
+  end
+
+end

data/lib/picky/backend/file/text.rb

@@ -0,0 +1,56 @@
+module Backend
+
+  module File
+
+    # Index data dumped in the text format.
+    #
+    class Text < Basic
+
+      # Uses the extension "txt".
+      #
+      def extension
+        :txt
+      end
+      # Text files are used exclusively for
+      # prepared data files.
+      #
+      def load
+        raise "Can't load from text file. Use JSON or Marshal."
+      end
+      # Text files are used exclusively for
+      # prepared data files.
+      #
+      def dump hash
+        raise "Can't dump to text file. Use JSON or Marshal."
+      end
+
+      # Retrieves prepared index data in the form
+      # * id,data\n
+      # * id,data\n
+      # * id,data\n
+      #
+      # Yields an id string and a symbol token.
+      #
+      def retrieve
+        id    = nil
+        token = nil
+        ::File.open(cache_path, 'r:binary') do |file|
+          file.each_line do |line|
+            id, token = line.split ?,, 2
+            yield id, (token.chomp! || token).to_sym
+          end
+        end
+      end
+
+      #
+      #
+      def open_for_indexing &block
+        ::File.open cache_path, 'w:binary', &block
+      end
+
+
+    end
+
+  end
+
+end

data/lib/picky/backend/files.rb

@@ -0,0 +1,30 @@
+module Backend
+
+  class Files < Backend
+
+    def initialize bundle_name, category
+      super bundle_name, category
+
+      # Note: We marshal the similarity, as the
+      #       Yajl json lib cannot load symbolized
+      #       values, just keys.
+      #
+      @index         = File::JSON.new    category.index_path(bundle_name, :index)
+      @weights       = File::JSON.new    category.index_path(bundle_name, :weights)
+      @similarity    = File::Marshal.new category.index_path(bundle_name, :similarity)
+      @configuration = File::JSON.new    category.index_path(bundle_name, :configuration)
+    end
+
+    def to_s
+      <<-FILES
+Files:
+#{"Index:      #{@index}".indented_to_s}
+#{"Weights:    #{@weights}".indented_to_s}
+#{"Similarity: #{@similarity}".indented_to_s}
+#{"Config:     #{@configuration}".indented_to_s}
+FILES
+    end
+
+  end
+
+end

data/lib/picky/backend/redis/basic.rb

@@ -0,0 +1,85 @@
+module Backend
+
+  class Redis
+
+    # Redis Backend Accessor.
+    #
+    # Provides necessary helper methods for its
+    # subclasses.
+    # Not directly useable, as it does not provide
+    # dump/load methods.
+    #
+    class Basic
+
+      attr_reader :namespace, :backend
+
+      # An index cache takes a path, without file extension,
+      # which will be provided by the subclasses.
+      #
+      def initialize namespace
+        @namespace = namespace
+
+        # TODO Turn this inside out such that people can pass in
+        # their own Redis instance.
+        #
+        # TODO Make the :db a real option.
+        #
+        @backend = ::Redis.new :db => 15
+      end
+
+      # Does nothing.
+      #
+      def load
+        # Nothing.
+      end
+      # We do not use Redis to retrieve data.
+      #
+      def retrieve
+        # Nothing.
+      end
+
+      # Redis does not backup.
+      #
+      def backup
+        # Nothing.
+      end
+
+      # Deletes the Redis index namespace.
+      #
+      def delete
+        # Not implemented here.
+        # Note: backend.flushdb might be the way to go,
+        #       but since we cannot delete by key pattern,
+        #       we don't do anything.
+      end
+
+      # Checks.
+      #
+
+      # Is this cache suspiciously small?
+      #
+      def cache_small?
+        size < 1
+      end
+      # Is the cache ok?
+      #
+      # A small cache is still ok.
+      #
+      def cache_ok?
+        size > 0
+      end
+      # Extracts the size of the file in Bytes.
+      #
+      # Note: This is a very forgiving implementation.
+      #       But as long as Redis does not implement
+      #       DBSIZE KEYPATTERN, we are stuck with this.
+      #
+      def size
+        backend.dbsize
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/backend/redis/list_hash.rb

@@ -0,0 +1,49 @@
+module Backend
+
+  class Redis
+
+    class ListHash < Basic
+
+      # Writes the hash into Redis.
+      #
+      def dump hash
+        clear
+        hash.each_pair do |key, values|
+          redis_key = "#{namespace}:#{key}"
+          i = 0
+          values.each do |value|
+            i += 1
+            backend.zadd redis_key, i, value
+          end
+        end
+      end
+      
+      # Clear the index for this list.
+      #
+      # Note: Perhaps we can use a server only command.
+      #       This is not the optimal way to do it.
+      #
+      def clear
+        redis_key = "#{namespace}:*"
+        backend.keys(redis_key).each do |key|
+          backend.del key
+        end
+      end
+
+      # Get a collection.
+      #
+      def collection sym
+        backend.lrange "#{namespace}:#{sym}", 0, -1
+      end
+
+      # Get a single value.
+      #
+      def member sym
+        raise "Can't retrieve a single value from a Redis ListHash. Use Index::Redis::StringHash."
+      end
+
+    end
+
+  end
+
+end