picky 1.2.4 → 1.3.0

data/lib/picky/adapters/rack/base.rb

@@ -0,0 +1,23 @@
+module Adapters
+  # Adapter that is plugged into a Rack outlet.
+  #
+  module Rack
+    
+    # Subclasses of this class should respond to
+    # * to_app(options)
+    #
+    class Base
+      
+      # Puts together an appropriately structured Rack response.
+      #
+      # Note: Bytesize is needed to have special characters not trip up Rack.
+      #
+      def respond_with response, content_type = 'application/json'
+        [200, { 'Content-Type' => content_type, 'Content-Length' => response.bytesize.to_s }, [response]]
+      end
+      
+    end
+    
+  end
+  
+end

data/lib/picky/adapters/rack/live_parameters.rb

@@ -0,0 +1,33 @@
+module Adapters
+  
+  #
+  #
+  module Rack
+    
+    class LiveParameters < Base
+
+      def initialize live_parameters
+        @live_parameters = live_parameters
+      end
+      
+      #
+      #
+      def to_app options = {}
+        # For capturing by the lambda block.
+        #
+        live_parameters = @live_parameters
+        
+        lambda do |env|
+          params = ::Rack::Request.new(env).params
+          
+          results = live_parameters.parameters params
+          
+          respond_with results.to_json
+        end
+      end
+      
+    end
+    
+  end
+  
+end

data/lib/picky/adapters/rack/query.rb

@@ -0,0 +1,59 @@
+module Adapters
+  # This is an adapter that is plugged into a Rack outlet.
+  #
+  # It looks at what is given to it and generate an appropriate
+  # adapter for it.
+  #
+  # For example, if you give it a query, it will extract the query param etc.
+  # and call search_with_text on it if it is called by Rack.
+  #
+  module Rack
+    
+    class Query < Base
+      
+      @@defaults = {
+        query_key:    'query'.freeze,
+        offset_key:   'offset'.freeze,
+        content_type: 'application/json'.freeze
+      }
+
+      def initialize query
+        @query    = query
+        @defaults = @@defaults.dup
+      end
+      
+      def to_app options = {}
+        # For capturing in the lambda.
+        #
+        query        = @query
+        query_key    = options[:query_key]    || @defaults[:query_key]
+        content_type = options[:content_type] || @defaults[:content_type]
+
+        lambda do |env|
+          params  = ::Rack::Request.new(env).params
+
+          results = query.search_with_text *extracted(params)
+
+          PickyLog.log results.to_log(params[query_key])
+
+          respond_with results.to_response, content_type
+        end
+      end
+      
+      # Helper method to extract the params
+      #
+      UTF8_STRING = 'UTF-8'.freeze
+      def extracted params
+        [
+          # query is encoded in ASCII
+          #
+          params[@defaults[:query_key]]  && params[@defaults[:query_key]].force_encoding(UTF8_STRING),
+          params[@defaults[:offset_key]] && params[@defaults[:offset_key]].to_i || 0
+        ]
+      end
+      
+    end
+    
+  end
+  
+end

data/lib/picky/adapters/rack.rb

@@ -0,0 +1,28 @@
+module Adapters
+  # This is an adapter that is plugged into a Rack outlet.
+  #
+  # It looks at what is given to it and generate an appropriate
+  # adapter for it.
+  #
+  # For example, if you give it a query, it will extract the query param etc.
+  # and call search_with_text on it if it is called by Rack.
+  #
+  # Usage:
+  #   Adapters::Rack.app_for(thing, options)
+  #
+  module Rack
+    
+    # Generates the appropriate app for Rack.
+    #
+    @@mapping = {
+      :search_with_text => Query,
+      :parameters       => LiveParameters
+    }
+    def self.app_for thing, options = {}
+      @@mapping.each_pair do |method, adapter|
+        return adapter.new(thing).to_app(options) if thing.respond_to?(method)
+      end
+    end
+    
+  end
+end

data/lib/picky/alias_instances.rb

@@ -1,4 +1,6 @@
 # This provides a nice accessor for users
 # who want to use the index API.
 #
+# TODO Rename to API::Indexes?
+#
 Indexes = IndexesAPI.new

data/lib/picky/application.rb

@@ -176,7 +176,7 @@ class Application
     #   - to identify an index (e.g. by you in Rake tasks).
     #   - in the frontend to describe which index a result came from.
     #   - index directory naming (index/development/the_identifier/<lots of indexes>)
-    # * source: The source the data comes from. See Sources::Base. # TODO Sources (all).
+    # * source: The source the data comes from. See Sources::Base.
     #
     # Options:
     # * result_identifier: Use if you'd like a different identifier/name in the results JSON than the name of the index. 
@@ -187,7 +187,7 @@ class Application
     
     # Routes.
     #
-    delegate :route, :root, :to => :routing
+    delegate :route, :root, :to => :rack_adapter
     
     #
     # API
@@ -198,10 +198,10 @@ class Application
     # Delegates to its routing to handle a request.
     #
     def call env
-      routing.call env
+      rack_adapter.call env
     end
-    def routing # :nodoc:
-      @routing ||= Routing.new
+    def rack_adapter # :nodoc:
+      @rack_adapter ||= FrontendAdapters::Rack.new
     end
     
     # Finalize the subclass as soon as it
@@ -223,7 +223,7 @@ class Application
     #
     def finalize # :nodoc:
       check
-      routing.freeze
+      rack_adapter.finalize
     end
     # Checks app for missing things.
     #
@@ -237,14 +237,15 @@ class Application
       puts "\n#{warnings.join(?\n)}\n\n" unless warnings.all? &:nil?
     end
     def check_external_interface
-      "WARNING: No routes defined for application configuration in #{self.class}." if routing.empty?
+      "WARNING: No routes defined for application configuration in #{self.class}." if rack_adapter.empty?
     end
     
     # TODO Add more info if possible.
     #
     def to_s # :nodoc:
-      "#{self.name}:\n#{routing}"
+      "#{self.name}:\n#{rack_adapter}"
     end
     
   end
+  
 end

data/lib/picky/cli.rb

@@ -27,8 +27,8 @@ module Picky
     end
     class Statistics < Base
       def execute name, args, params
-        relative_log_file            = args.shift
-        port                         = args.shift
+        relative_log_file = args.shift
+        port              = args.shift
         
         usage(name, params) || exit(1) unless relative_log_file
         
@@ -46,6 +46,27 @@ module Picky
         require 'picky-statistics/application/app'
       end
     end
+    class Live < Base
+      def execute name, args, params
+        url  = args.shift
+        port = args.shift
+        
+        usage(name, params) || exit(1) unless args.empty?
+        
+        ENV['PICKY_LIVE_URL']  = url
+        ENV['PICKY_LIVE_PORT'] = port
+        
+        begin
+          require 'picky-live'
+        rescue LoadError => e
+          require 'picky/extensions/object'
+          puts_gem_missing 'picky-live', 'the Picky Live Interface'
+          exit 1
+        end
+        
+        require 'picky-live/application/app'
+      end
+    end
     class Generate < Base
       def execute name, args, params
         system "picky-generate #{args.join(' ')}"
@@ -69,7 +90,8 @@ module Picky
     @@mapping = {
       :generate => [Generate, 'sinatra_client | unicorn_server | empty_unicorn_server', 'app_directory_name (optional)'],
       :help     => [Help],
-      :stats    => [Statistics, 'logfile, e.g. log/search.log', 'port (optional)']
+      :stats    => [Statistics, 'logfile, e.g. log/search.log', 'port (optional)'],
+      :live     => [Live, 'host:port/path (optional, default: localhost:8080/admin)', 'port (optional)']
     }
     def self.mapping
       @@mapping

data/lib/picky/frontend_adapters/rack.rb

@@ -0,0 +1,150 @@
+require 'rack/mount'
+
+module FrontendAdapters
+
+  # TODO Rename to Routing again. Push everything back into appropriate Adapters.
+  #
+  class Rack # :nodoc:all
+  
+    @@defaults = {
+      query_key:    'query'.freeze,
+      offset_key:   'offset'.freeze,
+      content_type: 'application/octet-stream'.freeze # TODO Wrong.
+    }
+  
+    def initialize
+      @defaults = @@defaults.dup
+    end
+  
+    # 
+    #
+    def reset_routes
+      @routes = ::Rack::Mount::RouteSet.new
+    end
+    def routes
+      @routes || reset_routes
+    end
+    def finalize
+      routes.freeze
+    end
+  
+    # Routing simply delegates to the route set to handle a request.
+    #
+    def call env
+      routes.call env
+    end
+  
+    # API method.
+    #
+    def route options = {}
+      mappings, route_options = split options
+      mappings.each do |url, query|
+        route_one url, query, route_options
+      end
+    end
+    # Splits the route method options
+    # into real options and route options (/regexp/ => thing or 'some/path' => thing).
+    #
+    def split options
+      mappings      = {}
+      route_options = {}
+      options.each_pair do |key, value|
+        if Regexp === key or String === key
+          mappings[key] = value
+        else
+          route_options[key] = value
+        end
+      end
+      [mappings, route_options]
+    end
+    def route_one url, query, route_options = {}
+      raise RouteTargetNilError.new(url) unless query
+      routes.add_route Adapters::Rack.app_for(query, route_options), default_options(url, route_options)
+    end
+    class RouteTargetNilError < StandardError
+      def initialize url
+        @url = url
+      end
+      def to_s
+        "Routing for #{@url.inspect} was defined with a nil target object, i.e. #{@url.inspect} => nil."
+      end
+    end
+    #
+    #
+    def root status
+      answer %r{^/$}, STATUSES[status]
+    end
+    #
+    #
+    def default status
+      answer nil, STATUSES[status]
+    end
+  
+  
+  
+    # TODO Can Rack handle this for me?
+    #
+    # Note: Rack-mount already handles the 404.
+    #
+    STATUSES = {
+      200 => lambda { |_| [200, { 'Content-Type' => 'text/html', 'Content-Length' => '0' }, ['']] },
+      404 => lambda { |_| [404, { 'Content-Type' => 'text/html', 'Content-Length' => '0' }, ['']] }
+    }
+  
+    #
+    #
+    def default_options url, route_options = {}
+      url = normalized url
+    
+      options = { request_method: 'GET' }.merge route_options
+    
+      options[:path_info] = url if url
+    
+      options.delete :content_type
+      
+      query_params = options.delete :query
+      options[:query_string] = %r{#{generate_query_string(query_params)}} if query_params
+    
+      options
+    end
+    #
+    #
+    def generate_query_string query_params
+      raise "At least one query string condition is needed." if query_params.size.zero?
+      raise "Too many query param conditions (only 1 allowed): #{query_params}" if query_params.size > 1
+      k, v = query_params.first
+      "#{k}=#{v}"
+    end
+    
+    # Setup a route that answers using the given app.
+    #
+    def answer url = nil, app = nil
+      routes.add_route (app || STATUSES[200]), default_options(url)
+    end
+  
+    # Returns a regular expression for the url if it is given a String-like object.
+    #
+    def normalized url
+      url.respond_to?(:to_str) ? %r{#{url}} : url
+    end
+  
+    # Returns true if there are no routes defined.
+    #
+    def empty?
+      routes.length.zero?
+    end
+  
+    # TODO Beautify.
+    #
+    def to_s
+      routes.instance_variable_get(:@routes).map do |route|
+        path_info = route.conditions[:path_info]
+        anchored = ::Rack::Mount::Utils.regexp_anchored?(path_info)
+        anchored_ok = anchored ? "\u2713" : " "
+        "#{anchored_ok}  #{path_info.source}"
+      end.join "\n"
+    end
+  
+  end
+  
+end

data/lib/picky/helpers/measuring.rb

@@ -1,7 +1,5 @@
 # Helper methods for measuring, benchmarking, logging.
 #
-# TODO Not used anymore? Remove.
-#
 module Helpers
   module Measuring
     

data/lib/picky/index_api.rb

@@ -157,7 +157,7 @@ class IndexAPI
   # * precision: Default 1 (20% error margin, very fast), up to 5 (5% error margin, slower) makes sense.
   # * from: The data category to take the data for this category from.
   #
-  # TODO Redo. Probably extract into define_latitude_category, define_longitude_category.
+  # TODO Redo. Will have to write a wrapper that combines two categories that are indexed simultaneously.
   #
   def define_map_location name, radius, options = {} # :nodoc:
     # The radius is given as if all the locations were on the equator.

data/lib/picky/indexed/categories.rb

@@ -7,29 +7,57 @@ module Indexed
     each_delegate :load_from_cache,
                   :to => :categories
     
+    # A list of indexed categories.
+    #
+    # Options:
+    #  * ignore_unassigned_tokens: Ignore the given token if it cannot be matched to a category.
+    #                              The default behaviour is that if a token does not match to
+    #                              any category, the query will not return anything (since a
+    #                              single token cannot be matched). If you set this option to
+    #                              true, any token that cannot be matched to a category will be
+    #                              simply ignored.
+    #                              Use this if only a few matched words are important, like for
+    #                              example of the query "Jonathan Myers 86455 Las Cucarachas"
+    #                              you only want to match the zipcode, to have the search engine
+    #                              display advertisements on the side for the zipcode.
+    #                              Nifty! :)
+    #
     def initialize options = {}
       clear
       
       @ignore_unassigned_tokens = options[:ignore_unassigned_tokens] || false
     end
     
+    # Clears both the array of categories and the hash of categories.
+    #
     def clear
       @categories    = []
       @category_hash = {}
     end
     
+    # Add the given category to the list of categories.
+    #
     def << category
       categories << category
-      category_hash[category.name] = [category] # TODO Why an Array?
+      # Note: [category] is an optimization, since I need an array
+      #       of categories.
+      #       It's faster to just package it in an array on loading
+      #       Picky than doing it over and over with each query.
+      #
+      category_hash[category.name] = [category]
     end
     
+    # Return all possible combinations for the given token.
     #
+    # This checks if it needs to also search through similar
+    # tokens, if for example, the token is one with ~.
+    # If yes, it puts together all solutions.
     #
     def possible_combinations_for token
       token.similar? ? similar_possible_for(token) : possible_for(token)
     end
-    
-    # 
+    # Gets all similar tokens and puts together the possible combinations
+    # for each found similar token.
     # 
     def similar_possible_for token
       # Get as many similar tokens as necessary
@@ -43,9 +71,12 @@ module Indexed
       text = token.text
       categories.inject([]) do |result, category|
         next_token = token
-        # TODO adjust either this or the amount of similar in index
+        # Note: We could also break off here if not all the available
+        #       similars are needed.
+        #       Wait for a concrete case that needs this before taking
+        #       action.
         #
-        while next_token = next_token.next(category)
+        while next_token = next_token.next_similar_token(category)
           result << next_token if next_token && next_token.text != text
         end
         result
@@ -60,13 +91,12 @@ module Indexed
     
     # Returns possible Combinations for the token.
     #
-    # The preselected_categories param is an optimization.
-    #
-    # TODO Return [RemovedCategory(token, nil)]
-    #      If the search is ...
+    # Note: The preselected_categories param is an optimization.
     #
-    # TODO Return [] if not ok, nil if needs to be removed?
-    #      Somehow unnice, but…
+    # Note: Returns [] if no categories matched (will produce no result).
+    #       Returns nil if this token needs to be removed from the query.
+    #       (Also none of the categories matched, but the ignore unassigned
+    #       tokens option is true)
     #
     def possible_for token, preselected_categories = nil
       possible = (preselected_categories || possible_categories(token)).map { |category| category.combination_for(token) }
@@ -76,14 +106,21 @@ module Indexed
       return if ignore_unassigned_tokens && possible.empty?
       possible # wrap in combinations
     end
+    # This returns the possible categories for this token.
+    # If the user has already preselected a category for this token,
+    # like "artist:moby", if not just return all for the given token,
+    # since all are possible.
     #
-    #
-    # TODO too many calls?
+    # Note: Once I thought this was called too often. But it is not (18.01.2011).
     #
     def possible_categories token
       user_defined_categories(token) || categories
     end
-    # Returns nil if there is no user defined category, the category else.
+    # This returns the array of categories if the user has defined
+    # an existing category.
+    #
+    # Note: Returns nil if the user did not define one
+    #       or if he/she has defined a non-existing one. 
     #
     def user_defined_categories token
       category_hash[token.user_defined_category_name]

data/lib/picky/indexers/solr.rb

@@ -2,7 +2,7 @@
 #
 require 'rsolr'
 module Indexers
-  # TODO Totally deprecated. This should be a source.
+  # TODO Deprecated. This should be handled in a special bundle which goes through Solr.
   #
   class Solr
 
@@ -16,14 +16,10 @@ module Indexers
       @solr   = RSolr.connect
     end
 
-    # TODO Rewrite such that it works in batches.
-    #
     def index
       timed_exclaim "Indexing solr for #{type.name}:#{fields.join(', ')}"
       statement = "SELECT indexed_id, #{fields.join(',')} FROM #{type.snapshot_table_name}"
       
-      # TODO Rewrite.
-      #
       DB.connect
       results   = DB.connection.execute statement
       

data/lib/picky/indexing/indexes.rb

@@ -39,8 +39,14 @@ module Indexing
       # Run in parallel.
       #
       timed_exclaim "INDEXING USING #{Cores.max_processors} PROCESSORS, IN #{randomly ? 'RANDOM' : 'GIVEN'} ORDER."
+      
+      # TODO Think about having serial work units.
+      #
       Cores.forked self.indexes, { randomly: randomly } do |an_index|
         an_index.index
+      # TODO
+      # end
+      # Cores.forked self.indexes, { randomly: randomly } do |an_index|
         an_index.cache
       end
       timed_exclaim "INDEXING FINISHED."