picky 0.0.0 → 0.0.2

data/lib/picky/query/tokens.rb

@@ -0,0 +1,102 @@
+# encoding: utf-8
+#
+module Query
+
+  # This class primarily handles switching through similar token constellations.
+  #
+  class Tokens
+
+    #
+    #
+    cattr_accessor :maximum
+    self.maximum = 5
+
+    # Basically delegates to its internal tokens array.
+    #
+    self.delegate *[Enumerable.instance_methods, :slice!, :[], :uniq!, :last, :reject!, :length, :size, :empty?, :each, :exit, { :to => :@tokens }].flatten
+
+    #
+    #
+    def initialize tokens = []
+      @tokens = tokens
+    end
+
+    #
+    #
+    def tokenize_with tokenizer
+      @tokens.each { |token| token.tokenize_with(tokenizer) }
+    end
+
+    # Generates an array in the form of
+    # [
+    #  [combination],                           # of token 1
+    #  [combination, combination, combination], # of token 2
+    #  [combination, combination]               # of token 3
+    # ]
+    #
+    # TODO If we want token behaviour defined per Query, we can
+    #      compact! here
+    #
+    def possible_combinations_in type
+      @tokens.inject([]) do |combinations, token|
+        combinations << token.possible_combinations_in(type)
+      end
+      # TODO compact! if ignore_unassigned_tokens
+    end
+
+    # Makes the last of the tokens partial.
+    #
+    def partialize_last
+      @tokens.last.partial = true unless empty?
+    end
+
+    # Caps the tokens to the maximum.
+    #
+    # TODO parametrize?
+    #
+    def cap
+      @tokens.slice!(@@maximum..-1) if cap?
+    end
+    def cap?
+      @tokens.size > @@maximum
+    end
+
+    # Rejects blank tokens.
+    #
+    def reject
+      @tokens.reject! &:blank?
+    end
+
+    # Switches the tokens
+    #
+    # TODO
+    #
+    def next_similar
+      @tokens.first.next_similar unless empty?
+    end
+
+    # Returns a solr query.
+    #
+    def to_solr_query
+      @tokens.map(&:to_solr).join ' '
+    end
+
+    #
+    #
+    def originals
+      @tokens.map(&:original)
+    end
+
+    # Just join the token original texts.
+    #
+    def to_s
+      originals.join ' '
+    end
+    
+    # def to_a
+    #   @tokens
+    # end
+
+  end
+
+end

data/lib/picky/query/weigher.rb

@@ -0,0 +1,159 @@
+module Query
+
+  # Weighs the given tokens, generates Allocations -> Allocation -> Combinations.
+  #
+  class Weigher
+
+    attr_reader :indexes
+
+    # A weigher has a number of typed indexes, for which it generates allocations.
+    #
+    def initialize types
+      @indexes = types
+    end
+
+    #
+    #
+    def allocations_for tokens
+      Allocations.new(indexes.inject([]) do |previous_allocations, index|
+        # Expand the combinations.
+        #
+        possible_combinations = tokens.possible_combinations_in index
+        # Optimization for ignoring tokens that allocate to nothing and
+        # can be ignored.
+        # For example in a context search, where "florian" is not
+        # mapped to city, zip, or category.
+        #
+        possible_combinations.compact!
+        expanded_combinations = expand_combinations_from possible_combinations
+
+        # TODO Rewrite.
+        #
+        # expanded_combinations.map! { |expanded_combination| Combinations.new(index, expanded_combination) }
+
+        if expanded_combinations.empty?
+          previous_allocations
+        else
+          # The recombination part, where
+          # [
+          #  [a,a,b,b,c,c]
+          #  [d,e,d,e,d,e]
+          # ]
+          # becomes
+          # [
+          #  [a,d],
+          #  [a,e],
+          #  [b,d],
+          #  [b,e],
+          #  [c,d],
+          #  [c,e]
+          # ]
+          #
+          expanded_combinations = expanded_combinations.shift.zip(*expanded_combinations)
+          
+          # Wrap into a real combination.
+          #
+          expanded_combinations.map! { |expanded_combination| Combinations.new(index, expanded_combination) }
+          
+          # Add the possible allocations to the ones we already have.
+          #
+          previous_allocations + expanded_combinations.map(&:pack_into_allocation)
+        end
+      end)
+    end
+    
+    # This is the core of the search engine.
+    #
+    # Gets an array of
+    # [
+    #  [<combinations for token1>],
+    #  [<combinations for token2>],
+    #  [<combinations for token3>]
+    # ]
+    #
+    # Generates all possible allocations of combinations.
+    # [
+    #  [first combination of token1, first c of t2, first c of t3],
+    #  [first combination of token1, first c of t2, second c of t3]
+    #  ...
+    # ]
+    #
+    # Generates all possible combinations of array elements:
+    # [1,2,3] x [a,b,c] x [k,l,m] => [[1,a,k], [1,a,l], [1,a,m], [1,b,k], [1,b,l], [1,b,m], [1,c,k], ..., [3,c,m]]
+    # Note: Also calculates the weights and sorts them accordingly.
+    #
+    # Note: This is a heavily optimized ruby version.
+    #
+    # Works like this:
+    # [1,2,3], [a,b,c], [k,l,m] are expanded to
+    #  group mult: 1
+    #  <- single mult ->
+    # [1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3] = 27 elements
+    #  group mult: 3
+    #  <- -> s/m
+    # [a,a,a,b,b,b,c,c,c,a,a,a,b,b,b,c,c,c,a,a,a,b,b,b,c,c,c] = 27 elements
+    #  group mult: 9
+    #  <> s/m
+    # [k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m] = 27 elements
+    # The array elements are then combined by index (i.e. vertically) to get all combinations.
+    #
+    # Note: Of course I could split this method up into smaller
+    #       ones, but I guess I am a bit sentimental.
+    #
+    def expand_combinations_from possible_combinations
+      # Generate the first multiplicator "with which" (well, not quite) to multiply the smallest amount of combinations.
+      #
+      # TODO How does this work if an element has size 0? Since below we account for size 0.
+      #      Should we even continue if an element has size 0?
+      #      This means one of the tokens cannot be allocated.
+      #
+      single_mult = possible_combinations.inject(1) { |total, combinations| total * combinations.size }
+
+      # Initialize a group multiplicator.
+      #
+      group_mult = 1
+
+      possible_combinations.reject!(&:empty?)
+
+      # The expanding part to line up the combinations
+      # for later combination in allocations.
+      #
+      possible_combinations.collect! do |combinations|
+
+        # Get the size of the combinations of the first token.
+        #
+        combinations_size = combinations.size
+
+        # Special case: If there is no combination for one of the tokens.
+        #               In that case, we just use the same single mult for
+        #               the next iteration.
+        #               If there are combinations, we divide the single mult
+        #               by the number of combinations.
+        #
+        single_mult /= combinations_size unless combinations_size.zero?
+        
+        # Expand each combination by the single mult:
+        # [a,b,c]
+        # [a,a,a,  b,b,b,  c,c,c]
+        # Then, expand the result by the group mult:
+        # [a,a,a,b,b,b,c,c,c,  a,a,a,b,b,b,c,c,c,  a,a,a,b,b,b,c,c,c]
+        #
+        combinations = combinations.inject([]) do |total, combination|
+          total + [combination]*single_mult
+        end * group_mult
+        
+        # Multiply the group mult by the combinations size,
+        # since the next combinations' single mult is smaller
+        # and we need to adjust for that.
+        #
+        group_mult = group_mult * combinations_size
+        
+        # Return the combinations.
+        #
+        combinations
+      end
+    end
+    
+  end
+  
+end

data/lib/picky/query/weights.rb

@@ -0,0 +1,55 @@
+module Query
+
+  # Calculates weights for certain combinations.
+  #
+  class Weights
+    
+    #
+    #
+    def initialize weights = {}
+      @weights_cache = {}
+      @weights = prepare weights
+    end
+    
+    # Get the category indexes for the given bonuses.
+    #
+    def prepare weights
+      weights
+    end
+    
+    # Get the weight of an allocation.
+    #
+    # TODO Add a block to evaluate?
+    #
+    def weight_for clustered
+      @weights[clustered] || 0
+    end
+    
+    # Returns an energy term E for allocation. this turns into a probability
+    # by P(allocation) = 1/Z * exp (-1/T * E(allocation)),
+    # where Z is the normalizing partition function
+    # sum_allocations exp(-1/T *E(allocation)), and T is a temperature constant.
+    # If T is high the distribution will be close to equally distributed.
+    # If T is low, the distribution will be the indicator function
+    # for min (E(allocation))…
+    #
+    # ...
+    #
+    # Just kidding. It's far more complicated than that. Ha ha ha ha ;)
+    #
+    include Helpers::Cache
+    def score combinations
+      # TODO Rewrite to use the category
+      #
+      categories = combinations.map { |combination| combination.bundle.category }.clustered_uniq
+      
+      # TODO Caching even necessary?
+      #
+      cached @weights_cache, categories do
+        categories.map! &:name
+        weight_for categories
+      end
+    end
+
+  end
+end

data/lib/picky/rack/harakiri.rb

@@ -0,0 +1,37 @@
+# Simple Rack Middleware to kill Unicorns after X requests.
+#
+# Use as follows in e.g. your rackup File:
+#
+# Rack::Harakiri.after = 50
+# use Rack::Harakiri
+#
+module Rack
+  class Harakiri
+    
+    # Set the amount of requests before the Unicorn commits Harakiri.
+    #
+    cattr_accessor :after
+    
+    def initialize app
+      @app = app
+      
+      @requests            = 0
+      @quit_after_requests = @@after || 50
+    end
+    
+    def call env
+      harakiri
+      @app.call env
+    end
+    
+    # Checks to see if it is time to honorably retire.
+    #
+    # If yes, kills itself (Unicorn will answer the request, honorably).
+    #
+    def harakiri
+      @requests = @requests + 1
+      Process.kill(:QUIT, Process.pid) if @requests > @quit_after_requests
+    end
+    
+  end
+end

data/lib/picky/results/base.rb

@@ -0,0 +1,103 @@
+module Results
+
+  # This is the internal results object. Usually, to_marshal, or to_json is called on it.
+  #
+  class Base
+
+    # Duration is set by the query.
+    #
+    attr_writer :duration
+    attr_reader :allocations, :offset
+
+    # Takes instances of Query::Allocations as param.
+    #
+    def initialize allocations = nil
+      @allocations = allocations || Query::Allocations.new
+      @offset = 0
+    end
+
+    def add more_results
+      @added = more_results
+      self
+    end
+    def added
+      @added || {}
+    end
+
+    #
+    #
+    def serialize
+      added.merge(:allocations => allocations.to_result,
+                  :offset      => offset,
+                  :duration    => duration,
+                  :total       => total)
+    end
+    # Convert to marshal format.
+    #
+    def to_marshal
+      Marshal.dump serialize
+    end
+    # Convert to json format.
+    #
+    def to_json options = {}
+      serialize.to_json options
+    end
+
+    # This starts the actual processing.
+    #
+    # Without this, the allocations are not processed,
+    # and no ids are calculated.
+    #
+    def prepare! offset = 0
+      @offset = offset
+      allocations.process! self.max_results, offset
+    end
+
+    # Duration default is 0.
+    #
+    def duration
+      @duration || 0
+    end
+    # The total results. Delegates to the allocations.
+    #
+    # Caches.
+    #
+    def total
+      @total || @total = allocations.total || 0
+    end
+
+    # How many results are returned.
+    #
+    # Set in config using
+    #   Results::Full.max_results = 20
+    #
+    class_inheritable_accessor :max_results
+    def max_results
+      self.class.max_results
+    end
+
+    # Convenience methods.
+    #
+
+    # Delegates to allocations.
+    #
+    def ids amount = 20
+      allocations.ids amount
+    end
+    # Gets an amout of random ids from the allocations.
+    #
+    # Note: Basically delegates to the allocations.
+    #
+    def random_ids amount = 1
+      allocations.random_ids amount
+    end
+
+    # Human readable log.
+    #
+    def to_log query
+      "|#{Time.now.to_s(:db)}|#{'%8f' % duration}|#{'%-50s' % query}|#{'%8d' % total}|#{'%4d' % offset}|#{'%2d' % allocations.size}|"
+    end
+    
+  end
+
+end

data/lib/picky/results/full.rb

@@ -0,0 +1,19 @@
+module Results
+  # Full results are limited to maximally 20 results (by default).
+  #
+  class Full < Base
+    
+    self.max_results = 20
+    
+    def to_log *args
+      ?> + super
+    end
+    
+    # The default response style for full results is to_marshal.
+    #
+    def to_response
+      to_marshal
+    end
+    
+  end
+end

data/lib/picky/results/live.rb

@@ -0,0 +1,19 @@
+module Results
+  # Live results are not returning any results.
+  #
+  class Live < Base
+    
+    self.max_results = 0
+    
+    def to_log *args
+      ?. + super
+    end
+    
+    # The default response style for live results is to_json.
+    #
+    def to_response
+      to_json
+    end
+    
+  end
+end

data/lib/picky/routing.rb

@@ -0,0 +1,165 @@
+require 'rack/mount'
+
+# 
+#
+class Routing
+  
+  @@defaults = {
+    :query_key    => 'query'.freeze,
+    :offset_key   => 'offset'.freeze,
+    :content_type => 'application/octet-stream'.freeze
+  }
+  
+  def initialize
+    @defaults = @@defaults.dup
+  end
+  
+  # #
+  # #
+  # def define_using &block
+  #   reset_routes
+  #   instance_eval &block
+  #   routes.freeze
+  # end
+  
+  # 
+  #
+  def reset_routes
+    @routes = Rack::Mount::RouteSet.new
+  end
+  def routes
+    @routes || reset_routes
+  end
+  def freeze
+    routes.freeze
+  end
+  
+  # Routing simply delegates to the route set to handle a request.
+  #
+  def call env
+    routes.call env
+  end
+  
+  # Set the defaults.
+  # 
+  # Options are:
+  #  * :query_key  => :query # default
+  #  * :offset_key => :offset # default
+  #  
+  #  * :tokenizer  => Tokenizers::Query.new # default
+  #
+  def defaults options = {}
+    @defaults[:query_key]    = options[:query_key].to_s  if options[:query_key]
+    @defaults[:offset_key]   = options[:offset_key].to_s if options[:offset_key]
+    
+    @defaults[:tokenizer]    = options[:tokenizer]       if options[:tokenizer]
+    @defaults[:content_type] = options[:content_type]    if options[:content_type]
+    
+    @defaults
+  end
+  
+  #
+  #
+  def route url, query, route_options = {}
+    query.tokenizer = @defaults[:tokenizer] if @defaults[:tokenizer]
+    routes.add_route generate_app(query, route_options), default_options(url, route_options)
+  end
+  #
+  #
+  def live url, *indexes_and_options
+    route_options = Hash === indexes_and_options.last ? indexes_and_options.pop : {}
+    route url, Query::Live.new(*indexes_and_options), route_options
+  end
+  #
+  #
+  def full url, *indexes_and_options
+    route_options = Hash === indexes_and_options.last ? indexes_and_options.pop : {}
+    route url, Query::Full.new(*indexes_and_options), route_options
+  end
+  #
+  #
+  def root status
+    answer %r{^/$}, STATUSES[status]
+  end
+  #
+  #
+  def default status
+    answer nil, STATUSES[status]
+  end
+  
+  
+  
+  # TODO Can Rack handle this for me?
+  #
+  # Note: Rack-mount already handles the 404.
+  #
+  STATUSES = {
+    200 => lambda { |_| [200, { 'Content-Type' => 'text/html', 'Content-Length' => '0' }, ['']] },
+    404 => lambda { |_| [404, { 'Content-Type' => 'text/html', 'Content-Length' => '0' }, ['']] }
+  }
+  
+  #
+  #
+  def default_options url, route_options = {}
+    url = normalized url
+    
+    options = { :request_method => 'GET' }.merge route_options
+    
+    options[:path_info] = url if url
+    
+    options.delete :content_type # TODO
+    
+    query_params = options.delete :query
+    options[:query_string] = %r{#{generate_query_string(query_params)}} if query_params
+    
+    options
+  end
+  
+  #
+  #
+  def generate_query_string query_params
+    raise "At least one query string condition is needed." if query_params.size.zero?
+    raise "Too many query param conditions (only 1 allowed): #{query_params}" if query_params.size > 1
+    k, v = query_params.first
+    "#{k}=#{v}"
+  end
+  
+  # Generates a rack app for the given query.
+  #
+  def generate_app query, options = {}
+    query_key    = options[:query_key]    || @defaults[:query_key]
+    content_type = options[:content_type] || @defaults[:content_type]
+    lambda do |env|
+      params  = Rack::Request.new(env).params
+      
+      results = query.search_with_text *extracted(params)
+      
+      PickyLog.log results.to_log(params[query_key]) # TODO Save the original query in the results object.
+      
+      respond_with results.to_response, content_type
+    end
+  end
+  UTF8_STRING = 'UTF-8'.freeze
+  def extracted params
+    [
+      # query is encoded in ASCII
+      #
+      params[@defaults[:query_key]]  && params[@defaults[:query_key]].force_encoding(UTF8_STRING),
+      params[@defaults[:offset_key]] && params[@defaults[:offset_key]].to_i || 0
+    ]
+  end
+  def respond_with response, content_type
+    [200, { 'Content-Type' => content_type, 'Content-Length' => response.size.to_s, }, [response]]
+  end
+  
+  # Setup a route that answers using the given app.
+  #
+  def answer url = nil, app = nil
+    routes.add_route (app || STATUSES[200]), default_options(url)
+  end
+  
+  def normalized url
+    String === url ? %r{#{url}} : url
+  end
+  
+end

data/lib/picky/signals.rb

@@ -0,0 +1,11 @@
+# TODO Cleanup and move to prototype_project.
+#
+# Signal.trap 'USR1' do
+#   Indexes.reload
+# end
+# Signal.trap 'USR2' do
+#   Loader.reload
+# end
+# Signal.trap 'INT' do
+#   exit!
+# end