picky 1.5.4 → 2.0.0.pre1

data/lib/picky/application.rb

@@ -50,26 +50,24 @@
 #
 # So now we have indexed data (the title), but nobody to ask the index anything.
 #
-# == Query::Full.new(*indexes, options = {})
+# == Search.new(*indexes, options = {})
 #
 # We need somebody who asks the index (a Query object, also see http://github.com/floere/picky/wiki/Queries-Configuration). That works like this:
-#    full_books_query = Query::Full.new books
-# Full just means that the ids are returned with the results.
-# Picky also offers a Query that returns live results, Query::Live. But that's not important right now.
+#    books_search = Search.new books
 #
 # Now we have somebody we can ask about the index. But no external interface.
 #
-# == route(/regexp1/ => query1, /regexp2/ => query2, ...)
+# == route(/regexp1/ => search1, /regexp2/ => search2, ...)
 #
 # Let's add a URL path (a Route, see http://github.com/floere/picky/wiki/Routing-configuration) to which we can send our queries. We do that with the route method:
-#   route %r{^/books/full$} => full_books_query
+#   route %r{^/books$} => books_query
 # In full glory:
 #   class MyGreatSearch < Application
 #
 #     books = index :books, Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
 #     books.define_category :title
 #
-#     route %r{^/books/full$} => Query::Full.new(books)
+#     route %r{^/books$} => Search.new(books)
 #
 #   end
 # That's it!
@@ -137,10 +135,9 @@
 #                           partial: Partial::Substring.new(:from => -2)
 #     books.define_category :isbn
 #
-#     query_options = { :weights => { [:title, :author] => +3, [:author, :title] => -1 } }
+#     options = { :weights => { [:title, :author] => +3, [:author, :title] => -1 } }
 #
-#     route %r{^/books/full$} => Query::Full.new(books, query_options)
-#     route %r{^/books/live$} => Query::Live.new(books, query_options)
+#     route %r{^/books$} => Search.new(books, options)
 #
 #   end
 # That's actually already a full-blown Picky App!
@@ -166,25 +163,6 @@ class Application
       Internals::Tokenizers::Query.default = Internals::Tokenizers::Query.new(options)
     end
 
-    # Create a new index for indexing and for querying.
-    #
-    # Parameters:
-    # * name: The identifier of the index. Used:
-    #   - to identify an index (e.g. by you in Rake tasks).
-    #   - in the frontend to describe which index a result came from.
-    #   - index directory naming (index/development/the_identifier/<lots of indexes>)
-    # * source: The source the data comes from. See Sources::Base.
-    #
-    # Options:
-    # * result_identifier: Use if you'd like a different identifier/name in the results JSON than the name of the index.
-    #
-    # TODO Remove in 1.6.
-    #
-    def index name, source, options = {}
-      raise "the Picky application method #index is deprecated, please use Index::Memory.new instead."
-      Index::Memory.new name, source, options
-    end
-
     # Routes.
     #
     delegate :route, :root, :to => :rack_adapter

data/lib/picky/internals/adapters/rack/query.rb

@@ -10,11 +10,12 @@ module Internals
     # and call search_with_text on it if it is called by Rack.
     #
     module Rack
-    
+
       class Query < Base
-      
+
         @@defaults = {
           query_key:    'query'.freeze,
+          ids_key:      'ids'.freeze,
           offset_key:   'offset'.freeze,
           content_type: 'application/json'.freeze
         }
@@ -23,7 +24,7 @@ module Internals
           @query    = query
           @defaults = @@defaults.dup
         end
-      
+
         def to_app options = {}
           # For capturing in the lambda.
           #
@@ -41,23 +42,26 @@ module Internals
             respond_with results.to_response, content_type
           end
         end
-      
+
         # Helper method to extract the params
         #
+        # Defaults are 20 ids, offset 0.
+        #
         UTF8_STRING = 'UTF-8'.freeze
         def extracted params
           [
             # query is encoded in ASCII
             #
             params[@defaults[:query_key]]  && params[@defaults[:query_key]].force_encoding(UTF8_STRING),
+            params[@defaults[:ids_key]]    && params[@defaults[:ids_key]].to_i || 20,
             params[@defaults[:offset_key]] && params[@defaults[:offset_key]].to_i || 0
           ]
         end
-      
+
       end
-    
+
     end
-  
+
   end
-  
+
 end

data/lib/picky/internals/query/indexes.rb

@@ -1,20 +1,20 @@
 module Internals
 
   module Query
-  
+
     # The query indexes class bundles indexes given to a query.
     #
     # Example:
     #   # If you call
-    #   Query::Full.new dvd_index, mp3_index, video_index
-    #   
+    #   Search.new dvd_index, mp3_index, video_index
+    #
     #   # What it does is take the three given (API-) indexes and
     #   # bundle them in an index bundle.
     #
     class Indexes
-    
+
       attr_reader :indexes
-    
+
       # Creates a new Query::Indexes.
       #
       # Its job is to generate all possible combinations, but also
@@ -33,22 +33,22 @@ module Internals
           # Expand the combinations.
           #
           possible_combinations = tokens.possible_combinations_in index
-        
+
           # Optimization for ignoring tokens that allocate to nothing and
           # can be ignored.
           # For example in a special search, where "florian" is not
           # mapped to any category.
           #
           possible_combinations.compact!
-        
+
           # Generate all possible combinations.
           #
           expanded_combinations = expand_combinations_from possible_combinations
-        
+
           # If there are none, try the next allocation.
           #
           next previous_allocations unless expanded_combinations
-        
+
           # Add the wrapped possible allocations to the ones we already have.
           #
           previous_allocations + expanded_combinations.map! do |expanded_combination|
@@ -56,7 +56,7 @@ module Internals
           end
         end)
       end
-    
+
       # This is the core of the search engine.
       #
       # Gets an array of
@@ -118,7 +118,7 @@ module Internals
         # tokens could not be allocated.
         #
         return if possible_combinations.any?(&:empty?)
-      
+
         # Generate the first multiplicator "with which" (well, not quite) to multiply the smallest amount of combinations.
         #
         single_mult = possible_combinations.inject(1) { |total, combinations| total * combinations.size }
@@ -131,7 +131,7 @@ module Internals
         # for later combination in allocations.
         #
         possible_combinations.collect! do |combinations|
-        
+
           # Get the size of the combinations of the first token.
           #
           combinations_size = combinations.size
@@ -143,7 +143,7 @@ module Internals
           #               by the number of combinations.
           #
           single_mult /= combinations_size unless combinations_size.zero?
-        
+
           # Expand each combination by the single mult:
           # [a,b,c]
           # [a,a,a,  b,b,b,  c,c,c]
@@ -153,25 +153,25 @@ module Internals
           combinations = combinations.inject([]) do |total, combination|
             total + Array.new(single_mult, combination)
           end * group_mult
-        
+
           # Multiply the group mult by the combinations size,
           # since the next combinations' single mult is smaller
           # and we need to adjust for that.
           #
           group_mult = group_mult * combinations_size
-        
+
           # Return the combinations.
           #
           combinations
         end
-      
+
         return if possible_combinations.empty?
-      
+
         possible_combinations.shift.zip *possible_combinations
       end
-    
+
     end
-  
+
   end
-  
+
 end

data/lib/picky/loader.rb

@@ -214,12 +214,6 @@ module Loader # :nodoc:all
 
     load_internals 'query/indexes'
 
-    # Results.
-    #
-    load_internals 'results/base'
-    load_internals 'results/full'
-    load_internals 'results/live'
-
     # Configuration.
     #
     load_internals 'configuration/index'
@@ -274,11 +268,13 @@ module Loader # :nodoc:all
     load_relative 'index_bundle'
     load_relative 'aliases'
 
-    # Query.
+    # Results.
+    #
+    load_relative 'results'
+
+    # Search.
     #
-    load_relative 'query/base'
-    load_relative 'query/live'
-    load_relative 'query/full'
+    load_relative 'search'
     #
     # load_relative 'query/solr'
 

data/lib/picky/results.rb

@@ -0,0 +1,93 @@
+module Internals
+
+  # This is the internal results object. Usually, to_marshal, or to_json
+  # is called on it to get a string for the answer.
+  #
+  class Results
+
+    # Duration is set externally by the query.
+    #
+    attr_writer :duration
+    attr_reader :allocations, :offset, :amount
+
+    # Takes instances of Query::Allocations as param.
+    #
+    def initialize amount = 0, offset = 0, allocations = Query::Allocations.new
+      @offset      = offset
+      @amount      = amount
+      @allocations = allocations
+    end
+    # Create new results and calculate the ids.
+    #
+    def self.from amount, offset, allocations
+      results = new amount, offset, allocations
+      results.prepare!
+      results
+    end
+
+    # Returns a hash with the allocations, offset, duration and total.
+    #
+    def serialize
+      { allocations: allocations.to_result,
+        offset:      offset,
+        duration:    duration,
+        total:       total }
+    end
+    # The default format is json.
+    #
+    def to_response options = {}
+      to_json options
+    end
+    # Convert to json format.
+    #
+    def to_json options = {}
+      serialize.to_json options
+    end
+
+    # This starts the actual processing.
+    #
+    # Without this, the allocations are not processed,
+    # and no ids are calculated.
+    #
+    def prepare!
+      allocations.process! amount, offset
+    end
+
+    # Duration default is 0.
+    #
+    def duration
+      @duration || 0
+    end
+    # The total results. Delegates to the allocations.
+    #
+    # Caches.
+    #
+    def total
+      @total || @total = allocations.total || 0
+    end
+
+    # Convenience methods.
+    #
+
+    # Delegates to allocations.
+    #
+    def ids amount = 20
+      allocations.ids amount
+    end
+
+    # Human readable log.
+    #
+    def to_log query
+      "#{log_type}|#{Time.now.to_s(:db)}|#{'%8f' % duration}|#{'%-50s' % query}|#{'%8d' % total}|#{'%4d' % offset}|#{'%2d' % allocations.size}|"
+    end
+    # The first character in the blog designates what type of query it is.
+    #
+    # No calculated ids means: No results.
+    #
+    def log_type
+      amount.zero?? :'.' : :'>'
+    end
+
+  end
+
+end

data/lib/picky/search.rb

@@ -0,0 +1,180 @@
+# = Picky Queries
+#
+# A Picky Search is an object which:
+# * holds one or more indexes
+# * offers an interface to query these indexes.
+#
+# You connect URL paths to indexes via a Query.
+#
+# We recommend not to use this directly, but connect it to an URL and query through one of these
+# (Protip: Use "curl 'localhost:8080/query/path?query=exampletext')" in a Terminal.
+#
+class Search
+
+  include Helpers::Measuring
+
+  attr_reader   :indexes
+  attr_writer   :tokenizer, :identifiers_to_remove
+  attr_accessor :reduce_to_amount, :weights
+
+  # Takes:
+  # * A number of indexes
+  # * Options hash (optional) with:
+  #   * tokenizer: Tokenizers::Query.default by default.
+  #   * weights:   A hash of weights, or a Query::Weights object.
+  #
+  def initialize *index_definitions
+    options      = Hash === index_definitions.last ? index_definitions.pop : {}
+
+    @indexes     = Internals::Query::Indexes.new *index_definitions, combinations_type_for(index_definitions)
+    @tokenizer   = options[:tokenizer] || Internals::Tokenizers::Query.default
+    weights      = options[:weights] || Query::Weights.new
+    @weights     = Hash === weights ? Query::Weights.new(weights) : weights
+  end
+
+  # Returns the right combinations strategy for
+  # a number of query indexes.
+  #
+  # Currently it isn't possible using Memory and Redis etc.
+  # indexes in the same query index group.
+  #
+  # Picky will raise a Query::Indexes::DifferentTypesError.
+  #
+  @@mapping = {
+    Index::Memory => Internals::Query::Combinations::Memory,
+    Index::Redis  => Internals::Query::Combinations::Redis
+  }
+  def combinations_type_for index_definitions_ary
+    index_types = index_definitions_ary.map(&:class)
+    index_types.uniq!
+    raise_different(index_types) if index_types.size > 1
+    !index_types.empty? && @@mapping[*index_types] || Internals::Query::Combinations::Memory
+  end
+  # Currently it isn't possible using Memory and Redis etc.
+  # indexes in the same query index group.
+  #
+  class DifferentTypesError < StandardError
+    def initialize types
+      @types = types
+    end
+    def to_s
+      "Currently it isn't possible to mix #{@types.join(" and ")} Indexes in the same Search instance."
+    end
+  end
+  def raise_different index_types
+    raise DifferentTypesError.new(index_types)
+  end
+
+  # This is the main entry point for a query.
+  # Use this in specs and also for running queries.
+  #
+  # Parameters:
+  # * text: The search text.
+  # * ids = 20: _optional_ The amount of ids to calculate (with offset).
+  # * offset = 0: _optional_ The offset from which position to return the ids. Useful for pagination.
+  #
+  # Note: The Rack adapter calls this method after unravelling the HTTP request.
+  #
+  def search_with_text text, ids = 20, offset = 0
+    search tokenized(text), ids, offset
+  end
+
+  # Runs the actual search using Query::Tokens.
+  #
+  # Note: Internal method, use #search_with_text.
+  #
+  def search tokens, ids = 20, offset = 0
+    results = nil
+
+    duration = timed do
+      results = execute tokens, ids, offset
+    end
+    results.duration = duration.round 6
+
+    results
+  end
+
+  # Execute a search using Query::Tokens.
+  #
+  # Note: Internal method, use #search_with_text.
+  #
+  def execute tokens, ids, offset
+    Internals::Results.from ids, offset, sorted_allocations(tokens)
+  end
+
+  # Delegates the tokenizing to the query tokenizer.
+  #
+  # Parameters:
+  # * text: The text to tokenize.
+  #
+  def tokenized text
+    @tokenizer.tokenize text
+  end
+
+  # Gets sorted allocations for the tokens.
+  #
+  # This generates the possible allocations, sorted.
+  #
+  # TODO Smallify.
+  #
+  # TODO Rename: allocations
+  #
+  def sorted_allocations tokens # :nodoc:
+    # Get the allocations.
+    #
+    # TODO Pass in reduce_to_amount (aka max_allocations)
+    #
+    # TODO uniq, score, sort in there
+    #
+    allocations = @indexes.allocations_for tokens
+
+    # Callbacks.
+    #
+    # TODO Reduce before sort?
+    #
+    reduce allocations
+    remove_from allocations
+
+    # Remove double allocations.
+    #
+    allocations.uniq
+
+    # Score the allocations using weights as bias.
+    #
+    allocations.calculate_score weights
+
+    # Sort the allocations.
+    # (allocations are sorted according to score, highest to lowest)
+    #
+    allocations.sort
+
+    # Return the allocations.
+    #
+    allocations
+  end
+  def reduce allocations # :nodoc:
+    allocations.reduce_to reduce_to_amount if reduce_to_amount
+  end
+
+  #
+  #
+  def remove_from allocations # :nodoc:
+    allocations.remove identifiers_to_remove
+  end
+  #
+  #
+  def identifiers_to_remove # :nodoc:
+    @identifiers_to_remove ||= []
+  end
+
+  # Display some nice information for the user.
+  #
+  def to_s
+    s = "#{self.class}("
+    s << @indexes.indexes.map(&:name).join(', ')
+    s << ", weights: #{@weights}" unless @weights.empty?
+    s << ")"
+    s
+  end
+
+end

data/lib/picky/sources/base.rb

@@ -48,9 +48,11 @@ module Sources
     # be sure to <tt>yield(id, text_for_id)</tt> (or <tt>block.call(id, text_for_id)</tt>)
     # for the given type symbol and category symbol.
     #
-    def harvest index, category # :yields: id, text_for_id
-      # This concrete implementation yields "nothing", override in subclasses.
-    end
+    # Note: Since harvest needs to be implemented, it has no default impementation.
+    #
+    # def harvest index, category # :yields: id, text_for_id
+    #
+    # end
 
     # Used to take a snapshot of your data if it is fast changing.
     #

data/lib/tasks/todo.rake

@@ -0,0 +1,8 @@
+desc "Finds where Picky still needs input from you."
+task :todo do
+  if system "grep -e 'TODO.*' -n --color=always -R *"
+    puts "Picky needs a bit of input from you there. Thanks."
+  else
+    puts "Picky seems to be fine (no TODOs found)."
+  end
+end

data/spec/lib/application_spec.rb

@@ -11,13 +11,9 @@ describe Application do
           books = Index::Memory.new :books, Sources::DB.new('SELECT id, title FROM books', :file => 'app/db.yml')
           books.define_category :title
           
-          full = Query::Full.new books
-          live = Query::Live.new books
-          
           rack_adapter.stub! :exclaim # Stopping it from exclaiming.
           
-          route %r{^/books/full} => full
-          route %r{^/books/live} => live
+          route %r{^/books} => Search.new(books)
         end
         Internals::Tokenizers::Index.default.tokenize 'some text'
         Internals::Tokenizers::Query.default.tokenize 'some text'
@@ -56,13 +52,9 @@ describe Application do
           geo_index.define_ranged_category(:north1, 1, precision: 3, from: :north)
                    .define_ranged_category(:east1,  1, precision: 3, from: :east)
           
-          full = Query::Full.new books_index
-          live = Query::Live.new books_index
-          
           rack_adapter.stub! :exclaim # Stopping it from exclaiming.
           
-          route %r{^/books/full} => full
-          route %r{^/books/live} => live
+          route %r{^/books} => Search.new(books_index)
         end
       }.should_not raise_error
     end

data/spec/lib/character_substituters/west_european_spec.rb

@@ -19,6 +19,20 @@ describe CharacterSubstituters do
       @substituter.substitute(special_character).should == special_character
     end
   end
+  
+  # Speed spec at the top since the order of the describes made the
+  # speed spec trip. And not on mushrooms either.
+  #
+  describe "speed" do
+    it "is fast" do
+      result = performance_of { @substituter.substitute('ä') }
+      result.should < 0.00009
+    end
+    it "is fast" do
+      result = performance_of { @substituter.substitute('abcdefghijklmnopqrstuvwxyz1234567890') }
+      result.should < 0.00015
+    end
+  end
 
   describe "normal characters" do
     it_should_not_substitute('abcdefghijklmnopqrstuvwxyz1234567890')
@@ -91,16 +105,5 @@ describe CharacterSubstituters do
   describe "diacritic" do
     it_should_substitute 'ñ', 'n'
   end
-  
-  describe "speed" do
-    it "is fast" do
-      result = performance_of { @substituter.substitute('ä') }
-      result.should < 0.00009
-    end
-    it "is fast" do
-      result = performance_of { @substituter.substitute('abcdefghijklmnopqrstuvwxyz1234567890') }
-      result.should < 0.00015
-    end
-  end
 
 end

data/spec/lib/internals/adapters/rack/query_spec.rb

@@ -22,11 +22,17 @@ describe Internals::Adapters::Rack::Query do
     it 'extracts the query' do
       @adapter.extracted('query' => 'some_query')[0].should == 'some_query'
     end
+    it 'extracts the default ids amount' do
+      @adapter.extracted('query' => 'some_query')[1].should == 20
+    end
     it 'extracts the default offset' do
-      @adapter.extracted('query' => 'some_query')[1].should == 0
+      @adapter.extracted('query' => 'some_query')[2].should == 0
+    end
+    it 'extracts a given ids amount' do
+      @adapter.extracted('query' => 'some_query', 'ids' => '123')[1].should == 123
     end
     it 'extracts a given offset' do
-      @adapter.extracted('query' => 'some_query', 'offset' => '123')[1].should == 123
+      @adapter.extracted('query' => 'some_query', 'offset' => '123')[2].should == 123
     end
   end