picky 0.12.3 → 1.0.0

data/lib/deployment.rb

@@ -1,7 +1,7 @@
 require File.expand_path '../constants', __FILE__
 
-module Picky
-  module Capistrano
+module Picky # :nodoc:
+  module Capistrano # :nodoc:all
     
     # Include all 
     #

data/lib/picky/application.rb

@@ -1,4 +1,152 @@
-# The Picky application wherein the indexing and querying is defined.
+# = Picky Applications
+#
+# A Picky Application is where you configure the whole search engine.
+#
+# This is a step-by-step description on how to configure your Picky app.
+#
+# Start by subclassing Application:
+#   class MyGreatSearch < Application
+#     # Your configuration goes here.
+#   end
+# The generator
+#   $ picky project project_name
+# will generate an example <tt>project_name/app/application.rb</tt> file for you
+# with some example code inside.
+#
+# == index(name, source)
+#
+# Next, define where your data comes from. You use the <tt>index</tt> method for that:
+#   my_index = index :some_index_name, some_source
+# You give the index a name (or identifier), and a source (see Sources), where its data comes from. Let's do that:
+#   class MyGreatSearch < Application
+#     
+#     books = index :books, Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
+#     
+#   end
+# Now we have an index <tt>books</tt>.
+# 
+# That on itself won't do much good.
+#
+# == index.define_category(identifier, options = {})
+#
+# Picky needs us to define categories on the data.
+#
+# Categories help your user find data.
+# It's best if you look at an example yourself: http://floere.github.com/picky/examples.html
+#
+# Let's go ahead and define a category:
+#   class MyGreatSearch < Application
+#     
+#     books = index :books, Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
+#     books.define_category :title
+#
+#   end
+# Now we could already run the indexer:
+#   $ rake index
+#
+# (You can define similarity or partial search capabilities on a category, see http://github.com/floere/picky/wiki/Categories-configuration for info)
+#
+# So now we have indexed data (the title), but nobody to ask the index anything.
+#
+# == Query::Full.new(*indexes, options = {})
+#
+# We need somebody who asks the index (a Query object, also see http://github.com/floere/picky/wiki/Queries-Configuration). That works like this:
+#    full_books_query = Query::Full.new books
+# Full just means that the ids are returned with the results.
+# Picky also offers a Query that returns live results, Query::Live. But that's not important right now.
+# 
+# Now we have somebody we can ask about the index. But no external interface.
+# 
+# == route(/regexp1/ => query1, /regexp2/ => query2, ...)
+#
+# Let's add a URL path (a Route, see http://github.com/floere/picky/wiki/Routing-configuration) to which we can send our queries. We do that with the route method:
+#   route %r{^/books/full$} => full_books_query
+# In full glory:
+#   class MyGreatSearch < Application
+#     
+#     books = index :books, Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
+#     books.define_category :title
+#
+#     full_books_query = Query::Full.new books
+#
+#     route %r{^/books/full$} => full_books_query
+#
+#   end
+# That's it!
+#
+# Now run the indexer and server:
+#   $ rake index
+#   $ rake start
+# Run your first query:
+#   $ curl 'localhost:8080/books/full?query=hello server'
+#
+# Nice, right? Your first query!
+#
+# Maybe you don't find everything. We need to process the data before it goes into the index.
+#
+# == default_indexing(options = {})
+#
+# That's what the <tt>default_indexing</tt> method is for:
+#   default_indexing options
+# Read more about the options here: http://github.com/floere/picky/wiki/Indexing-configuration
+#
+# Same thing with the search text – we need to process that as well.
+#
+# == default_querying(options = {})
+#
+# Analog to the default_indexing method, we use the <tt>default_querying</tt> method.
+#   default_querying options
+# Read more about the options here: http://github.com/floere/picky/wiki/Querying-Configuration
+#
+# And that's all there is. It's incredibly powerful though, as you can combine, weigh, refine to the max.
+#
+# == Wiki
+#
+# Read more in the Wiki: http://github.com/floere/picky/wiki
+#
+# Have fun!
+#
+# == Full example
+#
+# Our example, fully fleshed out with indexing, querying, and weights:
+#   class MyGreatSearch < Application
+#     
+#     default_indexing removes_characters: /[^a-zA-Z0-9\.]/,
+#                      stopwords: /\b(and|or|in|on|is|has)\b/,
+#                      splits_text_on: /\s/,
+#                      removes_characters_after_splitting: /\./,
+#                      substitutes_characters_with: CharacterSubstituters::WestEuropean.new,
+#                      normalizes_words: [
+#                        [/(.*)hausen/, 'hn'],
+#                        [/\b(\w*)str(eet)?/, 'st']
+#                      ]
+#
+#     default_querying removes_characters: /[^a-zA-Z0-9\s\/\-\,\&\"\~\*\:]/,
+#                      stopwords: /\b(and|the|of|it|in|for)\b/,
+#                      splits_text_on: /[\s\/\-\,\&]+/,
+#                      removes_characters_after_splitting: /\./,
+#                      substitutes_characters_with: CharacterSubstituters::WestEuropean.new,
+#                      maximum_tokens: 4
+#     
+#     books = index :books, Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
+#     books.define_category :title,
+#                           qualifiers: [:t, :title, :titre],
+#                           partial:    Partial::Substring.new(:from => 1),
+#                           similarity: Similarity::Phonetic.new(2)
+#     books.define_category :author,
+#                           partial: Partial::Substring.new(:from => -2)
+#     books.define_category :isbn
+#     
+#     query_options = { :weights => { [:title, :author] => +3, [:author, :title] => -1 } }
+#     
+#     full_books_query = Query::Full.new books, query_options
+#     live_books_query = Query::Full.new books, query_options
+#     
+#     route %r{^/books/full$} => full_books_query
+#     route %r{^/books/live$} => live_books_query
+#     
+#   end
+# That's actually already a full-blown Picky App!
 #
 class Application
   
@@ -21,10 +169,20 @@ class Application
       Tokenizers::Query.default = Tokenizers::Query.new(options)
     end
     
-    # Returns a new index frontend for configuring the index.
+    # Create a new index for indexing and for querying.
+    #
+    # Parameters:
+    # * name: The identifier of the index. Used:
+    #   - to identify an index (e.g. by you in Rake tasks).
+    #   - in the frontend to describe which index a result came from.
+    #   - index directory naming (index/development/the_identifier/<lots of indexes>)
+    # * source: The source the data comes from. See Sources::Base. # TODO Sources (all).
     #
-    def index *args
-      IndexAPI.new *args
+    # Options:
+    # * result_type: # TODO Rename.
+    #
+    def index name, source, options = {}
+      IndexAPI.new name, source, options
     end
     
     # Routes.
@@ -35,39 +193,41 @@ class Application
     # API
     
     
-    # An application simply delegates to the routing to handle a request.
+    # A Picky application implements the Rack interface.
+    #
+    # Delegates to its routing to handle a request.
     #
     def call env
       routing.call env
     end
-    def routing
+    def routing # :nodoc:
       @routing ||= Routing.new
     end
     
     # Finalize the subclass as soon as it
     # has finished loading.
     #
-    attr_reader :apps
-    def initialize_apps
+    attr_reader :apps # :nodoc:
+    def initialize_apps # :nodoc:
       @apps ||= []
     end
-    def inherited app
+    def inherited app # :nodoc:
       initialize_apps
       apps << app
     end
-    def finalize_apps
+    def finalize_apps # :nodoc:
       initialize_apps
       apps.each &:finalize
     end
     # Finalizes the routes.
     #
-    def finalize
+    def finalize # :nodoc:
       routing.freeze
     end
     
     # TODO Add more info if possible.
     #
-    def to_s
+    def to_s # :nodoc:
       "#{self.name}:\n#{routing}"
     end
     

data/lib/picky/cacher/generator.rb

@@ -1,4 +1,4 @@
-module Cacher
+module Cacher # :nodoc:all
   
   # A cache generator holds an index.
   #

data/lib/picky/calculations/location.rb

@@ -1,4 +1,4 @@
-module Calculations
+module Calculations # :nodoc:all
   
   # A location calculation recalculates a 1-d location
   # to the Picky internal 1-d "grid".
@@ -18,7 +18,15 @@ module Calculations
     end
     
     def minimum= minimum
+      # Add a margin of 1 user grid.
+      #
       minimum -= @user_grid
+      
+      # Add plus 1 grid so that the index key never falls on 0.
+      # Why? to_i maps by default to 0.
+      #
+      minimum -= @grid
+      
       @minimum = minimum
     end
     

data/lib/picky/character_substituters/west_european.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 #
-module CharacterSubstituters
+module CharacterSubstituters # :nodoc:all
   # Substitutes Umlauts like
   # ä, ö, ü => ae, oe, ue.
   # (and more, see specs)

data/lib/picky/configuration/index.rb

@@ -1,4 +1,4 @@
-module Configuration
+module Configuration # :nodoc:all
   
   # Holds the configuration for a
   # index/category combination.

data/lib/picky/cores.rb

@@ -2,7 +2,7 @@ Infinity = 1.0/0
 
 # Handles processing over multiple cores.
 #
-class Cores
+class Cores # :nodoc:all
   
   # Pass it an ary or generator.
   #

data/lib/picky/extensions/array.rb

@@ -1,6 +1,6 @@
 # The Array class we all know and love.
 #
-class Array
+class Array # :nodoc:all
 
   # Cluster-uniqs equal neighborly elements.
   #

data/lib/picky/extensions/hash.rb

@@ -1,6 +1,6 @@
 # Extensions for the Hash.
 #
-class Hash
+class Hash # :nodoc:all
   
   # Dumps jsonized self to the path given. Minus extension.
   #

data/lib/picky/extensions/module.rb

@@ -1,6 +1,6 @@
 # The original Module class.
 #
-class Module
+class Module # :nodoc:all
 
   def each_delegate *methods
     options = methods.pop

data/lib/picky/extensions/object.rb

@@ -1,4 +1,4 @@
-class Object
+class Object # :nodoc:all
   
   # Puts a text in the form:
   #   12:34:56: text here

data/lib/picky/extensions/symbol.rb

@@ -1,6 +1,6 @@
 # Extending the Symbol class.
 #
-class Symbol
+class Symbol # :nodoc:all
   
   # :keys.each_subtoken    # => yields each of [:keys, :key, :ke, :k]
   # :keys.each_subtoken(2) # => yields each of [:keys, :key, :ke]

data/lib/picky/generator.rb

@@ -8,7 +8,7 @@ module Picky
   #   picky <command> <options>
   # is found.
   #
-  class NoGeneratorError < StandardError
+  class NoGeneratorError < StandardError # :nodoc:all
     
     def initialize generator
       super usage + possible_commands(generator.types)
@@ -38,7 +38,7 @@ module Picky
   #
   # Basically copies a prototype project into a newly generated directory.
   #
-  class Generator
+  class Generator # :nodoc:all
     
     attr_reader :types
     

data/lib/picky/helpers/cache.rb

@@ -1,6 +1,7 @@
+# TODO Not used anymore? Remove.
 #
-#
-module Helpers
+module Helpers # :nodoc:all
+  
   module Cache
     # This is a simple cache.
     # The store needs to be able to answer to [] and []=.
@@ -10,14 +11,15 @@ module Helpers
       #
       results = store[key]
       return results if results
-
+      
       results = lambda(&block).call
-
+      
       # Store results
       #
       store[key] = results
-
+      
       results
     end
   end
+  
 end

data/lib/picky/helpers/gc.rb

@@ -1,3 +1,5 @@
+# TODO Not used anymore? Remove.
+#
 module Helpers
   module GC
     def gc_disabled &block

data/lib/picky/helpers/measuring.rb

@@ -1,5 +1,7 @@
 # Helper methods for measuring, benchmarking, logging.
 #
+# TODO Not used anymore? Remove.
+#
 module Helpers
   module Measuring
     

data/lib/picky/index/bundle.rb

@@ -1,4 +1,4 @@
-module Index
+module Index # :nodoc:all
   # A Bundle is a number of indexes
   # per [index, category] combination.
   #

data/lib/picky/index_api.rb

@@ -1,14 +1,14 @@
-# This class defines the indexing and index API.
+# This class defines the indexing and index API that is exposed to the user.
+# It provides a single front for both indexing and index options.
 #
 # Note: An Index holds both an *Indexed*::*Index* and an *Indexing*::*Type*.
 #
-class IndexAPI
-  
-  # TODO Delegation.
-  #
+class IndexAPI # :nodoc:all
   
   attr_reader :name, :indexing, :indexed
   
+  # TODO Doc.
+  #
   def initialize name, source, options = {}
     @name     = name
     @indexing = Indexing::Index.new name, source, options
@@ -19,15 +19,13 @@ class IndexAPI
     Indexes.register self
   end
   
-  # API.
-  #
-  # TODO Spec! Doc!
+  # TODO Doc.
   #
   def define_category category_name, options = {}
     category_name = category_name.to_sym
     
-    indexing_category = indexing.add_category category_name, options
-    indexed_category  = indexed.add_category  category_name, options
+    indexing_category = indexing.define_category category_name, options
+    indexed_category  = indexed.define_category  category_name, options
     
     yield indexing_category, indexed_category if block_given?
     
@@ -35,22 +33,42 @@ class IndexAPI
   end
   alias category define_category
   
-  # TODO Rewrite wrap_exact, wrap_source ?
+  # 
   #
   def define_location name, options = {}
-    grid      = options[:grid] || raise("Grid size needs to be given to a location")
+    grid      = options[:radius] || raise("Option :radius needs to be set on define_location, it defines the search radius.")
     precision = options[:precision]
     
+    options = { partial: Partial::None.new }.merge options
+    
     define_category name, options do |indexing, indexed|
       indexing.source    = Sources::Wrappers::Location.new indexing, grid: grid, precision: precision
       indexing.tokenizer = Tokenizers::Index.new
-      # indexing.partial = Partial::None.new
       
-      exact_bundle    = Indexed::Wrappers::Bundle::Location.new indexed.exact, grid: grid
+      exact_bundle    = Indexed::Wrappers::Bundle::Location.new indexed.exact, grid: grid, precision: precision
       indexed.exact   = exact_bundle
-      indexed.partial = exact_bundle
+      indexed.partial = exact_bundle # A partial token also uses the exact index.
     end
   end
   alias location define_location
   
+  # Options
+  # * radius (in km).
+  #
+  def define_map_location name, options = {}
+    radius = options[:radius] || raise("Option :radius needs to be set on define_map_location, it defines the search radius.")
+    
+    # The radius is given as if all the locations were on the equator.
+    #
+    # TODO Need to recalculate since not many locations are on the equator ;) This is just a prototype.
+    #
+    # This calculates km -> longitude (degrees).
+    #
+    # A degree on the equator is equal to ~111,319.9 meters.
+    # So a km on the equator is equal to 0.00898312 degrees.
+    #
+    options[:radius] = radius * 0.00898312
+    
+    define_location name, options
+  end
 end

data/lib/picky/indexed/bundle.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 #
-module Indexed
+module Indexed # :nodoc:all
 
   # This is the _actual_ index.
   #

data/lib/picky/indexed/index.rb

@@ -20,7 +20,7 @@ module Indexed
     
     # TODO Spec. Doc.
     #
-    def add_category category_name, options = {}
+    def define_category category_name, options = {}
       new_category = Category.new category_name, self, options
       categories << new_category
       new_category

data/lib/picky/indexed/wrappers/bundle/location.rb

@@ -28,7 +28,7 @@ module Indexed
           # Load first the bundle, then extract the config.
           #
           bundle.load
-          minimum = bundle[:location_minimum] || raise("Configuration :location_minimum for #{bundle.identifier} missing.")
+          minimum = bundle[:location_minimum] || raise("Configuration :location_minimum for #{bundle.identifier} missing. Did you run rake index already?")
           @calculation.minimum = minimum
         end
         

data/lib/picky/indexers/no_source_specified_error.rb

@@ -1,4 +1,4 @@
-module Indexers
+module Indexers # :nodoc:all
   
   # Raised if no source is available on a category.
   #

data/lib/picky/indexes_api.rb

@@ -1,6 +1,6 @@
 # Comfortable API convenience class, splits methods to indexes.
 #
-class IndexesAPI
+class IndexesAPI # :nodoc:all
   
   attr_reader :indexes, :index_mapping
   

data/lib/picky/indexing/bundle.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 #
-module Indexing
+module Indexing # :nodoc:all
 
   # This is the indexing bundle.
   # It does all menial tasks that have nothing to do

data/lib/picky/indexing/index.rb

@@ -30,7 +30,7 @@ module Indexing
     
     # TODO Spec. Doc.
     #
-    def add_category category_name, options = {}
+    def define_category category_name, options = {}
       options = default_category_options.merge options
       
       new_category = Category.new category_name, self, options

data/lib/picky/loader.rb

@@ -1,6 +1,6 @@
 # Loads the search engine and itself.
 #
-module Loader
+module Loader # :nodoc:all
   
   # Reloads the whole app.
   # First itself, then the app.

data/lib/picky/loggers/search.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 #
-module Loggers
+module Loggers # :nodoc:all
   # Log Proxy
   #
   class Search

data/lib/picky/performant.rb

@@ -0,0 +1,3 @@
+module Performant # :nodoc:all
+  # C Code here.
+end

data/lib/picky/query/allocation.rb

@@ -2,7 +2,7 @@ module Query
   # An allocation has a number of combinations:
   # [token, index] [other_token, other_index], ...
   #
-  class Allocation
+  class Allocation # :nodoc:all
 
     attr_reader :count, :ids, :score, :combinations, :result_type
     

data/lib/picky/query/allocations.rb

@@ -1,7 +1,7 @@
 module Query
   # Container class for allocations.
   #
-  class Allocations
+  class Allocations # :nodoc:all
 
     # TODO Remove size
     #