picky 1.4.1 → 1.4.2

data/lib/picky/{alias_instances.rb → aliases.rb}

@@ -1,6 +1,4 @@
 # This provides a nice accessor for users
 # who want to use the index API.
 #
-# TODO Rename to API::Indexes?
-#
-Indexes = IndexesAPI.new
+Indexes = IndexBundle.new

data/lib/picky/application.rb

@@ -13,21 +13,23 @@
 # will generate an example <tt>project_name/app/application.rb</tt> file for you
 # with some example code inside.
 #
-# == index(name, source)
+# == Index::Memory.new(name, source)
 #
-# Next, define where your data comes from. You use the <tt>index</tt> method for that:
-#   my_index = index :some_index_name, some_source
+# Next, define where your data comes from. You use the <tt>Index::Memory.new</tt> method for that:
+#   my_index = Index::Memory.new :some_index_name, some_source
 # You give the index a name (or identifier), and a source (see Sources), where its data comes from. Let's do that:
 #   class MyGreatSearch < Application
 #     
-#     books = index :books, Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
+#     books = Index::Memory.new :books, Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
 #     
 #   end
 # Now we have an index <tt>books</tt>.
 # 
 # That on itself won't do much good.
 #
-# == index.define_category(identifier, options = {})
+# Note that a Redis index is also available: Index::Redis.new.
+#
+# == index_instance.define_category(identifier, options = {})
 #
 # Picky needs us to define categories on the data.
 #
@@ -37,7 +39,7 @@
 # Let's go ahead and define a category:
 #   class MyGreatSearch < Application
 #     
-#     books = index :books, Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
+#     books = Index::Memory.new :books, Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
 #     books.define_category :title
 #
 #   end
@@ -67,9 +69,7 @@
 #     books = index :books, Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
 #     books.define_category :title
 #
-#     full_books_query = Query::Full.new books
-#
-#     route %r{^/books/full$} => full_books_query
+#     route %r{^/books/full$} => Query::Full.new(books)
 #
 #   end
 # That's it!
@@ -128,7 +128,7 @@
 #                      substitutes_characters_with: CharacterSubstituters::WestEuropean.new,
 #                      maximum_tokens: 4
 #     
-#     books = index :books, Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
+#     books = Index::Memory.new :books, Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
 #     books.define_category :title,
 #                           qualifiers: [:t, :title, :titre],
 #                           partial:    Partial::Substring.new(:from => 1),
@@ -139,11 +139,8 @@
 #     
 #     query_options = { :weights => { [:title, :author] => +3, [:author, :title] => -1 } }
 #     
-#     full_books_query = Query::Full.new books, query_options
-#     live_books_query = Query::Full.new books, query_options
-#     
-#     route %r{^/books/full$} => full_books_query
-#     route %r{^/books/live$} => live_books_query
+#     route %r{^/books/full$} => Query::Full.new(books, query_options)
+#     route %r{^/books/live$} => Query::Live.new(books, query_options)
 #     
 #   end
 # That's actually already a full-blown Picky App!
@@ -159,14 +156,14 @@ class Application
     # is used for indexing by default.
     # 
     def default_indexing options = {}
-      Tokenizers::Index.default = Tokenizers::Index.new(options)
+      Internals::Tokenizers::Index.default = Internals::Tokenizers::Index.new(options)
     end
     
     # Returns a configured tokenizer that
     # is used for querying by default.
     # 
     def default_querying options = {}
-      Tokenizers::Query.default = Tokenizers::Query.new(options)
+      Internals::Tokenizers::Query.default = Internals::Tokenizers::Query.new(options)
     end
     
     # Create a new index for indexing and for querying.
@@ -181,8 +178,10 @@ class Application
     # Options:
     # * result_identifier: Use if you'd like a different identifier/name in the results JSON than the name of the index. 
     #
+    # TODO Obsolete. Phase out.
+    #
     def index name, source, options = {}
-      IndexAPI.new name, source, options
+      Index::Memory.new name, source, options
     end
     
     # Routes.
@@ -201,7 +200,7 @@ class Application
       rack_adapter.call env
     end
     def rack_adapter # :nodoc:
-      @rack_adapter ||= FrontendAdapters::Rack.new
+      @rack_adapter ||= Internals::FrontendAdapters::Rack.new
     end
     
     # Finalize the subclass as soon as it

data/lib/picky/cores.rb

@@ -83,7 +83,7 @@ class Cores # :nodoc:all
   #   os_name => lambda_which_returns_a_number_of_cores
   #
   @@number_of_cores = {
-    'darwin' => lambda { `system_profiler SPHardwareDataType | grep 'Total Number Of Cores'`.gsub(/[^\d]/, '') },
+    'darwin' => lambda { `system_profiler SPHardwareDataType | grep -i 'Total Number Of Cores'`.gsub(/[^\d]/, '') },
     'linux'  => lambda { `grep -ci ^processor /proc/cpuinfo` }
   }
   def self.os_to_core_mapping

data/lib/picky/generators/aliases.rb

@@ -0,0 +1,3 @@
+Partial    = Internals::Generators::Partial
+Similarity = Internals::Generators::Similarity
+Weights    = Internals::Generators::Weights

data/lib/picky/index/base.rb

@@ -0,0 +1,179 @@
+module Index
+  
+  # This class defines the indexing and index API that is exposed to the user
+  # as the #index method inside the Application class.
+  #
+  # It provides a single front for both indexing and index options. We suggest to always use the index API.
+  #
+  # Note: An Index holds both an *Indexed*::*Index* and an *Indexing*::*Type*.
+  #  
+  class Base
+
+    attr_reader :name, :indexing, :indexed
+
+    # Create a new index with a given source.
+    #
+    # === Parameters
+    # * name: A name that will be used for the index directory and in the Picky front end.
+    # * source: Where the data comes from, e.g. Sources::CSV.new(...)
+    #
+    # === Options
+    # * result_identifier: Use if you'd like a different identifier/name in the results than the name of the index.
+    # * after_indexing: As of this writing only used in the db source. Executes the given after_indexing as SQL after the indexing process.
+    #
+    def initialize name, source, options = {}
+      @name     = name
+      @indexing = Internals::Indexing::Index.new name, source, options
+      @indexed  = Internals::Indexed::Index.new  name, options
+  
+      # Centralized registry.
+      #
+      Indexes.register self
+    end
+
+    # Defines a searchable category on the index.
+    #
+    # === Parameters
+    # * category_name: This identifier is used in the front end, but also to categorize query text. For example, “title:hobbit” will narrow the hobbit query on categories with the identifier :title.
+    #
+    # === Options
+    # * partial: Partial::None.new or Partial::Substring.new(from: starting_char, to: ending_char). Default is Partial::Substring.new(from: -3, to: -1).
+    # * similarity: Similarity::None.new or Similarity::Phonetic.new(similar_words_searched). Default is Similarity::None.new.
+    # * qualifiers: An array of qualifiers with which you can define which category you’d like to search, for example “title:hobbit” will search for hobbit in just title categories. Example: qualifiers: [:t, :titre, :title] (use it for example with multiple languages). Default is the name of the category.
+    # * qualifier: Convenience options if you just need a single qualifier, see above. Example: qualifiers => :title. Default is the name of the category.
+    # * source: Use a different source than the index uses. If you think you need that, there might be a better solution to your problem. Please post to the mailing list first with your application.rb :)
+    # * from: Take the data from the data category with this name. Example: You have a source Sources::CSV.new(:title, file:'some_file.csv') but you want the category to be called differently. The you use from: define_category(:similar_title, :from => :title).
+    #
+    def define_category category_name, options = {}
+      category_name = category_name.to_sym
+  
+      indexing_category = indexing.define_category category_name, options
+      indexed_category  = indexed.define_category  category_name, options
+  
+      yield indexing_category, indexed_category if block_given?
+  
+      self
+    end
+    alias category define_category
+
+    #  HIGHLY EXPERIMENTAL Try if you feel "beta" ;)
+    #
+    # Make this category range searchable with a fixed range. If you need other ranges, define another category with a different range value.
+    #
+    # Example:
+    # You have data values inside 1..100, and you want to have Picky return
+    # not only the results for 47 if you search for 47, but also results for
+    # 45, 46, or 47.2, 48.9, in a range of 2 around 47, so (45..49).
+    #
+    # Then you use:
+    #  my_index.define_ranged_category :values_inside_1_100, 2
+    #
+    # Optionally, you give it a precision value to reduce the error margin
+    # around 47 (Picky is a bit liberal).
+    #  my_index.define_ranged_category :values_inside_1_100, 2, precision: 5
+    #
+    # This will force Picky to maximally be wrong 5% of the given range value
+    # (5% of 2 = 0.1) instead of the default 20% (20% of 2 = 0.4).
+    #
+    # We suggest not to use much more than 5 as a higher precision is more performance intensive for less and less precision gain.
+    #
+    # == Protip 1
+    #
+    # Create two ranged categories to make an area search:
+    #   index.define_ranged_category :x, 1
+    #   index.define_ranged_category :y, 1
+    #
+    # Search for it using for example:
+    #   x:133, y:120
+    #
+    # This will search this square area (* = 133, 120: The "search" point entered):
+    # 
+    #    132       134
+    #     |         |
+    #   --|---------|-- 121
+    #     |         |
+    #     |    *    |
+    #     |         |
+    #   --|---------|-- 119
+    #     |         | 
+    #
+    # Note: The area does not need to be square, but can be rectangular.
+    #
+    # == Protip 2
+    #
+    # Create three ranged categories to make a volume search.
+    #
+    # Or go crazy and use 4 ranged categories for a space/time search! ;)
+    #
+    # === Parameters
+    # * category_name: The category_name as used in #define_category.
+    # * range: The range (in the units of your data values) around the query point where we search for results.
+    #
+    #  -----|<- range  ->*------------|-----
+    #
+    # === Options
+    # * precision: Default is 1 (20% error margin, very fast), up to 5 (5% error margin, slower) makes sense.
+    # * ... all options of #define_category.
+    #
+    def define_ranged_category category_name, range, options = {}
+      precision = options[:precision]
+  
+      options = { partial: Partial::None.new }.merge options
+  
+      define_category category_name, options do |indexing, indexed|
+        indexing.source    = Sources::Wrappers::Location.new indexing, grid: range, precision: precision
+        indexing.tokenizer = Internals::Tokenizers::Index.new
+    
+        exact_bundle    = Indexed::Wrappers::Bundle::Location.new indexed.exact, grid: range, precision: precision
+        indexed.exact   = exact_bundle
+        indexed.partial = exact_bundle # A partial token also uses the exact index.
+      end
+    end
+    alias ranged_category define_ranged_category
+
+    #  HIGHLY EXPERIMENTAL Not correctly working yet. Try it if you feel "beta".
+    #
+    # Also a range search see #define_ranged_category, but on the earth's surface.
+    #
+    # Parameters:
+    # * name: The name as used in #define_category.
+    # * radius: The distance (in km) around the query point which we search for results.
+    #
+    # Note: Picky uses a square, not a circle. We hope that's ok for most usages.
+    #
+    #  -----------------------------
+    #  |                           |
+    #  |                           |
+    #  |                           |
+    #  |                           |
+    #  |                           |
+    #  |             *<-  radius ->|
+    #  |                           |
+    #  |                           |
+    #  |                           |
+    #  |                           |
+    #  |                           |
+    #  -----------------------------
+    #
+    # Options
+    # * precision: Default 1 (20% error margin, very fast), up to 5 (5% error margin, slower) makes sense.
+    # * from: The data category to take the data for this category from.
+    #
+    # TODO Redo. Will have to write a wrapper that combines two categories that are indexed simultaneously.
+    #
+    def define_map_location name, radius, options = {} # :nodoc:
+      # The radius is given as if all the locations were on the equator.
+      #
+      # TODO Need to recalculate since not many locations are on the equator ;) This is just a prototype.
+      #
+      # This calculates km -> longitude (degrees).
+      #
+      # A degree on the equator is equal to ~111,319.9 meters.
+      # So a km on the equator is equal to 0.00898312 degrees.
+      #
+      define_ranged_category name, radius * 0.00898312, options
+    end
+    alias map_location define_map_location
+  end
+  
+end

data/lib/picky/index/memory.rb

@@ -0,0 +1,28 @@
+module Index
+  
+  # An index that is persisted in files, loaded at startup and kept in memory at runtime.
+  #
+  class Memory < Base
+    
+    # Create a new memory index for indexing and for querying.
+    #
+    # Parameters:
+    # * name: The identifier of the index. Used:
+    #   - to identify an index (e.g. by you in Rake tasks: Indexes[:the_identifier]).
+    #   - in the frontend to describe which index a result came from.
+    #   - index directory naming (index/development/the_identifier/<lots of indexes>)
+    # * source: The source the data comes from. See Sources::Base.
+    #
+    # Options:
+    # * result_identifier: Use if you'd like a different identifier/name in the results JSON than the name of the index. 
+    #
+    def initialize name, source, options = {}
+      options[:indexing_bundle_class] ||= Internals::Indexing::Bundle::Memory
+      options[:indexed_bundle_class]  ||= Internals::Indexed::Bundle::Memory
+      
+      super name, source, options
+    end
+    
+  end
+  
+end

data/lib/picky/index/redis.rb

@@ -0,0 +1,28 @@
+module Index
+  
+  # An index that is persisted in Redis.
+  #
+  class Redis < Base
+    
+    # Create a new Redis index for indexing and for querying.
+    #
+    # Parameters:
+    # * name: The identifier of the index. Used:
+    #   - to identify an index (e.g. by you in Rake tasks: Indexes[:the_identifier]).
+    #   - in the frontend to describe which index a result came from.
+    #   - index directory naming (index/development/the_identifier/<lots of indexes>)
+    # * source: The source the data comes from. See Sources::Base.
+    #
+    # Options:
+    # * result_identifier: Use if you'd like a different identifier/name in the results JSON than the name of the index. 
+    #
+    def initialize name, source, options = {}
+      options[:indexing_bundle_class] ||= Internals::Indexing::Bundle::Redis
+      options[:indexed_bundle_class]  ||= Internals::Indexed::Bundle::Redis
+      
+      super name, source, options
+    end
+    
+  end
+  
+end

data/lib/picky/{indexes_api.rb → index_bundle.rb}

@@ -1,41 +1,41 @@
 # Comfortable API convenience class, splits methods to indexes.
 #
-class IndexesAPI # :nodoc:all
-  
-  attr_reader :indexes, :index_mapping
-  
+class IndexBundle # :nodoc:all
+
+  attr_reader :indexes, :index_mapping, :indexing, :indexed
+
   delegate :reload,
            :load_from_cache,
-           :to => :@indexed
-  
+           :to => :indexed
+
   delegate :check_caches,
            :find,
            :generate_cache_only,
            :generate_index_only,
            :index,
            :index_for_tests,
-           :to => :@indexing
-  
+           :to => :indexing
+
   def initialize
     @indexes = []
     @index_mapping = {}
-    
+  
     @indexed  = Indexed::Indexes.new
     @indexing = Indexing::Indexes.new
   end
-  
+
   def register index
     self.indexes << index
     self.index_mapping[index.name] = index
-    
-    @indexing.register index.indexing
-    @indexed.register  index.indexed
-  end
   
+    indexing.register index.indexing
+    indexed.register  index.indexed
+  end
+
   def [] name
     name = name.to_sym
-    
+  
     self.index_mapping[name]
   end
-  
+
 end

data/lib/picky/indexed/indexes.rb

@@ -1,5 +1,7 @@
 module Indexed
   
+  # Registers the indexes held at runtime, for queries.
+  #
   class Indexes
     
     attr_reader :indexes, :index_mapping
@@ -11,29 +13,31 @@ module Indexed
       clear
     end
     
-    # TODO Spec.
+    # Clears the indexes and the mapping.
     #
     def clear
       @indexes       = []
       @index_mapping = {}
     end
     
-    # TODO Spec.
+    # Reloads all indexes, one after another,
+    # in the order they were added.
     #
     def reload
       load_from_cache
     end
     
-    # TODO Spec
+    # Registers an index with the indexes.
     #
     def register index
       self.indexes << index
       self.index_mapping[index.name] = index
     end
-    def [] name
-      name = name.to_sym
-      
-      index_mapping[name]
+    
+    # Extracts an index, given its identifier.
+    #
+    def [] identifier
+      index_mapping[identifier.to_sym]
     end
     
   end

data/lib/picky/indexing/indexes.rb

@@ -1,5 +1,7 @@
 module Indexing
   
+  # Registers the indexes held at index time, for indexing.
+  #
   class Indexes
     
     attr_reader :indexes
@@ -17,13 +19,13 @@ module Indexing
       clear
     end
     
-    # TODO Doc.
+    # Clears the array of indexes.
     #
     def clear
       @indexes = []
     end
     
-    # TODO Spec. Superclass?
+    # Registers an index with the indexes.
     #
     def register index
       self.indexes << index
@@ -52,7 +54,7 @@ module Indexing
       timed_exclaim "INDEXING FINISHED."
     end
     
-    # For testing.
+    # For integration testing – indexes for the tests without forking and shouting ;)
     #
     def index_for_tests
       take_snapshot
@@ -63,25 +65,29 @@ module Indexing
       end
     end
     
-    # TODO Spec
+    # Generate only the index for the given index:category pair.
     #
-    def generate_index_only index_name, category_name
+    def generate_index_only index_name, category_name = nil
       found = find index_name, category_name
       found.index if found
     end
-    def generate_cache_only index_name, category_name
+    # Generate only the cache for the given index:category pair.
+    #
+    def generate_cache_only index_name, category_name = nil
       found = find index_name, category_name
       found.generate_caches if found
     end
     
-    # TODO Spec
+    # Find a given index:category pair.
     #
     def find index_name, category_name
-      index_name     = index_name.to_sym
+      index_name = index_name.to_sym
       
       indexes.each do |index|
         next unless index.name == index_name
         
+        return index unless category_name
+        
         found = index.categories.find category_name
         return found if found
       end

data/lib/picky/internals/adapters/rack/base.rb

@@ -0,0 +1,27 @@
+module Internals
+
+  module Adapters
+    # Adapter that is plugged into a Rack outlet.
+    #
+    module Rack
+    
+      # Subclasses of this class should respond to
+      # * to_app(options)
+      #
+      class Base
+      
+        # Puts together an appropriately structured Rack response.
+        #
+        # Note: Bytesize is needed to have special characters not trip up Rack.
+        #
+        def respond_with response, content_type = 'application/json'
+          [200, { 'Content-Type' => content_type, 'Content-Length' => response.bytesize.to_s }, [response]]
+        end
+      
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/internals/adapters/rack/live_parameters.rb

@@ -0,0 +1,37 @@
+module Internals
+
+  module Adapters
+  
+    #
+    #
+    module Rack
+    
+      class LiveParameters < Base
+
+        def initialize live_parameters
+          @live_parameters = live_parameters
+        end
+      
+        #
+        #
+        def to_app options = {}
+          # For capturing by the lambda block.
+          #
+          live_parameters = @live_parameters
+        
+          lambda do |env|
+            params = ::Rack::Request.new(env).params
+          
+            results = live_parameters.parameters params
+          
+            respond_with results.to_json
+          end
+        end
+      
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/internals/adapters/rack/query.rb

@@ -0,0 +1,63 @@
+module Internals
+
+  module Adapters
+    # This is an adapter that is plugged into a Rack outlet.
+    #
+    # It looks at what is given to it and generate an appropriate
+    # adapter for it.
+    #
+    # For example, if you give it a query, it will extract the query param etc.
+    # and call search_with_text on it if it is called by Rack.
+    #
+    module Rack
+    
+      class Query < Base
+      
+        @@defaults = {
+          query_key:    'query'.freeze,
+          offset_key:   'offset'.freeze,
+          content_type: 'application/json'.freeze
+        }
+
+        def initialize query
+          @query    = query
+          @defaults = @@defaults.dup
+        end
+      
+        def to_app options = {}
+          # For capturing in the lambda.
+          #
+          query        = @query
+          query_key    = options[:query_key]    || @defaults[:query_key]
+          content_type = options[:content_type] || @defaults[:content_type]
+
+          lambda do |env|
+            params  = ::Rack::Request.new(env).params
+
+            results = query.search_with_text *extracted(params)
+
+            PickyLog.log results.to_log(params[query_key])
+
+            respond_with results.to_response, content_type
+          end
+        end
+      
+        # Helper method to extract the params
+        #
+        UTF8_STRING = 'UTF-8'.freeze
+        def extracted params
+          [
+            # query is encoded in ASCII
+            #
+            params[@defaults[:query_key]]  && params[@defaults[:query_key]].force_encoding(UTF8_STRING),
+            params[@defaults[:offset_key]] && params[@defaults[:offset_key]].to_i || 0
+          ]
+        end
+      
+      end
+    
+    end
+  
+  end
+  
+end