picky 3.0.1 → 3.1.0

data/lib/picky/index.rb

@@ -0,0 +1,329 @@
+module Picky
+
+  # = Picky Indexes
+  #
+  # A Picky Index defines
+  # * what backend it uses.
+  # * where its data comes from (a data source).
+  # * how this data it is indexed.
+  # * a number of categories that may or may not map directly to data categories.
+  #
+  # == Howto
+  #
+  # This is a step-by-step description on how to create an index.
+  #
+  # Start by choosing an <tt>Index</tt> or an <tt>Index</tt>.
+  # In the example, we will be using an in-memory index, <tt>Index</tt>.
+  #
+  #   books = Index.new(:books)
+  #
+  # That in itself won't do much good, that's why we add a data source:
+  #
+  #   books = Index.new(:books) do
+  #     source Sources::CSV.new(:title, :author, file: 'data/books.csv')
+  #   end
+  #
+  # In the example, we use an explicit <tt>Sources::CSV</tt> of Picky.
+  # However, anything that responds to <tt>#each</tt>, and returns an object that
+  # answers to <tt>#id</tt>, works.
+  #
+  # For example, a 3.0 ActiveRecord class:
+  #
+  #   books = Index.new(:books) do
+  #     source Book.order('isbn ASC')
+  #   end
+  #
+  # Now we know where the data comes from, but not, how to categorize it.
+  #
+  # Let's add a few categories:
+  #
+  #   books = Index.new(:books) do
+  #     source   Book.order('isbn ASC')
+  #     category :title
+  #     category :author
+  #     category :isbn
+  #   end
+  #
+  # Categories offer quite a few options, see <tt>Indexes::Base#category</tt> for details.
+  #
+  # After adding more options, it might look like this:
+  #
+  #   books = Index.new(:books) do
+  #     source   Book.order('isbn ASC')
+  #     category :title,
+  #              partial: Partial::Substring.new(from: 1),
+  #              similarity: Similarity::DoubleMetaphone.new(3),
+  #              qualifiers: [:t, :title, :titulo]
+  #     category :author,
+  #              similarity: Similarity::Metaphone.new(2)
+  #     category :isbn,
+  #              partial: Partial::None.new,
+  #              from: :legacy_isbn_name
+  #   end
+  #
+  # For this to work, a <tt>Book</tt> should support methods <tt>#title</tt>, <tt>#author</tt> and <tt>#legacy_isbn_name</tt>.
+  #
+  # If it uses <tt>String</tt> ids, use <tt>#key_format</tt> to define a formatting method:
+  #
+  #   books = Index.new(:books) do
+  #     key_format :to_s
+  #     source     Book.order('isbn ASC')
+  #     category   :title
+  #     category   :author
+  #     category   :isbn
+  #   end
+  #
+  # Finally, use the index for a <tt>Search</tt>:
+  #
+  #   route %r{^/media$} => Search.new(books, dvds, mp3s)
+  #
+  # This class defines the indexing and index API that is exposed to the user
+  # as the #index method inside the Application class.
+  #
+  # It provides a single front for both indexing and index options. We suggest to always use the index API.
+  #
+  # Note: An Index holds both an *Indexed*::*Index* and an *Indexing*::*Index*.
+  #
+  class Index
+
+    attr_reader :name,
+                :categories
+
+    delegate :[],
+             :each_category,
+             :to => :categories
+
+    # Create a new index with a given source.
+    #
+    # === Parameters
+    # * name: A name that will be used for the index directory and in the Picky front end.
+    #
+    # === Options (all are used in the block, see examples)
+    # * source: Where the data comes from, e.g. Sources::CSV.new(...). Optional, can be defined in the block using #source.
+    # * result_identifier: Use if you'd like a different identifier/name in the results than the name of the index.
+    # * after_indexing: As of this writing only used in the db source. Executes the given after_indexing as SQL after the indexing process.
+    # * tokenizer: Call and pass either a tokenizer (responds to #tokenize) or the options for a tokenizer..
+    # * key_format: Call and pass in a format method for the ids (default is #to_i).
+    #
+    # Example:
+    #   my_index = Index.new(:my_index) do
+    #     source            Sources::CSV.new(file: 'data/index.csv')
+    #     key_format        :to_sym
+    #     category          :bla
+    #     result_identifier :my_special_results
+    #   end
+    #
+    def initialize name, options = {}
+      @name              = name.to_sym
+
+      # TODO Move ignore_unassigned_tokens to query, somehow. Then, remove options.
+      #
+      @categories = Categories.new ignore_unassigned_tokens: (options[:ignore_unassigned_tokens] || false)
+
+      # Centralized registry.
+      #
+      Indexes.register self
+
+      instance_eval(&Proc.new) if block_given?
+    end
+
+    # API method.
+    #
+    # Sets/returns the backend used.
+    # Default is @Backends::Memory.new@.
+    #
+    def backend backend = nil
+      if backend
+        @backend = backend
+      else
+        @backend || Backends::Memory.new
+      end
+    end
+
+    # Defines a searchable category on the index.
+    #
+    # === Parameters
+    # * category_name: This identifier is used in the front end, but also to categorize query text. For example, “title:hobbit” will narrow the hobbit query on categories with the identifier :title.
+    #
+    # === Options
+    # * partial: Partial::None.new or Partial::Substring.new(from: starting_char, to: ending_char). Default is Partial::Substring.new(from: -3, to: -1).
+    # * similarity: Similarity::None.new or Similarity::DoubleMetaphone.new(similar_words_searched). Default is Similarity::None.new.
+    # * qualifiers: An array of qualifiers with which you can define which category you’d like to search, for example “title:hobbit” will search for hobbit in just title categories. Example: qualifiers: [:t, :titre, :title] (use it for example with multiple languages). Default is the name of the category.
+    # * qualifier: Convenience options if you just need a single qualifier, see above. Example: qualifiers => :title. Default is the name of the category.
+    # * source: Use a different source than the index uses. If you think you need that, there might be a better solution to your problem. Please post to the mailing list first with your application.rb :)
+    # * from: Take the data from the data category with this name. Example: You have a source Sources::CSV.new(:title, file:'some_file.csv') but you want the category to be called differently. The you use from: define_category(:similar_title, :from => :title).
+    #
+    def category category_name, options = {}
+      new_category = Category.new category_name.to_sym, self, options
+      categories << new_category
+
+      new_category = yield new_category if block_given?
+
+      new_category
+    end
+    alias define_category category
+
+    # Make this category range searchable with a fixed range. If you need other
+    # ranges, define another category with a different range value.
+    #
+    # Example:
+    # You have data values inside 1..100, and you want to have Picky return
+    # not only the results for 47 if you search for 47, but also results for
+    # 45, 46, or 47.2, 48.9, in a range of 2 around 47, so (45..49).
+    #
+    # Then you use:
+    #  ranged_category :values_inside_1_100, 2
+    #
+    # Optionally, you give it a precision value to reduce the error margin
+    # around 47 (Picky is a bit liberal).
+    #   Index.new :range do
+    #     ranged_category :values_inside_1_100, 2, precision: 5
+    #   end
+    #
+    # This will force Picky to maximally be wrong 5% of the given range value
+    # (5% of 2 = 0.1) instead of the default 20% (20% of 2 = 0.4).
+    #
+    # We suggest not to use much more than 5 as a higher precision is more
+    # performance intensive for less and less precision gain.
+    #
+    # == Protip 1
+    #
+    # Create two ranged categories to make an area search:
+    #   Index.new :area do
+    #     ranged_category :x, 1
+    #     ranged_category :y, 1
+    #   end
+    #
+    # Search for it using for example:
+    #   x:133, y:120
+    #
+    # This will search this square area (* = 133, 120: The "search" point entered):
+    #
+    #    132       134
+    #     |         |
+    #   --|---------|-- 121
+    #     |         |
+    #     |    *    |
+    #     |         |
+    #   --|---------|-- 119
+    #     |         |
+    #
+    # Note: The area does not need to be square, but can be rectangular.
+    #
+    # == Protip 2
+    #
+    # Create three ranged categories to make a volume search.
+    #
+    # Or go crazy and use 4 ranged categories for a space/time search! ;)
+    #
+    # === Parameters
+    # * category_name: The category_name as used in #define_category.
+    # * range: The range (in the units of your data values) around the query point where we search for results.
+    #
+    #  -----|<- range  ->*------------|-----
+    #
+    # === Options
+    # * precision: Default is 1 (20% error margin, very fast), up to 5 (5% error margin, slower) makes sense.
+    # * ... all options of #define_category.
+    #
+    def ranged_category category_name, range, options = {}
+      precision = options[:precision] || 1 # THINK options.delete?
+
+      # Note: :key_format => :to_f ?
+      #
+      options = { partial: Partial::None.new }.merge options
+
+      define_category category_name, options do |category|
+        Indexing::Wrappers::Category::Location.install_on category, range, precision
+        Indexed::Wrappers::Category::Location.install_on category, range, precision
+      end
+    end
+    alias define_ranged_category ranged_category
+
+    # HIGHLY EXPERIMENTAL Not correctly working yet. Try it if you feel "beta".
+    #
+    # Also a range search see #ranged_category, but on the earth's surface.
+    #
+    # Parameters:
+    # * lat_name: The latitude's name as used in #define_category.
+    # * lng_name: The longitude's name as used in #define_category.
+    # * radius: The distance (in km) around the query point which we search for results.
+    #
+    # Note: Picky uses a square, not a circle. That should be ok for most usages.
+    #
+    #  -----------------------------
+    #  |                           |
+    #  |                           |
+    #  |                           |
+    #  |                           |
+    #  |                           |
+    #  |             *<-  radius ->|
+    #  |                           |
+    #  |                           |
+    #  |                           |
+    #  |                           |
+    #  |                           |
+    #  -----------------------------
+    #
+    # Options
+    # * precision: Default 1 (20% error margin, very fast), up to 5 (5% error margin, slower) makes sense.
+    # * lat_from: The data category to take the data for the latitude from.
+    # * lng_from: The data category to take the data for the longitude from.
+    #
+    # TODO Will have to write a wrapper that combines two categories that are
+    #      indexed simultaneously, since lat/lng are correlated.
+    #
+    def geo_categories lat_name, lng_name, radius, options = {} # :nodoc:
+
+      # Extract lat/lng specific options.
+      #
+      lat_from = options.delete :lat_from
+      lng_from = options.delete :lng_from
+
+      # One can be a normal ranged_category.
+      #
+      ranged_category lat_name, radius*0.00898312, options.merge(from: lat_from)
+
+      # The other needs to adapt the radius depending on the one.
+      #
+      # Depending on the latitude, the radius of the longitude
+      # needs to enlarge, the closer we get to the pole.
+      #
+      # In our simplified case, the radius is given as if all the
+      # locations were on the 45 degree line.
+      #
+      # This calculates km -> longitude (degrees).
+      #
+      # A degree on the 45 degree line is equal to ~222.6398 km.
+      # So a km on the 45 degree line is equal to 0.01796624 degrees.
+      #
+      ranged_category lng_name, radius*0.01796624, options.merge(from: lng_from)
+
+    end
+    alias define_geo_categories geo_categories
+
+    def to_stats # :nodoc:
+      stats = <<-INDEX
+#{name} (#{self.class}):
+#{"source:            #{source}".indented_to_s}
+#{"categories:        #{categories.map(&:name).join(', ')}".indented_to_s}
+INDEX
+      stats << "  result identifier: \"#{result_identifier}\"".indented_to_s unless result_identifier.to_s == name.to_s
+      stats
+    end
+
+    # Identifier used for technical output.
+    #
+    def identifier
+      "#{PICKY_ENVIRONMENT}:#{name}"
+    end
+
+    #
+    #
+    def to_s
+      "#{self.class}(#{name}, result_id: #{result_identifier}, source: #{@source}, categories: #{categories})"
+    end
+
+  end
+
+end

data/lib/picky/index_indexed.rb

@@ -0,0 +1,31 @@
+module Picky
+
+  #
+  #
+  class Index
+
+    attr_reader :combinator
+
+    delegate :load_from_cache,
+             :analyze,
+             :reindex,
+             :possible_combinations,
+             :to => :categories
+
+    alias reload load_from_cache
+
+    # Define how the results of this index are identified.
+    # (Shown in the client, for example)
+    #
+    # Default is the name of the index.
+    #
+    def result_identifier result_identifier = nil
+      result_identifier ? define_result_identifier(result_identifier) : (@result_identifier || @name)
+    end
+    def define_result_identifier result_identifier
+      @result_identifier = result_identifier
+    end
+
+  end
+
+end

data/lib/picky/index_indexing.rb

@@ -0,0 +1,161 @@
+module Picky
+
+  #
+  #
+  class Index
+
+    attr_reader :bundle_class
+
+    # Delegators for indexing.
+    #
+    delegate :cache,
+             :check,
+             :clear,
+             :backup,
+             :restore,
+             :to => :categories
+
+    # Calling index on an index will call index
+    # on every category.
+    #
+    # Decides whether to use a parallel indexer or whether to
+    # delegate to each category to index themselves.
+    #
+    def index
+      if source.respond_to?(:each)
+        check_source_empty
+        index_in_parallel
+      else
+        with_data_snapshot do
+          categories.each &:index
+        end
+      end
+    end
+
+    # Define an index tokenizer on the index.
+    #
+    # Parameters are the exact same as for indexing.
+    #
+    def indexing options = {}
+      @tokenizer = if options.respond_to?(:tokenize)
+        options
+      else
+        options && Tokenizer.new(options)
+      end
+    end
+    alias define_indexing indexing
+
+    # Check if the given enumerable source is empty.
+    #
+    # Note: Checking as early as possible to tell the
+    #       user as early as possible.
+    #
+    def check_source_empty
+      warn %Q{\n\033[1mWarning\033[m, source for index "#{name}" is empty: #{source} (responds true to empty?).\n} if source.respond_to?(:empty?) && source.empty?
+    end
+
+    # Note: Duplicated in category_indexing.rb.
+    #
+    # Take a data snapshot if the source offers it.
+    #
+    def with_data_snapshot
+      if source.respond_to? :with_snapshot
+        source.with_snapshot(self) do
+          yield
+        end
+      else
+        yield
+      end
+    end
+
+    # Indexes the categories in parallel.
+    #
+    # Only use where the category does have a #each source defined.
+    #
+    def index_in_parallel
+      indexer = Indexers::Parallel.new self
+      indexer.index categories
+      categories.each &:cache
+    end
+
+    # Returns the installed tokenizer or the default.
+    #
+    def tokenizer
+      @tokenizer || Indexes.tokenizer
+    end
+
+    # Define a source on the index.
+    #
+    # Parameter is a source, either one of the standard sources or
+    # anything responding to #each and returning objects that
+    # respond to id and the category names (or the category from option).
+    #
+    def source some_source = nil, &block
+      some_source ||= block
+      some_source ? define_source(some_source) : (@source && extract_source || raise_no_source)
+    end
+    # Extract the actual source if it is wrapped in a time
+    # capsule, i.e. a block/lambda.
+    #
+    # TODO Extract into module.
+    #
+    def extract_source
+      @source = @source.respond_to?(:call) ? @source.call : @source
+    end
+    def define_source source
+      check_source source
+      @source = source
+    end
+    def raise_no_source
+      raise NoSourceSpecifiedException.new(<<-NO_SOURCE
+
+
+No source given for index #{name}. An index needs a source.
+Example:
+Index.new(:with_source) do
+  source   Sources::CSV.new(:title, file: 'data/books.csv')
+  category :title
+  category :author
+end
+
+      NO_SOURCE
+)
+    end
+    def check_source source # :nodoc:
+      raise ArgumentError.new(<<-SOURCE
+
+
+The index "#{name}" should use a data source that responds to either the method #each, or the method #harvest, which yields(id, text), OR it can be a lambda/block, returning such a source.
+Or it could use one of the built-in sources:
+  Sources::#{(Sources.constants - [:Base, :Wrappers, :NoCSVFileGiven, :NoCouchDBGiven]).join(',
+  Sources::')}
+
+
+SOURCE
+) unless source.respond_to?(:each) || source.respond_to?(:harvest) || source.respond_to?(:call)
+    end
+
+    # Define a key_format on the index.
+    #
+    # Parameter is a method name to use on the key (e.g. :to_i, :to_s, :strip).
+    #
+    def key_format format = nil
+      format ? define_key_format(format) : @key_format
+    end
+    def define_key_format key_format
+      @key_format = key_format
+    end
+
+    # Define what to do after indexing.
+    # (Only used in the Sources::DB)
+    #
+    def after_indexing after_indexing = nil
+      after_indexing ? define_after_indexing(after_indexing) : @after_indexing
+    end
+    def define_after_indexing after_indexing
+      @after_indexing = after_indexing
+    end
+
+  end
+
+end

data/lib/picky/indexed/bundle.rb

@@ -0,0 +1,112 @@
+module Picky
+
+  module Indexed # :nodoc:all
+
+    # An indexed bundle is a number of memory/redis
+    # indexes that compose the indexes for a single category:
+    #  * core (inverted) index
+    #  * weights index
+    #  * similarity index
+    #  * index configuration
+    #
+    # Indexed refers to them being indexed.
+    # This class notably offers the methods:
+    #  * load
+    #  * clear
+    #
+    # To (re)load or clear the current indexes.
+    #
+    class Bundle < Picky::Bundle
+
+      # Get the ids for the given symbol.
+      #
+      # Returns a (potentially empty) array of ids.
+      #
+      def ids sym
+        @inverted[sym] || []
+      end
+
+      # Get a weight for the given symbol.
+      #
+      # Returns a number, or nil.
+      #
+      def weight sym
+        @weights[sym]
+      end
+
+      # Get settings for this bundle.
+      #
+      # Returns an object.
+      #
+      def [] sym
+        @configuration[sym]
+      end
+
+      # Loads all indexes.
+      #
+      # Loading loads index objects from the backend.
+      # They should each respond to [] and return something appropriate.
+      #
+      def load
+        load_inverted
+        load_weights
+        load_similarity
+        load_configuration
+      end
+
+      # Loads the core index.
+      #
+      def load_inverted
+        self.inverted = @backend_inverted.load
+      end
+      # Loads the weights index.
+      #
+      def load_weights
+        self.weights = @backend_weights.load
+      end
+      # Loads the similarity index.
+      #
+      def load_similarity
+        self.similarity = @backend_similarity.load
+      end
+      # Loads the configuration.
+      #
+      def load_configuration
+        self.configuration = @backend_configuration.load
+      end
+
+      # Clears all indexes.
+      #
+      def clear
+        clear_inverted
+        clear_weights
+        clear_similarity
+        clear_configuration
+      end
+
+      # Clears the core index.
+      #
+      def clear_inverted
+        inverted.clear
+      end
+      # Clears the weights index.
+      #
+      def clear_weights
+        weights.clear
+      end
+      # Clears the similarity index.
+      #
+      def clear_similarity
+        similarity.clear
+      end
+      # Clears the configuration.
+      #
+      def clear_configuration
+        configuration.clear
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/indexed/wrappers/exact_first.rb

@@ -9,7 +9,7 @@ module Picky
       # This index combines an exact and partial index.
       # It serves to order the results such that exact hits are found first.
       #
-      class ExactFirst < Indexed::Bundle::Base
+      class ExactFirst < Indexed::Bundle
 
         delegate :similar,
                  :identifier,

data/lib/picky/indexers/parallel.rb

@@ -45,7 +45,8 @@ module Picky
           # Is it a good idea that not the tokenizer has control over when he gets the next text?
           #
           combined.each do |category, cache, _, tokenizer|
-            tokenizer.tokenize(object.send(category.from).to_s).each do |token_text|
+            tokens, _ = tokenizer.tokenize object.send(category.from).to_s # Note: Originals not needed.
+            tokens.each do |token_text|
               next unless token_text
               cache << id << comma << token_text << newline
             end

data/lib/picky/indexers/serial.rb

@@ -28,7 +28,8 @@ module Picky
             result = []
 
             source.harvest(category) do |indexed_id, text|
-              tokenizer.tokenize(text).each do |token_text|
+              tokens, _ = tokenizer.tokenize text # Note: Originals not needed.
+              tokens.each do |token_text|
                 next unless token_text
                 result << indexed_id << comma << token_text << newline
               end

data/lib/picky/indexes_indexing.rb

@@ -42,7 +42,7 @@ module Picky
     #
     #
     def tokenizer
-      Tokenizers::Index.default
+      Tokenizer.index_default
     end
 
   end