picky 3.0.1 → 3.1.0

data/lib/picky/query/tokens.rb

@@ -20,17 +20,16 @@ module Picky
 
       # Creates a new Tokens object from a number of Strings.
       #
-      # Options:
-      #  * downcase: Whether to downcase the passed strings (default is true)
-      #
-      def self.processed words, downcase = true
-        new words.collect! { |word| Token.processed word, downcase }
+      def self.processed words, originals
+        new words.zip(originals).collect! { |word, original| Token.processed word, original }
       end
 
       # Tokenizes each token.
       #
       # Note: Passed tokenizer needs to offer #normalize(text).
       #
+      # TODO Still needed?
+      #
       def tokenize_with tokenizer
         @tokens.each { |token| token.tokenize_with(tokenizer) }
       end
@@ -63,21 +62,6 @@ module Picky
         @tokens.last.partial = true unless empty?
       end
 
-      # Caps the tokens to the maximum.
-      #
-      def cap maximum
-        @tokens.slice!(maximum..-1) if cap?(maximum)
-      end
-      def cap? maximum
-        @tokens.size > maximum
-      end
-
-      # Rejects blank tokens.
-      #
-      def reject
-        @tokens.reject! &:blank?
-      end
-
       #
       #
       def categorize mapper

data/lib/picky/search.rb

@@ -18,7 +18,8 @@ module Picky
     include Helpers::Measuring
 
     attr_reader   :indexes
-    attr_accessor :tokenizer, :weights
+    attr_accessor :tokenizer,
+                  :weights
 
     # Takes:
     # * A number of indexes
@@ -29,16 +30,17 @@ module Picky
     # It is also possible to define the tokenizer and weights like so.
     # Example:
     #   search = Search.new(index1, index2, index3) do
-    #     searching removes_characters: /[^a-z]/, etc.
-    #     weights [:author, :title] => +3, [:title, :isbn] => +1
+    #     searching removes_characters: /[^a-z]/ # etc.
+    #     weights [:author, :title] => +3,
+    #             [:title, :isbn] => +1
     #   end
     #
     def initialize *index_definitions
-      @indexes = Query::Indexes.new *index_definitions, combinations_type_for(index_definitions)
+      @indexes = Query::Indexes.new *index_definitions
 
       instance_eval(&Proc.new) if block_given?
 
-      @tokenizer ||= Tokenizers::Query.default
+      @tokenizer ||= Tokenizer.query_default # THINK Not dynamic. Ok?
       @weights   ||= Query::Weights.new
 
       self
@@ -58,28 +60,50 @@ module Picky
       @tokenizer = if options.respond_to?(:tokenize)
         options
       else
-        options && Tokenizers::Query.new(options)
+        options && Tokenizer.new(options)
       end
     end
 
-    # Example:
+    # Examples:
     #   search = Search.new(books_index, dvd_index, mp3_index) do
     #     boost [:author, :title] => +3,
     #           [:title, :isbn]   => +1
     #   end
     #
+    # or
+    #
+    #   # Explicitly add a random number (0...1) to the weights.
+    #   #
+    #   my_weights = Class.new do
+    #     # Instance only needs to implement
+    #     #   score_for combinations
+    #     # and return a number that is
+    #     # added to the weight.
+    #     #
+    #     def score_for combinations
+    #       rand
+    #     end
+    #   end.new
+    #
+    #   search = Search.new(books_index, dvd_index, mp3_index) do
+    #     boost my_weights
+    #   end
+    #
     def boost weights
-      weights  ||= Query::Weights.new
-      @weights = Hash === weights ? Query::Weights.new(weights) : weights
+      @weights = if weights.respond_to?(:score_for)
+        weights
+      else
+        Query::Weights.new weights
+      end
     end
 
     # This is the main entry point for a query.
     # Use this in specs and also for running queries.
     #
     # Parameters:
-    # * text: The search text.
-    # * ids = 20: _optional_ The amount of ids to calculate (with offset).
-    # * offset = 0: _optional_ The offset from which position to return the ids. Useful for pagination.
+    # * text:       The search text.
+    # * ids = 20:   The amount of ids to calculate (with offset).
+    # * offset = 0: The offset from which position to return the ids. Useful for pagination.
     #
     # Note: The Rack adapter calls this method after unravelling the HTTP request.
     #
@@ -89,7 +113,7 @@ module Picky
 
     # Runs the actual search using Query::Tokens.
     #
-    # Note: Internal method, use #search
+    # Note: Internal method, use #search to search.
     #
     def search_with tokens, ids = 20, offset = 0, original_text = nil
       results = nil
@@ -104,7 +128,7 @@ module Picky
 
     # Execute a search using Query::Tokens.
     #
-    # Note: Internal method, use #search.
+    # Note: Internal method, use #search to search.
     #
     def execute tokens, ids, offset, original_text = nil
       Results.from original_text, ids, offset, sorted_allocations(tokens)
@@ -113,10 +137,16 @@ module Picky
     # Delegates the tokenizing to the query tokenizer.
     #
     # Parameters:
-    # * text: The text to tokenize.
+    # * text: The string to tokenize.
+    #
+    # Returns:
+    # * A Picky::Query::Tokens instance.
     #
     def tokenized text
-      tokenizer.tokenize text
+      tokens, originals = tokenizer.tokenize text
+      tokens = Query::Tokens.processed tokens, originals || tokens
+      tokens.partialize_last # Note: In the standard Picky search, the last token is always partial.
+      tokens
     end
 
     # Gets sorted allocations for the tokens.
@@ -125,52 +155,15 @@ module Picky
       indexes.prepared_allocations_for tokens, weights
     end
 
-    # Returns the right combinations strategy for
-    # a number of query indexes.
-    #
-    # Currently it isn't possible using Memory and Redis etc.
-    # indexes in the same query index group.
-    #
-    # Picky will raise a Query::Indexes::DifferentTypesError.
-    #
-    @@mapping = {
-      Indexes::Memory => Query::Combinations::Memory,
-      Indexes::Redis  => Query::Combinations::Redis
-    }
-    def combinations_type_for index_definitions_ary
-      index_types = extract_index_types index_definitions_ary
-      !index_types.empty? && @@mapping[*index_types] || Query::Combinations::Memory
-    end
-    def extract_index_types index_definitions_ary
-      index_types = index_definitions_ary.map(&:class)
-      index_types.uniq!
-      check_index_types index_types
-      index_types
-    end
-    def check_index_types index_types
-      raise_different index_types if index_types.size > 1
-    end
-    # Currently it isn't possible using Memory and Redis etc.
-    # indexes in the same query index group.
-    #
-    class DifferentTypesError < StandardError
-      def initialize types
-        @types = types
-      end
-      def to_s
-        "Currently it isn't possible to mix #{@types.join(" and ")} Indexes in the same Search instance."
-      end
-    end
-    def raise_different index_types
-      raise DifferentTypesError.new(index_types)
-    end
-
     # Display some nice information for the user.
     #
     def to_s
       s = "#{self.class}("
-      s << @indexes.indexes.map(&:name).join(', ')
-      s << ", weights: #{@weights}" unless @weights.empty?
+      unless @indexes.indexes.empty?
+        s << @indexes.indexes.map(&:name).join(', ')
+        s << ", "
+      end
+      s << "weights: #{@weights}"
       s << ")"
       s
     end

data/lib/picky/tokenizer.rb

@@ -0,0 +1,231 @@
+# encoding: utf-8
+#
+module Picky
+
+  # Defines tokenizing processes used both in indexing and querying.
+  #
+  class Tokenizer
+
+    def self.index_default= new_default
+      @index_default = new_default
+    end
+    def self.index_default
+      @index_default ||= new
+    end
+
+    def self.query_default= new_default
+      @query_default = new_default
+    end
+    def self.query_default
+      @query_default ||= new
+    end
+
+    # TODO Move EMPTY_STRING top level.
+    #
+    EMPTY_STRING = ''.freeze
+
+    def to_s
+      reject_condition_location = @reject_condition.to_s[/:(\d+) \(lambda\)/, 1]
+      <<-TOKENIZER
+Removes characters: #{@removes_characters_regexp ? "/#{@removes_characters_regexp.source}/" : '-'}
+Stopwords:          #{@remove_stopwords_regexp ? "/#{@remove_stopwords_regexp.source}/" : '-'}
+Splits text on:     #{@splits_text_on.respond_to?(:source) ? "/#{@splits_text_on.source}/" : (@splits_text_on ? @splits_text_on : '-')}
+Normalizes words:   #{@normalizes_words_regexp_replaces ? @normalizes_words_regexp_replaces : '-'}
+Rejects tokens?     #{reject_condition_location ? "Yes, see line #{reject_condition_location} in app/application.rb" : '-'}
+Substitutes chars?  #{@substituter ? "Yes, using #{@substituter}." : '-' }
+Case sensitive?     #{@case_sensitive ? "Yes." : "-"}
+      TOKENIZER
+    end
+
+    # Stopwords.
+    #
+    # We only allow regexps (even if string would be okay
+    # too for gsub! - it's too hard to understand)
+    #
+    def stopwords regexp
+      check_argument_in __method__, Regexp, regexp
+      @remove_stopwords_regexp = regexp
+    end
+    def remove_stopwords text
+      text.gsub! @remove_stopwords_regexp, EMPTY_STRING if @remove_stopwords_regexp
+      text
+    end
+    @@non_single_stopword_regexp = /^\b[\w:]+?\b[\.\*\~]?\s?$/
+    def remove_non_single_stopwords text
+      return text unless @remove_stopwords_regexp
+      return text if text.match @@non_single_stopword_regexp
+      remove_stopwords text
+    end
+
+    # Illegals.
+    #
+    # We only allow regexps (even if string would be okay
+    # too for gsub! - it's too hard to understand)
+    #
+    def removes_characters regexp
+      check_argument_in __method__, Regexp, regexp
+      @removes_characters_regexp = regexp
+    end
+    def remove_illegals text
+      text.gsub! @removes_characters_regexp, EMPTY_STRING if @removes_characters_regexp
+      text
+    end
+
+    # Splitting.
+    #
+    # We allow Strings and Regexps.
+    # Note: We do not test against to_str since symbols do not work with String#split.
+    #
+    def splits_text_on regexp_or_string
+      raise ArgumentError.new "#{__method__} takes a Regexp or String as argument, not a #{regexp_or_string.class}." unless Regexp === regexp_or_string || String === regexp_or_string
+      @splits_text_on = regexp_or_string
+    end
+    def split text
+      text.split @splits_text_on
+    end
+
+    # Normalizing.
+    #
+    # We only allow arrays.
+    #
+    def normalizes_words regexp_replaces
+      raise ArgumentError.new "#{__method__} takes an Array of replaces as argument, not a #{regexp_replaces.class}." unless regexp_replaces.respond_to?(:to_ary)
+      @normalizes_words_regexp_replaces = regexp_replaces
+    end
+    def normalize_with_patterns text
+      return text unless @normalizes_words_regexp_replaces
+
+      @normalizes_words_regexp_replaces.each do |regex, replace|
+        # This should be sufficient
+        #
+        text.gsub!(regex, replace) and break
+      end
+
+      text
+    end
+    def normalize_with_patterns?
+      @normalizes_words_regexp_replaces
+    end
+
+    # Substitute Characters with this substituter.
+    #
+    # Default is European Character substitution.
+    #
+    def substitutes_characters_with substituter = CharacterSubstituters::WestEuropean.new
+      raise ArgumentError.new "The substitutes_characters_with option needs a character substituter, which responds to #substitute." unless substituter.respond_to?(:substitute)
+      @substituter = substituter
+    end
+    def substitute_characters text
+      substituter?? substituter.substitute(text) : text
+    end
+
+    # Reject tokens after tokenizing based on the given criteria.
+    #
+    def rejects_token_if &condition
+      @reject_condition = condition
+    end
+    def reject tokens
+      tokens.reject! &@reject_condition
+    end
+
+    def case_sensitive case_sensitive
+      @case_sensitive = case_sensitive
+    end
+    def downcase?
+      !@case_sensitive
+    end
+
+    def maximum_tokens amount
+      @maximum_tokens = amount
+    end
+    def cap words
+      words.slice!(@maximum_tokens..-1) if cap?(words)
+    end
+    def cap? words
+      @maximum_tokens && words.size > @maximum_tokens
+    end
+
+    # Checks if the right argument type has been given.
+    #
+    def check_argument_in method, type, argument, &condition
+      raise ArgumentError.new "Application##{method} takes a #{type} as argument, not a #{argument.class}." unless type === argument
+    end
+
+    attr_reader :substituter
+    alias substituter? substituter
+
+    def initialize options = {}
+      substitutes_characters_with options[:substitutes_characters_with] if options[:substitutes_characters_with]
+      removes_characters options[:removes_characters]                   if options[:removes_characters]
+      stopwords options[:stopwords]                                     if options[:stopwords]
+      splits_text_on options[:splits_text_on]                           || /\s/
+      normalizes_words options[:normalizes_words]                       if options[:normalizes_words]
+      maximum_tokens options[:maximum_tokens]
+      rejects_token_if &(options[:rejects_token_if]                     || :blank?)
+      case_sensitive options[:case_sensitive]                           unless options[:case_sensitive].nil?
+    end
+
+    # Returns a number of tokens, generated from the given text,
+    # based on the parameters given.
+    #
+    # Returns:
+    #  [[:token1, :token2], ["Original1", "Original2"]]
+    #
+    def tokenize text
+      text   = preprocess text  # processing the text
+      return empty_tokens if text.blank?
+      words  = pretokenize text # splitting and preparations for tokenizing
+      return empty_tokens if words.empty?
+      tokens = tokens_for words # creating tokens / strings
+      [tokens, words]
+    end
+
+    # Default preprocessing hook.
+    #
+    # Does:
+    # 1. Character substitution.
+    # 2. Remove illegal expressions.
+    # 3. Remove non-single stopwords. (Stopwords that occur with other words)
+    #
+    def preprocess text
+      text = substitute_characters text
+      remove_illegals text
+      # We do not remove single stopwords e.g. in the indexer for
+      # an entirely different reason than in the query tokenizer.
+      # An indexed thing with just name "UND" (a possible stopword)
+      # should not lose its name.
+      #
+      remove_non_single_stopwords text
+      text
+    end
+
+    # Pretokenizing.
+    #
+    # Does:
+    #  * Split the text into words.
+    #  * Cap the amount of tokens if maximum_tokens is set.
+    #
+    def pretokenize text
+      words = split text
+      words.collect! { |word| normalize_with_patterns word } if normalize_with_patterns?
+      reject words
+      cap words if cap?(words)
+      words
+    end
+
+    # Downcases.
+    #
+    def tokens_for words
+      words.collect! { |word| word.downcase!; word } if downcase?
+      words
+    end
+
+    # Returns empty tokens.
+    #
+    def empty_tokens
+      [[], []]
+    end
+
+  end
+
+end

data/lib/picky/tokenizers/location.rb

@@ -2,7 +2,7 @@ module Picky
 
   module Tokenizers
 
-    class Location < Base
+    class Location < Picky::Tokenizer # TODO Still needed?
 
       attr_reader :calculation
 

data/lib/tasks/try.rake

@@ -2,18 +2,10 @@
 #
 desc "Try the given text in the indexer/query (index and category optional)."
 task :try, [:text, :index, :category] => :application do |_, options|
-  text, index, category = options.text, options.index, options.category
-
   puts
-  fail "\x1b[31mrake try needs a text to try indexing and query preparation\x1b[m, e.g. rake 'try[yourtext]'." unless text
-
-  specific = Picky::Indexes
-  specific = specific[index]    if index
-  specific = specific[category] if category
+  fail "\x1b[31mrake try needs a text to try indexing and query preparation\x1b[m, e.g. rake 'try[yourtext]'." unless options.text
 
-  puts "\"#{text}\" is saved in the #{specific.identifier} index as #{specific.tokenizer.tokenize(text.dup).to_a}"
-
-  puts "\"#{text}\" as a search will be tokenized into #{Picky::Tokenizers::Query.default.tokenize(text.dup).to_a.map(&:to_s).map(&:to_sym)}"
-  puts
-  puts "(category qualifiers, e.g. title: are removed if they do not exist as a qualifier, so 'toitle:bla' -> 'bla')"
+  require File.expand_path '../try', __FILE__
+  try = Picky::Try.new options.text, options.index, options.category
+  try.to_stdout
 end

data/lib/tasks/try.rb

@@ -0,0 +1,37 @@
+module Picky
+
+  class Try
+
+    attr_reader :text, :specific
+
+    def initialize text, index = nil, category = nil
+      @text = text
+      @specific = Picky::Indexes
+      @specific = @specific[index.to_sym]    if index
+      @specific = @specific[category.to_sym] if category
+    end
+
+    def saved
+      specific.tokenizer.tokenize(text.dup).first
+    end
+
+    def searched
+      Picky::Tokenizer.query_default.tokenize(text.dup).first
+    end
+
+    def output
+      <<-OUTPUT
+\"#{text}\" is saved in the #{specific.identifier} index as #{saved}
+\"#{text}\" as a search will be tokenized as #{searched}
+
+(category qualifiers, e.g. title: are removed if they do not exist as a qualifier, so 'toitle:bla' -> 'bla')
+      OUTPUT
+    end
+
+    def to_stdout
+      puts output
+    end
+
+  end
+
+end

data/spec/lib/application_spec.rb

@@ -8,7 +8,7 @@ describe Picky::Application do
     it "should run ok" do
       lambda {
         class MinimalTestApplication < described_class
-          books = Picky::Indexes::Memory.new :books do
+          books = Picky::Index.new :books do
                   source Picky::Sources::DB.new(
                     'SELECT id, title FROM books',
                     :file => 'app/db.yml'
@@ -20,8 +20,8 @@ describe Picky::Application do
           
           route %r{^/books} => Picky::Search.new(books)
         end
-        Picky::Tokenizers::Index.default.tokenize 'some text'
-        Picky::Tokenizers::Query.default.tokenize 'some text'
+        Picky::Tokenizer.index_default.tokenize 'some text'
+        Picky::Tokenizer.query_default.tokenize 'some text'
       }.should_not raise_error
     end
     it "should run ok" do
@@ -44,7 +44,7 @@ describe Picky::Application do
                     substitutes_characters_with: Picky::CharacterSubstituters::WestEuropean.new,
                     maximum_tokens: 5
           
-          books_index = Picky::Indexes::Memory.new :books do
+          books_index = Picky::Index.new :books do
             source Picky::Sources::DB.new(
               'SELECT id, title, author, isbn13 as isbn FROM books',
               :file => 'app/db.yml'
@@ -58,7 +58,7 @@ describe Picky::Application do
           books_index.define_category :isbn,
                                       partial: Picky::Partial::None.new # Partially searching on an ISBN makes not much sense.
           
-          geo_index = Picky::Indexes::Memory.new :geo do
+          geo_index = Picky::Index.new :geo do
             source          Picky::Sources::CSV.new(:location, :north, :east, file: 'data/ch.csv', col_sep: ',')
             indexing        removes_characters: /[^a-z]/
             category        :location,

data/spec/lib/{backend → backends}/file/basic_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe Picky::Backend::File::Basic do
+describe Picky::Backends::File::Basic do
   
   let(:file) { described_class.new 'some/cache/path/to/file' }
   
@@ -18,7 +18,7 @@ describe Picky::Backend::File::Basic do
   
   describe 'to_s' do
     it 'returns the cache path with the default file extension' do
-      file.to_s.should == 'Picky::Backend::File::Basic(some/cache/path/to/file.index)'
+      file.to_s.should == 'Picky::Backends::File::Basic(some/cache/path/to/file.index)'
     end
   end
   

data/spec/lib/{backend → backends}/file/json_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe Picky::Backend::File::JSON do
+describe Picky::Backends::File::JSON do
   
   let(:file) { described_class.new 'some/cache/path/to/file' }
   
@@ -24,7 +24,7 @@ describe Picky::Backend::File::JSON do
   
   describe 'to_s' do
     it 'returns the cache path with the default file extension' do
-      file.to_s.should == 'Picky::Backend::File::JSON(some/cache/path/to/file.json)'
+      file.to_s.should == 'Picky::Backends::File::JSON(some/cache/path/to/file.json)'
     end
   end
   

data/spec/lib/{backend → backends}/file/marshal_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe Picky::Backend::File::Marshal do
+describe Picky::Backends::File::Marshal do
   
   let(:file) { described_class.new 'some/cache/path/to/file' }
   
@@ -24,7 +24,7 @@ describe Picky::Backend::File::Marshal do
   
   describe 'to_s' do
     it 'returns the cache path with the default file extension' do
-      file.to_s.should == 'Picky::Backend::File::Marshal(some/cache/path/to/file.dump)'
+      file.to_s.should == 'Picky::Backends::File::Marshal(some/cache/path/to/file.dump)'
     end
   end
   

data/spec/lib/{backend → backends}/file/text_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe Picky::Backend::File::Text do
+describe Picky::Backends::File::Text do
   
   before(:each) do
     @file = described_class.new "some_cache_path"