picky 2.7.0 → 3.0.0.pre1

data/lib/picky/sources/wrappers/location.rb

@@ -1,48 +1,52 @@
-module Sources
+module Picky
 
-  module Wrappers
+  module Sources
 
-    # Should this actually just be a tokenizer?
-    #
-    class Location < Base
+    module Wrappers
 
-      attr_reader :calculation
-
-      def initialize source, grid, precision = 1
-        super source
-        @calculation = Calculations::Location.new grid, precision
-      end
-
-      # Yield the data (id, text for id) for the given category.
+      # Should this actually just be a tokenizer?
       #
-      def harvest category
-        minimum = 1.0/0
+      class Location < Base
 
-        # Cache. TODO Make option?
-        #
-        locations = []
+        attr_reader :calculation
 
-        # Gather min/max.
-        #
-        source.harvest category do |indexed_id, location|
-          location = location.to_f
-          minimum = location if location < minimum
-          locations << [indexed_id, location]
+        def initialize source, grid, precision = 1
+          super source
+          @calculation = Calculations::Location.new grid, precision
         end
 
-        calculation.minimum = minimum
-
-        # Recalculate locations.
+        # Yield the data (id, text for id) for the given category.
         #
-        locations.each do |indexed_id, location|
-          calculation.recalculated_range(location).each do |new_location|
-            yield indexed_id, new_location.to_s
+        def harvest category
+          minimum = 1.0/0
+
+          # Cache.
+          #
+          locations = []
+
+          # Gather min/max.
+          #
+          source.harvest category do |indexed_id, location|
+            location = location.to_f
+            minimum = location if location < minimum
+            locations << [indexed_id, location]
           end
+
+          calculation.minimum = minimum
+
+          # Recalculate locations.
+          #
+          locations.each do |indexed_id, location|
+            calculation.recalculated_range(location).each do |new_location|
+              yield indexed_id, new_location.to_s
+            end
+          end
+
+          # TODO Move to the right place.
+          #
+          category.indexing_exact[:location_minimum] = minimum
         end
 
-        # TODO Move to the right place.
-        #
-        category.indexing_exact[:location_minimum] = minimum
       end
 
     end

data/lib/picky/statistics.rb

@@ -1,60 +1,63 @@
 # encoding: utf-8
 #
+module Picky
 
-# Gathers various statistics.
-#
-class Statistics # :nodoc:all
+  # Gathers various statistics.
+  #
+  class Statistics # :nodoc:all
 
-  def initialize
-    @indexes = ["\033[1mIndexes analysis\033[m:"]
-  end
+    def initialize
+      @indexes = ["\033[1mIndexes analysis\033[m:"]
+    end
 
-  def preamble
-    loc = lines_of_code File.open('app/application.rb').read
+    def preamble
+      loc = lines_of_code File.open('app/application.rb').read
 
-    @preamble ||= <<-PREAMBLE
-\033[1mApplication(s)\033[m
-  Definition LOC:  #{"%4d" % loc}
-  Indexes defined: #{"%4d" % Indexes.size}
-PREAMBLE
-  end
+      @preamble ||= <<-PREAMBLE
+  \033[1mApplication(s)\033[m
+    Definition LOC:  #{"%4d" % loc}
+    Indexes defined: #{"%4d" % Indexes.size}
+  PREAMBLE
+    end
 
-  # Gathers information about the application.
-  #
-  def application
-    preamble
-    @application = Application.apps.map &:indented_to_s
-  end
+    # Gathers information about the application.
+    #
+    def application
+      preamble
+      @application = Application.apps.map &:indented_to_s
+    end
 
-  # Gathers information about the indexes.
-  #
-  def analyze object
-    object.each_category do |category|
-      @indexes << <<-ANALYSIS
-#{"#{category.index_name}".indented_to_s}\n
-#{"#{category.name}".indented_to_s(4)}\n
-#{"exact\n#{Analyzer.new.analyze(category.indexed_exact).indented_to_s}".indented_to_s(6)}\n
-#{"partial\n#{Analyzer.new.analyze(category.indexed_partial).indented_to_s}".indented_to_s(6)}
-ANALYSIS
+    # Gathers information about the indexes.
+    #
+    def analyze object
+      object.each_category do |category|
+        @indexes << <<-ANALYSIS
+  #{"#{category.index_name}".indented_to_s}\n
+  #{"#{category.name}".indented_to_s(4)}\n
+  #{"exact\n#{Analyzer.new.analyze(category.indexed_exact).indented_to_s}".indented_to_s(6)}\n
+  #{"partial\n#{Analyzer.new.analyze(category.indexed_partial).indented_to_s}".indented_to_s(6)}
+  ANALYSIS
+      end
     end
-  end
 
-  # Outputs all gathered statistics.
-  #
-  def to_s
-    <<-STATS
+    # Outputs all gathered statistics.
+    #
+    def to_s
+      <<-STATS
 
-Picky Configuration:
+  Picky Configuration:
 
-#{[@preamble, @application, @indexes.join("\n")].compact.join("\n")}
-STATS
-  end
+  #{[@preamble, @application, @indexes.join("\n")].compact.join("\n")}
+  STATS
+    end
 
-  # Internal methods.
-  #
+    # Internal methods.
+    #
+
+    def lines_of_code text
+      text.scan(/^\s*[^#\s].*$/).size
+    end
 
-  def lines_of_code text
-    text.scan(/^\s*[^#\s].*$/).size
   end
 
 end

data/lib/picky/tasks.rb

@@ -0,0 +1,3 @@
+all_rake_files = File.expand_path '../../tasks/*.rake', __FILE__
+
+Dir[all_rake_files].each { |rakefile| load rakefile }

data/lib/picky/tokenizers/base.rb

@@ -1,16 +1,18 @@
-module Tokenizers # :nodoc:all
+module Picky
 
-  # Defines tokenizing processes used both in indexing and querying.
-  #
-  class Base
+  module Tokenizers # :nodoc:all
 
-    # TODO Move EMPTY_STRING top level.
+    # Defines tokenizing processes used both in indexing and querying.
     #
-    EMPTY_STRING = ''.freeze
+    class Base
 
-    def to_s
-      reject_condition_location = @reject_condition.to_s[/:(\d+) \(lambda\)/, 1]
-      <<-TOKENIZER
+      # TODO Move EMPTY_STRING top level.
+      #
+      EMPTY_STRING = ''.freeze
+
+      def to_s
+        reject_condition_location = @reject_condition.to_s[/:(\d+) \(lambda\)/, 1]
+        <<-TOKENIZER
 Removes characters:        #{@removes_characters_regexp ? "/#{@removes_characters_regexp.source}/" : '-'}
 Stopwords:                 #{@remove_stopwords_regexp ? "/#{@remove_stopwords_regexp.source}/" : '-'}
 Splits text on:            #{@splits_text_on.respond_to?(:source) ? "/#{@splits_text_on.source}/" : (@splits_text_on ? @splits_text_on : '-')}
@@ -19,204 +21,207 @@ Normalizes words:          #{@normalizes_words_regexp_replaces ? @normalizes_wor
 Rejects tokens?            #{reject_condition_location ? "Yes, see line #{reject_condition_location} in app/application.rb" : '-'}
 Substitutes chars?         #{@substituter ? "Yes, using #{@substituter}." : '-' }
 Case sensitive?            #{@case_sensitive ? "Yes." : "-"}
-      TOKENIZER
-    end
+        TOKENIZER
+      end
 
-    # Stopwords.
-    #
-    # We only allow regexps (even if string would be okay
-    # too for gsub! - it's too hard to understand)
-    #
-    def stopwords regexp
-      check_argument_in __method__, Regexp, regexp
-      @remove_stopwords_regexp = regexp
-    end
-    def remove_stopwords text
-      text.gsub! @remove_stopwords_regexp, EMPTY_STRING if @remove_stopwords_regexp
-      text
-    end
-    @@non_single_stopword_regexp = /^\b[\w:]+?\b[\.\*\~]?\s?$/
-    def remove_non_single_stopwords text
-      return text if text.match @@non_single_stopword_regexp
-      remove_stopwords text
-    end
+      # Stopwords.
+      #
+      # We only allow regexps (even if string would be okay
+      # too for gsub! - it's too hard to understand)
+      #
+      def stopwords regexp
+        check_argument_in __method__, Regexp, regexp
+        @remove_stopwords_regexp = regexp
+      end
+      def remove_stopwords text
+        text.gsub! @remove_stopwords_regexp, EMPTY_STRING if @remove_stopwords_regexp
+        text
+      end
+      @@non_single_stopword_regexp = /^\b[\w:]+?\b[\.\*\~]?\s?$/
+      def remove_non_single_stopwords text
+        return text if text.match @@non_single_stopword_regexp
+        remove_stopwords text
+      end
 
-    # Illegals.
-    #
-    # We only allow regexps (even if string would be okay
-    # too for gsub! - it's too hard to understand)
-    #
-    def removes_characters regexp
-      check_argument_in __method__, Regexp, regexp
-      @removes_characters_regexp = regexp
-    end
-    def remove_illegals text
-      text.gsub! @removes_characters_regexp, EMPTY_STRING if @removes_characters_regexp
-      text
-    end
+      # Illegals.
+      #
+      # We only allow regexps (even if string would be okay
+      # too for gsub! - it's too hard to understand)
+      #
+      def removes_characters regexp
+        check_argument_in __method__, Regexp, regexp
+        @removes_characters_regexp = regexp
+      end
+      def remove_illegals text
+        text.gsub! @removes_characters_regexp, EMPTY_STRING if @removes_characters_regexp
+        text
+      end
 
-    # Splitting.
-    #
-    # We allow Strings and Regexps.
-    # Note: We do not test against to_str since symbols do not work with String#split.
-    #
-    def splits_text_on regexp_or_string
-      raise ArgumentError.new "#{__method__} takes a Regexp or String as argument, not a #{regexp_or_string.class}." unless Regexp === regexp_or_string || String === regexp_or_string
-      @splits_text_on = regexp_or_string
-    end
-    def split text
-      text.split @splits_text_on
-    end
+      # Splitting.
+      #
+      # We allow Strings and Regexps.
+      # Note: We do not test against to_str since symbols do not work with String#split.
+      #
+      def splits_text_on regexp_or_string
+        raise ArgumentError.new "#{__method__} takes a Regexp or String as argument, not a #{regexp_or_string.class}." unless Regexp === regexp_or_string || String === regexp_or_string
+        @splits_text_on = regexp_or_string
+      end
+      def split text
+        text.split @splits_text_on
+      end
 
-    # Normalizing.
-    #
-    # We only allow arrays.
-    #
-    def normalizes_words regexp_replaces
-      raise ArgumentError.new "#{__method__} takes an Array of replaces as argument, not a #{regexp_replaces.class}." unless regexp_replaces.respond_to?(:to_ary)
-      @normalizes_words_regexp_replaces = regexp_replaces
-    end
-    def normalize_with_patterns text
-      return text unless @normalizes_words_regexp_replaces
+      # Normalizing.
+      #
+      # We only allow arrays.
+      #
+      def normalizes_words regexp_replaces
+        raise ArgumentError.new "#{__method__} takes an Array of replaces as argument, not a #{regexp_replaces.class}." unless regexp_replaces.respond_to?(:to_ary)
+        @normalizes_words_regexp_replaces = regexp_replaces
+      end
+      def normalize_with_patterns text
+        return text unless @normalizes_words_regexp_replaces
 
-      @normalizes_words_regexp_replaces.each do |regex, replace|
-        # This should be sufficient
-        #
-        text.gsub!(regex, replace) and break
+        @normalizes_words_regexp_replaces.each do |regex, replace|
+          # This should be sufficient
+          #
+          text.gsub!(regex, replace) and break
+        end
+
+        remove_after_normalizing_illegals text
+        text
       end
-      remove_after_normalizing_illegals text
-      text
-    end
 
-    # Illegal after normalizing.
-    #
-    # We only allow regexps (even if string would be okay
-    # too for gsub! - it's too hard to understand)
-    #
-    def removes_characters_after_splitting regexp
-      check_argument_in __method__, Regexp, regexp
-      @removes_characters_after_splitting_regexp = regexp
-    end
-    def remove_after_normalizing_illegals text
-      text.gsub! @removes_characters_after_splitting_regexp, EMPTY_STRING if @removes_characters_after_splitting_regexp
-    end
+      # Illegal after normalizing.
+      #
+      # We only allow regexps (even if string would be okay
+      # too for gsub! - it's too hard to understand)
+      #
+      def removes_characters_after_splitting regexp
+        check_argument_in __method__, Regexp, regexp
+        @removes_characters_after_splitting_regexp = regexp
+      end
+      def remove_after_normalizing_illegals text
+        text.gsub! @removes_characters_after_splitting_regexp, EMPTY_STRING if @removes_characters_after_splitting_regexp
+      end
 
-    # Substitute Characters with this substituter.
-    #
-    # Default is European Character substitution.
-    #
-    def substitutes_characters_with substituter = CharacterSubstituters::WestEuropean.new
-      raise ArgumentError.new "The substitutes_characters_with option needs a character substituter, which responds to #substitute." unless substituter.respond_to?(:substitute)
-      @substituter = substituter
-    end
-    def substitute_characters text
-      substituter?? substituter.substitute(text) : text
-    end
+      # Substitute Characters with this substituter.
+      #
+      # Default is European Character substitution.
+      #
+      def substitutes_characters_with substituter = CharacterSubstituters::WestEuropean.new
+        raise ArgumentError.new "The substitutes_characters_with option needs a character substituter, which responds to #substitute." unless substituter.respond_to?(:substitute)
+        @substituter = substituter
+      end
+      def substitute_characters text
+        substituter?? substituter.substitute(text) : text
+      end
 
-    # Reject tokens after tokenizing based on the given criteria.
-    #
-    # Note: Currently only for indexing.
-    #
-    def reject_token_if &condition
-      @reject_condition = condition
-    end
-    def reject tokens
-      tokens.reject! &@reject_condition
-    end
+      # Reject tokens after tokenizing based on the given criteria.
+      #
+      # Note: Currently only for indexing.
+      #
+      def reject_token_if &condition
+        @reject_condition = condition
+      end
+      def reject tokens
+        tokens.reject! &@reject_condition
+      end
 
-    def case_sensitive case_sensitive
-      @case_sensitive = case_sensitive
-    end
-    def downcase?
-      !@case_sensitive
-    end
+      def case_sensitive case_sensitive
+        @case_sensitive = case_sensitive
+      end
+      def downcase?
+        !@case_sensitive
+      end
 
-    # Checks if the right argument type has been given.
-    #
-    def check_argument_in method, type, argument, &condition
-      raise ArgumentError.new "Application##{method} takes a #{type} as argument, not a #{argument.class}." unless type === argument
-    end
+      # Checks if the right argument type has been given.
+      #
+      def check_argument_in method, type, argument, &condition
+        raise ArgumentError.new "Application##{method} takes a #{type} as argument, not a #{argument.class}." unless type === argument
+      end
 
 
-    # Returns a number of tokens, generated from the given text.
-    #
-    # Note:
-    #  * preprocess, pretokenize are hooks
-    #
-    def tokenize text
-      text   = preprocess text  # processing the text
-      return empty_tokens if text.blank?
-      words  = pretokenize text # splitting and preparations for tokenizing
-      return empty_tokens if words.empty?
-      tokens = tokens_for words # creating tokens / strings
-               process tokens   # processing tokens / strings
-    end
+      # Returns a number of tokens, generated from the given text.
+      #
+      # Note:
+      #  * preprocess, pretokenize are hooks
+      #
+      def tokenize text
+        text   = preprocess text  # processing the text
+        return empty_tokens if text.blank?
+        words  = pretokenize text # splitting and preparations for tokenizing
+        return empty_tokens if words.empty?
+        tokens = tokens_for words # creating tokens / strings
+                 process tokens   # processing tokens / strings
+      end
+
+      attr_reader :substituter
+      alias substituter? substituter
 
-    attr_reader :substituter
-    alias substituter? substituter
+      def initialize options = {}
+        removes_characters options[:removes_characters]                                 if options[:removes_characters]
+        contracts_expressions *options[:contracts_expressions]                          if options[:contracts_expressions]
+        stopwords options[:stopwords]                                                   if options[:stopwords]
+        normalizes_words options[:normalizes_words]                                     if options[:normalizes_words]
+        removes_characters_after_splitting options[:removes_characters_after_splitting] if options[:removes_characters_after_splitting]
+        substitutes_characters_with options[:substitutes_characters_with]               if options[:substitutes_characters_with]
+        case_sensitive options[:case_sensitive]                                         unless options[:case_sensitive].nil?
 
-    def initialize options = {}
-      removes_characters options[:removes_characters]                                 if options[:removes_characters]
-      contracts_expressions *options[:contracts_expressions]                          if options[:contracts_expressions]
-      stopwords options[:stopwords]                                                   if options[:stopwords]
-      normalizes_words options[:normalizes_words]                                     if options[:normalizes_words]
-      removes_characters_after_splitting options[:removes_characters_after_splitting] if options[:removes_characters_after_splitting]
-      substitutes_characters_with options[:substitutes_characters_with]               if options[:substitutes_characters_with]
-      case_sensitive options[:case_sensitive]                                         unless options[:case_sensitive].nil?
+        # Defaults.
+        #
+        splits_text_on options[:splits_text_on] || /\s/
+        reject_token_if &(options[:reject_token_if] || options[:rejects_token_if] || :blank?) # TODO Decide on using an s or not.
+      end
 
-      # Defaults.
+      # Default preprocessing hook.
       #
-      splits_text_on options[:splits_text_on] || /\s/
-      reject_token_if &(options[:reject_token_if] || options[:rejects_token_if] || :blank?) # TODO Decide on using an s or not.
-    end
+      # Does:
+      # 1. Character substitution.
+      # 2. Remove illegal expressions.
+      # 3. Remove non-single stopwords. (Stopwords that occur with other words)
+      #
+      def preprocess text
+        text = substitute_characters text
+        remove_illegals text
+        # We do not remove single stopwords e.g. in the indexer for
+        # an entirely different reason than in the query tokenizer.
+        # An indexed thing with just name "UND" (a possible stopword)
+        # should not lose its name.
+        #
+        remove_non_single_stopwords text
+        text
+      end
+      # Pretokenizing.
+      #
+      # Does:
+      # 1. Split the text into words.
+      # 2. Normalize each word.
+      #
+      def pretokenize text
+        words = split text
+        words.collect! do |word|
+          normalize_with_patterns word
+          word
+        end
+      end
+      # Basic postprocessing (overridden in both query/index tokenizers).
+      #
+      def process tokens
+        reject tokens # Reject any tokens that don't meet criteria
+        tokens
+      end
 
-    # Default preprocessing hook.
-    #
-    # Does:
-    # 1. Character substitution.
-    # 2. Remove illegal expressions.
-    # 3. Remove non-single stopwords. (Stopwords that occur with other words)
-    #
-    def preprocess text
-      text = substitute_characters text
-      remove_illegals text
-      # We do not remove single stopwords e.g. in the indexer for
-      # an entirely different reason than in the query tokenizer.
-      # An indexed thing with just name "UND" (a possible stopword)
-      # should not lose its name.
-      #
-      remove_non_single_stopwords text
-      text
-    end
-    # Pretokenizing.
-    #
-    # Does:
-    # 1. Split the text into words.
-    # 2. Normalize each word.
-    #
-    def pretokenize text
-      words = split text
-      words.collect! do |word|
-        normalize_with_patterns word
-        word
+      # # Converts words into real tokens.
+      # #
+      # def tokens_for words
+      #   Query::Tokens.new words.collect! { |word| token_for word }
+      # end
+      # Turns non-blank text into symbols.
+      #
+      def symbolize text
+        text.blank? ? nil : text.to_sym
       end
-    end
-    # Basic postprocessing (overridden in both query/index tokenizers).
-    #
-    def process tokens
-      reject tokens # Reject any tokens that don't meet criteria
-      tokens
-    end
 
-    # # Converts words into real tokens.
-    # #
-    # def tokens_for words
-    #   Query::Tokens.new words.collect! { |word| token_for word }
-    # end
-    # Turns non-blank text into symbols.
-    #
-    def symbolize text
-      text.blank? ? nil : text.to_sym
     end
 
   end

data/lib/picky/tokenizers/index.rb

@@ -1,28 +1,32 @@
-module Tokenizers
+module Picky
 
-  # The base indexing tokenizer.
-  #
-  # Override in indexing subclasses and define in configuration.
-  #
-  class Index < Base
+  module Tokenizers
 
-    def self.default= new_default
-      @default = new_default
-    end
-    def self.default
-      @default ||= new
-    end
-
-    # Does not actually return a token, but a
-    # symbol "token".
+    # The base indexing tokenizer.
     #
-    def tokens_for words
-      words.collect! { |word| word.downcase! if downcase?; word.to_sym }
-    end
-    # Returns empty tokens.
+    # Override in indexing subclasses and define in configuration.
     #
-    def empty_tokens
-      []
+    class Index < Base
+
+      def self.default= new_default
+        @default = new_default
+      end
+      def self.default
+        @default ||= new
+      end
+
+      # Does not actually return a token, but a
+      # symbol "token".
+      #
+      def tokens_for words
+        words.collect! { |word| word.downcase! if downcase?; word.to_sym }
+      end
+      # Returns empty tokens.
+      #
+      def empty_tokens
+        []
+      end
+
     end
 
   end