picky 1.4.1 → 1.4.2

data/lib/picky/internals/query/token.rb

@@ -0,0 +1,215 @@
+module Internals
+
+  module Query
+  
+    # This is a query token. Together with other tokens it makes up a query.
+    #
+    # It remembers the original form, and and a normalized form.
+    #
+    # It also knows whether it needs to look for similarity (bla~), or whether it is a partial (bla*).
+    #
+    # TODO Make partial / similarity char configurable.
+    #
+    class Token # :nodoc:all
+
+      attr_reader :text, :original
+      attr_writer :similar
+
+      delegate :blank?, :to => :text
+
+      # Normal initializer.
+      #
+      # Note: Use this if you do not want a qualified and normalized token.
+      #
+      def initialize text
+        @text = text
+      end
+
+      # Returns a qualified and normalized token.
+      #
+      # Note: Use this in the search engine if you need a qualified
+      #       and normalized token. I.e. one prepared for a search.
+      #
+      def self.processed text
+        token = new text
+        token.qualify
+        token.extract_original
+        token.partialize
+        token.similarize
+        token.remove_illegals
+        token
+      end
+
+      # This returns a predefined category name if the user has given one.
+      #
+      def user_defined_category_name
+        @qualifier
+      end
+
+      # Extracts a qualifier for this token and pre-assigns an allocation.
+      #
+      # Note: Removes the qualifier if it is not allowed.
+      #
+      def qualify
+        @qualifier, @text = split @text
+        @qualifier = Query::Qualifiers.instance.normalize @qualifier
+      end
+      def extract_original
+        @original = @text.dup
+      end
+
+      # Partial is a conditional setter.
+      #
+      # It is only settable if it hasn't been set yet.
+      #
+      def partial= partial
+        @partial = partial if @partial.nil?
+      end
+      def partial?
+        !@similar && @partial
+      end
+
+      # If the text ends with *, partialize it. If with ", don't.
+      #
+      @@no_partial = /\"\Z/
+      @@partial    = /\*\Z/
+      def partialize
+        self.partial = false and return if @text =~ @@no_partial
+        self.partial = true if @text =~ @@partial
+      end
+
+      # If the text ends with ~ similarize it. If with ", don't.
+      #
+      @@no_similar = /\"\Z/
+      @@similar    = /\~\Z/
+      def similarize
+        self.similar = false and return if @text =~ @@no_similar
+        self.similar = true if @text =~ @@similar
+      end
+
+      def similar?
+        @similar
+      end
+
+      # Normalizes this token's text.
+      #
+      @@illegals = /["*~]/
+      def remove_illegals
+        @text.gsub! @@illegals, '' unless @text.blank?
+      end
+    
+      # Visitor for tokenizer.
+      #
+      # TODO Rewrite!!!
+      #
+      def tokenize_with tokenizer
+        @text = tokenizer.normalize @text
+      end
+      # TODO spec!
+      #
+      # TODO Rewrite!!
+      #
+      def tokenized tokenizer
+        tokenizer.tokenize(@text.to_s).each do |text|
+          yield text
+        end
+      end
+
+      # Returns an array of possible combinations.
+      #
+      def possible_combinations_in type
+        type.possible_combinations self
+      end
+    
+      # Returns a token with the next similar text.
+      #
+      # TODO Rewrite this. It is hard to understand. Also spec performance.
+      #
+      def next_similar_token category
+        token = self.dup
+        token if token.next_similar category.bundle_for(token)
+      end
+      # Sets and returns the next similar word.
+      #
+      # Note: Also overrides the original.
+      #
+      def next_similar bundle
+        @text = @original = (similarity(bundle).shift || return) if similar?
+      end
+      # Lazy similar reader.
+      #
+      def similarity bundle = nil
+        @similarity || @similarity = generate_similarity_for(bundle)
+      end
+      # Returns an enumerator that traverses over the similar.
+      #
+      # Note: The dup isn't too nice – since it is needed on account of the shift, above.
+      #       (We avoid a StopIteration exception. Which of both is less evil?)
+      #
+      def generate_similarity_for bundle
+        bundle.similar(@text).dup || []
+      end
+
+      # Generates a solr term from this token.
+      #
+      # E.g. "name:heroes~0.75"
+      #
+      @@solr_fuzzy_mapping = {
+        1 => :'',
+        2 => :'',
+        3 => :'',
+        4 => :'~0.74',
+        5 => :'~0.78',
+        6 => :'~0.81',
+        7 => :'~0.83',
+        8 => :'~0.85',
+        9 => :'~0.87',
+       10 => :'~0.89'
+      }
+      @@solr_fuzzy_mapping.default = :'~0.9'
+      def to_solr
+        blank? ? '' : (to_s + @@solr_fuzzy_mapping[@text.size].to_s)
+      end
+    
+      #
+      #
+      def to_result
+        [@original, @text]
+      end
+    
+      # Internal identifier.
+      #
+      # TODO Uh.
+      #
+      def identifier
+        "#{similar?? :similarity : :index}:#{@text}"
+      end
+    
+      # Displays the qualifier text and the text, joined.
+      #
+      # e.g. name:meier
+      #
+      def to_s
+        [@qualifier, @text].compact.join ':'
+      end
+    
+      private
+      
+        # Splits text into a qualifier and text.
+        #
+        # Returns [qualifier, text].
+        #
+        def split unqualified_text
+          qualifier, text = (unqualified_text || '').split(':', 2)
+          if text.blank?
+            [nil, (qualifier || '')]
+          else
+            [qualifier, text]
+          end
+        end
+
+    end
+    
+  end
+  
+end

data/lib/picky/internals/query/tokens.rb

@@ -0,0 +1,89 @@
+# encoding: utf-8
+#
+module Internals
+  
+  #
+  #
+  module Query
+  
+    # This class primarily handles switching through similar token constellations.
+    #
+    class Tokens # :nodoc:all
+    
+      # Basically delegates to its internal tokens array.
+      #
+      self.delegate *[Enumerable.instance_methods, :slice!, :[], :uniq!, :last, :reject!, :length, :size, :empty?, :each, :exit, { :to => :@tokens }].flatten
+    
+      #
+      #
+      def initialize tokens = []
+        @tokens = tokens
+      end
+    
+      #
+      #
+      def tokenize_with tokenizer
+        @tokens.each { |token| token.tokenize_with(tokenizer) }
+      end
+    
+      # Generates an array in the form of
+      # [
+      #  [combination],                           # of token 1
+      #  [combination, combination, combination], # of token 2
+      #  [combination, combination]               # of token 3
+      # ]
+      #
+      # TODO If we want token behaviour defined per Query, we can
+      #      compact! here
+      #
+      def possible_combinations_in type
+        @tokens.inject([]) do |combinations, token|
+          combinations << token.possible_combinations_in(type)
+        end
+        # TODO compact! if ignore_unassigned_tokens
+      end
+
+      # Makes the last of the tokens partial.
+      #
+      def partialize_last
+        @tokens.last.partial = true unless empty?
+      end
+
+      # Caps the tokens to the maximum.
+      #
+      def cap maximum
+        @tokens.slice!(maximum..-1) if cap?(maximum)
+      end
+      def cap? maximum
+        @tokens.size > maximum
+      end
+    
+      # Rejects blank tokens.
+      #
+      def reject
+        @tokens.reject! &:blank?
+      end
+    
+      # Returns a solr query.
+      #
+      def to_solr_query
+        @tokens.map(&:to_solr).join ' '
+      end
+    
+      #
+      #
+      def originals
+        @tokens.map(&:original)
+      end
+    
+      # Just join the token original texts.
+      #
+      def to_s
+        originals.join ' '
+      end
+    
+    end
+  
+  end
+  
+end

data/lib/picky/internals/results/base.rb

@@ -0,0 +1,106 @@
+module Internals
+
+  module Results # :nodoc:all
+
+    # This is the internal results object. Usually, to_marshal, or to_json
+    # is called on it to get a string for the answer.
+    #
+    class Base
+
+      # Duration is set externally by the query.
+      #
+      attr_writer :duration
+      attr_reader :allocations, :offset
+
+      # Takes instances of Query::Allocations as param.
+      #
+      def initialize offset = 0, allocations = Query::Allocations.new
+        @offset = offset
+        @allocations = allocations # || Query::Allocations.new
+      end
+      # Create new results and calculate the ids.
+      #
+      def self.from offset, allocations
+        results = new offset, allocations
+        results.prepare!
+        results
+      end
+
+      #
+      #
+      def serialize
+        { allocations: allocations.to_result,
+          offset:      offset,
+          duration:    duration,
+          total:       total }
+      end
+      # The default format is json.
+      #
+      def to_response options = {}
+        to_json options
+      end
+      # Convert to json format.
+      #
+      def to_json options = {}
+        serialize.to_json options
+      end
+
+      # This starts the actual processing.
+      #
+      # Without this, the allocations are not processed,
+      # and no ids are calculated.
+      #
+      def prepare!
+        allocations.process! self.max_results, self.offset
+      end
+
+      # Duration default is 0.
+      #
+      def duration
+        @duration || 0
+      end
+      # The total results. Delegates to the allocations.
+      #
+      # Caches.
+      #
+      def total
+        @total || @total = allocations.total || 0
+      end
+
+      # How many results are returned.
+      #
+      # Set in config using
+      #   Results::Full.max_results = 20
+      #
+      class_inheritable_accessor :max_results
+      def max_results
+        self.class.max_results
+      end
+
+      # Convenience methods.
+      #
+
+      # Delegates to allocations.
+      #
+      def ids amount = 20
+        allocations.ids amount
+      end
+      # Gets an amout of random ids from the allocations.
+      #
+      # Note: Basically delegates to the allocations.
+      #
+      def random_ids amount = 1
+        allocations.random_ids amount
+      end
+
+      # Human readable log.
+      #
+      def to_log query
+        "|#{Time.now.to_s(:db)}|#{'%8f' % duration}|#{'%-50s' % query}|#{'%8d' % total}|#{'%4d' % offset}|#{'%2d' % allocations.size}|"
+      end
+    
+    end
+
+  end
+  
+end

data/lib/picky/internals/results/full.rb

@@ -0,0 +1,17 @@
+module Internals
+
+  module Results
+    # Full results are limited to maximally 20 results (by default).
+    #
+    class Full < Base
+    
+      self.max_results = 20
+    
+      def to_log *args
+        ?> + super
+      end
+    
+    end
+  end
+  
+end

data/lib/picky/internals/results/live.rb

@@ -0,0 +1,17 @@
+module Internals
+
+  module Results
+    # Live results are not returning any results.
+    #
+    class Live < Base
+    
+      self.max_results = 0
+    
+      def to_log *args
+        ?. + super
+      end
+    
+    end
+  end
+  
+end

data/lib/picky/internals/tokenizers/base.rb

@@ -0,0 +1,166 @@
+module Internals
+  
+  module Tokenizers # :nodoc:all
+  
+    # Defines tokenizing processes used both in indexing and querying.
+    #
+    class Base
+    
+      # TODO Move EMPTY_STRING top level.
+      #
+      EMPTY_STRING = ''.freeze
+    
+      # Stopwords.
+      #
+      def stopwords regexp
+        @remove_stopwords_regexp = regexp
+      end
+      def remove_stopwords text
+        text.gsub! @remove_stopwords_regexp, EMPTY_STRING if @remove_stopwords_regexp
+        text
+      end
+      @@non_single_stopword_regexp = /^\b[\w:]+?\b[\.\*\~]?\s?$/
+      def remove_non_single_stopwords text
+        return text if text.match @@non_single_stopword_regexp
+        remove_stopwords text
+      end
+    
+      # Illegals.
+      #
+      # TODO Should there be a legal?
+      #
+      def removes_characters regexp
+        @removes_characters_regexp = regexp
+      end
+      def remove_illegals text
+        text.gsub! @removes_characters_regexp, EMPTY_STRING if @removes_characters_regexp
+        text
+      end
+    
+      # Splitting.
+      #
+      def splits_text_on regexp
+        @splits_text_on_regexp = regexp
+      end
+      def split text
+        text.split @splits_text_on_regexp
+      end
+    
+      # Normalizing.
+      #
+      def normalizes_words regexp_replaces
+        @normalizes_words_regexp_replaces = regexp_replaces
+      end
+      def normalize_with_patterns text
+        return text unless @normalizes_words_regexp_replaces
+      
+        @normalizes_words_regexp_replaces.each do |regex, replace|
+          # This should be sufficient
+          #
+          text.gsub!(regex, replace) and break
+        end
+        remove_after_normalizing_illegals text
+        text
+      end
+    
+      # Illegal after normalizing.
+      #
+      def removes_characters_after_splitting regexp
+        @removes_characters_after_splitting_regexp = regexp
+      end
+      def remove_after_normalizing_illegals text
+        text.gsub! @removes_characters_after_splitting_regexp, EMPTY_STRING if @removes_characters_after_splitting_regexp
+      end
+    
+      # Substitute Characters with this substituter.
+      #
+      # Default is European Character substitution.
+      #
+      def substitutes_characters_with substituter = CharacterSubstituters::WestEuropean.new
+        # TODO Raise if it doesn't quack substitute?
+        @substituter = substituter
+      end
+      def substitute_characters text
+        substituter?? substituter.substitute(text) : text 
+      end
+    
+      # Reject tokens after tokenizing based on the given criteria.
+      #
+      # Note: Currently only for indexing. TODO Redesign and write for both!
+      #
+      def reject_token_if &condition
+        @reject_condition = condition
+      end
+      def reject tokens
+        tokens.reject! &@reject_condition
+      end
+    
+    
+      # Returns a number of tokens, generated from the given text.
+      #
+      # Note:
+      #  * preprocess, pretokenize are hooks
+      #
+      def tokenize text
+        text   = preprocess text  # processing the text
+        return empty_tokens if text.blank?
+        words  = pretokenize text # splitting and preparations for tokenizing
+        return empty_tokens if words.empty?
+        tokens = tokens_for words # creating tokens / strings
+                 process tokens   # processing tokens / strings
+      end
+    
+      attr_reader :substituter
+      alias substituter? substituter
+    
+      def initialize options = {}
+        removes_characters options[:removes_characters]                                 if options[:removes_characters]
+        contracts_expressions *options[:contracts_expressions]                          if options[:contracts_expressions]
+        stopwords options[:stopwords]                                                   if options[:stopwords]
+        normalizes_words options[:normalizes_words]                                     if options[:normalizes_words]
+        removes_characters_after_splitting options[:removes_characters_after_splitting] if options[:removes_characters_after_splitting]
+        substitutes_characters_with options[:substitutes_characters_with]               if options[:substitutes_characters_with]
+      
+        # Defaults.
+        #
+        splits_text_on options[:splits_text_on] || /\s/
+        reject_token_if &(options[:reject_token_if] || :blank?)
+      end
+    
+      # Hooks.
+      #
+    
+      # Preprocessing.
+      #
+      def preprocess text; end
+      # Pretokenizing.
+      #
+      def pretokenize text; end
+      # Postprocessing.
+      #
+      def process tokens
+        reject tokens # Reject any tokens that don't meet criteria
+        tokens
+      end
+    
+      # Converts words into real tokens.
+      #
+      def tokens_for words
+        Internals::Query::Tokens.new words.collect! { |word| token_for word }
+      end
+      # Turns non-blank text into symbols.
+      #
+      def symbolize text
+        text.blank? ? nil : text.to_sym
+      end
+      # Returns a tokens object.
+      #
+      def empty_tokens
+        Internals::Query::Tokens.new
+      end
+    
+    end
+    
+  end
+  
+end

data/lib/picky/internals/tokenizers/index.rb

@@ -0,0 +1,63 @@
+module Internals
+
+  module Tokenizers
+  
+    # The base indexing tokenizer.
+    #
+    # Override in indexing subclasses and define in configuration.
+    #
+    class Index < Base
+    
+      def self.default= new_default
+        @default = new_default
+      end
+      def self.default
+        @default ||= new
+      end
+    
+      # Default indexing preprocessing hook.
+      #
+      # Does:
+      # 1. Character substitution.
+      # 2. Downcasing.
+      # 3. Remove illegal expressions.
+      # 4. Remove non-single stopwords. (Stopwords that occur with other words)
+      #
+      def preprocess text
+        text = substitute_characters text
+        text.downcase!
+        remove_illegals text
+        # we do not remove single stopwords for an entirely different
+        # reason than in the query tokenizer.
+        # An indexed thing with just name "UND" (a possible stopword) should not lose its name.
+        #
+        remove_non_single_stopwords text
+        text
+      end
+    
+      # Default indexing pretokenizing hook.
+      #
+      # Does:
+      # 1. Split the text into words.
+      # 2. Normalize each word.
+      #
+      def pretokenize text
+        words = split text
+        words.collect! do |word|
+          normalize_with_patterns word
+          word
+        end
+      end
+    
+      # Does not actually return a token, but a
+      # symbol "token".
+      #
+      def token_for text
+        symbolize text
+      end
+    
+    end
+    
+  end
+  
+end