picky 1.4.1 → 1.4.2

data/lib/picky/query/weigher.rb

@@ -1,165 +0,0 @@
-module Query
-
-  # Weighs the given tokens, generates Allocations -> Allocation -> Combinations.
-  #
-  class Weigher # :nodoc:all
-
-    attr_reader :indexes
-
-    # A weigher has a number of typed indexes, for which it generates allocations.
-    #
-    def initialize types
-      @indexes = types
-    end
-
-    #
-    #
-    def allocations_for tokens
-      Allocations.new(indexes.inject([]) do |previous_allocations, index|
-        # Expand the combinations.
-        #
-        possible_combinations = tokens.possible_combinations_in index
-        
-        # Optimization for ignoring tokens that allocate to nothing and
-        # can be ignored.
-        # For example in a special search, where "florian" is not
-        # mapped to city, zip, or category.
-        #
-        possible_combinations.compact!
-        expanded_combinations = expand_combinations_from possible_combinations
-        
-        #
-        #
-        next previous_allocations if expanded_combinations.empty?
-        
-        # The recombination part, where
-        # [
-        #  [a,a,b,b,c,c]
-        #  [d,e,d,e,d,e]
-        # ]
-        # becomes
-        # [
-        #  [a,d],
-        #  [a,e],
-        #  [b,d],
-        #  [b,e],
-        #  [c,d],
-        #  [c,e]
-        # ]
-        #
-        # TODO Use transpose?
-        #
-        expanded_combinations = expanded_combinations.shift.zip *expanded_combinations
-        
-        # Wrap into a real combination.
-        #
-        # expanded_combinations.map! { |expanded_combination| Combinations.new(expanded_combination).pack_into_allocation(index.result_identifier) }
-        
-        # Add the possible allocations to the ones we already have.
-        #
-        # previous_allocations + expanded_combinations.map(&:pack_into_allocation)
-        
-        
-        # Add the wrapped possible allocations to the ones we already have.
-        #
-        previous_allocations + expanded_combinations.map! do |expanded_combination|
-          Combinations.new(expanded_combination).pack_into_allocation(index.result_identifier) # TODO Do not extract result_identifier. Remove pack_into_allocation.
-        end
-      end)
-    end
-    
-    # This is the core of the search engine.
-    #
-    # Gets an array of
-    # [
-    #  [<combinations for token1>],
-    #  [<combinations for token2>],
-    #  [<combinations for token3>]
-    # ]
-    #
-    # Generates all possible allocations of combinations.
-    # [
-    #  [first combination of token1, first c of t2, first c of t3],
-    #  [first combination of token1, first c of t2, second c of t3]
-    #  ...
-    # ]
-    #
-    # Generates all possible combinations of array elements:
-    # [1,2,3] x [a,b,c] x [k,l,m] => [[1,a,k], [1,a,l], [1,a,m], [1,b,k], [1,b,l], [1,b,m], [1,c,k], ..., [3,c,m]]
-    # Note: Also calculates the weights and sorts them accordingly.
-    #
-    # Note: This is a heavily optimized ruby version.
-    #
-    # Works like this:
-    # [1,2,3], [a,b,c], [k,l,m] are expanded to
-    #  group mult: 1
-    #  <- single mult ->
-    # [1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3] = 27 elements
-    #  group mult: 3
-    #  <- -> s/m
-    # [a,a,a,b,b,b,c,c,c,a,a,a,b,b,b,c,c,c,a,a,a,b,b,b,c,c,c] = 27 elements
-    #  group mult: 9
-    #  <> s/m
-    # [k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m,k,l,m] = 27 elements
-    # The array elements are then combined by index (i.e. vertically) to get all combinations.
-    #
-    # Note: Of course I could split this method up into smaller
-    #       ones, but I guess I am a bit sentimental.
-    #
-    def expand_combinations_from possible_combinations
-      # Generate the first multiplicator "with which" (well, not quite) to multiply the smallest amount of combinations.
-      #
-      # TODO How does this work if an element has size 0? Since below we account for size 0.
-      #      Should we even continue if an element has size 0?
-      #      This means one of the tokens cannot be allocated.
-      #
-      single_mult = possible_combinations.inject(1) { |total, combinations| total * combinations.size }
-
-      # Initialize a group multiplicator.
-      #
-      group_mult = 1
-
-      possible_combinations.reject!(&:empty?)
-
-      # The expanding part to line up the combinations
-      # for later combination in allocations.
-      #
-      possible_combinations.collect! do |combinations|
-
-        # Get the size of the combinations of the first token.
-        #
-        combinations_size = combinations.size
-
-        # Special case: If there is no combination for one of the tokens.
-        #               In that case, we just use the same single mult for
-        #               the next iteration.
-        #               If there are combinations, we divide the single mult
-        #               by the number of combinations.
-        #
-        single_mult /= combinations_size unless combinations_size.zero?
-        
-        # Expand each combination by the single mult:
-        # [a,b,c]
-        # [a,a,a,  b,b,b,  c,c,c]
-        # Then, expand the result by the group mult:
-        # [a,a,a,b,b,b,c,c,c,  a,a,a,b,b,b,c,c,c,  a,a,a,b,b,b,c,c,c]
-        #
-        combinations = combinations.inject([]) do |total, combination|
-          total + [combination]*single_mult
-        end * group_mult
-        
-        # Multiply the group mult by the combinations size,
-        # since the next combinations' single mult is smaller
-        # and we need to adjust for that.
-        #
-        group_mult = group_mult * combinations_size
-        
-        # Return the combinations.
-        #
-        combinations
-      end
-    end
-    
-  end
-  
-end

data/lib/picky/results/base.rb

@@ -1,102 +0,0 @@
-module Results # :nodoc:all
-
-  # This is the internal results object. Usually, to_marshal, or to_json
-  # is called on it to get a string for the answer.
-  #
-  class Base
-
-    # Duration is set externally by the query.
-    #
-    attr_writer :duration
-    attr_reader :allocations, :offset
-
-    # Takes instances of Query::Allocations as param.
-    #
-    def initialize offset = 0, allocations = Query::Allocations.new
-      @offset = offset
-      @allocations = allocations # || Query::Allocations.new
-    end
-    # Create new results and calculate the ids.
-    #
-    def self.from offset, allocations
-      results = new offset, allocations
-      results.prepare!
-      results
-    end
-
-    #
-    #
-    def serialize
-      { allocations: allocations.to_result,
-        offset:      offset,
-        duration:    duration,
-        total:       total }
-    end
-    # The default format is json.
-    #
-    def to_response options = {}
-      to_json options
-    end
-    # Convert to json format.
-    #
-    def to_json options = {}
-      serialize.to_json options
-    end
-
-    # This starts the actual processing.
-    #
-    # Without this, the allocations are not processed,
-    # and no ids are calculated.
-    #
-    def prepare!
-      allocations.process! self.max_results, self.offset
-    end
-
-    # Duration default is 0.
-    #
-    def duration
-      @duration || 0
-    end
-    # The total results. Delegates to the allocations.
-    #
-    # Caches.
-    #
-    def total
-      @total || @total = allocations.total || 0
-    end
-
-    # How many results are returned.
-    #
-    # Set in config using
-    #   Results::Full.max_results = 20
-    #
-    class_inheritable_accessor :max_results
-    def max_results
-      self.class.max_results
-    end
-
-    # Convenience methods.
-    #
-
-    # Delegates to allocations.
-    #
-    def ids amount = 20
-      allocations.ids amount
-    end
-    # Gets an amout of random ids from the allocations.
-    #
-    # Note: Basically delegates to the allocations.
-    #
-    def random_ids amount = 1
-      allocations.random_ids amount
-    end
-
-    # Human readable log.
-    #
-    def to_log query
-      "|#{Time.now.to_s(:db)}|#{'%8f' % duration}|#{'%-50s' % query}|#{'%8d' % total}|#{'%4d' % offset}|#{'%2d' % allocations.size}|"
-    end
-    
-  end
-
-end

data/lib/picky/results/full.rb

@@ -1,13 +0,0 @@
-module Results
-  # Full results are limited to maximally 20 results (by default).
-  #
-  class Full < Base
-    
-    self.max_results = 20
-    
-    def to_log *args
-      ?> + super
-    end
-    
-  end
-end

data/lib/picky/results/live.rb

@@ -1,13 +0,0 @@
-module Results
-  # Live results are not returning any results.
-  #
-  class Live < Base
-    
-    self.max_results = 0
-    
-    def to_log *args
-      ?. + super
-    end
-    
-  end
-end

data/lib/picky/tokenizers/base.rb

@@ -1,161 +0,0 @@
-module Tokenizers # :nodoc:all
-  
-  # Defines tokenizing processes used both in indexing and querying.
-  #
-  class Base
-    
-    # TODO Move EMPTY_STRING top level.
-    #
-    EMPTY_STRING = ''.freeze
-    
-    # Stopwords.
-    #
-    def stopwords regexp
-      @remove_stopwords_regexp = regexp
-    end
-    def remove_stopwords text
-      text.gsub! @remove_stopwords_regexp, EMPTY_STRING if @remove_stopwords_regexp
-      text
-    end
-    @@non_single_stopword_regexp = /^\b[\w:]+?\b[\.\*\~]?\s?$/
-    def remove_non_single_stopwords text
-      return text if text.match @@non_single_stopword_regexp
-      remove_stopwords text
-    end
-    
-    # Illegals.
-    #
-    # TODO Should there be a legal?
-    #
-    def removes_characters regexp
-      @removes_characters_regexp = regexp
-    end
-    def remove_illegals text
-      text.gsub! @removes_characters_regexp, EMPTY_STRING if @removes_characters_regexp
-      text
-    end
-    
-    # Splitting.
-    #
-    def splits_text_on regexp
-      @splits_text_on_regexp = regexp
-    end
-    def split text
-      text.split @splits_text_on_regexp
-    end
-    
-    # Normalizing.
-    #
-    def normalizes_words regexp_replaces
-      @normalizes_words_regexp_replaces = regexp_replaces
-    end
-    def normalize_with_patterns text
-      return text unless @normalizes_words_regexp_replaces
-      
-      @normalizes_words_regexp_replaces.each do |regex, replace|
-        # This should be sufficient
-        #
-        text.gsub!(regex, replace) and break
-      end
-      remove_after_normalizing_illegals text
-      text
-    end
-    
-    # Illegal after normalizing.
-    #
-    def removes_characters_after_splitting regexp
-      @removes_characters_after_splitting_regexp = regexp
-    end
-    def remove_after_normalizing_illegals text
-      text.gsub! @removes_characters_after_splitting_regexp, EMPTY_STRING if @removes_characters_after_splitting_regexp
-    end
-    
-    # Substitute Characters with this substituter.
-    #
-    # Default is European Character substitution.
-    #
-    def substitutes_characters_with substituter = CharacterSubstituters::WestEuropean.new
-      # TODO Raise if it doesn't quack substitute?
-      @substituter = substituter
-    end
-    def substitute_characters text
-      substituter?? substituter.substitute(text) : text 
-    end
-    
-    # Reject tokens after tokenizing based on the given criteria.
-    #
-    # Note: Currently only for indexing. TODO Redesign and write for both!
-    #
-    def reject_token_if &condition
-      @reject_condition = condition
-    end
-    def reject tokens
-      tokens.reject! &@reject_condition
-    end
-    
-    
-    # Returns a number of tokens, generated from the given text.
-    #
-    # Note:
-    #  * preprocess, pretokenize are hooks
-    #
-    def tokenize text
-      text   = preprocess text  # processing the text
-      return empty_tokens if text.blank?
-      words  = pretokenize text # splitting and preparations for tokenizing
-      return empty_tokens if words.empty?
-      tokens = tokens_for words # creating tokens / strings
-               process tokens   # processing tokens / strings
-    end
-    
-    attr_reader :substituter
-    alias substituter? substituter
-    
-    def initialize options = {}
-      removes_characters options[:removes_characters]                                 if options[:removes_characters]
-      contracts_expressions *options[:contracts_expressions]                          if options[:contracts_expressions]
-      stopwords options[:stopwords]                                                   if options[:stopwords]
-      normalizes_words options[:normalizes_words]                                     if options[:normalizes_words]
-      removes_characters_after_splitting options[:removes_characters_after_splitting] if options[:removes_characters_after_splitting]
-      substitutes_characters_with options[:substitutes_characters_with]               if options[:substitutes_characters_with]
-      
-      # Defaults.
-      #
-      splits_text_on options[:splits_text_on] || /\s/
-      reject_token_if &(options[:reject_token_if] || :blank?)
-    end
-    
-    # Hooks.
-    #
-    
-    # Preprocessing.
-    #
-    def preprocess text; end
-    # Pretokenizing.
-    #
-    def pretokenize text; end
-    # Postprocessing.
-    #
-    def process tokens
-      reject tokens # Reject any tokens that don't meet criteria
-      tokens
-    end
-    
-    # Converts words into real tokens.
-    #
-    def tokens_for words
-      ::Query::Tokens.new words.collect! { |word| token_for word }
-    end
-    # Turns non-blank text into symbols.
-    #
-    def symbolize text
-      text.blank? ? nil : text.to_sym
-    end
-    # Returns a tokens object.
-    #
-    def empty_tokens
-      ::Query::Tokens.new
-    end
-    
-  end
-end

data/lib/picky/tokenizers/index.rb

@@ -1,58 +0,0 @@
-module Tokenizers
-  
-  # The base indexing tokenizer.
-  #
-  # Override in indexing subclasses and define in configuration.
-  #
-  class Index < Base
-    
-    def self.default= new_default
-      @default = new_default
-    end
-    def self.default
-      @default ||= new
-    end
-    
-    # Default indexing preprocessing hook.
-    #
-    # Does:
-    # 1. Character substitution.
-    # 2. Downcasing.
-    # 3. Remove illegal expressions.
-    # 4. Remove non-single stopwords. (Stopwords that occur with other words)
-    #
-    def preprocess text
-      text = substitute_characters text
-      text.downcase!
-      remove_illegals text
-      # we do not remove single stopwords for an entirely different
-      # reason than in the query tokenizer.
-      # An indexed thing with just name "UND" (a possible stopword) should not lose its name.
-      #
-      remove_non_single_stopwords text
-      text
-    end
-    
-    # Default indexing pretokenizing hook.
-    #
-    # Does:
-    # 1. Split the text into words.
-    # 2. Normalize each word.
-    #
-    def pretokenize text
-      words = split text
-      words.collect! do |word|
-        normalize_with_patterns word
-        word
-      end
-    end
-    
-    # Does not actually return a token, but a
-    # symbol "token".
-    #
-    def token_for text
-      symbolize text
-    end
-    
-  end
-end

data/lib/picky/tokenizers/query.rb

@@ -1,74 +0,0 @@
-# encoding: utf-8
-#
-module Tokenizers
-  
-  # There are a few class methods that you can use to configure how a query works.
-  #
-  # removes_characters regexp
-  # illegal_after_normalizing regexp
-  # stopwords regexp
-  # contracts_expressions regexp, to_string
-  # splits_text_on regexp
-  # normalizes_words [[/regexp1/, 'replacement1'], [/regexp2/, 'replacement2']]
-  #
-  class Query < Base
-    
-    def self.default= new_default
-      @default = new_default
-    end
-    def self.default
-      @default ||= new
-    end
-    
-    attr_reader :maximum_tokens
-    
-    def initialize options = {}
-      super options
-      @maximum_tokens = options[:maximum_tokens] || 5
-    end
-    
-    def preprocess text
-      remove_illegals text             # Remove illegal characters
-      remove_non_single_stopwords text # remove stop words
-      text
-    end
-    
-    # Split the text and put some back together.
-    #
-    # TODO Make the same as in indexing?
-    #
-    def pretokenize text
-      split text
-    end
-    
-    # Let each token process itself.
-    # Reject, limit, and partialize tokens.
-    #
-    def process tokens
-      tokens.tokenize_with self
-      tokens.reject              # Reject any tokens that don't meet criteria
-      tokens.cap maximum_tokens  # Cut off superfluous tokens
-      tokens.partialize_last     # Set certain tokens as partial
-      tokens
-    end
-    
-    # Called by the token.
-    #
-    # TODO Perhaps move to Normalizer?
-    #
-    def normalize text
-      text = substitute_characters text # Substitute special characters
-      text.downcase!                    # Downcase all text
-      normalize_with_patterns text      # normalize
-      text.to_sym                       # symbolize
-    end
-    
-    # Returns a token for a word.
-    # The basic query tokenizer uses new tokens.
-    #
-    def token_for word
-      ::Query::Token.processed word
-    end
-    
-  end
-end

data/spec/lib/cacher/partial/default_spec.rb

@@ -1,15 +0,0 @@
-require 'spec_helper'
-
-describe Cacher::Partial::Default do
-  
-  it "should be a subtoken" do
-    Cacher::Partial::Default.should be_kind_of(Cacher::Partial::Substring)
-  end
-  it "should be a the right down to" do
-    Cacher::Partial::Default.from.should == -3
-  end
-  it "should be a the right starting at" do
-    Cacher::Partial::Default.to.should == -1
-  end
-  
-end

data/spec/lib/cacher/partial/none_spec.rb

@@ -1,17 +0,0 @@
-require 'spec_helper'
-
-describe Cacher::Partial::None do
-  
-  it "has the right superclass" do
-    Cacher::Partial::None.should < Cacher::Partial::Strategy
-  end
-  it "returns an empty index" do
-    Cacher::Partial::None.new.generate_from(:unimportant).should == {}
-  end
-  describe 'use_exact_for_partial?' do
-    it 'returns true' do
-      Cacher::Partial::None.new.use_exact_for_partial?.should == true
-    end
-  end
-  
-end

data/spec/lib/cacher/weights_generator_spec.rb

@@ -1,21 +0,0 @@
-require 'spec_helper'
-
-describe Cacher::WeightsGenerator do
-
-  context 'integration' do
-    it 'should generate the correct values' do
-      generator = Cacher::WeightsGenerator.new :a => Array.new(0),
-                                               :b => Array.new(1),
-                                               :c => Array.new(10),
-                                               :d => Array.new(100),
-                                               :e => Array.new(1000)
-
-      result = generator.generate
-
-      result[:c].should be_close 2.3, 0.011
-      result[:d].should be_close 4.6, 0.011
-      result[:e].should be_close 6.9, 0.011
-    end
-  end
-
-end