picky 2.5.2 → 2.6.0

data/lib/picky/internals/query/tokens.rb

@@ -1,109 +0,0 @@
-# encoding: utf-8
-#
-module Internals
-
-  #
-  #
-  module Query
-
-    # This class primarily handles switching through similar token constellations.
-    #
-    class Tokens # :nodoc:all
-
-      # Basically delegates to its internal tokens array.
-      #
-      self.delegate *[Enumerable.instance_methods, :slice!, :[], :uniq!, :last, :reject!, :length, :size, :empty?, :each, :exit, { :to => :@tokens }].flatten
-
-      # Create a new Tokens object with the array of tokens passed in.
-      #
-      def initialize tokens = []
-        @tokens = tokens
-      end
-
-      # Creates a new Tokens object from a number of Strings.
-      #
-      # Options:
-      #  * downcase: Whether to downcase the passed strings (default is true)
-      #
-      def self.processed words, downcase = true
-        new words.collect! { |word| Token.processed word, downcase }
-      end
-
-      # Tokenizes each token.
-      #
-      # Note: Passed tokenizer needs to offer #normalize(text).
-      #
-      def tokenize_with tokenizer
-        @tokens.each { |token| token.tokenize_with(tokenizer) }
-      end
-
-      # Generates an array in the form of
-      # [
-      #  [combination],                           # of token 1
-      #  [combination, combination, combination], # of token 2
-      #  [combination, combination]               # of token 3
-      # ]
-      #
-      def possible_combinations_in index
-        @tokens.inject([]) do |combinations, token|
-          possible_combinations = token.possible_combinations_in index
-
-          # TODO Could move the ignore_unassigned_tokens here!
-          #
-          # Note: Optimization for ignoring tokens that allocate to nothing and
-          # can be ignored.
-          # For example in a special search, where "florian" is not
-          # mapped to any category.
-          #
-          possible_combinations ? combinations << possible_combinations : combinations
-        end
-      end
-
-      # Makes the last of the tokens partial.
-      #
-      def partialize_last
-        @tokens.last.partial = true unless empty?
-      end
-
-      # Caps the tokens to the maximum.
-      #
-      def cap maximum
-        @tokens.slice!(maximum..-1) if cap?(maximum)
-      end
-      def cap? maximum
-        @tokens.size > maximum
-      end
-
-      # Rejects blank tokens.
-      #
-      def reject
-        @tokens.reject! &:blank?
-      end
-
-      # Returns a solr query.
-      #
-      def to_solr_query
-        @tokens.map(&:to_solr).join ' '
-      end
-
-      #
-      #
-      def originals
-        @tokens.map(&:original)
-      end
-
-      def == other
-        self.tokens == other.tokens
-      end
-
-      # Just join the token original texts.
-      #
-      def to_s
-        originals.join ' '
-      end
-
-    end
-
-  end
-
-end

data/lib/picky/internals/shared/category.rb

@@ -1,52 +0,0 @@
-module Internals
-  module Shared
-
-    module Category
-
-      def index_name
-        index.name
-      end
-      def category_name
-        name
-      end
-
-      # Path and partial filename of a specific index on this category.
-      #
-      def index_path bundle_name, type
-        "#{index_directory}/#{name}_#{bundle_name}_#{type}"
-      end
-
-      #
-      #
-      def prepared_index_path
-        @prepared_index_path ||= "#{index_directory}/prepared_#{name}_index"
-      end
-      def prepared_index_file &block
-        @prepared_index_file ||= Internals::Index::File::Text.new prepared_index_path
-        @prepared_index_file.open_for_indexing &block
-      end
-
-      # Identifier for internal use.
-      #
-      def identifier
-        @identifier ||= "#{index.name}:#{name}"
-      end
-      def to_s
-        "#{index.name} #{name}"
-      end
-
-      # The index directory for this category.
-      #
-      def index_directory
-        @index_directory ||= "#{PICKY_ROOT}/index/#{PICKY_ENVIRONMENT}/#{index.name}"
-      end
-      # Creates the index directory including all necessary paths above it.
-      #
-      def prepare_index_directory
-        FileUtils.mkdir_p index_directory
-      end
-
-    end
-
-  end
-end

data/lib/picky/internals/tokenizers/base.rb

@@ -1,228 +0,0 @@
-module Internals
-
-  module Tokenizers # :nodoc:all
-
-    # Defines tokenizing processes used both in indexing and querying.
-    #
-    class Base
-
-      # TODO Move EMPTY_STRING top level.
-      #
-      EMPTY_STRING = ''.freeze
-
-      def to_s
-        reject_condition_location = @reject_condition.to_s[/:(\d+) \(lambda\)/, 1]
-        <<-TOKENIZER
-Removes characters:        #{@removes_characters_regexp ? "/#{@removes_characters_regexp.source}/" : '-'}
-Stopwords:                 #{@remove_stopwords_regexp ? "/#{@remove_stopwords_regexp.source}/" : '-'}
-Splits text on:            #{@splits_text_on.respond_to?(:source) ? "/#{@splits_text_on.source}/" : (@splits_text_on ? @splits_text_on : '-')}
-Removes chars after split: #{@removes_characters_after_splitting_regexp ? "/#{@removes_characters_after_splitting_regexp.source}/" : '-'}
-Normalizes words:          #{@normalizes_words_regexp_replaces ? @normalizes_words_regexp_replaces : '-'}
-Rejects tokens?            #{reject_condition_location ? "Yes, see line #{reject_condition_location} in app/application.rb" : '-'}
-Substitutes chars?         #{@substituter ? "Yes, using #{@substituter}." : '-' }
-Case sensitive?            #{@case_sensitive ? "Yes." : "-"}
-        TOKENIZER
-      end
-
-      # Stopwords.
-      #
-      # We only allow regexps (even if string would be okay
-      # too for gsub! - it's too hard to understand)
-      #
-      def stopwords regexp
-        check_argument_in __method__, Regexp, regexp
-        @remove_stopwords_regexp = regexp
-      end
-      def remove_stopwords text
-        text.gsub! @remove_stopwords_regexp, EMPTY_STRING if @remove_stopwords_regexp
-        text
-      end
-      @@non_single_stopword_regexp = /^\b[\w:]+?\b[\.\*\~]?\s?$/
-      def remove_non_single_stopwords text
-        return text if text.match @@non_single_stopword_regexp
-        remove_stopwords text
-      end
-
-      # Illegals.
-      #
-      # We only allow regexps (even if string would be okay
-      # too for gsub! - it's too hard to understand)
-      #
-      def removes_characters regexp
-        check_argument_in __method__, Regexp, regexp
-        @removes_characters_regexp = regexp
-      end
-      def remove_illegals text
-        text.gsub! @removes_characters_regexp, EMPTY_STRING if @removes_characters_regexp
-        text
-      end
-
-      # Splitting.
-      #
-      # We allow Strings and Regexps.
-      # Note: We do not test against to_str since symbols do not work with String#split.
-      #
-      def splits_text_on regexp_or_string
-        raise ArgumentError.new "#{__method__} takes a Regexp or String as argument, not a #{regexp_or_string.class}." unless Regexp === regexp_or_string || String === regexp_or_string
-        @splits_text_on = regexp_or_string
-      end
-      def split text
-        text.split @splits_text_on
-      end
-
-      # Normalizing.
-      #
-      # We only allow arrays.
-      #
-      def normalizes_words regexp_replaces
-        raise ArgumentError.new "#{__method__} takes an Array of replaces as argument, not a #{regexp_replaces.class}." unless regexp_replaces.respond_to?(:to_ary)
-        @normalizes_words_regexp_replaces = regexp_replaces
-      end
-      def normalize_with_patterns text
-        return text unless @normalizes_words_regexp_replaces
-
-        @normalizes_words_regexp_replaces.each do |regex, replace|
-          # This should be sufficient
-          #
-          text.gsub!(regex, replace) and break
-        end
-        remove_after_normalizing_illegals text
-        text
-      end
-
-      # Illegal after normalizing.
-      #
-      # We only allow regexps (even if string would be okay
-      # too for gsub! - it's too hard to understand)
-      #
-      def removes_characters_after_splitting regexp
-        check_argument_in __method__, Regexp, regexp
-        @removes_characters_after_splitting_regexp = regexp
-      end
-      def remove_after_normalizing_illegals text
-        text.gsub! @removes_characters_after_splitting_regexp, EMPTY_STRING if @removes_characters_after_splitting_regexp
-      end
-
-      # Substitute Characters with this substituter.
-      #
-      # Default is European Character substitution.
-      #
-      def substitutes_characters_with substituter = CharacterSubstituters::WestEuropean.new
-        raise ArgumentError.new "The substitutes_characters_with option needs a character substituter, which responds to #substitute." unless substituter.respond_to?(:substitute)
-        @substituter = substituter
-      end
-      def substitute_characters text
-        substituter?? substituter.substitute(text) : text
-      end
-
-      # Reject tokens after tokenizing based on the given criteria.
-      #
-      # Note: Currently only for indexing.
-      #
-      def reject_token_if &condition
-        @reject_condition = condition
-      end
-      def reject tokens
-        tokens.reject! &@reject_condition
-      end
-
-      def case_sensitive case_sensitive
-        @case_sensitive = case_sensitive
-      end
-      def downcase?
-        !@case_sensitive
-      end
-
-      # Checks if the right argument type has been given.
-      #
-      def check_argument_in method, type, argument, &condition
-        raise ArgumentError.new "Application##{method} takes a #{type} as argument, not a #{argument.class}." unless type === argument
-      end
-
-
-      # Returns a number of tokens, generated from the given text.
-      #
-      # Note:
-      #  * preprocess, pretokenize are hooks
-      #
-      def tokenize text
-        text   = preprocess text  # processing the text
-        return empty_tokens if text.blank?
-        words  = pretokenize text # splitting and preparations for tokenizing
-        return empty_tokens if words.empty?
-        tokens = tokens_for words # creating tokens / strings
-                 process tokens   # processing tokens / strings
-      end
-
-      attr_reader :substituter
-      alias substituter? substituter
-
-      def initialize options = {}
-        removes_characters options[:removes_characters]                                 if options[:removes_characters]
-        contracts_expressions *options[:contracts_expressions]                          if options[:contracts_expressions]
-        stopwords options[:stopwords]                                                   if options[:stopwords]
-        normalizes_words options[:normalizes_words]                                     if options[:normalizes_words]
-        removes_characters_after_splitting options[:removes_characters_after_splitting] if options[:removes_characters_after_splitting]
-        substitutes_characters_with options[:substitutes_characters_with]               if options[:substitutes_characters_with]
-        case_sensitive options[:case_sensitive]                                         unless options[:case_sensitive].nil?
-
-        # Defaults.
-        #
-        splits_text_on options[:splits_text_on] || /\s/
-        reject_token_if &(options[:reject_token_if] || options[:rejects_token_if] || :blank?) # TODO Decide on using an s or not.
-      end
-
-      # Default preprocessing hook.
-      #
-      # Does:
-      # 1. Character substitution.
-      # 2. Remove illegal expressions.
-      # 3. Remove non-single stopwords. (Stopwords that occur with other words)
-      #
-      def preprocess text
-        text = substitute_characters text
-        remove_illegals text
-        # We do not remove single stopwords e.g. in the indexer for
-        # an entirely different reason than in the query tokenizer.
-        # An indexed thing with just name "UND" (a possible stopword)
-        # should not lose its name.
-        #
-        remove_non_single_stopwords text
-        text
-      end
-      # Pretokenizing.
-      #
-      # Does:
-      # 1. Split the text into words.
-      # 2. Normalize each word.
-      #
-      def pretokenize text
-        words = split text
-        words.collect! do |word|
-          normalize_with_patterns word
-          word
-        end
-      end
-      # Basic postprocessing (overridden in both query/index tokenizers).
-      #
-      def process tokens
-        reject tokens # Reject any tokens that don't meet criteria
-        tokens
-      end
-
-      # # Converts words into real tokens.
-      # #
-      # def tokens_for words
-      #   Internals::Query::Tokens.new words.collect! { |word| token_for word }
-      # end
-      # Turns non-blank text into symbols.
-      #
-      def symbolize text
-        text.blank? ? nil : text.to_sym
-      end
-
-    end
-
-  end
-
-end

data/lib/picky/internals/tokenizers/index.rb

@@ -1,34 +0,0 @@
-module Internals
-
-  module Tokenizers
-
-    # The base indexing tokenizer.
-    #
-    # Override in indexing subclasses and define in configuration.
-    #
-    class Index < Base
-
-      def self.default= new_default
-        @default = new_default
-      end
-      def self.default
-        @default ||= new
-      end
-
-      # Does not actually return a token, but a
-      # symbol "token".
-      #
-      def tokens_for words
-        words.collect! { |word| word.downcase! if downcase?; word.to_sym }
-      end
-      # Returns empty tokens.
-      #
-      def empty_tokens
-        []
-      end
-
-    end
-
-  end
-
-end

data/lib/picky/internals/tokenizers/location.rb

@@ -1,54 +0,0 @@
-module Internals
-
-  module Tokenizers
-
-
-    class Location < Base
-
-      attr_reader :calculation
-
-      def initialize options = {}
-        super options
-
-        grid      = options[:grid]
-        precision = options[:precision] || 1
-
-        @calculation = Internals::Calculations::Location.new grid, precision
-
-        @minimum = 1.0 / 0
-
-        @locations = []
-      end
-
-      # TODO Work on this!
-      #
-      def tokenize text
-
-        # Gather min/max.
-        #
-        source.harvest category do |indexed_id, location|
-          location = location.to_f
-          minimum = location if location < minimum
-          locations << [indexed_id, location]
-        end
-
-        calculation.minimum = minimum
-
-        # Recalculate locations.
-        #
-        locations.each do |indexed_id, location|
-          calculation.recalculated_range(location).each do |new_location|
-            yield indexed_id, new_location.to_s
-          end
-        end
-
-        # TODO Move to the right place.
-        #
-        category.exact[:location_minimum] = minimum
-      end
-
-    end
-
-  end
-
-end

data/lib/picky/internals/tokenizers/query.rb

@@ -1,59 +0,0 @@
-# encoding: utf-8
-#
-module Internals
-
-  module Tokenizers
-
-    # There are a few class methods that you can use to configure how a query works.
-    #
-    # removes_characters regexp
-    # illegal_after_normalizing regexp
-    # stopwords regexp
-    # contracts_expressions regexp, to_string
-    # splits_text_on regexp
-    # normalizes_words [[/regexp1/, 'replacement1'], [/regexp2/, 'replacement2']]
-    #
-    class Query < Base
-
-      def self.default= new_default
-        @default = new_default
-      end
-      def self.default
-        @default ||= new
-      end
-
-      attr_reader :maximum_tokens
-
-      def initialize options = {}
-        super options
-        @maximum_tokens = options[:maximum_tokens] || 5
-      end
-
-      # Let each token process itself.
-      # Reject, limit, and partialize tokens.
-      #
-      # In querying we work with real tokens (in indexing it's just symbols).
-      #
-      def process tokens
-        tokens.reject                # Reject any tokens that don't meet criteria.
-        tokens.cap maximum_tokens    # Cut off superfluous tokens.
-        tokens.partialize_last       # Set certain tokens as partial.
-        tokens
-      end
-
-      # Converts words into real tokens.
-      #
-      def tokens_for words
-        Internals::Query::Tokens.processed words, downcase?
-      end
-      # Returns a tokens object.
-      #
-      def empty_tokens
-        Internals::Query::Tokens.new
-      end
-
-    end
-
-  end
-
-end

data/lib/picky/internals.rb

@@ -1,2 +0,0 @@
-module Internals # :nodoc:all
-end

data/spec/lib/aliases_spec.rb

@@ -1,9 +0,0 @@
-# encoding: utf-8
-#
-require 'spec_helper'
-
-describe 'aliases' do
-  it 'exists an Indexes class that is an instance of API::Indexes' do
-    Indexes.class.should == IndexBundle
-  end
-end

data/spec/lib/index_bundle_spec.rb

@@ -1,69 +0,0 @@
-# encoding: utf-8
-#
-require 'spec_helper'
-
-describe IndexBundle do
-  
-  let(:some_index) { stub :index, :name => :some_index, :internal_indexed => :indexed_index, :internal_indexing => :indexing_index }
-  let(:indexes) { described_class.new }
-  let(:indexed) { stub :indexed, :register => nil }
-  let(:indexing) { stub :indexing, :register => nil }
-  
-  before(:each) do
-    indexes.stub! :indexing => indexing
-    indexes.stub! :indexed  => indexed
-  end
-  
-  def self.it_delegates method, receiver
-    it "delegates #{method} to #{receiver}" do
-      indexes.send(receiver).should_receive(method.to_sym).once
-      
-      indexes.send method
-    end
-  end
-  
-  describe 'delegation' do
-    it_delegates :reload, :indexed
-    it_delegates :load_from_cache, :indexed
-    
-    it_delegates :check_caches, :indexing
-    it_delegates :find, :indexing
-    it_delegates :index, :indexing
-    it_delegates :index_for_tests, :indexing
-  end
-  
-  describe '[]' do
-    before(:each) do
-      indexes.register some_index
-    end
-    it 'takes strings' do
-      indexes['some_index'].should == some_index
-    end
-    it 'takes symbols' do
-      indexes[:some_index].should == some_index
-    end
-  end
-  
-  describe 'register' do
-    it 'registers with the indexes' do
-      indexes.register some_index
-      
-      indexes.indexes.should == [some_index]
-    end
-    it 'registers with the index map' do
-      indexes.register some_index
-      
-      indexes[some_index.name].should == some_index
-    end
-    it 'registers with the indexing' do
-      indexing.should_receive(:register).once.with :indexing_index
-      
-      indexes.register some_index
-    end
-    it 'registers with the indexed' do
-      indexed.should_receive(:register).once.with :indexed_index
-      
-      indexes.register some_index
-    end
-  end
-end