picky 3.0.1 → 3.1.0

data/lib/picky/tokenizers/base.rb

@@ -1,231 +0,0 @@
-# encoding: utf-8
-#
-module Picky
-
-  module Tokenizers # :nodoc:all
-
-    # Defines tokenizing processes used both in indexing and querying.
-    #
-    class Base
-
-      # TODO Move EMPTY_STRING top level.
-      #
-      EMPTY_STRING = ''.freeze
-
-      def to_s
-        reject_condition_location = @reject_condition.to_s[/:(\d+) \(lambda\)/, 1]
-        <<-TOKENIZER
-Removes characters:        #{@removes_characters_regexp ? "/#{@removes_characters_regexp.source}/" : '-'}
-Stopwords:                 #{@remove_stopwords_regexp ? "/#{@remove_stopwords_regexp.source}/" : '-'}
-Splits text on:            #{@splits_text_on.respond_to?(:source) ? "/#{@splits_text_on.source}/" : (@splits_text_on ? @splits_text_on : '-')}
-Removes chars after split: #{@removes_characters_after_splitting_regexp ? "/#{@removes_characters_after_splitting_regexp.source}/" : '-'}
-Normalizes words:          #{@normalizes_words_regexp_replaces ? @normalizes_words_regexp_replaces : '-'}
-Rejects tokens?            #{reject_condition_location ? "Yes, see line #{reject_condition_location} in app/application.rb" : '-'}
-Substitutes chars?         #{@substituter ? "Yes, using #{@substituter}." : '-' }
-Case sensitive?            #{@case_sensitive ? "Yes." : "-"}
-        TOKENIZER
-      end
-
-      # Stopwords.
-      #
-      # We only allow regexps (even if string would be okay
-      # too for gsub! - it's too hard to understand)
-      #
-      def stopwords regexp
-        check_argument_in __method__, Regexp, regexp
-        @remove_stopwords_regexp = regexp
-      end
-      def remove_stopwords text
-        text.gsub! @remove_stopwords_regexp, EMPTY_STRING if @remove_stopwords_regexp
-        text
-      end
-      @@non_single_stopword_regexp = /^\b[\w:]+?\b[\.\*\~]?\s?$/
-      def remove_non_single_stopwords text
-        return text if text.match @@non_single_stopword_regexp
-        remove_stopwords text
-      end
-
-      # Illegals.
-      #
-      # We only allow regexps (even if string would be okay
-      # too for gsub! - it's too hard to understand)
-      #
-      def removes_characters regexp
-        check_argument_in __method__, Regexp, regexp
-        @removes_characters_regexp = regexp
-      end
-      def remove_illegals text
-        text.gsub! @removes_characters_regexp, EMPTY_STRING if @removes_characters_regexp
-        text
-      end
-
-      # Splitting.
-      #
-      # We allow Strings and Regexps.
-      # Note: We do not test against to_str since symbols do not work with String#split.
-      #
-      def splits_text_on regexp_or_string
-        raise ArgumentError.new "#{__method__} takes a Regexp or String as argument, not a #{regexp_or_string.class}." unless Regexp === regexp_or_string || String === regexp_or_string
-        @splits_text_on = regexp_or_string
-      end
-      def split text
-        text.split @splits_text_on
-      end
-
-      # Normalizing.
-      #
-      # We only allow arrays.
-      #
-      def normalizes_words regexp_replaces
-        raise ArgumentError.new "#{__method__} takes an Array of replaces as argument, not a #{regexp_replaces.class}." unless regexp_replaces.respond_to?(:to_ary)
-        @normalizes_words_regexp_replaces = regexp_replaces
-      end
-      def normalize_with_patterns text
-        return text unless @normalizes_words_regexp_replaces
-
-        @normalizes_words_regexp_replaces.each do |regex, replace|
-          # This should be sufficient
-          #
-          text.gsub!(regex, replace) and break
-        end
-
-        remove_after_normalizing_illegals text
-        text
-      end
-
-      # Illegal after normalizing.
-      #
-      # We only allow regexps (even if string would be okay
-      # too for gsub! - it's too hard to understand)
-      #
-      def removes_characters_after_splitting regexp
-        check_argument_in __method__, Regexp, regexp
-        @removes_characters_after_splitting_regexp = regexp
-      end
-      def remove_after_normalizing_illegals text
-        text.gsub! @removes_characters_after_splitting_regexp, EMPTY_STRING if @removes_characters_after_splitting_regexp
-      end
-
-      # Substitute Characters with this substituter.
-      #
-      # Default is European Character substitution.
-      #
-      def substitutes_characters_with substituter = CharacterSubstituters::WestEuropean.new
-        raise ArgumentError.new "The substitutes_characters_with option needs a character substituter, which responds to #substitute." unless substituter.respond_to?(:substitute)
-        @substituter = substituter
-      end
-      def substitute_characters text
-        substituter?? substituter.substitute(text) : text
-      end
-
-      # Reject tokens after tokenizing based on the given criteria.
-      #
-      # Note: Currently only for indexing. TODO Put into searching as well.
-      #
-      def rejects_token_if &condition
-        @reject_condition = condition
-      end
-      def reject tokens
-        tokens.reject! &@reject_condition
-      end
-
-      def case_sensitive case_sensitive
-        @case_sensitive = case_sensitive
-      end
-      def downcase?
-        !@case_sensitive
-      end
-
-      # Checks if the right argument type has been given.
-      #
-      def check_argument_in method, type, argument, &condition
-        raise ArgumentError.new "Application##{method} takes a #{type} as argument, not a #{argument.class}." unless type === argument
-      end
-
-
-      # Returns a number of tokens, generated from the given text.
-      #
-      # Note:
-      #  * preprocess, pretokenize are hooks
-      #
-      def tokenize text
-        text   = preprocess text  # processing the text
-        return empty_tokens if text.blank?
-        words  = pretokenize text # splitting and preparations for tokenizing
-        return empty_tokens if words.empty?
-        tokens = tokens_for words # creating tokens / strings
-                 process tokens   # processing tokens / strings
-      end
-
-      attr_reader :substituter
-      alias substituter? substituter
-
-      def initialize options = {}
-        removes_characters options[:removes_characters]                                 if options[:removes_characters]
-        contracts_expressions *options[:contracts_expressions]                          if options[:contracts_expressions]
-        stopwords options[:stopwords]                                                   if options[:stopwords]
-        normalizes_words options[:normalizes_words]                                     if options[:normalizes_words]
-        removes_characters_after_splitting options[:removes_characters_after_splitting] if options[:removes_characters_after_splitting]
-        substitutes_characters_with options[:substitutes_characters_with]               if options[:substitutes_characters_with]
-        case_sensitive options[:case_sensitive]                                         unless options[:case_sensitive].nil?
-
-        # Defaults.
-        #
-        splits_text_on options[:splits_text_on] || /\s/
-        rejects_token_if &(options[:rejects_token_if] || :blank?)
-      end
-
-      # Default preprocessing hook.
-      #
-      # Does:
-      # 1. Character substitution.
-      # 2. Remove illegal expressions.
-      # 3. Remove non-single stopwords. (Stopwords that occur with other words)
-      #
-      def preprocess text
-        text = substitute_characters text
-        remove_illegals text
-        # We do not remove single stopwords e.g. in the indexer for
-        # an entirely different reason than in the query tokenizer.
-        # An indexed thing with just name "UND" (a possible stopword)
-        # should not lose its name.
-        #
-        remove_non_single_stopwords text
-        text
-      end
-      # Pretokenizing.
-      #
-      # Does:
-      # 1. Split the text into words.
-      # 2. Normalize each word.
-      #
-      def pretokenize text
-        words = split text
-        words.collect! do |word|
-          normalize_with_patterns word
-          word
-        end
-      end
-      # Basic postprocessing (overridden in both query/index tokenizers).
-      #
-      def process tokens
-        reject tokens # Reject any tokens that don't meet criteria
-        tokens
-      end
-
-      # # Converts words into real tokens.
-      # #
-      # def tokens_for words
-      #   Query::Tokens.new words.collect! { |word| token_for word }
-      # end
-      # Turns non-blank text into symbols.
-      #
-      def symbolize text
-        text.blank? ? nil : text.to_sym
-      end
-
-    end
-
-  end
-
-end

data/lib/picky/tokenizers/index.rb

@@ -1,34 +0,0 @@
-module Picky
-
-  module Tokenizers
-
-    # The base indexing tokenizer.
-    #
-    # Override in indexing subclasses and define in configuration.
-    #
-    class Index < Base
-
-      def self.default= new_default
-        @default = new_default
-      end
-      def self.default
-        @default ||= new
-      end
-
-      # Does not actually return a token, but a
-      # symbol "token".
-      #
-      def tokens_for words
-        words.collect! { |word| word.downcase! if downcase?; word.to_sym }
-      end
-      # Returns empty tokens.
-      #
-      def empty_tokens
-        []
-      end
-
-    end
-
-  end
-
-end

data/lib/picky/tokenizers/query.rb

@@ -1,61 +0,0 @@
-# encoding: utf-8
-#
-module Picky
-
-  module Tokenizers
-
-    # There are a few class methods that you can use to configure how a query works.
-    #
-    # removes_characters regexp
-    # illegal_after_normalizing regexp
-    # stopwords regexp
-    # contracts_expressions regexp, to_string
-    # splits_text_on regexp
-    # normalizes_words [[/regexp1/, 'replacement1'], [/regexp2/, 'replacement2']]
-    #
-    class Query < Base
-
-      attr_reader :qualifiers
-
-      def self.default= new_default
-        @default = new_default
-      end
-      def self.default
-        @default ||= new
-      end
-
-      attr_reader :maximum_tokens
-
-      def initialize options = {}
-        super options
-        @maximum_tokens = options[:maximum_tokens] || 5
-      end
-
-      # Let each token process itself.
-      # Reject, limit, and partialize tokens.
-      #
-      # In querying we work with real tokens (in indexing it's just symbols).
-      #
-      def process tokens
-        tokens.reject                # Reject any tokens that don't meet criteria.
-        tokens.cap maximum_tokens    # Cut off superfluous tokens.
-        tokens.partialize_last       # Set certain tokens as partial.
-        tokens
-      end
-
-      # Converts words into real tokens.
-      #
-      def tokens_for words
-        Picky::Query::Tokens.processed words, downcase?
-      end
-      # Returns a tokens object.
-      #
-      def empty_tokens
-        Picky::Query::Tokens.new
-      end
-
-    end
-
-  end
-
-end

data/spec/lib/backend/files_spec.rb

@@ -1,189 +0,0 @@
-require 'spec_helper'
-
-describe Picky::Backend::Files do
-
-  before(:each) do
-    index         = Picky::Indexes::Memory.new :some_index
-    category      = Picky::Category.new :some_category, index
-    bundle        = Picky::Indexing::Bundle::Base.new :some_bundle, category, nil, nil, nil
-    
-    @files         = described_class.new bundle
-    
-    @index         = @files.inverted
-    @weights       = @files.weights
-    @similarity    = @files.similarity
-    @configuration = @files.configuration
-  end
-  
-  describe "dump indexes" do
-    before(:each) do
-      @files.stub! :timed_exclaim
-    end
-    describe "dump_index" do
-      it "uses the right file" do
-        @index.should_receive(:dump).once.with :some_hash
-        
-        @files.dump_inverted :some_hash
-      end
-    end
-    describe "dump_weights" do
-      it "uses the right file" do
-        @weights.should_receive(:dump).once.with :some_hash
-        
-        @files.dump_weights :some_hash
-      end
-    end
-    describe "dump_similarity" do
-      it "uses the right file" do
-        @similarity.should_receive(:dump).once.with :some_hash
-        
-        @files.dump_similarity :some_hash
-      end
-    end
-    describe "dump_configuration" do
-      it "uses the right file" do
-        @configuration.should_receive(:dump).once.with :some_hash
-        
-        @files.dump_configuration :some_hash
-      end
-    end
-  end
-  
-  describe "loading indexes" do
-    before(:each) do
-      @files.stub! :timed_exclaim
-    end
-    describe "load_index" do
-      it "uses the right file" do
-        Yajl::Parser.stub! :parse
-        
-        File.should_receive(:open).once.with 'spec/test_directory/index/test/some_index/some_category_some_bundle_inverted.json', 'r'
-        
-        @files.load_inverted
-      end
-    end
-    describe "load_weights" do
-      it "uses the right file" do
-        Yajl::Parser.stub! :parse
-        
-        File.should_receive(:open).once.with 'spec/test_directory/index/test/some_index/some_category_some_bundle_weights.json', 'r'
-        
-        @files.load_weights
-      end
-    end
-    describe "load_similarity" do
-      it "uses the right file" do
-        Marshal.stub! :load
-        
-        File.should_receive(:open).once.with 'spec/test_directory/index/test/some_index/some_category_some_bundle_similarity.dump', 'r:binary'
-        
-        @files.load_similarity
-      end
-    end
-    describe "load_configuration" do
-      it "uses the right file" do
-        Yajl::Parser.stub! :parse
-        
-        File.should_receive(:open).once.with 'spec/test_directory/index/test/some_index/some_category_some_bundle_configuration.json', 'r'
-        
-        @files.load_configuration
-      end
-    end
-  end
-  
-  describe "dump indexes" do
-    describe "index_cache_ok?" do
-      it 'uses the right method' do
-        @index.should_receive(:cache_ok?).once.with
-        
-        @files.inverted_cache_ok?
-      end
-    end
-    describe "weights_cache_ok?" do
-      it 'uses the right method' do
-        @weights.should_receive(:cache_ok?).once.with
-        
-        @files.weights_cache_ok?
-      end
-    end
-    describe "similarity_cache_ok?" do
-      it 'uses the right method' do
-        @similarity.should_receive(:cache_ok?).once.with
-        
-        @files.similarity_cache_ok?
-      end
-    end
-  end
-  
-  describe 'dump indexes' do
-    describe 'inverted_cache_small?' do
-      it 'uses the right method' do
-        @index.should_receive(:cache_small?).once.with
-        
-        @files.inverted_cache_small?
-      end
-    end
-    describe 'weights_cache_small?' do
-      it 'uses the right method' do
-        @weights.should_receive(:cache_small?).once.with
-        
-        @files.weights_cache_small?
-      end
-    end
-    describe 'similarity_cache_small?' do
-      it 'uses the right method' do
-        @similarity.should_receive(:cache_small?).once.with
-        
-        @files.similarity_cache_small?
-      end
-    end
-  end
-  
-  describe 'backup' do
-    it 'should call backup on all' do
-      @index.should_receive(:backup).once.with
-      @weights.should_receive(:backup).once.with
-      @similarity.should_receive(:backup).once.with
-      @configuration.should_receive(:backup).once.with
-      
-      @files.backup
-    end
-  end
-  describe 'restore' do
-    it 'should call delete on all' do
-      @index.should_receive(:restore).once.with
-      @weights.should_receive(:restore).once.with
-      @similarity.should_receive(:restore).once.with
-      @configuration.should_receive(:restore).once.with
-      
-      @files.restore
-    end
-  end
-  describe 'delete' do
-    it 'should call delete on all' do
-      @index.should_receive(:delete).once.with
-      @weights.should_receive(:delete).once.with
-      @similarity.should_receive(:delete).once.with
-      @configuration.should_receive(:delete).once.with
-      
-      @files.delete
-    end
-  end
-  
-  describe 'initialization' do
-    it 'should initialize the name correctly' do
-      @files.bundle.name.should == :some_bundle
-    end
-  end
-  
-  describe 'to_s' do
-    it 'returns the right value' do
-      bundle   = stub :bundle,
-                      :index_path => 'index/path',
-                      :prepared_index_path => 'prepared/index/path'
-      
-      described_class.new(bundle).to_s.should == "Picky::Backend::Files(Picky::Backend::File::JSON(index/path.json), Picky::Backend::File::JSON(index/path.json), Picky::Backend::File::Marshal(index/path.dump), Picky::Backend::File::JSON(index/path.json))"
-    end
-  end
-
-end

data/spec/lib/backend/redis/list_hash_spec.rb

@@ -1,40 +0,0 @@
-require 'spec_helper'
-
-describe Picky::Backend::Redis::ListHash do
-  
-  let(:index) { described_class.new :some_namespace }
-  
-  describe 'member' do
-    it 'raises an error' do
-      expect {
-        index.member :some_sym
-      }.to raise_error("Can't retrieve single value :some_sym from a Redis ListHash. Use Indexes::Redis::StringHash.")
-    end
-  end
-  
-  describe 'collection' do
-    it 'returns whatever comes back from the backend' do
-      backend = stub :backend
-      index.should_receive(:backend).and_return backend
-      
-      backend.stub! :zrange => :some_lrange_result
-      
-      index.collection(:anything).should == :some_lrange_result
-    end
-    it 'calls the right method on the backend' do
-      backend = stub :backend
-      index.should_receive(:backend).and_return backend
-      
-      backend.should_receive(:zrange).once.with "some_namespace:some_sym", 0, -1
-      
-      index.collection :some_sym
-    end
-  end
-  
-  describe 'to_s' do
-    it 'returns the cache path with the default file extension' do
-      index.to_s.should == 'Picky::Backend::Redis::ListHash(some_namespace:*)'
-    end
-  end
-  
-end

data/spec/lib/backend/redis/string_hash_spec.rb

@@ -1,47 +0,0 @@
-require 'spec_helper'
-
-describe Picky::Backend::Redis::StringHash do
-  
-  let(:index) { described_class.new :some_namespace }
-    
-  describe 'dump' do
-    let(:backend) { index.backend }
-    it 'dumps correctly' do
-      backend.should_receive(:hset).once.ordered.with :some_namespace, :a, 1
-      backend.should_receive(:hset).once.ordered.with :some_namespace, :b, 2
-      backend.should_receive(:hset).once.ordered.with :some_namespace, :c, 3
-      
-      index.dump a: 1, b: 2, c: 3
-    end
-  end
-  
-  describe 'collection' do
-    it 'raises' do
-      expect { index.collection :anything }.to raise_error("Can't retrieve collection for :anything from a StringHash. Use Indexes::Redis::ListHash.")
-    end
-  end
-  
-  describe 'member' do
-    before(:each) do
-      @backend = stub :backend
-      index.stub! :backend => @backend
-    end
-    it 'delegates to the backend' do
-      @backend.should_receive(:hget).once.with :some_namespace, :some_symbol
-      
-      index.member :some_symbol
-    end
-    it 'returns whatever it gets from the backend' do
-      @backend.should_receive(:hget).any_number_of_times.and_return :some_result
-      
-      index.member(:anything).should == :some_result
-    end
-  end
-  
-  describe 'to_s' do
-    it 'returns the cache path with the default file extension' do
-      index.to_s.should == 'Picky::Backend::Redis::StringHash(some_namespace:*)'
-    end
-  end
-  
-end