picky 0.0.0 → 0.0.2

data/spec/lib/tokenizers/base_spec.rb

@@ -0,0 +1,61 @@
+# coding: utf-8
+require 'spec_helper'
+
+describe Tokenizers::Base do
+
+  before(:each) do
+    @tokenizer = Tokenizers::Base.new
+  end
+
+  context 'stopwords' do
+    describe '.stopwords' do
+      context 'without stopwords given' do
+        it 'should define a method remove_stopwords' do
+          lambda { @tokenizer.remove_stopwords('from this text') }.should_not raise_error
+        end
+        it 'should define a method remove_stopwords that does nothing' do
+          @tokenizer.remove_stopwords('from this text').should == nil
+        end
+        it 'should not define a method remove_non_single_stopwords' do
+          lambda { @tokenizer.remove_non_single_stopwords('from this text') }.should raise_error(NoMethodError)
+        end
+      end
+      context 'with stopwords given' do
+        before(:each) do
+          class << @tokenizer
+            stopwords(/r|e/)
+          end
+        end
+        it 'should define a method remove_stopwords' do
+          lambda { @tokenizer.remove_stopwords('from this text') }.should_not raise_error
+        end
+        it 'should define a method stopwords that removes stopwords' do
+          @tokenizer.remove_stopwords('from this text').should == 'fom this txt'
+        end
+        it 'should define a method remove_non_single_stopwords' do
+          lambda { @tokenizer.remove_non_single_stopwords('from this text') }.should_not raise_error
+        end
+        it 'should define a method remove_non_single_stopwords that removes non-single stopwords' do
+          @tokenizer.remove_non_single_stopwords('rerere rerere').should == ' '
+        end
+        it 'should define a method remove_non_single_stopwords that does not single stopwords' do
+          @tokenizer.remove_non_single_stopwords('rerere').should == 'rerere'
+        end
+      end
+      context 'error case' do
+        before(:each) do
+          class << @tokenizer
+            stopwords(/any/)
+          end
+        end
+        it 'should not remove non-single stopwords with a star' do
+          @tokenizer.remove_non_single_stopwords('a*').should == 'a*'
+        end
+        it 'should not remove non-single stopwords with a tilde' do
+          @tokenizer.remove_non_single_stopwords('a~').should == 'a~'
+        end
+      end
+    end
+  end
+
+end

data/spec/lib/tokenizers/index_spec.rb

@@ -0,0 +1,51 @@
+# encoding: utf-8
+#
+require 'spec_helper'
+
+# TODO CLEAN UP.
+#
+describe Tokenizers::Index do
+  
+  before(:each) do
+    @tokenizer = Tokenizers::Index.new
+  end
+  
+  describe "remove_illegal_characters" do
+    it "should not remove ' from a query by default" do
+      @tokenizer.remove_illegals("Lugi's").should == "Lugi's"
+    end
+  end
+
+  describe "reject!" do
+    it "should reject tokens if blank" do
+      t1 = stub(:token, :to_s => '')
+      t2 = stub(:token, :to_s => 'not blank')
+      t3 = stub(:token, :to_s => '')
+
+      @tokenizer.reject([t1, t2, t3]).should == [t2]
+    end
+  end
+
+  describe "tokenize" do
+    describe "normalizing" do
+      def self.it_should_normalize_token(text, expected)
+        it "should handle the #{text} case" do
+          @tokenizer.tokenize(text).to_a.should == [expected].compact
+        end
+      end
+      # defaults
+      it_should_normalize_token 'it_should_not_normalize_by_default', :it_should_not_normalize_by_default
+    end
+    describe "tokenizing" do
+      def self.it_should_tokenize_token(text, expected)
+        it "should handle the #{text} case" do
+          @tokenizer.tokenize(text).to_a.should == expected
+        end
+      end
+      # defaults
+      it_should_tokenize_token "splitting on \\s", [:splitting, :on, :"\\s"]
+      it_should_tokenize_token 'und', [:und]
+    end
+  end
+
+end

data/spec/lib/tokenizers/query_spec.rb

@@ -0,0 +1,105 @@
+# coding: utf-8
+require 'spec_helper'
+
+describe Tokenizers::Query do
+
+  before(:each) do
+    @tokenizer = Tokenizers::Query.new
+  end
+  
+  describe 'preprocess' do
+    it 'should call methods in order' do
+      text = stub :text
+      
+      @tokenizer.should_receive(:remove_illegals).once.ordered.with text
+      @tokenizer.should_receive(:remove_non_single_stopwords).once.ordered.with text
+      @tokenizer.should_receive(:contract).once.ordered
+      
+      @tokenizer.preprocess text
+    end
+    it 'should return the text unchanged by default' do
+      text = "some text"
+      
+      @tokenizer.preprocess(text).should == text
+    end
+  end
+  
+  describe 'process' do
+    before(:each) do
+      @tokens = mock :tokens, :null_object => true
+    end
+    it 'should tokenize the tokens' do
+      @tokens.should_receive(:tokenize_with).once.with @tokenizer
+
+      @tokenizer.process @tokens
+    end
+    it 'should call methods on the tokens in order' do
+      @tokens.should_receive(:tokenize_with).once.ordered
+      @tokens.should_receive(:reject).once.ordered
+      @tokens.should_receive(:cap).once.ordered
+      @tokens.should_receive(:partialize_last).once.ordered
+
+      @tokenizer.process @tokens
+    end
+    it 'should return the tokens' do
+      @tokenizer.process(@tokens).should == @tokens
+    end
+  end
+
+  describe 'pretokenize' do
+    def self.it_should_pretokenize text, expected
+      it "should pretokenize #{text} as #{expected}" do
+        @tokenizer.pretokenize(text).should == expected
+      end
+    end
+    it_should_pretokenize 'test miau test', ['test', 'miau', 'test']
+  end
+
+  describe "tokenizing" do
+    def self.it_should_tokenize_token(text, expected)
+      it "should handle the #{text} case" do
+        @tokenizer.tokenize(text).map(&:text).should == expected
+      end
+    end
+    it_should_tokenize_token 'simple tokenizing on \s', [:simple, :tokenizing, :on, :'\s']
+  end
+  
+  describe 'normalize_with_patterns' do
+    def self.it_should_pattern_normalize original, expected
+      it "should normalize #{original} with pattern into #{expected}" do
+        @tokenizer.normalize_with_patterns(original).should == expected
+      end
+    end
+    it_should_pattern_normalize 'no pattern normalization', 'no pattern normalization'
+  end
+  
+  describe 'reject' do
+    it 'should reject blank tokens' do
+      @tokenizer.reject(["some token answering to blank?", nil, nil]).should == ["some token answering to blank?"]
+    end
+  end
+  
+  describe "last token" do
+    it "should be partial" do
+      @tokenizer.tokenize("First Second Third Last").last.instance_variable_get(:@partial).should be_true
+    end
+  end
+  
+  describe ".tokenize" do
+    it "should return an Array of tokens" do
+      @tokenizer.tokenize('test test').to_a.should be_instance_of(Array)
+    end
+    it "should return an empty tokenized query if the query string is blank or empty" do
+      @tokenizer.tokenize('').map(&:to_s).should == []
+    end
+  end
+  describe "token_for" do
+    it "should get a preprocessed token" do
+      text = stub(:text)
+      Query::Token.should_receive(:processed).with text
+      
+      @tokenizer.token_for text
+    end
+  end
+  
+end

data/spec/lib/umlaut_substituter_spec.rb

@@ -0,0 +1,84 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe UmlautSubstituter do
+  include UmlautSubstituter
+
+  # A bit of metaprogramming to help with the myriads of its.
+  #
+  def self.it_should_substitute(special_character, normal_character)
+    it "should substitute #{special_character} with #{normal_character}" do
+      substitute_umlauts(special_character).should == normal_character
+    end
+  end
+  def self.it_should_not_substitute(special_character)
+    it "should not substitute #{special_character}" do
+      substitute_umlauts(special_character).should == special_character
+    end
+  end
+
+  describe "normal characters" do
+    it_should_not_substitute('abcdefghijklmnopqrstuvwxyz1234567890')
+  end
+
+  describe "situations" do
+    it_should_substitute 'Peter Müller', 'Peter Mueller'
+    it_should_substitute 'Lüchinger', 'Luechinger'
+    # it_should_substitute 'LÜCHINGER', 'LUECHINGER'
+  end
+
+  describe "umlauts" do
+    it_should_substitute 'ä', 'ae'
+    it_should_substitute 'Ä', 'Ae'
+    it_should_substitute 'ë', 'e'
+    it_should_substitute 'Ë', 'E'
+    it_should_substitute 'ï', 'i'
+    it_should_substitute 'Ï', 'I'
+    it_should_substitute 'ö', 'oe'
+    it_should_substitute 'Ö', 'Oe'
+    it_should_substitute 'ü', 'ue'
+    it_should_substitute 'Ü', 'Ue'
+  end
+
+  describe "acute" do
+    it_should_substitute 'é', 'e'
+    it_should_substitute 'É', 'E'
+  end
+
+  describe "grave" do
+    it_should_substitute 'à', 'a'
+    it_should_substitute 'À', 'A'
+    it_should_substitute 'è', 'e'
+    it_should_substitute 'È', 'E'
+    it_should_substitute 'ì', 'i'
+    it_should_substitute 'ò', 'o'
+  end
+
+  describe "circonflex" do
+    it_should_substitute 'â', 'a'
+    it_should_substitute 'ê', 'e'
+    it_should_substitute 'Ê', 'E'
+    it_should_substitute 'î', 'i'
+    it_should_substitute 'Î', 'I'
+    it_should_substitute 'ô', 'o'
+    it_should_substitute 'Ô', 'O'
+    it_should_substitute 'û', 'u'
+  end
+
+  describe "cedilla" do
+    it_should_substitute 'ç', 'c'
+    it_should_substitute 'Ç', 'C'
+  end
+
+  describe "ligatures" do
+    it_should_substitute 'ß', 'ss'
+    # it_should_substitute 'Æ', 'AE'
+  end
+
+  describe "norse" do
+    # it_should_substitute 'ø', 'o'
+    it_should_substitute 'å', 'a'
+    it_should_substitute 'Å', 'A'
+  end
+
+end

data/spec/specific/speed_spec.rb

@@ -0,0 +1,55 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe "Speccing Ruby for speed" do
+  
+  describe "various versions for allocation id concatenating" do
+    before(:each) do
+      @allocs = [:hello, :speed, :test]
+      @ids = {
+        :hello => (1..100_000).to_a,
+        :speed => (1..5_000).to_a,
+        :test => (1..1_000).to_a
+      }
+      GC.disable
+    end
+    after(:each) do
+      GC.enable
+      GC.start # start the GC to minimize the chance that it will run again during the speed spec
+    end
+    describe "+" do
+      it "should be fast" do
+        Benchmark.realtime do
+          @allocs.inject([]) do |total, alloc|
+            total + @ids[alloc]
+          end
+        end.should <= 0.0025
+      end
+    end
+    describe "map and flatten!(1)" do
+      it "should be fast" do
+        Benchmark.realtime do
+          @allocs.map { |alloc| @ids[alloc] }.flatten!(1)
+        end.should <= 0.02
+      end
+    end
+    describe "<< and flatten!(1)" do
+      it "should be fast" do
+        Benchmark.realtime do
+          @allocs.inject([]) do |total, alloc|
+            total << @ids[alloc]
+          end.flatten!(1)
+        end.should <= 0.02
+      end
+    end
+    describe "<< and flatten!" do
+      it "should be fast" do
+        Benchmark.realtime do
+          @allocs.inject([]) do |total, alloc|
+            total << @ids[alloc]
+          end.flatten!
+        end.should <= 0.02
+      end
+    end
+  end
+  
+end