wordlist 0.1.1 → 1.0.1

data/spec/lexer_spec.rb

@@ -0,0 +1,718 @@
+require 'spec_helper'
+require 'wordlist/lexer'
+
+describe Wordlist::Lexer do
+  let(:text) { "foo bar baz qux" }
+
+  describe "#initialize" do
+    let(:default_lang) { Wordlist::Lexer::Lang.default }
+
+    it "must default #lang to Lang.default_lang" do
+      expect(subject.lang).to eq(default_lang)
+    end
+
+    it "must set #stop_words to the stop words for :en" do
+      expect(subject.stop_words).to eq(Wordlist::Lexer::StopWords[default_lang])
+    end
+
+    it "must default #ignore_words to []" do
+      expect(subject.ignore_words).to eq([])
+    end
+
+    it "must default #digits? to true" do
+      expect(subject.digits?).to be(true)
+    end
+
+    it "must default #special_chars to SPECIAL_CHARS" do
+      expect(subject.special_chars).to eq(described_class::SPECIAL_CHARS)
+    end
+
+    it "must default #numbers? to false" do
+      expect(subject.numbers?).to be(false)
+    end
+
+    it "must default #acroynyms? to true" do
+      expect(subject.acronyms?).to be(true)
+    end
+
+    it "must default #normalize_case? to false" do
+      expect(subject.normalize_case?).to be(false)
+    end
+
+    it "must default #normalize_apostrophes? to false" do
+      expect(subject.normalize_apostrophes?).to be(false)
+    end
+
+    it "must default #normalize_acroynyms? to false" do
+      expect(subject.normalize_acronyms?).to be(false)
+    end
+
+    context "when the lang: keyword is given" do
+      let(:lang) { :es }
+
+      subject { described_class.new(lang: lang) }
+
+      it "must set #lang" do
+        expect(subject.lang).to eq(lang)
+      end
+
+      it "must set #stop_words to the stop words for that language" do
+        expect(subject.stop_words).to eq(Wordlist::Lexer::StopWords[lang])
+      end
+    end
+
+    context "when ignore_words: keyword argument is given" do
+      let(:ignore_words) { %w[foo bar] }
+
+      subject { described_class.new(ignore_words: ignore_words) }
+
+      it "must set #ignore_words" do
+        expect(subject.ignore_words).to eq(ignore_words)
+      end
+
+      context "and it contains an object other than a String or Regexp" do
+        it do
+          expect {
+            described_class.new(ignore_words: [Object.new])
+          }.to raise_error(ArgumentError,"ignore_words: must contain only Strings or Regexps")
+        end
+      end
+    end
+
+    context "when digits: false is given" do
+      subject { described_class.new(digits: false) }
+
+      it "must set #digits? to false" do
+        expect(subject.digits?).to be(false)
+      end
+    end
+
+    context "when special_chars: keyword is given" do
+      let(:special_chars) { %w[_ -] }
+
+      subject { described_class.new(special_chars: special_chars) }
+
+      it "must set #special_chars" do
+        expect(subject.special_chars).to eq(special_chars)
+      end
+    end
+
+    context "when numbers: true is given" do
+      subject { described_class.new(numbers: true) }
+
+      it "must set #numbers? to true" do
+        expect(subject.numbers?).to be(true)
+      end
+    end
+
+    context "when acronyms: true is given" do
+      subject { described_class.new(acronyms: true) }
+
+      it "must set #acronyms? to true" do
+        expect(subject.acronyms?).to be(true)
+      end
+    end
+
+    context "when normalize_case: true is given" do
+      subject { described_class.new(normalize_case: true) }
+
+      it "must set #normalize_case? to true" do
+        expect(subject.normalize_case?).to be(true)
+      end
+    end
+
+    context "when normalize_apostrophes: true is given" do
+      subject { described_class.new(normalize_apostrophes: true) }
+
+      it "must set #normalize_apostrophes? to true" do
+        expect(subject.normalize_apostrophes?).to be(true)
+      end
+    end
+
+    context "when normalize_acronyms: true is given" do
+      subject { described_class.new(normalize_acronyms: true) }
+
+      it "must set #normalize_acronyms? to true" do
+        expect(subject.normalize_acronyms?).to be(true)
+      end
+    end
+  end
+
+  describe "#parse" do
+    let(:expected_words) { %w[foo bar baz qux]      }
+    let(:text)           { expected_words.join(' ') }
+
+    context "when a block is given" do
+      it "must yield each scanned word from the text" do
+        expect { |b|
+          subject.parse(text,&b)
+        }.to yield_successive_args(*expected_words)
+      end
+
+      context "and when the text is empty" do
+        let(:text) { '' }
+
+        it "must not yield any words" do
+          expect { |b|
+            subject.parse(text,&b)
+          }.to_not yield_control
+        end
+      end
+
+      context "and when the text is only whitespace" do
+        let(:text) { " \t\r\n" }
+
+        it "must not yield any words" do
+          expect { |b|
+            subject.parse(text,&b)
+          }.to_not yield_control
+        end
+      end
+
+      context "and when the text contains newlines" do
+        let(:text) { expected_words.join("\n") }
+
+        it "must parse each line" do
+          expect { |b|
+            subject.parse(text,&b)
+          }.to yield_successive_args(*expected_words)
+        end
+      end
+
+      context "and when the text is only unicode whitespace" do
+        let(:text) { "\u{a0}" }
+
+        it "must not yield any words" do
+          expect { |b|
+            subject.parse(text,&b)
+          }.to_not yield_control
+        end
+      end
+
+      context "and when the text is only unicode symbols" do
+        let(:text) { "©" }
+
+        it "must not yield any words" do
+          expect { |b|
+            subject.parse(text,&b)
+          }.to_not yield_control
+        end
+      end
+
+      context "and when the words are separated by unicode whitespace" do
+        let(:text) { expected_words.join("\u{a0}") }
+
+        it "must skip the unicode whitespace and yield the words" do
+          expect { |b|
+            subject.parse(text,&b)
+          }.to yield_successive_args(*expected_words)
+        end
+      end
+
+      context "and when the text contains unicode symbols" do
+        let(:text) { "© #{super()}" }
+
+        it "must skip any unicode symbols and yield the words" do
+          expect { |b|
+            subject.parse(text,&b)
+          }.to yield_successive_args(*expected_words)
+        end
+      end
+
+      context "when the words contain unicode letters" do
+        let(:expected_words) { %w[Hervé Schäfer Ålesund] }
+
+        it "must parse words containing unicode letters" do
+          expect { |b|
+            subject.parse(text,&b)
+          }.to yield_successive_args(*expected_words)
+        end
+      end
+
+      context "when the words contain uppercase letters" do
+        let(:expected_words) { %w[foo Bar baZ QUX] }
+
+        it "must parse words containing uppercase letters" do
+          expect { |b|
+            subject.parse(text,&b)
+          }.to yield_successive_args(*expected_words)
+        end
+
+        context "but when initialized with normalize_case: true" do
+          let(:expected_words) { %w[foo bar baz qux] }
+          let(:text)           { "foo Bar baZ QUX"   }
+
+          subject { described_class.new(normalize_case: true) }
+
+          it "must convert all words to lowercase" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+        end
+      end
+
+      context "and when the text contains single letters" do
+        let(:letters)        { %w[x y z]         }
+        let(:expected_words) { super() + letters }
+
+        it "must parse single letter words" do
+          expect { |b|
+            subject.parse(text,&b)
+          }.to yield_successive_args(*expected_words)
+        end
+
+        context "when the text also contains single letter stop words" do
+          let(:letters)        { %w[a b c i j k] }
+          let(:stop_words)     { %w[a i]         }
+          let(:expected_words) { super() - stop_words }
+          let(:text)           { "#{super()} #{stop_words.join(' ')}" }
+
+          it "must parse single letter words" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+        end
+      end
+
+      context "and when the text contains punctuation" do
+        let(:text) { expected_words.join(", ") + '.' }
+
+        it "must ignore all punctuation" do
+          expect { |b|
+            subject.parse(text,&b)
+          }.to yield_successive_args(*expected_words)
+        end
+
+        context "and the words start with a '\\'' characters" do
+          let(:expected_words) { %w[foo bar baz] }
+          let(:text)           { "foo 'bar baz"  }
+
+          it "must skip the leading '\\' character'" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+        end
+
+        context "and the words contain '\\'' characters" do
+          let(:expected_words) { super() + %w[O'Brian] }
+
+          it "must parse the words containing a '\\''" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+
+          context "and when initialized with normalize_apostrophes: true" do
+            let(:text)           { "foo bar's baz" }
+            let(:expected_words) { %w[foo bar baz] }
+
+            subject { described_class.new(normalize_apostrophes: true) }
+
+            it "must remove any trailing \"'s\" from the words" do
+              expect { |b|
+                subject.parse(text,&b)
+              }.to yield_successive_args(*expected_words)
+            end
+          end
+        end
+
+        context "and the words end with a '\\'' characters" do
+          let(:expected_words) { %w[foo bar baz] }
+          let(:text)           { "foo bar' baz"  }
+
+          it "must skip the trailing '\\' character'" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+        end
+
+        context "and the words start with a '-' characters" do
+          let(:expected_words) { %w[foo bar baz] }
+          let(:text)           { "foo -bar baz"  }
+
+          it "must skip the leading '-' character'" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+        end
+
+        context "and the words contain '-' characters" do
+          let(:expected_words) { %w[foo-bar baz-qux] }
+
+          it "must parse words containing a '-'" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+
+          context "and when initialized with chars: keyword argument" do
+            context "and it does not include '-'" do
+              let(:text)           { "foo bar-baz qux"   }
+              let(:expected_words) { %w[foo bar baz qux] }
+
+              subject { described_class.new(special_chars: ['_']) }
+
+              it "must split hyphenated words into multiple words" do
+                expect { |b|
+                  subject.parse(text,&b)
+                }.to yield_successive_args(*expected_words)
+              end
+            end
+          end
+        end
+
+        context "and the words end with a '-' characters" do
+          let(:expected_words) { %w[foo bar baz] }
+          let(:text)           { "foo bar- baz"  }
+
+          it "must skip the trailing '-' character'" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+        end
+
+        context "and the words start with a '_' characters" do
+          let(:expected_words) { %w[foo bar baz] }
+          let(:text)           { "foo _bar baz"  }
+
+          it "must skip the leading '_' character'" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+        end
+
+        context "and the words contain '_' characters" do
+          let(:expected_words) { %w[foo_bar baz_qux] }
+
+          it "must treat the words containing a '_' as a single word" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+
+          context "and when initialized with chars: keyword argument" do
+            context "and it does not include '_'" do
+              let(:text)           { "foo bar_baz qux"   }
+              let(:expected_words) { %w[foo bar baz qux] }
+
+              subject { described_class.new(special_chars: ['-']) }
+
+              it "must split hyphenated words into multiple words" do
+                expect { |b|
+                  subject.parse(text,&b)
+                }.to yield_successive_args(*expected_words)
+              end
+            end
+          end
+        end
+
+        context "and the words end with a '_' characters" do
+          let(:expected_words) { %w[foo bar baz] }
+          let(:text)           { "foo bar_ baz"  }
+
+          it "must skip the trailing '_' character'" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+        end
+
+        context "and the words start with a '.' characters" do
+          let(:expected_words) { %w[foo bar baz] }
+          let(:text)           { "foo .bar baz"  }
+
+          it "must skip the leading '.' character'" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+        end
+
+        context "and the words contain '.' characters" do
+          let(:text)           { "foo.bar baz.qux"  }
+          let(:expected_words) { %w[foo bar baz qux] }
+
+          it "must split words containing '.' into multiple words" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+
+          context "and when initialized with chars: keyword argument" do
+            context "and it does include '.'" do
+              let(:expected_words) { %w[foo.bar baz.qux] }
+
+              subject { described_class.new(special_chars: ['.']) }
+
+              it "must treat words containing a '.' as a single word" do
+                expect { |b|
+                  subject.parse(text,&b)
+                }.to yield_successive_args(*expected_words)
+              end
+            end
+          end
+        end
+
+        context "and the words end with a '.' characters" do
+          let(:expected_words) { %w[foo bar baz] }
+          let(:text)           { "foo bar. baz"  }
+
+          it "must skip the trailing '.' character'" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+
+          context "but the word is an acronym" do
+            let(:expected_words) { %w[foo B.A.R. baz] }
+            let(:text)           { "foo B.A.R. baz"  }
+
+            it "must parse whole acronyms'" do
+              expect { |b|
+                subject.parse(text,&b)
+              }.to yield_successive_args(*expected_words)
+            end
+
+            context "but was initialized with acronyms: false" do
+              let(:expected_words) { %w[foo baz] }
+
+              subject { described_class.new(acronyms: false) }
+
+              it "must skip the whole acronyms" do
+                expect { |b|
+                  subject.parse(text,&b)
+                }.to yield_successive_args(*expected_words)
+              end
+            end
+
+            context "and when initialized with normalize_acronyms: true" do
+              let(:expected_words) { %w[foo BAR baz] }
+              let(:text)           { "foo B.A.R. baz"  }
+
+              subject { described_class.new(normalize_acronyms: true) }
+
+              it "must remove the '.' characters from acronyms" do
+                expect { |b|
+                  subject.parse(text,&b)
+                }.to yield_successive_args(*expected_words)
+              end
+            end
+          end
+        end
+      end
+
+      context "and when the text contains numbers" do
+        let(:text) { expected_words.join(" 1234 ") }
+
+        it "must ignore whole numbers" do
+          expect { |b|
+            subject.parse(text,&b)
+          }.to yield_successive_args(*expected_words)
+        end
+
+        context "when initialized with numbers: true" do
+          let(:expected_words) { %w[foo 1234 bar 000 baz 0123] }
+          let(:text)           { expected_words.join(' ')      }
+
+          subject { described_class.new(numbers: true) }
+
+          it "must parse whole numbers" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+        end
+
+        context "but the text also contains words that start with digits" do
+          let(:text) { expected_words.map { |word| "123#{word}" }.join(' ') }
+
+          it "must ignore the leading digits of words" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+        end
+
+        context "but the text also contains words that contain digits" do
+          let(:expected_words) { %w[foo bar1baz qux] }
+
+          it "must not ignore the digits within words" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+
+          context "but when initialized with digits: false" do
+            let(:expected_words) { %w[foo bar baz qux] }
+            let(:text)           { "foo bar2baz qux"   }
+
+            subject { described_class.new(digits: false) }
+
+            it "must ignore the leading digits within words" do
+              expect { |b|
+                subject.parse(text,&b)
+              }.to yield_successive_args(*expected_words)
+            end
+          end
+        end
+
+        context "but the text also contains words that end in digits" do
+          let(:expected_words) { super().map { |word| "#{word}123" } }
+
+          it "must not ignore the digits within words" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+
+          context "but when initialized with digits: false" do
+            let(:expected_words) { %w[foo bar baz qux] }
+            let(:text)           { "foo bar2 baz qux4" }
+
+            subject { described_class.new(digits: false) }
+
+            it "must ignore the leading digits within words" do
+              expect { |b|
+                subject.parse(text,&b)
+              }.to yield_successive_args(*expected_words)
+            end
+          end
+        end
+      end
+
+      context "and when the text contains new-lines" do
+        let(:text) { expected_words.join($/) }
+
+        it "must ignore new-line characters" do
+          expect { |b|
+            subject.parse(text,&b)
+          }.to yield_successive_args(*expected_words)
+        end
+      end
+
+      context "and when the text contains stop-words" do
+        let(:stop_words) { %w[a the is be] }
+        let(:text)       { expected_words.zip(stop_words).flatten.join(' ') }
+        
+        it "must ignore the stop-words and parse the non-stop-words" do
+          expect { |b|
+            subject.parse(text,&b)
+          }.to yield_successive_args(*expected_words)
+        end
+
+        context "and when the stop words are capitlized" do
+          let(:stop_words) { super().map(&:capitalize) }
+
+          it "must ignore the capitlized stop-words" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+        end
+
+        context "and when the stop words are uppercase" do
+          let(:stop_words) { super().map(&:upcase) }
+
+          it "must ignore the uppercase stop-words" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+        end
+
+        context "and when the text ends with a stop word" do
+          let(:text) { "#{super()} is" }
+
+          it "must ignore the last stop word" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+        end
+
+        context "and when a stop word is followed by other letters" do
+          let(:stop_word)      { "be" }
+          let(:expected_words) { super() + ["#{stop_word}tter"] }
+
+          it "must not ignore stop words followed by other letters" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+        end
+
+        context "and when a stop word is followed by digits" do
+          let(:stop_word)      { "a" }
+          let(:expected_words) { super() + ["#{stop_word}1234"] }
+
+          it "must not ignore stop words followed by digits" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+        end
+
+        context "and when a stop word is followed by punctuation" do
+          let(:stop_words) { %w[is. be, the?] }
+
+          it "must not ignore stop words followed by punctuation" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+        end
+
+        context "and when the text contains multiple successive stop-words" do
+          let(:text) { (stop_words + expected_words).join(' ') }
+        
+          it "must ignore multiple successive stop-words" do
+            expect { |b|
+              subject.parse(text,&b)
+            }.to yield_successive_args(*expected_words)
+          end
+        end
+      end
+
+      context "and when #ignore_words contains a String" do
+        let(:ignore_words)   { %w[foo baz] }
+        let(:expected_words) { %w[bar qux] }
+        let(:text)           { "foo bar baz qux" }
+
+        subject { described_class.new(ignore_words: ignore_words) }
+
+        it "must filter out words matching that String" do
+          expect { |b|
+            subject.parse(text,&b)
+          }.to yield_successive_args(*expected_words)
+        end
+      end
+
+      context "and when #ignore_words contains a Regexp" do
+        let(:ignore_words)   { [/ba[a-z]/] }
+        let(:expected_words) { %w[foo qux] }
+        let(:text)           { "foo bar baz qux" }
+
+        subject { described_class.new(ignore_words: ignore_words) }
+
+        it "must filter out words matching that Regexp" do
+          expect { |b|
+            subject.parse(text,&b)
+          }.to yield_successive_args(*expected_words)
+        end
+      end
+    end
+
+    context "when no block is given" do
+      it "must return an Array of the parsed words" do
+        expect(subject.parse(text)).to eq(expected_words)
+      end
+    end
+  end
+end