corpus-processor 0.2.0 → 0.3.0

data/spec/unit/processor.rb

@@ -1,37 +0,0 @@
-require "spec_helper"
-
-describe CorpusProcessor::Processor do
-  subject(:processor) { CorpusProcessor::Processor.new(parser, generator) }
-
-  describe "#process" do
-    subject { processor.process(corpus) }
-
-    let(:corpus) { "Some corpus" }
-    let(:processed_corpus) {
-<<-CORPUS
-Some	O
-corpus	O
-CORPUS
-    }
-    let(:tokens) {
-      [
-        CorpusProcessor::Token.new("Some"),
-        CorpusProcessor::Token.new("corpus"),
-      ]
-    }
-    let(:parser)    { double :parser    }
-    let(:generator) { double :generator }
-
-    specify {
-      parser.should_receive(:parse)
-            .with(corpus)
-            .and_return(tokens)
-
-      generator.should_receive(:generate)
-            .with(tokens)
-            .and_return(processed_corpus)
-
-      subject.should == processed_corpus
-    }
-  end
-end

data/spec/unit/token_spec.rb

@@ -1,8 +0,0 @@
-require "spec_helper"
-
-describe CorpusProcessor::Token do
-  subject { CorpusProcessor::Token.new }
-
-  it { should respond_to(:word)     }
-  it { should respond_to(:category) }
-end

data/spec/unit/tokenizer_spec.rb

@@ -1,121 +0,0 @@
-require "spec_helper"
-
-describe CorpusProcessor::Tokenizer do
-  subject(:tokenizer) { CorpusProcessor::Tokenizer.new }
-
-  describe "#tokenize" do
-    subject { tokenizer.tokenize(text, category) }
-
-    let(:category) { nil }
-
-    context "empty string" do
-      let(:text) { "" }
-
-      it { should == [] }
-    end
-
-    context "one word" do
-      let(:text) { "banana" }
-
-      it { should == [CorpusProcessor::Token.new("banana")] }
-    end
-
-    context "two words" do
-      let(:text) { "good banana" }
-
-      it { should == [
-        CorpusProcessor::Token.new("good"),
-        CorpusProcessor::Token.new("banana"),
-      ] }
-    end
-
-    context "ponctuation" do
-      let(:text) { "good, banana" }
-
-      it { should == [
-        CorpusProcessor::Token.new("good"),
-        CorpusProcessor::Token.new("banana"),
-      ] }
-    end
-
-    context "default category" do
-      let(:text)     { "Google" }
-      let(:category) { :organization }
-
-      it { should == [
-        CorpusProcessor::Token.new("Google", :organization),
-      ] }
-    end
-
-    context "with tags" do
-      let(:text) { "good<lalala/>, banana" }
-
-      it { should == [
-        CorpusProcessor::Token.new("good"),
-        CorpusProcessor::Token.new("banana"),
-      ] }
-    end
-  end
-
-  describe "#strip_tags" do
-    subject { tokenizer.strip_tags(text) }
-
-    context "empty text" do
-      let(:text) { "" }
-
-      it { should == "" }
-    end
-
-    context "self closed tag" do
-      let(:text) { "<br/>" }
-
-      it { should == "" }
-    end
-
-    context "tag with content" do
-      let(:text) { "<p>Some text</p>" }
-
-      it { should == "Some text" }
-    end
-
-    context "content after tag" do
-      let(:text) { "<p>Some<br/>text</p>" }
-
-      it { should == "Some text" }
-    end
-  end
-
-  describe "#join_lines" do
-    subject { tokenizer.join_lines(text) }
-
-    context "empty text" do
-      let(:text) { "" }
-
-      it { should == "" }
-    end
-
-    context "one word" do
-      let(:text) { "banana" }
-
-      it { should == "banana" }
-    end
-
-    context "two lines" do
-      let(:text) { "banana\nquiabo" }
-
-      it { should == "banana quiabo" }
-    end
-
-    context "line with empty space" do
-      let(:text) { "banana\n   \nquiabo" }
-
-      it { should == "banana quiabo" }
-    end
-
-    context "leading spaces" do
-      let(:text) { "  \n  banana\n   \nquiabo  \n" }
-
-      it { should == "banana quiabo" }
-    end
-  end
-end

data/spec/unit/traverser_spec.rb

@@ -1,68 +0,0 @@
-require "spec_helper"
-
-describe CorpusProcessor::Traverser do
-  subject(:traverser) { CorpusProcessor::Traverser.new }
-
-  describe "#traverse" do
-    subject { traverser.traverse(text, regexp) }
-
-    context "empty text" do
-      let(:text)   { "" }
-      let(:regexp) { // }
-
-      specify {
-        expect { |mock_block|
-          traverser.traverse(text, regexp, &mock_block)
-        }.not_to yield_control
-      }
-    end
-
-    context "simple text" do
-      let(:text)   { "abc" }
-      let(:regexp) { /b/ }
-
-      specify {
-        expect { |mock_block|
-          traverser.traverse(text, regexp, &mock_block)
-        }.to yield_successive_args "a", text.match(regexp), "c"
-      }
-    end
-
-    context "two matches" do
-      let(:text)   { "abcbd" }
-      let(:regexp) { /b/ }
-
-      specify {
-        expect { |mock_block|
-          traverser.traverse(text, regexp, &mock_block)
-        }.to yield_successive_args "a",
-                                   text.match(regexp),
-                                   "c",
-                                   text[2..-1].match(regexp),
-                                   "d"
-      }
-    end
-
-    context "match in beginning" do
-      let(:text)   { "bc" }
-      let(:regexp) { /b/ }
-
-      specify {
-        expect { |mock_block|
-          traverser.traverse(text, regexp, &mock_block)
-        }.to yield_successive_args text.match(regexp), "c"
-      }
-    end
-
-    context "match in ending" do
-      let(:text)   { "bc" }
-      let(:regexp) { /c/ }
-
-      specify {
-        expect { |mock_block|
-          traverser.traverse(text, regexp, &mock_block)
-        }.to yield_successive_args "b", text.match(regexp)
-      }
-    end
-  end
-end