wikipedia-vandalism_detection 0.0.1

data/spec/vandalism_detection/features/inserted_size_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+
+describe Wikipedia::VandalismDetection::Features::InsertedSize do
+
+  before do
+    @feature = Wikipedia::VandalismDetection::Features::InsertedSize.new
+  end
+
+  it { should be_a Wikipedia::VandalismDetection::Features::Base }
+
+  describe "#calculate" do
+
+    it "returns the size of the edit's new revisions inserted text sizes" do
+      old_revision_text = Wikipedia::VandalismDetection::Text.new '123'
+      new_revision_text = Wikipedia::VandalismDetection::Text.new '123 456789' # 6 inserted
+
+      old_revision = build(:old_revision, text: old_revision_text)
+      new_revision = build(:new_revision, text: new_revision_text)
+      edit = build(:edit, old_revision: old_revision, new_revision: new_revision)
+
+      expect(@feature.calculate(edit)).to eq 6
+    end
+
+    it "returns 0 if no inserted text " do
+      old_revision_text = Wikipedia::VandalismDetection::Text.new '123 456789'
+      new_revision_text = Wikipedia::VandalismDetection::Text.new '123' # 0 inserted
+
+      old_revision = build(:old_revision, text: old_revision_text)
+      new_revision = build(:new_revision, text: new_revision_text)
+      edit = build(:edit, old_revision: old_revision, new_revision: new_revision)
+
+      expect(@feature.calculate(edit)).to eq 0
+    end
+  end
+end

data/spec/vandalism_detection/features/inserted_words_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+
+describe Wikipedia::VandalismDetection::Features::InsertedWords do
+
+  before do
+    @feature = Wikipedia::VandalismDetection::Features::InsertedWords.new
+  end
+
+  it { should be_a Wikipedia::VandalismDetection::Features::Base }
+
+  describe "#calculate" do
+
+    it "returns the number of the inserted words" do
+      old_revision_text = Wikipedia::VandalismDetection::Text.new 'zero'
+      new_revision_text = Wikipedia::VandalismDetection::Text.new 'zero one two three four five six' # 6 inserted
+
+      old_revision = build(:old_revision, text: old_revision_text)
+      new_revision = build(:new_revision, text: new_revision_text)
+      edit = build(:edit, old_revision: old_revision, new_revision: new_revision)
+
+      expect(@feature.calculate(edit)).to eq 6
+    end
+
+    it "returns 0 if no inserted text" do
+      old_revision_text = Wikipedia::VandalismDetection::Text.new 'zero one'
+      new_revision_text = Wikipedia::VandalismDetection::Text.new 'zero' # 0 inserted
+
+      old_revision = build(:old_revision, text: old_revision_text)
+      new_revision = build(:new_revision, text: new_revision_text)
+      edit = build(:edit, old_revision: old_revision, new_revision: new_revision)
+
+      expect(@feature.calculate(edit)).to eq 0
+    end
+  end
+end

data/spec/vandalism_detection/features/longest_word_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+
+describe Wikipedia::VandalismDetection::Features::LongestWord do
+
+  before do
+    @feature = Wikipedia::VandalismDetection::Features::LongestWord.new
+  end
+
+  it { should be_a Wikipedia::VandalismDetection::Features::Base }
+
+  describe "#calculate" do
+
+    it "returns the length of the edit's new revisions text's longest word" do
+      old_text = Wikipedia::VandalismDetection::Text.new "1 7777777"
+      new_text = Wikipedia::VandalismDetection::Text.new "1 7777777 22 a2c4e 4444 55555\n\n======head======\nfffff"
+
+      old_revision = build(:old_revision, text: old_text)
+      new_revision = build(:new_revision, text: new_text)
+      edit = build(:edit, old_revision: old_revision, new_revision: new_revision)
+
+      expect(@feature.calculate(edit)).to eq 5
+    end
+
+    it "returns 0 on non inserted clean text" do
+      old_text = Wikipedia::VandalismDetection::Text.new "1 22"
+      new_text = Wikipedia::VandalismDetection::Text.new "1 22 {{speedy deletion}}"
+
+      old_revision = build(:old_revision, text: old_text)
+      new_revision = build(:new_revision, text: new_text)
+      edit = build(:edit, old_revision: old_revision, new_revision: new_revision)
+
+      expect(@feature.calculate(edit)).to eq 0
+    end
+  end
+end

data/spec/vandalism_detection/features/markup_frequency_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe Wikipedia::VandalismDetection::Features::MarkupFrequency do
+
+  before do
+    @feature = Wikipedia::VandalismDetection::Features::MarkupFrequency.new
+  end
+
+  it { should be_a Wikipedia::VandalismDetection::Features::Base }
+
+  describe "#calculate" do
+
+    it "returns the number of markup related words relative to all inserted words count" do
+      # total 4 words, 3 markup
+      old_text = Wikipedia::VandalismDetection::Text.new 'Old whatever.'
+      new_text = Wikipedia::VandalismDetection::Text.new "Old whatever. {{template}} <ref>list</ref> [[heading]] boy."
+
+      old_revision = build(:old_revision, text: old_text)
+      new_revision = build(:new_revision, text: new_text)
+      edit = build(:edit, old_revision: old_revision, new_revision: new_revision)
+
+      expect(@feature.calculate(edit)).to eq 3.0 / 4.0
+    end
+
+    it "returns 0.0 on no inserted text" do
+      old_text = Wikipedia::VandalismDetection::Text.new 'Old guy.'
+      new_text = Wikipedia::VandalismDetection::Text.new "Old guy."
+
+      old_revision = build(:old_revision, text: old_text)
+      new_revision = build(:new_revision, text: new_text)
+      edit = build(:edit, old_revision: old_revision, new_revision: new_revision)
+
+      expect(@feature.calculate(edit)).to eq 0.0
+    end
+  end
+end

data/spec/vandalism_detection/features/markup_impact_spec.rb

@@ -0,0 +1,59 @@
+require 'spec_helper'
+
+describe Wikipedia::VandalismDetection::Features::MarkupImpact do
+
+  before do
+    @feature = Wikipedia::VandalismDetection::Features::MarkupImpact.new
+  end
+
+  it { should be_a Wikipedia::VandalismDetection::Features::Base }
+
+  describe "#calculate" do
+
+    it "returns the impact of markup words of the edit's new revision text" do
+      # 3 markup words
+      old_text = "{{template}} <ref>reference</ref> [[hello]] hello"
+
+      # 4 markup words
+      new_text = "{{template}} <ref>reference</ref> [[hello]] cite dude"
+
+      old_revision = build(:old_revision, text: old_text)
+      new_revision = build(:new_revision, text: new_text)
+      edit = build(:edit, old_revision: old_revision, new_revision: new_revision)
+
+      expect(@feature.calculate(edit)).to eq 3.0 / (3.0 + 4.0)
+    end
+
+    it "returns 0.5 on both no terms in text revisions" do
+      text = ""
+
+      old_revision = build(:old_revision, text: text)
+      new_revision = build(:new_revision, text: text)
+      edit = build(:edit, new_revision: new_revision, old_revision: old_revision)
+
+      expect(@feature.calculate(edit)).to eq 0.5
+    end
+
+    it "returns 0.0 on emtpy text of old revision" do
+      old_text = ""
+      new_text = "{{template}}"
+
+      old_revision = build(:old_revision, text: old_text)
+      new_revision = build(:new_revision, text: new_text)
+      edit = build(:edit, new_revision: new_revision, old_revision: old_revision)
+
+      expect(@feature.calculate(edit)).to eq 0.0
+    end
+
+    it "returns 1.0 on emtpy text of new revision" do
+      old_text = "{{template}}"
+      new_text = ""
+
+      old_revision = build(:old_revision, text: old_text)
+      new_revision = build(:new_revision, text: new_text)
+      edit = build(:edit, new_revision: new_revision, old_revision: old_revision)
+
+      expect(@feature.calculate(edit)).to eq 1.0
+    end
+  end
+end

data/spec/vandalism_detection/features/non_alphanumeric_ratio_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+
+describe Wikipedia::VandalismDetection::Features::NonAlphanumericRatio do
+
+  before do
+    @feature = Wikipedia::VandalismDetection::Features::NonAlphanumericRatio.new
+  end
+
+  it { should be_a Wikipedia::VandalismDetection::Features::Base }
+
+  describe "#calculate" do
+
+    it "returns the non-alphanumeric to all letters ratio of the edit's new revision uncleaned inserted text" do
+      old_text = Wikipedia::VandalismDetection::Text.new('t$xt')
+      new_text = Wikipedia::VandalismDetection::Text.new 't$xt [[1A$% 4B6]] 8Cd?' # 7 non-alphanumeric letters of total 15 letters
+
+      old_revision = build(:old_revision, text: old_text)
+      new_revision = build(:new_revision, text: new_text)
+      edit = build(:edit, old_revision: old_revision, new_revision: new_revision)
+
+      expect(@feature.calculate(edit)).to eq (1.0 + 7) / (1.0 + 15)
+    end
+
+    it "returns 0.0 if no text inserted" do
+      old_text = Wikipedia::VandalismDetection::Text.new("deletion text")
+      new_text = Wikipedia::VandalismDetection::Text.new("text")
+
+      old_revision = build(:old_revision, text: old_text)
+      new_revision = build(:new_revision, text: new_text)
+      edit = build(:edit, new_revision: new_revision, old_revision: old_revision)
+
+      expect(@feature.calculate(edit)).to eq 0.0
+    end
+  end
+end

data/spec/vandalism_detection/features/personal_life_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe Wikipedia::VandalismDetection::Features::PersonalLife do
+
+  before do
+    @feature = Wikipedia::VandalismDetection::Features::PersonalLife.new
+  end
+
+  it { should be_a Wikipedia::VandalismDetection::Features::ContainsBase }
+
+  describe "#calculate" do
+
+    it "returns 1 if the edit comment includes 'personal life'" do
+      comment = Wikipedia::VandalismDetection::Text.new "/* Personal life */ edited"
+      edit = build :edit, new_revision: build(:new_revision, comment: comment)
+
+      expect(@feature.calculate(edit)).to eq 1
+    end
+
+    it "returns 0 on emtpy comment" do
+      edit = build :edit, new_revision: build(:new_revision, comment: "")
+
+      expect(@feature.calculate(edit)).to eq 0
+    end
+  end
+end

data/spec/vandalism_detection/features/pronoun_frequency_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe Wikipedia::VandalismDetection::Features::PronounFrequency do
+
+  before do
+    @feature = Wikipedia::VandalismDetection::Features::PronounFrequency.new
+  end
+
+  it { should be_a Wikipedia::VandalismDetection::Features::FrequencyBase }
+
+  describe "#calculate" do
+
+    it "returns the number of pronouns relative to all words count" do
+      # total 12 words, 7 pronouns
+      old_text = Wikipedia::VandalismDetection::Text.new 'Your old.'
+      new_text = Wikipedia::VandalismDetection::Text.new "Your old. I was you if You was Me and we are ourselves us."
+
+      old_revision = build(:old_revision, text: old_text)
+      new_revision = build(:new_revision, text: new_text)
+      edit = build(:edit, old_revision: old_revision, new_revision: new_revision)
+
+      expect(@feature.calculate(edit)).to eq 7.0/12.0
+    end
+
+    it "returns 0.0 on emtpy clean text revisions" do
+      old_text = Wikipedia::VandalismDetection::Text.new 'Your old.'
+      new_text = Wikipedia::VandalismDetection::Text.new "Your old. {{speedy deletion}}"
+
+      old_revision = build(:old_revision, text: old_text)
+      new_revision = build(:new_revision, text: new_text)
+      edit = build(:edit, old_revision: old_revision, new_revision: new_revision)
+
+      expect(@feature.calculate(edit)).to eq 0.0
+    end
+  end
+end

data/spec/vandalism_detection/features/pronoun_impact_spec.rb

@@ -0,0 +1,59 @@
+require 'spec_helper'
+
+describe Wikipedia::VandalismDetection::Features::PronounImpact do
+
+  before do
+    @feature = Wikipedia::VandalismDetection::Features::PronounImpact.new
+  end
+
+  it { should be_a Wikipedia::VandalismDetection::Features::ImpactBase }
+
+  describe "#calculate" do
+
+    it "returns the impact of pronouns of the edit's new revision text" do
+      # 3 pronouns
+      old_text = Wikipedia::VandalismDetection::Text.new "Your old text will be mine or Your's"
+
+      # 4 pronouns
+      new_text = Wikipedia::VandalismDetection::Text.new "My new text and your old text will be ours and mine"
+
+      old_revision = build(:old_revision, text: old_text)
+      new_revision = build(:new_revision, text: new_text)
+      edit = build(:edit, old_revision: old_revision, new_revision: new_revision)
+
+      expect(@feature.calculate(edit)).to eq 3.0 / (3.0 + 4.0)
+    end
+
+    it "returns 0.5 on both no terms in text revisions" do
+      text = Wikipedia::VandalismDetection::Text.new "{{speedy deletion}}"
+
+      old_revision = build(:old_revision, text: text)
+      new_revision = build(:new_revision, text: text)
+      edit = build(:edit, new_revision: new_revision, old_revision: old_revision)
+
+      expect(@feature.calculate(edit)).to eq 0.5
+    end
+
+    it "returns 0.0 on emtpy clean text of old revision" do
+      old_text = Wikipedia::VandalismDetection::Text.new "{{speedy deletion}}"
+      new_text = Wikipedia::VandalismDetection::Text.new "You"
+
+      old_revision = build(:old_revision, text: old_text)
+      new_revision = build(:new_revision, text: new_text)
+      edit = build(:edit, new_revision: new_revision, old_revision: old_revision)
+
+      expect(@feature.calculate(edit)).to eq 0.0
+    end
+
+    it "returns 1.0 on emtpy clean text of new revision" do
+      old_text = Wikipedia::VandalismDetection::Text.new "You"
+      new_text = Wikipedia::VandalismDetection::Text.new "{{speedy deletion}}"
+
+      old_revision = build(:old_revision, text: old_text)
+      new_revision = build(:new_revision, text: new_text)
+      edit = build(:edit, new_revision: new_revision, old_revision: old_revision)
+
+      expect(@feature.calculate(edit)).to eq 1.0
+    end
+  end
+end

data/spec/vandalism_detection/features/removed_all_wordlists_frequency_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe Wikipedia::VandalismDetection::Features::RemovedAllWordlistsFrequency do
+
+  before do
+    @feature = Wikipedia::VandalismDetection::Features::RemovedAllWordlistsFrequency.new
+  end
+
+  it { should be_a Wikipedia::VandalismDetection::Features::FrequencyBase }
+
+  describe "#calculate" do
+
+    it "returns the removed number of all wordlists words relative to all removed words count" do
+      # inserted: total 8 words, 1 vulgarism, 1 biased, 2 pronouns = 4 bad
+      old_text = Wikipedia::VandalismDetection::Text.new 'Your old shit. Fuck you great, you and all the others.'
+      new_text = Wikipedia::VandalismDetection::Text.new 'Your old shit.'
+
+      old_revision = build(:old_revision, text: old_text)
+      new_revision = build(:new_revision, text: new_text)
+      edit = build(:edit, old_revision: old_revision, new_revision: new_revision)
+
+      expect(@feature.calculate(edit)).to eq 4.0 / 8.0
+    end
+
+    it "returns 0.0 on empty removed clean text" do
+      old_text = Wikipedia::VandalismDetection::Text.new 'Your old shit. {{speedy deletion}}'
+      new_text = Wikipedia::VandalismDetection::Text.new 'Your old shit.'
+
+      old_revision = build(:old_revision, text: old_text)
+      new_revision = build(:new_revision, text: new_text)
+      edit = build(:edit, old_revision: old_revision, new_revision: new_revision)
+
+      expect(@feature.calculate(edit)).to eq 0.0
+    end
+  end
+end

data/spec/vandalism_detection/features/removed_bad_frequency_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe Wikipedia::VandalismDetection::Features::RemovedBadFrequency do
+
+  before do
+    @feature = Wikipedia::VandalismDetection::Features::RemovedBadFrequency.new
+  end
+
+  it { should be_a Wikipedia::VandalismDetection::Features::FrequencyBase }
+
+  describe "#calculate" do
+
+    it "returns the number of removed bad words relative to all removed words count" do
+      # inserted: total 10 words, 4 biased
+      old_text = Wikipedia::VandalismDetection::Text.new "666 old. It's 666 man, this is 666, 666 a whatever."
+      new_text = Wikipedia::VandalismDetection::Text.new '666 old.'
+
+      old_revision = build(:old_revision, text: old_text)
+      new_revision = build(:new_revision, text: new_text)
+      edit = build(:edit, old_revision: old_revision, new_revision: new_revision)
+
+      expect(@feature.calculate(edit)).to eq 4.0 / 9.0
+    end
+
+    it "returns 0.0 on emtpy removed clean text" do
+      old_text = Wikipedia::VandalismDetection::Text.new "whatever old. {{speedy deletion}}"
+      new_text = Wikipedia::VandalismDetection::Text.new 'whatever old. whatever new.'
+
+      old_revision = build(:old_revision, text: old_text)
+      new_revision = build(:new_revision, text: new_text)
+      edit = build(:edit, old_revision: old_revision, new_revision: new_revision)
+
+      expect(@feature.calculate(edit)).to eq 0.0
+    end
+  end
+end

data/spec/vandalism_detection/features/removed_biased_frequency_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe Wikipedia::VandalismDetection::Features::RemovedBiasedFrequency do
+
+  before do
+    @feature = Wikipedia::VandalismDetection::Features::RemovedBiasedFrequency.new
+  end
+
+  it { should be_a Wikipedia::VandalismDetection::Features::FrequencyBase }
+
+  describe "#calculate" do
+
+    it "returns the number of removed biased words relative to all removed words count" do
+      # inserted: total 10 words, 4 biased
+      old_text = Wikipedia::VandalismDetection::Text.new "Great old. It's Great man, this is amazing, really a classic."
+      new_text = Wikipedia::VandalismDetection::Text.new 'Great old.'
+
+      old_revision = build(:old_revision, text: old_text)
+      new_revision = build(:new_revision, text: new_text)
+      edit = build(:edit, old_revision: old_revision, new_revision: new_revision)
+
+      expect(@feature.calculate(edit)).to eq 4.0 / 9.0
+    end
+
+    it "returns 0.0 on emtpy removed clean text" do
+      old_text = Wikipedia::VandalismDetection::Text.new "Great old. {{speedy deletion}}"
+      new_text = Wikipedia::VandalismDetection::Text.new 'Great old. Great new.'
+
+      old_revision = build(:old_revision, text: old_text)
+      new_revision = build(:new_revision, text: new_text)
+      edit = build(:edit, old_revision: old_revision, new_revision: new_revision)
+
+      expect(@feature.calculate(edit)).to eq 0.0
+    end
+  end
+end