wikipedia-vandalism_detection 0.1.0-java

data/spec/vandalism_detection/instances_spec.rb

@@ -0,0 +1,146 @@
+require 'spec_helper'
+require 'wikipedia/vandalism_detection/instances'
+
+describe Wikipedia::VandalismDetection::Instances do
+  Instances = Wikipedia::VandalismDetection::Instances
+
+  it 'responds to #empty' do
+    expect(Instances).to respond_to :empty
+  end
+
+  describe '#empty' do
+    before do
+      use_test_configuration
+
+      @dataset         = Instances.empty
+      @attributes      = @dataset.attributes
+      @class_attribute = @dataset.class_attribute
+    end
+
+    it 'returns a weka dataset' do
+      expect(@dataset.class).to eq Java::WekaCore::Instances
+    end
+
+    it 'returns an empty dataset' do
+      expect(@dataset.size).to eq 0
+    end
+
+    it 'has all configured features and class as attributes' do
+      names = test_config.features.map { |name| name.tr(' ', '_') }
+      expect(@dataset.attribute_names).to eq names
+    end
+
+    it 'has feature attributes of type "numeric"' do
+      all_features_numeric = @attributes.reduce do |result, attribute|
+        result && attribute.numeric?
+      end
+
+      expect(all_features_numeric).to be true
+    end
+
+    it 'has a nominal class attribute' do
+      expect(@class_attribute).to be_nominal
+    end
+
+    it 'has a class attribute with values "vandalism" and "regular"' do
+      expect(@class_attribute.values).to eq %w[regular vandalism]
+    end
+  end
+
+  describe '#empty_for_feature' do
+    before do
+      @dataset         = Instances.empty_for_feature('comment length')
+      @attributes      = @dataset.attributes
+      @class_attribute = @dataset.class_attribute
+    end
+
+    it 'returns a weka dataset' do
+      expect(@dataset).to be_a Java::WekaCore::Instances
+    end
+
+    it 'returns an empty dataset' do
+      expect(@dataset).to be_empty
+    end
+
+    it 'has only given feature and class as attributes' do
+      expect(@dataset.attribute_names).to eq %w[comment_length]
+    end
+
+    it 'has numeric feature attributes' do
+      expect(@attributes.first).to be_numeric
+    end
+
+    it 'has a nominal class attribute' do
+      expect(@class_attribute).to be_nominal
+    end
+
+    it 'has a class attribute with values "vandalism" and "regular"' do
+      expect(@class_attribute.values).to eq %w[regular vandalism]
+    end
+  end
+
+  describe '#empty_for_test_feature' do
+    before do
+      @dataset = Instances.empty_for_test_feature('comment length')
+
+      @feature_attribute         = @dataset.attributes.first
+      @old_revision_id_attribute = @dataset.attributes[-2]
+      @new_revision_id_attribute = @dataset.attributes.last
+    end
+
+    it 'returns a weka dataset' do
+      expect(@dataset).to be_a Java::WekaCore::Instances
+    end
+
+    it 'returns an empty dataset' do
+      expect(@dataset).to be_empty
+    end
+
+    it 'has one given feature as attributes' do
+      expect(@feature_attribute.name).to eq 'comment_length'
+    end
+
+    it 'has numeric feature attributes' do
+      expect(@feature_attribute).to be_numeric
+    end
+
+    it 'has an attribute with name "oldrevisionid"' do
+      expect(@old_revision_id_attribute.name).to  eq 'oldrevisionid'
+    end
+
+    it 'has a numeric oldrevisionid attribute' do
+      expect(@old_revision_id_attribute).to be_numeric
+    end
+
+    it 'has an attribute with name "newrevisionid"' do
+      expect(@new_revision_id_attribute.name).to eq 'newrevisionid'
+    end
+
+    it 'has a numeric newrevisionid attribute' do
+      expect(@new_revision_id_attribute).to be_numeric
+    end
+  end
+
+  describe '#empty_for_test_class' do
+    before do
+      @dataset = Instances.empty_for_test_class
+      @class   = @dataset.attributes.first
+    end
+
+    it 'returns a weka dataset' do
+      expect(@dataset).to be_a Java::WekaCore::Instances
+    end
+
+    it 'returns an empty dataset' do
+      expect(@dataset).to be_empty
+    end
+
+    it 'has one given feature as attributes' do
+      expect(@class.name).to eq 'class'
+    end
+
+    it 'has nominal feature attributes' do
+      expect(@class).to be_nominal
+    end
+  end
+end

data/spec/vandalism_detection/page_parser_spec.rb

@@ -0,0 +1,190 @@
+require 'spec_helper'
+
+describe Wikipedia::VandalismDetection::PageParser do
+  let(:xml)  { load_file('vandalism_on_wikipedia.xml') }
+  let(:page) { subject.parse(xml) }
+
+  let(:simplified_xml) { load_file('vandalism_on_wikipedia_simplified.xml') }
+  let(:simplified_page) { subject.parse(simplified_xml) }
+
+  describe 'parser structure' do
+    describe '#parse' do
+      it 'returns a Wikipedia::Page object' do
+        expect(page).to be_a Wikipedia::VandalismDetection::Page
+      end
+
+      it 'returns a Wikipedia::Page with the right number of revisions' do
+        expect(page.revisions.count).to eq 5
+      end
+    end
+  end
+
+  describe 'a single page content parsing' do
+    let(:revision_a) { build(:empty_revision, id: '398880281') }
+    let(:revision_b) { build(:empty_revision, id: '398880502', parent_id: '398880281') }
+    let(:revision_c) { build(:empty_revision, id: '398883278', parent_id: '398880502') }
+    let(:revision_d) { build(:empty_revision, id: '398883675', parent_id: '398883278') }
+    let(:revision_e) { build(:empty_revision, id: '398885233', parent_id: '398883675') }
+
+    let(:revisions) do
+      {
+        revision_a.id => revision_a,
+        revision_b.id => revision_b,
+        revision_c.id => revision_c,
+        revision_d.id => revision_d,
+        revision_e.id => revision_e
+      }
+    end
+
+    it 'can read a single page dump text input' do
+      expect(page).to_not be_nil
+    end
+
+    it 'has a title' do
+      expect(page.title).to eq 'Vandalism on Wikipedia'
+    end
+
+    it 'has an id' do
+      expect(page.id).to eq '29753790'
+    end
+
+    describe 'page’s revisions' do
+      it 'has the right number of revisions' do
+        expect(page.revisions.count).to eq 5
+      end
+
+      it 'has revisions each with the right id' do
+        page_revisions = page.revisions
+
+        page_revisions.each do |key, value|
+          expect(value.id).to eq revisions[key].id
+        end
+      end
+
+      it 'has revisions each with the right parent_id' do
+        page_revisions = page.revisions
+
+        page_revisions.each do |key, value|
+          expect(value.parent_id).to eq revisions[key].parent_id
+        end
+      end
+
+      it 'has revisions each with the right text' do
+        revision_a = build(:empty_revision, id: '1', text: "text\n\n\n        1\n\n      ")
+        revision_b = build(:empty_revision, id: '2', text: 'text 2')
+        revision_c = build(:empty_revision, id: '3', text: 'text 3')
+        revision_d = build(:empty_revision, id: '4', text: 'text 4')
+        revision_e = build(:empty_revision, id: '5', text: 'text 5')
+
+        revisions = {
+          revision_a.id => revision_a,
+          revision_b.id => revision_b,
+          revision_c.id => revision_c,
+          revision_d.id => revision_d,
+          revision_e.id => revision_e
+        }
+
+        simplified_page.revisions.each do |key, value|
+          expect(value.text).to eq revisions[key].text
+        end
+      end
+
+      it 'has revisions each with the right comment' do
+        revision_a = build(:empty_revision, id: '1', comment: "comment\n\n        1\n\n      ")
+        revision_b = build(:empty_revision, id: '2', comment: 'comment 2')
+        revision_c = build(:empty_revision, id: '3')
+        revision_d = build(:empty_revision, id: '4')
+        revision_e = build(:empty_revision, id: '5', comment: 'comment 3')
+
+        revisions = {
+          revision_a.id => revision_a,
+          revision_b.id => revision_b,
+          revision_c.id => revision_c,
+          revision_d.id => revision_d,
+          revision_e.id => revision_e
+        }
+
+        simplified_page.revisions.each do |key, value|
+          expect(value.comment).to eq revisions[key].comment
+        end
+      end
+
+      describe 'contributor properties' do
+        let(:revision_a) { build(:empty_revision, id: '1', contributor: '1') }
+        let(:revision_b) { build(:empty_revision, id: '2', contributor: '10', contributor_username: 'user') }
+        let(:revision_c) { build(:empty_revision, id: '3', contributor: '11', contributor_username: 'user') }
+        let(:revision_d) { build(:empty_revision, id: '4', contributor: '12', contributor_username: 'user') }
+        let(:revision_e) { build(:empty_revision, id: '5', contributor: '2') }
+
+        let(:revisions) do
+          {
+            revision_a.id => revision_a,
+            revision_b.id => revision_b,
+            revision_c.id => revision_c,
+            revision_d.id => revision_d,
+            revision_e.id => revision_e
+          }
+        end
+
+        it 'has revisions each with the right contributor id' do
+          simplified_page.revisions.each do |key, value|
+            expect(value.contributor_id).to eq revisions[key].contributor_id
+          end
+        end
+
+        it 'has revisions each with the right contributor ip' do
+          simplified_page.revisions.each do |key, value|
+            expect(value.contributor_ip).to eq revisions[key].contributor_ip
+          end
+        end
+
+        it 'has revisions each with the right contributor username' do
+          simplified_page.revisions.each do |key, value|
+            username = revisions[key].contributor_username
+            expect(value.contributor_username).to eq username
+          end
+        end
+      end
+
+      it 'has revisions each with the right timestamp' do
+        revision_a = build :empty_revision, id: '1', timestamp: 'time 1'
+        revision_b = build :empty_revision, id: '2', timestamp: 'time 2'
+        revision_c = build :empty_revision, id: '3', timestamp: 'time 3'
+        revision_d = build :empty_revision, id: '4', timestamp: 'time 4'
+        revision_e = build :empty_revision, id: '5', timestamp: 'time 5'
+
+        revisions = {
+          revision_a.id => revision_a,
+          revision_b.id => revision_b,
+          revision_c.id => revision_c,
+          revision_d.id => revision_d,
+          revision_e.id => revision_e
+        }
+
+        simplified_page.revisions.each do |key, value|
+          expect(value.timestamp).to eq revisions[key].timestamp
+        end
+      end
+
+      it 'has revisions each with the right sha1 hash' do
+        revision_a = build :empty_revision, id: '1', sha1: 'hash1'
+        revision_b = build :empty_revision, id: '2', sha1: 'hash2'
+        revision_c = build :empty_revision, id: '3', sha1: 'hash3'
+        revision_d = build :empty_revision, id: '4', sha1: 'hash4'
+        revision_e = build :empty_revision, id: '5', sha1: 'hash5'
+
+        revisions = {
+          revision_a.id => revision_a,
+          revision_b.id => revision_b,
+          revision_c.id => revision_c,
+          revision_d.id => revision_d,
+          revision_e.id => revision_e
+        }
+
+        simplified_page.revisions.each do |key, value|
+          expect(value.sha1).to eq revisions[key].sha1
+        end
+      end
+    end
+  end
+end

data/spec/vandalism_detection/page_spec.rb

@@ -0,0 +1,134 @@
+require 'spec_helper'
+
+describe Wikipedia::VandalismDetection::Page do
+  describe 'constants' do
+    it 'has a START_TAG constant' do
+      expect(Page::START_TAG).to eq '<page>'
+    end
+
+    it 'has an END_Tag constant' do
+      expect(Page::END_TAG).to eq '</page>'
+    end
+  end
+
+  it 'has a title' do
+    expect(subject).to respond_to :title
+  end
+
+  it 'has an id' do
+    expect(subject).to respond_to :id
+  end
+
+  it 'has revisions defaulting to an empty Hash' do
+    expect(subject).to respond_to :revisions
+    expect(subject.revisions).to be_a Hash
+    expect(subject.revisions).to be_empty
+  end
+
+  describe '#edits' do
+    it { is_expected.to respond_to :edits }
+
+    it 'returns an empty array if no revision is available' do
+      expect(subject.revisions).to be_empty
+      expect(subject.edits).to be_an Array
+      expect(subject.edits).to be_empty
+    end
+
+    it 'resets the @revision_added flag to false' do
+      subject.add_revision build(:empty_revision, id: '1')
+      subject.edits
+      expect(subject.instance_variable_get(:@update_edits)).to be false
+    end
+
+    it 'computes edits from the page’s revisions' do
+      subject.add_revision build(:empty_revision, id: '1')
+      subject.add_revision build(:empty_revision, id: '3', parent_id: '2')
+      subject.add_revision build(:empty_revision, id: '2', parent_id: '1')
+
+      expect(subject.edits.count).to eq 2
+    end
+
+    it 'computes edits of which each holds the parent page as reference' do
+      subject.id = '1234'
+      subject.title = 'Article'
+
+      subject.add_revision build(:empty_revision, id: '1')
+      subject.add_revision build(:empty_revision, id: '3', parent_id: '2')
+      subject.add_revision build(:empty_revision, id: '2', parent_id: '1')
+
+      subject.edits.each { |edit| expect(edit.page).to eq subject }
+    end
+  end
+
+  describe '#add_revision' do
+    it { is_expected.to respond_to :add_revision }
+
+    it 'takes a revision and adds it to revisions' do
+      revision = build(:empty_revision)
+
+      expect { subject.add_revision(revision) }
+        .to change(subject.revisions, :count)
+        .by(1)
+    end
+
+    it 'sets the @update_edits flag to true after adding a revision' do
+      revision = build :empty_revision
+      subject.add_revision(revision)
+      expect(subject.instance_variable_get(:@update_edits)).to be true
+    end
+
+    it 'sets the @update_reverted_edits flag to true after adding a revision' do
+      revision = build :empty_revision
+      subject.add_revision(revision)
+      expect(subject.instance_variable_get(:@update_reverted_edits)).to be true
+    end
+  end
+
+  describe '#reverted_edits' do
+    it { is_expected.to respond_to :reverted_edits }
+
+    it 'returns reverted edits by comparing the sha1 values' do
+      # principle:
+      # in edit wars the in-between of the first revert triple which has another
+      # hash before can be seen as vandalism (here revision with id 2)
+
+      revision_a = build(:empty_revision, id: 1, parent_id: nil, sha1: 'hash0')
+      revision_b = build(:empty_revision, id: 2, parent_id: 1, sha1: 'hash1')
+      revision_c = build(:empty_revision, id: 3, parent_id: 2, sha1: 'hash2')
+      revision_d = build(:empty_revision, id: 4, parent_id: 3, sha1: 'hash1')
+      revision_e = build(:empty_revision, id: 5, parent_id: 4, sha1: 'hash2')
+      revision_f = build(:empty_revision, id: 6, parent_id: 5, sha1: 'hash3')
+
+      subject.add_revision(revision_c)
+      subject.add_revision(revision_f)
+      subject.add_revision(revision_a)
+      subject.add_revision(revision_e)
+      subject.add_revision(revision_d)
+      subject.add_revision(revision_b)
+
+      reverted_ids = subject.reverted_edits.map do |edit|
+        edit.new_revision.id
+      end
+
+      expect(reverted_ids).to eq [revision_c.id]
+    end
+
+    it 'returns reverted edit if no previous revision is available' do
+      revision_a = build(:empty_revision, id: 1, parent_id: nil, sha1: 'hash1')
+      revision_b = build(:empty_revision, id: 2, parent_id: 1, sha1: 'hash2')
+      revision_c = build(:empty_revision, id: 3, parent_id: 2, sha1: 'hash1')
+      revision_d = build(:empty_revision, id: 4, parent_id: 3, sha1: 'hash2')
+
+      subject.add_revision(revision_c)
+      subject.add_revision(revision_a)
+      subject.add_revision(revision_d)
+      subject.add_revision(revision_b)
+
+      reverted_ids = subject.reverted_edits.map do |edit|
+        edit.new_revision.id
+      end
+
+      expect(reverted_ids).to eq [revision_b.id]
+    end
+  end
+end

data/spec/vandalism_detection/revision_parser_spec.rb

@@ -0,0 +1,53 @@
+require 'spec_helper'
+
+describe Wikipedia::VandalismDetection::RevisionParser do
+  let(:xml) { load_file('revision_simplified.xml') }
+  let(:revision) { subject.parse(xml) }
+
+  let(:expected_revision) do
+    build(
+      :empty_revision,
+      id:          'id1',
+      parent_id:   'parentid1',
+      timestamp:   'time1',
+      contributor: 'ip1',
+      comment:     'comment 1',
+      text:        'text 1',
+      sha1:        'hash1'
+    )
+  end
+
+  describe '#parse' do
+    it 'returns a Wikipedia::Revision object' do
+      expect(revision).to be_a Wikipedia::VandalismDetection::Revision
+    end
+
+    it 'returns a revision with only the configured properties' do
+      revision = subject.parse(xml, only: %i[id parent_id])
+
+      %i[id parent_id].each do |attribute|
+        expect(revision.send(attribute)).not_to be_nil
+      end
+
+      %i[timestamp contributor sha1].each do |attribute|
+        expect(revision.send(attribute)).to be_nil
+      end
+
+      %i[comment text].each do |attribute|
+        expect(revision.send(attribute)).to eq ''
+      end
+    end
+  end
+
+  describe 'a single revison content parsing' do
+    it 'can read a single revsion dump text input' do
+      expect(revision).to_not be_nil
+    end
+
+    %i[id timestamp contributor comment text sha1].each do |attribute|
+      it "has the expected #{attribute}" do
+        expect(revision.send(attribute)).to eq expected_revision.send(attribute)
+      end
+    end
+  end
+end

data/spec/vandalism_detection/revision_spec.rb

@@ -0,0 +1,148 @@
+require 'spec_helper'
+
+describe Wikipedia::VandalismDetection::Revision do
+  Revision = Wikipedia::VandalismDetection::Revision
+
+  describe 'constants' do
+    it 'has a START_TAG constant' do
+      expect(Revision::START_TAG).to eq '<revision>'
+    end
+
+    it 'has an END_TAG constant' do
+      expect(Revision::END_TAG).to eq '</revision>'
+    end
+  end
+
+  let(:revision) { Revision.new }
+
+  describe '#anonymous_user?' do
+    it { is_expected.to respond_to :anonymous_contributor? }
+
+    it 'returns true in case of an anonymous user' do
+      anonymous_revision = build(:anonymous_revision)
+      expect(anonymous_revision.anonymous_contributor?).to be true
+    end
+  end
+
+  describe '#contributor=' do
+    it { is_expected.to respond_to :contributor= }
+
+    it 'sets the @contributor_id if contributor is no IPv4' do
+      id = '12345'
+      revision.contributor = id
+
+      expect(revision.instance_variable_get(:@contributor_id)).to eq id
+      expect(revision.instance_variable_get(:@contributor_ip)).to be_nil
+    end
+
+    it 'sets the @contributor_ip if contributor is an IPv4' do
+      ip = '127.0.0.1'
+      revision.contributor = ip
+
+      expect(revision.instance_variable_get(:@contributor_ip)).to eq ip
+      expect(revision.instance_variable_get(:@contributor_id)).to be_nil
+    end
+  end
+
+  describe '#contributor' do
+    it { is_expected.to respond_to :contributor }
+
+    it 'returns the contributor_id if set' do
+      id = '12345'
+      revision.contributor = id
+      contributor_id = revision.instance_variable_get(:@contributor_id)
+
+      expect(revision.contributor).to eq contributor_id
+    end
+
+    it 'returns the contributor_ip if set' do
+      ip = '127.0.0.1'
+      revision.contributor = ip
+      contributor_ip = revision.instance_variable_get(:@contributor_ip)
+
+      expect(revision.contributor).to eq contributor_ip
+    end
+  end
+
+  it 'has the revision attributes' do
+    instance_variables = %i[
+      id
+      parent_id
+      timestamp
+      comment
+      text
+      contributor_id
+      contributor_ip
+      sha1
+    ]
+
+    instance_variables.each do |name|
+      expect(revision).to respond_to name
+    end
+  end
+
+  it 'defaults its attributes to nil' do
+    instance_variables = %i[
+      id
+      parent_id
+      timestamp
+      contributor_id
+      contributor_ip
+      contributor_username
+      sha1
+    ]
+
+    instance_variables.each do |name|
+      expect(revision.send(name)).to be_nil
+    end
+  end
+
+  it 'does not allow setting read only attributes' do
+    attributes = %i[contributor_id contributor_ip]
+    attributes.each { |name| expect(revision).not_to respond_to "#{name}=" }
+  end
+
+  it 'has an empty default text' do
+    expect(revision.text).to be_empty
+  end
+
+  it 'sets an empty text if no String is assigned' do
+    revision.text = nil
+    expect(revision.text).to eq ''
+
+    revision.text = []
+    expect(revision.text).to eq ''
+  end
+
+  it 'has a text of type Wikipedia::VandalismDetection::Text' do
+    expect(revision.text).to be_a Text
+  end
+
+  it 'has an empty default comment' do
+    expect(revision.comment).to be_empty
+  end
+
+  it 'has a comment of type Wikipedia::VandalismDetection::Text' do
+    expect(revision.comment).to be_a Text
+  end
+
+  it 'sets an empty comment if no String is assigned' do
+    revision.comment = nil
+    expect(revision.comment).to eq ''
+
+    revision.comment = []
+    expect(revision.comment).to eq ''
+  end
+
+  it { is_expected.to respond_to :redirect? }
+
+  it 'is marked as redirect if #REDIRECT appears in its text' do
+    revision.text = "#REDIRECT [[Redirect Page Name]]\n"
+    expect(revision.redirect?).to be true
+  end
+
+  it 'is not marked as redirect if #REDIRECT does not appear in its text' do
+    revision.text = "''text''"
+    expect(revision.redirect?).to be false
+  end
+end