wikipedia-vandalism_detection 0.1.0-java

data/spec/vandalism_detection/test_dataset_spec.rb

@@ -0,0 +1,227 @@
+require 'spec_helper'
+require 'fileutils'
+require 'weka'
+
+describe Wikipedia::VandalismDetection::TestDataset do
+  subject { Wikipedia::VandalismDetection::TestDataset }
+
+  before do
+    use_test_configuration
+    @config = test_config
+
+    @arff_file  = @config.test_output_arff_file
+    @index_file = @config.test_output_index_file
+    @features   = @config.features
+
+    @arff_files_dir = File.join(@config.output_base_directory, 'test')
+  end
+
+  after do
+    if File.exist?(@arff_file)
+      File.delete(@arff_file)
+      directory = File.dirname(@arff_file)
+      FileUtils.rm_r(directory)
+    end
+
+    File.delete(@index_file) if File.exist?(@index_file)
+
+    # remove feature arff files
+    @config.features.each do |name|
+      file = File.join(@arff_files_dir, "#{name.tr(' ', '_')}.arff")
+
+      next unless File.exist?(file)
+
+      File.delete(file)
+      directory = File.dirname file
+      FileUtils.rm_r(directory)
+    end
+  end
+
+  describe '#build' do
+    describe 'exceptions' do
+      it 'raises an error if no edits file is configured' do
+        config = test_config
+        config.instance_variable_set(:@test_corpus_edits_file, nil)
+        use_configuration(config)
+
+        expect { subject.build }.to raise_error \
+          Wikipedia::VandalismDetection::EditsFileNotConfiguredError
+      end
+    end
+
+    it 'returns a weka instances' do
+      expect(subject.build).to be_a Java::WekaCore::Instances
+    end
+
+    Wikipedia::VandalismDetection::DefaultConfiguration::DEFAULTS['features'].each do |name|
+      it "creates an arff file for the feature '#{name}'" do
+        config = test_config
+        config.instance_variable_set(:@features, [name])
+        use_configuration(config)
+
+        file = File.join(@arff_files_dir, "#{name.tr(' ', '_')}.arff")
+
+        expect(File.exist?(file)).to be false
+        subject.build
+        expect(File.exist?(file)).to be true
+      end
+    end
+
+    it 'creates only feature files that are not available yet' do
+      config = test_config
+      config.instance_variable_set(:@features, ['anonymity', 'comment length'])
+      use_configuration(config)
+
+      anonymity_file = File.join(config.output_base_directory, 'test', 'anonymity.arff')
+
+      # create file manually, so it is existent when building dataset
+      data = [1, 2, 3]
+      anonymity = Wikipedia::VandalismDetection::Instances.empty_for_test_feature('anonymity')
+      6.times { anonymity.add_instance(data) }
+      anonymity.to_arff(anonymity_file)
+
+      Wikipedia::VandalismDetection::TestDataset.build
+
+      # anonymity should not be overwritten
+      values = Weka::Core::Instances.from_arff(anonymity_file).first.values
+      expect(values).to eq data
+    end
+
+    describe 'internal algorithm' do
+      it 'builds the right number of data lines' do
+        edits_count = File.open(@config.training_corpus_edits_file, 'r').lines.count - 1
+        additional_header_lines = 4 # without class
+        revision_id_lines       = 2 # old and new revision id attributes
+        class_line              = 1
+
+        lines_count = [
+          additional_header_lines,
+          edits_count,
+          @features.count,
+          revision_id_lines,
+          class_line
+        ].inject(:+)
+
+        dataset = subject.build
+
+        expect(dataset.to_s.lines.count).to eq lines_count
+      end
+
+      it 'builds the right number of data columns' do
+        old_and_new_edit_attr_count = 2
+        class_value      = 1
+        dataset          = subject.build
+
+        attributes_count = [
+          @features.count,
+          class_value,
+          old_and_new_edit_attr_count
+        ].inject(:+)
+
+        expect(dataset.attributes_count).to eq attributes_count
+      end
+
+      it 'builds a class attribute' do
+        dataset = subject.build
+        expect(dataset.attributes.last.name).to eq 'class'
+      end
+    end
+  end
+
+  describe '#instances' do
+    it 'is an alias method for #build' do
+      build     = subject.method(:build)
+      instances = subject.method(:instances)
+
+      expect(build).to eq instances
+    end
+  end
+
+  describe '#create_corpus_index_file!' do
+    it 'responds to #create_corpus_file_index!' do
+      expect(subject).to respond_to :create_corpus_file_index!
+    end
+
+    describe 'exceptions' do
+      it 'raises an error unless revisions directory is configured' do
+        config = test_config
+        config.instance_variable_set(:@test_corpus_revisions_directory, nil)
+        use_configuration(config)
+
+        expect { subject.create_corpus_file_index! }.to raise_error \
+          Wikipedia::VandalismDetection::RevisionsDirectoryNotConfiguredError
+      end
+    end
+
+    it 'creates a corpus_index.yml file in the build directory' do
+      expect(File.exist?(@index_file)).to be false
+      subject.create_corpus_file_index!
+      expect(File.exist?(@index_file)).to be true
+    end
+  end
+
+  describe '#build!' do
+    it { is_expected.to respond_to :build! }
+
+    it 'creates an .arff file in the configured irectory' do
+      expect(File.exist?(@arff_file)).to be false
+      subject.build!
+      expect(File.exist?(@arff_file)).to be true
+    end
+
+    it 'overwrites existing test arff file' do
+      use_test_configuration
+
+      # test config uses 3 features + 2 edit id columns + 1 class value = 6
+      subject.build!
+      first_parsed_dataset = Weka::Core::Instances.from_arff(@arff_file)
+      expect(first_parsed_dataset.attributes_count).to eq 6
+
+      config = test_config
+      config.instance_variable_set(:@features, ['anonymity'])
+      use_configuration(config)
+
+      # uses only 1 feature + 2 edit id columns + 1 class vlaue = 4
+      subject.build!
+      second_parsed_dataset = Weka::Core::Instances.from_arff(@arff_file)
+
+      expect(second_parsed_dataset.attributes_count).to eq 4
+    end
+  end
+
+  describe '#edit' do
+    it 'raises an error unless edits file is configured' do
+      config = test_config
+      config.instance_variable_set(:@test_corpus_edits_file, nil)
+      use_configuration(config)
+
+      expect { subject.edit('1', '2') }.to raise_error \
+        Wikipedia::VandalismDetection::EditsFileNotConfiguredError
+    end
+
+    it 'returns nil if Edit could not be found' do
+      edit = subject.edit('1', '2')
+      expect(edit).to be_nil
+    end
+
+    it 'returns an Edit' do
+      edit = subject.edit('307084144', '326873205')
+      expect(edit).to be_an Edit
+    end
+
+    it 'returns an edit whose parent page title is not nil' do
+      edit = subject.edit('307084144', '326873205')
+      expect(edit.page.title).to_not be_nil
+    end
+
+    it 'returns an edit whose parent page id is not nil' do
+      edit = subject.edit('307084144', '326873205')
+      expect(edit.page.id).to_not be_nil
+    end
+
+    it 'returns nil for a not annotated edit with given revision ids' do
+      edit = subject.edit('328774088', '328774188')
+      expect(edit).to be_nil
+    end
+  end
+end

data/spec/vandalism_detection/text_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+
+describe Text do
+  it { is_expected.to be_a String }
+  it { is_expected.to respond_to :clean }
+
+  describe '#initialze' do
+    it 'removes invalid byte sequences' do
+      text = Text.new("text \255".force_encoding('UTF-8'))
+      expect(text).to eq 'text '
+    end
+  end
+
+  describe '#clean' do
+    it 'raises an WikitextExtractionError if text cannot be parsed' do
+      text = Text.new("[[Image:img.jpg|\n{|\n|-\n|||| |}")
+
+      expect { text.clean }.to raise_error \
+        Wikipedia::VandalismDetection::WikitextExtractionError
+    end
+
+    it 'returns the text cleaned from wiki tags' do
+      wiki_text = Text.new(load_file('sample_revision.txt'))
+      clean_text = load_file('sample_revision_clean_text.txt')
+
+      expect(wiki_text.clean).to eq clean_text
+    end
+  end
+end

data/spec/vandalism_detection/training_dataset_spec.rb

@@ -0,0 +1,266 @@
+require 'spec_helper'
+require 'fileutils'
+
+describe Wikipedia::VandalismDetection::TrainingDataset do
+  before do
+    use_test_configuration
+    @config = test_config
+
+    @arff_file = @config.training_output_arff_file
+    @index_file = @config.training_output_index_file
+    @annotations_file = @config.training_corpus_annotations_file
+
+    @arff_files_dir = File.join(@config.output_base_directory, 'training')
+  end
+
+  after do
+    if File.exist?(@arff_file)
+      File.delete(@arff_file)
+      directory = File.dirname(@arff_file)
+      FileUtils.rm_r(directory)
+    end
+
+    File.delete(@index_file) if File.exist?(@index_file)
+
+    # remove feature arff files
+    @config.features.each do |name|
+      file = File.join(@arff_files_dir, "#{name.tr(' ', '_')}.arff")
+
+      next unless File.exist?(file)
+
+      File.delete(file)
+      directory = File.dirname(file)
+      FileUtils.rm_r(directory)
+    end
+  end
+
+  describe '#build' do
+    it 'returns a weka instances' do
+      dataset = TrainingDataset.build
+      expect(dataset).to be_a Java::WekaCore::Instances
+    end
+
+    describe 'exceptions' do
+      it 'raises error unless edits file is configured' do
+        config = test_config
+        config.instance_variable_set(:@training_corpus_edits_file, nil)
+        use_configuration(config)
+
+        expect { TrainingDataset.build }.to raise_error \
+          Wikipedia::VandalismDetection::EditsFileNotConfiguredError
+      end
+
+      it 'raises error unless annotations file is configured' do
+        config = test_config
+        config.instance_variable_set(:@training_corpus_annotations_file, nil)
+        use_configuration(config)
+
+        expect { TrainingDataset.build }.to raise_error \
+          Wikipedia::VandalismDetection::AnnotationsFileNotConfiguredError
+      end
+    end
+
+    Wikipedia::VandalismDetection::DefaultConfiguration::DEFAULTS['features'].each do |name|
+      it "creates an arff file for the feature '#{name}'" do
+        config = test_config
+        config.instance_variable_set(:@features, [name])
+        use_configuration(config)
+
+        file = File.join(@arff_files_dir, "#{name.tr(' ', '_')}.arff")
+
+        expect(File.exist?(file)).to be false
+        TrainingDataset.build
+        expect(File.exist?(file)).to be true
+      end
+    end
+
+    it 'creates only feature files that are not available yet' do
+      config = test_config
+      config.instance_variable_set(:@features, ['anonymity', 'comment length'])
+      use_configuration(config)
+
+      anonymity_file = File.join(config.output_base_directory, 'test', 'anonymity.arff')
+
+      # create file manually, so it is existent when building the dataset
+      data = [1, 2, 3]
+      anonymity = Instances.empty_for_test_feature('anonymity')
+      6.times { anonymity.add_instance(data) }
+      anonymity.to_arff(anonymity_file)
+
+      TrainingDataset.build
+
+      # anonymity should not be overwritten
+      values = Weka::Core::Instances.from_arff(anonymity_file).first.values
+      expect(values).to eq data
+    end
+
+    describe 'internal algorithm' do
+      let(:features_count) { @config.features.count }
+
+      it 'builds the right number of data lines' do
+        dataset = TrainingDataset.build
+        annotations_count = File.open(@annotations_file, 'r').lines.count - 1
+        additional_header_lines = 5
+
+        total_lines = additional_header_lines + annotations_count + features_count
+
+        expect(dataset.to_s.lines.count).to eq total_lines
+      end
+
+      it 'builds the right number of data columns' do
+        dataset = TrainingDataset.build
+        expect(dataset.attributes_count).to eq features_count + 1
+      end
+    end
+
+    describe 'replacing missing values' do
+      it 'replaces missing values if configured' do
+        config = test_config
+        config.instance_variable_set(:@replace_missing_values, 'true')
+        use_configuration(config)
+
+        dataset = TrainingDataset.build
+
+        filter = /weka\.filters\.unsupervised\.attribute\.ReplaceMissingValues/
+        expect(dataset.to_s).to match filter
+      end
+
+      it 'does not replace missing values if not configured' do
+        config = test_config
+        config.instance_variable_set(:@replace_missing_values, 'Nope')
+        use_configuration(config)
+
+        dataset = TrainingDataset.build
+
+        filter = /weka\.filters\.unsupervised\.attribute\.ReplaceMissingValues/
+        expect(dataset.to_s).not_to match filter
+      end
+    end
+  end
+
+  describe '#instances' do
+    it 'is an alias method for #build' do
+      build = TrainingDataset.method(:build)
+      instances = TrainingDataset.method(:instances)
+
+      expect(build).to eq instances
+    end
+  end
+
+  describe '#balanced_instances' do
+    before do
+      config = test_config
+      config.instance_variable_set(:@training_data_options, 'balanced')
+      use_configuration(config)
+
+      @dataset = TrainingDataset.balanced_instances
+    end
+
+    it 'returns a weka dataset' do
+      expect(@dataset).to be_a Java::WekaCore::Instances
+    end
+
+    it 'returns a dataset of rigth size built from the configured corpus' do
+      # 2 vandalism, 2 regular, see resources/corpora/training/annotations.csv
+      expect(@dataset.size).to eq 4
+    end
+
+    %i[VANDALISM REGULAR].each do |class_const|
+      it "has the right number of '#{class_const.downcase}' samples in its instances" do
+        class_count = @dataset.enumerate_instances.reduce(0) do |count, instance|
+          label = Instances::CLASSES[instance.class_value.to_i]
+          value = Instances.const_get(class_const)
+
+          label == value ? count + 1 : count
+        end
+
+        expect(class_count).to eq 2
+      end
+    end
+  end
+
+  describe '#oversampled_instances' do
+    describe 'with default options' do
+      before do
+        config = test_config
+        config.instance_variable_set(:@training_data_options, 'oversampled')
+        use_configuration(config)
+
+        # default -P 100 -U true
+        @dataset = TrainingDataset.oversampled_instances
+      end
+
+      it 'returns a weka dataset' do
+        expect(@dataset).to be_a Java::WekaCore::Instances
+      end
+
+      it 'returns a dataset of size 8 built from the configured corpus' do
+        # 4 vandalism, 4 regular, see resources/corpora/training/annotations.csv
+        expect(@dataset.size).to eq 8
+      end
+
+      %i[VANDALISM REGULAR].each do |class_const|
+        it "has the right number of '#{class_const.downcase}' samples in its instances" do
+          class_count = @dataset.enumerate_instances.reduce(0) do |count, instance|
+            label = Instances::CLASSES[instance.class_value.to_i]
+            value = Instances.const_get(class_const)
+
+            label == value ? count + 1 : count
+          end
+
+          expect(class_count).to eq 4
+        end
+      end
+
+      it 'returns the right-sized SMOTEd dataset from the configured corpus' do
+        # 4 vandalism, 4 regular, see resources/corpora/training/annotations.csv
+        dataset = TrainingDataset.oversampled_instances(percentage: 200)
+        expect(dataset.size).to eq 8
+      end
+    end
+
+    describe 'with custom options' do
+      before do
+        config = test_config
+        options = 'oversampled -p 300 -u false'
+        config.instance_variable_set(:@training_data_options, options)
+        use_configuration(config)
+
+        @dataset = TrainingDataset.oversampled_instances
+      end
+
+      it 'returns a weka dataset' do
+        expect(@dataset).to be_a Java::WekaCore::Instances
+      end
+
+      it 'returns the right dataset size built from the configured corpus' do
+        # 2 + 300 % = 8 vandalism, 4 regular,
+        # see resources/corpora/training/annotations.csv
+        expect(@dataset.size).to eq 12
+      end
+    end
+  end
+
+  describe '#create_corpus_index_file!' do
+    it 'responds to #create_corpus_file_index!' do
+      expect(TrainingDataset).to respond_to :create_corpus_file_index!
+    end
+
+    describe 'exceptions' do
+      it 'raises an error if no revisions directory is configured' do
+        config = test_config
+        config.instance_variable_set(:@training_corpus_revisions_directory, nil)
+        use_configuration(config)
+
+        expect { TrainingDataset.create_corpus_file_index! }.to raise_error \
+          Wikipedia::VandalismDetection::RevisionsDirectoryNotConfiguredError
+      end
+    end
+
+    it 'creates a corpus_index.yml file in the build directory' do
+      expect(File.exist?(@index_file)).to be false
+      TrainingDataset.create_corpus_file_index!
+      expect(File.exist?(@index_file)).to be true
+    end
+  end
+end

data/spec/vandalism_detection/wikitext_extractor_spec.rb

@@ -0,0 +1,97 @@
+require 'spec_helper'
+
+describe  Wikipedia::VandalismDetection::WikitextExtractor do
+  subject { Wikipedia::VandalismDetection::WikitextExtractor }
+
+  describe '.extract' do
+    it 'can handle invalid byte sequences' do
+      wiki_text = "text \255".force_encoding('UTF-8')
+      extracted_text = subject.extract(wiki_text)
+
+      expect(extracted_text).to eq 'text'
+    end
+
+    it 'returns an empty string if all the markup is extracted' do
+      wiki_text = '{{speedy deletion}}'
+      extracted_text = subject.extract(wiki_text)
+
+      expect(extracted_text).to be_empty
+    end
+
+    it 'removes #REDIRECT markup' do
+      wiki_text = '#REDIRECT [[Heading]]'
+      plain_text = 'Heading'
+      extracted_text = subject.extract(wiki_text)
+
+      expect(extracted_text).to eq plain_text
+    end
+
+    it 'can extract plaintext from wikitext' do
+      wiki_text = load_file('sample_revision.txt')
+      plain_text = load_file('sample_revision_plain_text.txt')
+      extracted_text = subject.extract(wiki_text)
+
+      expect(extracted_text << "\n").to eq plain_text
+    end
+
+    it 'raises a WikitextExtractionError when extracting unparsable text' do
+      unparsable_wiki_text = "[[Image:img.jpg|\n{|\n|-\n|||| |}"
+
+      expect { subject.extract(unparsable_wiki_text) }
+        .to raise_error Wikipedia::VandalismDetection::WikitextExtractionError
+    end
+  end
+
+  describe '.extract_clean' do
+    it 'can extract full cleaned text from wikitext' do
+      wiki_text = load_file('sample_revision.txt')
+      clean_text = load_file('sample_revision_clean_text.txt')
+      extracted_text = subject.extract_clean(wiki_text)
+
+      expect(extracted_text).to eq clean_text
+    end
+
+    it 'removes section numbering while cleaning wikitext' do
+      wiki_text = "1.1. header 1\n\n1.2. header 2"
+      clean_text = 'header 1 header 2'
+      extracted_text = subject.extract_clean(wiki_text)
+
+      expect(extracted_text).to eq clean_text
+    end
+
+    it 'removes line breaks while cleaning wikitext' do
+      wiki_text = "line 1\n\nline 2\nline 3"
+      clean_text = 'line 1 line 2 line 3'
+      extracted_text = subject.extract_clean(wiki_text)
+
+      expect(extracted_text).to eq clean_text
+    end
+
+    it 'removes multiple spaces while cleaning wikitext' do
+      wiki_text = "line    1  \n\nline   2   \nline       3   "
+      clean_text = 'line 1 line 2 line 3'
+      extracted_text = subject.extract_clean(wiki_text)
+
+      expect(extracted_text).to eq clean_text
+    end
+
+    it 'removes links from text while cleaning wikitext' do
+      wiki_text = "A link\nis here http://example.com/image.jpg not\nanymore." \
+                  "\n==Reference==\n" \
+                  '*[http://www.itis.usda.gov/servlet/SingleRpt/SingleRpt?' \
+                  'search_topic=TSN&amp;search_value=180211 ITIS 180211] 2002-12-14'
+
+      clean_text = 'A link is here not anymore. Reference ITIS 180211 2002-12-14'
+      extracted_text = subject.extract_clean(wiki_text)
+
+      expect(extracted_text).to eq clean_text
+    end
+
+    it 'raises a WikitextExtractionError while extracting unparsable text' do
+      unparsable_wiki_text = "[[Image:img.jpg|\n{|\n|-\n|||| |}"
+
+      expect { subject.extract_clean(unparsable_wiki_text) }
+        .to raise_error Wikipedia::VandalismDetection::WikitextExtractionError
+    end
+  end
+end

data/spec/weka/classifiers/meta/one_class_classifier_spec.rb

@@ -0,0 +1,82 @@
+require 'spec_helper'
+
+describe Weka::Classifiers::Meta::OneClassClassifier do
+  it { is_expected.to be_a Java::WekaClassifiersMeta::OneClassClassifier }
+
+  let(:classifier_type) { 'Meta::OneClassClassifier' }
+
+  before do
+    @config = test_config
+    classifier_options = '-W weka.classifiers.trees.RandomForest -- -I 100'
+    @w_options = "-W weka.classifiers.meta.Bagging -- #{classifier_options}"
+    vandalism = Wikipedia::VandalismDetection::Instances::VANDALISM
+    options = "-tcl #{vandalism} #{@w_options}"
+
+    @config.instance_variable_set(:@classifier_type, classifier_type)
+    @config.instance_variable_set(:@classifier_options, options)
+    @config.instance_variable_set(:@cross_validation_fold, 2)
+
+    use_configuration(@config)
+
+    # add more test instances because instances number must be higher than
+    # cross validation fold:
+    data = Wikipedia::VandalismDetection::TrainingDataset.instances.to_m.to_a
+    dataset = Wikipedia::VandalismDetection::Instances.empty
+
+    2.times do
+      data.each do |row|
+        values = row[0..-2]
+        index = rand((0..1))
+        class_value = Wikipedia::VandalismDetection::Instances::CLASSES[index]
+
+        dataset.add_instance([*values, class_value])
+      end
+    end
+
+    allow(Wikipedia::VandalismDetection::TrainingDataset)
+      .to receive(:instances)
+      .and_return(dataset)
+  end
+
+  after do
+    arff_file = @config.training_output_arff_file
+    build_dir = @config.output_base_directory
+
+    if File.exist?(arff_file)
+      File.delete(arff_file)
+      directory = File.dirname(arff_file)
+      FileUtils.rm_r(directory)
+    end
+
+    FileUtils.rm_r(build_dir) if Dir.exist?(build_dir)
+  end
+
+  it 'can be used to classify vandalism' do
+    classifier = Wikipedia::VandalismDetection::Classifier.new
+    features = [1.0, 2.0, 55.0]
+
+    expect(classifier.classify(features)).to be_between(0.0, 1.0)
+  end
+
+  it 'can be used to classify vandalism using regulars' do
+    regular = Wikipedia::VandalismDetection::Instances::REGULAR
+    options = "-tcl #{regular} #{@w_options}"
+
+    @config.instance_variable_set(:@classifier_type, classifier_type)
+    @config.instance_variable_set(:@classifier_options, options)
+
+    use_configuration(@config)
+
+    classifier = Wikipedia::VandalismDetection::Classifier.new
+    features = [1.0, 2.0, 8.0]
+
+    expect(classifier.classify(features)).to be_between(0.0, 1.0)
+  end
+
+  describe '#type' do
+    it 'returns the classifier’s type name' do
+      expect(Weka::Classifiers::Meta::OneClassClassifier.type)
+        .to eq 'Meta::OneClassClassifier'
+    end
+  end
+end