wikipedia-vandalism_detection 0.0.1

data/spec/vandalism_detection/text_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+
+describe Wikipedia::VandalismDetection::Text do
+
+  it { should be_a String }
+  it { should respond_to :clean }
+
+  describe "#initialze" do
+    it "can removes invalid byte sequences" do
+      text = Wikipedia::VandalismDetection::Text.new("text \255".force_encoding('UTF-8'))
+      expect { text.scan('text') }.not_to raise_error
+    end
+  end
+
+  describe "#clean" do
+
+    it "raises an WikitextExtractionError if text cannot be parsed" do
+      text = Wikipedia::VandalismDetection::Text.new "[[Image:img.jpg|\n{|\n|-\n|||| |}"
+      expect { text.clean }.to raise_error Wikipedia::VandalismDetection::WikitextExtractionError
+    end
+
+    it "returns the text cleaned from wiki tags" do
+      wiki_text = Wikipedia::VandalismDetection::Text.new load_file('sample_revision.txt')
+      clean_text = load_file('sample_revision_clean_text.txt')
+
+      expect(wiki_text.clean).to eq clean_text
+    end
+  end
+end

data/spec/vandalism_detection/training_dataset_spec.rb

@@ -0,0 +1,264 @@
+require 'spec_helper'
+require 'fileutils'
+require 'ruby-band'
+
+describe Wikipedia::VandalismDetection::TrainingDataset do
+
+  before do
+    use_test_configuration
+    @config = test_config
+
+    @arff_file = @config.training_output_arff_file
+    @index_file = @config.training_output_index_file
+    @annotations_file = @config.training_corpus_annotations_file
+
+    @arff_files_dir = File.join(@config.output_base_directory, 'training')
+  end
+
+  after do
+    if File.exists?(@arff_file)
+      File.delete(@arff_file)
+      FileUtils.rm_r(File.dirname @arff_file)
+    end
+
+    File.delete(@index_file) if File.exists?(@index_file)
+
+    # remove feature arff files
+    @config.features.each do |name|
+      file = File.join(@arff_files_dir, name.gsub(' ', '_') + '.arff')
+
+      if File.exists?(file)
+        File.delete(file)
+        FileUtils.rm_r(File.dirname file)
+      end
+    end
+  end
+
+  describe "#build" do
+
+    it "returns a weka instances" do
+      dataset = Wikipedia::VandalismDetection::TrainingDataset.build
+      expect(dataset.class).to be Java::WekaCore::Instances
+    end
+
+    describe "exceptions" do
+      it "raises an EditsFileNotConfiguredError if no edits file is configured" do
+        config = test_config
+        config.instance_variable_set(:@training_corpus_edits_file, nil)
+        use_configuration(config)
+
+        expect { Wikipedia::VandalismDetection::TrainingDataset.build }.to raise_error \
+          Wikipedia::VandalismDetection::EditsFileNotConfiguredError
+      end
+
+      it "raises an AnnotationsFileNotConfiguredError if no annotations file is configured" do
+        config = test_config
+        config.instance_variable_set(:@training_corpus_annotations_file, nil)
+        use_configuration(config)
+
+        expect { Wikipedia::VandalismDetection::TrainingDataset.build }.to raise_error \
+          Wikipedia::VandalismDetection::AnnotationsFileNotConfiguredError
+      end
+    end
+
+    Wikipedia::VandalismDetection::DefaultConfiguration::DEFAULTS['features'].each do |name|
+      it "creates an arff file for the feature '#{name}'" do
+        config = test_config
+        config.instance_variable_set :@features, [name]
+        use_configuration(config)
+
+        file = File.join(@arff_files_dir, name.gsub(' ', '_') + '.arff')
+
+        expect(File.exist?(file)).to be false
+        Wikipedia::VandalismDetection::TrainingDataset.build
+        expect(File.exist?(file)).to be true
+      end
+    end
+
+    it "creates only feature files that are not available yet" do
+      config = test_config
+      config.instance_variable_set :@features, ['anonymity', 'comment length']
+      use_configuration(config)
+
+      anonymity_file = File.join(config.output_base_directory, 'test', 'anonymity.arff')
+
+      # create file manually, so it is existent when building dataset
+      data = [10000, 123456, 234567]
+      anonymity = Wikipedia::VandalismDetection::Instances.empty_for_test_feature('anonymity')
+      6.times { anonymity.add_instance(data) }
+      anonymity.to_ARFF(anonymity_file)
+
+      Wikipedia::VandalismDetection::TrainingDataset.build
+
+      # anonymity should not be overwritten
+      expect(Core::Parser.parse_ARFF(anonymity_file).to_a2d.first).to eq data
+    end
+
+    describe "internal algorithm" do
+      before do
+        @features_num = @config.features.count
+      end
+
+      it "has builds the right number of data lines" do
+        dataset = Wikipedia::VandalismDetection::TrainingDataset.build
+        annotations_num = File.open(@annotations_file, 'r').lines.count - 1
+        additional_header_lines = 5
+
+        expect(dataset.to_s.lines.count).to eq additional_header_lines + annotations_num + @features_num
+      end
+
+      it "builds the right number of data columns" do
+        dataset = Wikipedia::VandalismDetection::TrainingDataset.build
+        expect(dataset.n_col).to eq @config.features.count + 1
+      end
+    end
+
+    describe "replacing missing values" do
+
+      it "replaces missing values if configured" do
+        config = test_config
+        config.instance_variable_set :@replace_missing_values, 'true'
+        use_configuration(config)
+
+        dataset = Wikipedia::VandalismDetection::TrainingDataset.build
+
+        expect(dataset.to_s).to match /weka\.filters\.unsupervised\.attribute\.ReplaceMissingValues/
+      end
+
+      it "does not replace missing values if not configured" do
+        config = test_config
+        config.instance_variable_set :@replace_missing_values, 'Nope'
+        use_configuration(config)
+
+        dataset = Wikipedia::VandalismDetection::TrainingDataset.build
+
+        expect(dataset.to_s).not_to match /weka\.filters\.unsupervised\.attribute\.ReplaceMissingValues/
+      end
+    end
+  end
+
+  describe "#instances" do
+    it "is an alias method for #build" do
+      build = Wikipedia::VandalismDetection::TrainingDataset.build
+      instances = Wikipedia::VandalismDetection::TrainingDataset.instances
+
+      expect(build.to_s).to eq instances.to_s
+    end
+  end
+
+  describe "#balanced_instances" do
+
+    before do
+      config = test_config
+      config.instance_variable_set(:@training_data_options, 'balanced')
+      use_configuration(config)
+
+      @dataset = Wikipedia::VandalismDetection::TrainingDataset.balanced_instances
+    end
+
+    it "returns a weka dataset" do
+      expect(@dataset.class).to eq Java::WekaCore::Instances
+    end
+
+    it "returns a dataset built from the configured corpus" do
+      # 2 vandalism, 2 regular, see resources/corpora/training/annotations.csv
+      expect(@dataset.n_rows).to eq 4
+    end
+
+    [:VANDALISM, :REGULAR].each do |class_const|
+      it "has 2 '#{class_const.downcase}' samples in its instances" do
+        class_count = @dataset.enumerate_instances.reduce(0) do |count, instance|
+          label = Wikipedia::VandalismDetection::Instances::CLASSES[instance.class_value.to_i]
+          (label == Wikipedia::VandalismDetection::Instances::const_get(class_const)) ? (count + 1) : count
+        end
+
+        expect(class_count).to eq 2
+      end
+    end
+  end
+
+  describe "#oversampled_instances" do
+    describe "with default options" do
+      before do
+        config = test_config
+        config.instance_variable_set(:@training_data_options, 'oversampled')
+        use_configuration(config)
+
+        @dataset = Wikipedia::VandalismDetection::TrainingDataset.oversampled_instances # default -P 100 -U true
+      end
+
+      it "returns a weka dataset" do
+        expect(@dataset.class).to be Java::WekaCore::Instances
+      end
+
+      it "returns a dataset of size 8 built from the configured corpus" do
+        # 4 vandalism, 4 regular, see resources/corpora/training/annotations.csv
+        expect(@dataset.n_rows).to eq 8
+      end
+
+      [:VANDALISM, :REGULAR].each do |class_const|
+        it "has 4 '#{class_const.downcase}' samples in its instances" do
+          class_count = @dataset.enumerate_instances.reduce(0) do |count, instance|
+            label = Wikipedia::VandalismDetection::Instances::CLASSES[instance.class_value.to_i]
+            (label == Wikipedia::VandalismDetection::Instances::const_get(class_const)) ? (count + 1) : count
+          end
+
+          expect(class_count).to eq 4
+        end
+      end
+
+      it "returns a dataset of size 8 for 200% 'SMOTEING' built from the configured corpus" do
+        # 4 vandalism, 4 regular, see resources/corpora/training/annotations.csv
+        dataset = Wikipedia::VandalismDetection::TrainingDataset.oversampled_instances(percentage: 200)
+        puts dataset
+
+        expect(dataset.n_rows).to eq 8
+      end
+    end
+
+    describe "with custom options" do
+      before do
+        config = test_config
+        config.instance_variable_set(:@training_data_options, 'oversampled -p 300 -u false')
+        use_configuration(config)
+
+        @dataset = Wikipedia::VandalismDetection::TrainingDataset.oversampled_instances
+      end
+
+      it "returns a weka dataset" do
+        expect(@dataset.class).to be Java::WekaCore::Instances
+      end
+
+      it "returns a dataset of size 12 built from the configured corpus" do
+        # 2 + 300 % = 8 vandalism, 4 regular, see resources/corpora/training/annotations.csv
+        puts @dataset
+        expect(@dataset.n_rows).to eq 12
+      end
+    end
+  end
+
+  describe "#create_corpus_index_file!" do
+
+    it "responds to #create_corpus_file_index!" do
+      expect(Wikipedia::VandalismDetection::TrainingDataset).to respond_to :create_corpus_file_index!
+    end
+
+    describe "exceptions" do
+
+      it "raises an RevisionsDirectoryNotConfiguredError if no revisions directory is configured" do
+        config = test_config
+        config.instance_variable_set :@training_corpus_revisions_directory, nil
+        use_configuration(config)
+
+        expect { Wikipedia::VandalismDetection::TrainingDataset.create_corpus_file_index! }.to raise_error \
+          Wikipedia::VandalismDetection::RevisionsDirectoryNotConfiguredError
+      end
+    end
+
+    it "creates a corpus_index.yml file in the build directory" do
+      expect(File.exist?(@index_file)).to be false
+      Wikipedia::VandalismDetection::TrainingDataset.create_corpus_file_index!
+      expect(File.exist?(@index_file)).to be true
+    end
+  end
+end

data/spec/vandalism_detection/wikitext_extractor_spec.rb

@@ -0,0 +1,72 @@
+require 'spec_helper'
+
+describe  Wikipedia::VandalismDetection::WikitextExtractor do
+
+  it "can handle invalid byte sequences" do
+    wiki_text = "{{speedy deletion}} \255".force_encoding('UTF-8')
+    expect { Wikipedia::VandalismDetection::WikitextExtractor.extract(wiki_text) }.not_to raise_error
+  end
+
+  it "returns an empty string if the all the markup is extracted" do
+    wiki_text = "{{speedy deletion}}"
+
+    expect(Wikipedia::VandalismDetection::WikitextExtractor.extract(wiki_text)).to be_empty
+  end
+
+  it "removes #REDIRECT markup" do
+    wiki_text = "#REDIRECT [[Heading]]"
+    plain_text = "Heading"
+
+    expect(Wikipedia::VandalismDetection::WikitextExtractor.extract(wiki_text)).to eq plain_text
+  end
+
+  it "can extract plaintext from wikitext" do
+    wiki_text = load_file('sample_revision.txt')
+    plain_text = load_file('sample_revision_plain_text.txt')
+
+    expect((Wikipedia::VandalismDetection::WikitextExtractor.extract(wiki_text) << "\n")).to eq plain_text
+  end
+
+  it "can extract full cleaned text from wikitext" do
+    wiki_text = load_file('sample_revision.txt')
+    clean_text = load_file('sample_revision_clean_text.txt')
+
+    expect(Wikipedia::VandalismDetection::WikitextExtractor.extract_clean(wiki_text)).to eq clean_text
+  end
+
+  it "removes section numbering while cleaning wikitext" do
+    wiki_text = "1.1. header 1\n\n1.2. header 2"
+    clean_text = "header 1 header 2"
+
+    expect(Wikipedia::VandalismDetection::WikitextExtractor.extract_clean(wiki_text)).to eq clean_text
+  end
+
+  it "removes line breaks while cleaning wikitext" do
+    wiki_text = "line 1\n\nline 2\nline 3"
+    clean_text = "line 1 line 2 line 3"
+
+    expect(Wikipedia::VandalismDetection::WikitextExtractor.extract_clean(wiki_text)).to eq clean_text
+  end
+
+  it "removes multiple spaces while cleaning wikitext" do
+    wiki_text = "line    1  \n\nline   2   \nline       3   "
+    clean_text = "line 1 line 2 line 3"
+
+    expect(Wikipedia::VandalismDetection::WikitextExtractor.extract_clean(wiki_text)).to eq clean_text
+  end
+
+  it "removes links from text while cleaning wikitext" do
+    wiki_text = "A link\nis here http://example.com/image.jpg not\nanymore." <<
+    "\n==Reference==\n" <<
+    "*[http://www.itis.usda.gov/servlet/SingleRpt/SingleRpt?search_topic=TSN&amp;search_value=180211 ITIS 180211] 2002-12-14"
+    clean_text = "A link is here not anymore. Reference ITIS 180211 2002-12-14"
+
+    expect(Wikipedia::VandalismDetection::WikitextExtractor.extract_clean(wiki_text)).to eq clean_text
+  end
+
+  it "raises a WikitextExtractionError while extracting unparsable wikitext" do
+    unparsable_wiki_text = "[[Image:img.jpg|\n{|\n|-\n|||| |}"
+    expect {  Wikipedia::VandalismDetection::WikitextExtractor.extract(unparsable_wiki_text) }.to raise_error
+  end
+
+end

data/spec/weka/classifiers/functions/lib_svm_spec.rb

@@ -0,0 +1,38 @@
+require 'spec_helper'
+
+describe Weka::Classifiers::Functions::LibSVM do
+
+  it { should be_a Java::WekaClassifiersFunctions::LibSVM}
+
+  before do
+    @config = test_config
+    @config.instance_variable_set :@classifier_type, 'Functions::LibSVM'
+    @config.instance_variable_set :@classifier_options, '-S 2 -N 0.001 -M 100.0 -C 1.0 -E 0.001 -P 0.1 -Z'
+    @config.instance_variable_set :@cross_validation_fold, '2'
+
+    use_configuration(@config)
+  end
+
+  after do
+    arff_file = @config.training_output_arff_file
+    build_dir = @config.output_base_directory
+
+    if File.exists?(arff_file)
+      File.delete(arff_file)
+      FileUtils.rm_r(File.dirname arff_file)
+    end
+
+    if Dir.exists?(build_dir)
+      FileUtils.rm_r(build_dir)
+    end
+  end
+
+  it "can be used to classifiy vandalism" do
+    expect {
+      classifier = Wikipedia::VandalismDetection::Classifier.new
+      features = [0.0, 25, 5]
+      confidence = classifier.classify(features)
+      puts "confidence: #{confidence}"
+    }.not_to raise_error
+  end
+end

data/spec/weka/classifiers/meta/one_class_classifier_spec.rb

@@ -0,0 +1,76 @@
+require 'spec_helper'
+
+describe Weka::Classifiers::Meta::OneClassClassifier do
+
+  it { should be_a Java::WekaClassifiersMeta::OneClassClassifier }
+
+  before do
+    @config = test_config
+    @w_options = "-W weka.classifiers.meta.Bagging -- -W weka.classifiers.trees.RandomForest -- -I 100"
+    options = "-tcl #{ Wikipedia::VandalismDetection::Instances::VANDALISM } #{@w_options}"
+
+    @config.instance_variable_set :@classifier_type, 'Meta::OneClassClassifier'
+    @config.instance_variable_set :@classifier_options, options
+    @config.instance_variable_set :@cross_validation_fold, 2
+
+    use_configuration(@config)
+
+    # add more test instances because instances number must higher than cross validation fold
+    instances = Wikipedia::VandalismDetection::TrainingDataset.instances.to_a2d
+    dataset = Wikipedia::VandalismDetection::Instances.empty
+
+    2.times do
+      instances.each do |row|
+        dataset.add_instance([*row, Wikipedia::VandalismDetection::Instances::CLASSES[rand((0..1))]])
+      end
+    end
+
+    Wikipedia::VandalismDetection::TrainingDataset.stub(instances: dataset)
+    puts Wikipedia::VandalismDetection::TrainingDataset.instances
+  end
+
+  after do
+    arff_file = @config.training_output_arff_file
+    build_dir = @config.output_base_directory
+
+    if File.exists?(arff_file)
+      File.delete(arff_file)
+      FileUtils.rm_r(File.dirname arff_file)
+    end
+
+    if Dir.exists?(build_dir)
+      FileUtils.rm_r(build_dir)
+    end
+  end
+
+  it "can be used to classify vandalism" do
+    expect {
+      classifier = Wikipedia::VandalismDetection::Classifier.new
+      features = [1.0, 2.0, 55.0]
+      confidence = classifier.classify features
+      puts "vandalism confidence: #{confidence}}"
+    }.not_to raise_error
+  end
+
+  it "can be used to classify vandalism using regulars" do
+    options = "-tcl #{ Wikipedia::VandalismDetection::Instances::REGULAR } #{@w_options}"
+
+    @config.instance_variable_set :@classifier_type, 'Meta::OneClassClassifier'
+    @config.instance_variable_set :@classifier_options, options
+
+    use_configuration(@config)
+
+    expect {
+      classifier = Wikipedia::VandalismDetection::Classifier.new
+      features = [1.0, 2.0, 8.0]
+      confidence = classifier.classify features
+      puts "regular confidence: #{confidence}}"
+    }.not_to raise_error
+  end
+
+  describe "#type" do
+    it "returns the classifier's type name" do
+      expect(Weka::Classifiers::Meta::OneClassClassifier.type).to eq 'Meta::OneClassClassifier'
+    end
+  end
+end