wikipedia-vandalism_detection 0.0.1

data/lib/wikipedia/vandalism_detection/exceptions.rb

@@ -0,0 +1,40 @@
+module Wikipedia
+  module VandalismDetection
+
+    # @abstract Exceptions raised by Wikipedia::VandalismDetection inherit from this Error
+    class Error < StandardError; end
+
+    # Exception is raised when trying to classify without a configured classifier
+    class ClassifierNotConfiguredError < Error; end
+
+    # Exception is raised when tyring to classifiy with an unknown classifier
+    class ClassifierUnknownError < Error; end
+
+    # Exception is raised when trying to use features without having configured some
+    class FeaturesNotConfiguredError < Error; end
+
+    # Exception is raised when trying to use edits file without having configured some
+    class EditsFileNotConfiguredError < Error; end
+
+    # Exception is raised when trying to use annotations file without having configured some
+    class AnnotationsFileNotConfiguredError < Error; end
+
+    # Exception is raised when trying to read revisions directory without having configured some
+    class RevisionsDirectoryNotConfiguredError < Error; end
+
+    # Exception is raised when trying to classify without a configured ground thruth test file
+    class GroundTruthFileNotConfiguredError < Error; end
+
+    # Exception is raises when there is no arff file available
+    class ArffFileNotFoundError < Error; end
+
+    # Exception is raises when there is no ground truth file available
+    class GroundTruthFileNotFoundError < Error; end
+
+    # Exceptoion is raised when an already available featture should be added to the arff file
+    class FeatureAlreadyUsedError < Error; end
+
+    # Exception is raised when a revisions text file cannot be found and loaded
+    class RevisionFileNotFound < Error; end
+  end
+end

data/lib/wikipedia/vandalism_detection/feature_calculator.rb

@@ -0,0 +1,89 @@
+require 'active_support/core_ext/string'
+require 'active_support/core_ext/array'
+
+require 'wikipedia/vandalism_detection/wikitext_extractor'
+require 'wikipedia/vandalism_detection/features'
+require 'wikipedia/vandalism_detection/edit'
+
+module Wikipedia
+  module VandalismDetection
+
+    # This class provides methods for calculating a feature set of an edit.
+    # The features that shall be used can be defined in the config/config.yml file
+    # under the 'features:' root attribute like this:
+    #
+    # features:
+    #   - anonymity
+    #   - character sequence
+    #   - ...
+    # etc.
+    class FeatureCalculator
+
+      def initialize
+        @features = Wikipedia::VandalismDetection.configuration.features
+        raise FeaturesNotConfiguredError if (@features.blank? || @features.empty?)
+        @feature_classes = build_feature_classes @features
+      end
+
+      # Calculates the configured festures for the given edit and
+      # returns an array of the computed values.
+      def calculate_features_for(edit)
+        raise ArgumentError, "Input has to be an Edit." unless edit.is_a? Edit
+
+        features = @feature_classes.map do |feature|
+          begin
+            feature.calculate(edit)
+          rescue WikitextExtractionError => e
+            $stderr.print %Q{
+              Edit (#{edit.old_revision.id}, #{edit.new_revision.id}) could not be parsed
+              by the WikitextExtractor and will be discarded.\n""}
+
+            Features::MISSING_VALUE
+          end
+        end
+
+        features
+      end
+
+      # Returns the calculated Numeric feature value for given edit and feature with given name
+      def calculate_feature_for(edit, feature_name)
+        raise ArgumentError, "First parameter has to be an Edit." unless edit.is_a? Edit
+        raise ArgumentError, "Second parameter has to be a feature name String (e.g. 'anonymity')." unless \
+          feature_name.is_a? String
+
+        value = Features::MISSING_VALUE
+
+        begin
+          feature = feature_class_from_name(feature_name)
+          value = feature.calculate(edit)
+        rescue WikitextExtractionError
+          $stderr.print %Q{
+            Edit (#{edit.old_revision.id}, #{edit.new_revision.id}) could not be parsed
+            by the WikitextExtractor and will be discarded.\n""}
+        end
+
+        value
+      end
+
+      # Returns the feature names as defined in conf/config.yml under 'features:'.
+      def used_features
+        @features
+      end
+
+      private
+
+      # Returns an array of all configured Feature class instances.
+      def build_feature_classes(feature_names)
+        feature_names.map do |name|
+          feature_class_from_name(name)
+        end
+      end
+
+      # Returns the Feature class of the given name
+      def feature_class_from_name(name)
+        camelcased_name = name.split(/[\s-]/).map{ |s| s.capitalize! }.join('')
+        "Wikipedia::VandalismDetection::Features::#{camelcased_name}".constantize.new
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features.rb

@@ -0,0 +1,67 @@
+require 'wikipedia/vandalism_detection/features/all_wordlists_frequency'
+require 'wikipedia/vandalism_detection/features/all_wordlists_impact'
+require 'wikipedia/vandalism_detection/features/anonymity'
+require 'wikipedia/vandalism_detection/features/anonymity_previous'
+require 'wikipedia/vandalism_detection/features/article_size'
+require 'wikipedia/vandalism_detection/features/bad_frequency'
+require 'wikipedia/vandalism_detection/features/bad_impact'
+require 'wikipedia/vandalism_detection/features/biased_frequency'
+require 'wikipedia/vandalism_detection/features/biased_impact'
+require 'wikipedia/vandalism_detection/features/blanking'
+require 'wikipedia/vandalism_detection/features/character_diversity'
+require 'wikipedia/vandalism_detection/features/character_sequence'
+require 'wikipedia/vandalism_detection/features/comment_length'
+require 'wikipedia/vandalism_detection/features/comment_bad_frequency'
+require 'wikipedia/vandalism_detection/features/comment_biased_frequency'
+require 'wikipedia/vandalism_detection/features/comment_markup_frequency'
+require 'wikipedia/vandalism_detection/features/comment_pronoun_frequency'
+require 'wikipedia/vandalism_detection/features/comment_sex_frequency'
+require 'wikipedia/vandalism_detection/features/comment_vulgarism_frequency'
+require 'wikipedia/vandalism_detection/features/compressibility'
+require 'wikipedia/vandalism_detection/features/copyedit'
+require 'wikipedia/vandalism_detection/features/digit_ratio'
+require 'wikipedia/vandalism_detection/features/edits_per_user'
+require 'wikipedia/vandalism_detection/features/emoticons_frequency'
+require 'wikipedia/vandalism_detection/features/emoticons_impact'
+require 'wikipedia/vandalism_detection/features/inserted_size'
+require 'wikipedia/vandalism_detection/features/inserted_words'
+require 'wikipedia/vandalism_detection/features/inserted_character_distribution'
+require 'wikipedia/vandalism_detection/features/inserted_external_links'
+require 'wikipedia/vandalism_detection/features/inserted_internal_links'
+require 'wikipedia/vandalism_detection/features/longest_word'
+require 'wikipedia/vandalism_detection/features/markup_frequency'
+require 'wikipedia/vandalism_detection/features/markup_impact'
+require 'wikipedia/vandalism_detection/features/non_alphanumeric_ratio'
+require 'wikipedia/vandalism_detection/features/personal_life'
+require 'wikipedia/vandalism_detection/features/pronoun_frequency'
+require 'wikipedia/vandalism_detection/features/pronoun_impact'
+require 'wikipedia/vandalism_detection/features/removed_all_wordlists_frequency'
+require 'wikipedia/vandalism_detection/features/removed_bad_frequency'
+require 'wikipedia/vandalism_detection/features/removed_biased_frequency'
+require 'wikipedia/vandalism_detection/features/removed_character_distribution'
+require 'wikipedia/vandalism_detection/features/removed_emoticons_frequency'
+require 'wikipedia/vandalism_detection/features/removed_markup_frequency'
+require 'wikipedia/vandalism_detection/features/removed_pronoun_frequency'
+require 'wikipedia/vandalism_detection/features/removed_sex_frequency'
+require 'wikipedia/vandalism_detection/features/removed_vulgarism_frequency'
+require 'wikipedia/vandalism_detection/features/removed_size'
+require 'wikipedia/vandalism_detection/features/removed_words'
+require 'wikipedia/vandalism_detection/features/replacement_similarity'
+require 'wikipedia/vandalism_detection/features/reverted'
+require 'wikipedia/vandalism_detection/features/revisions_character_distribution'
+require 'wikipedia/vandalism_detection/features/same_editor'
+require 'wikipedia/vandalism_detection/features/sex_frequency'
+require 'wikipedia/vandalism_detection/features/sex_impact'
+require 'wikipedia/vandalism_detection/features/size_increment'
+require 'wikipedia/vandalism_detection/features/size_ratio'
+require 'wikipedia/vandalism_detection/features/term_frequency'
+require 'wikipedia/vandalism_detection/features/time_interval'
+require 'wikipedia/vandalism_detection/features/time_of_day'
+require 'wikipedia/vandalism_detection/features/upper_case_ratio'
+require 'wikipedia/vandalism_detection/features/upper_case_words_ratio'
+require 'wikipedia/vandalism_detection/features/upper_to_lower_case_ratio'
+require 'wikipedia/vandalism_detection/features/user_reputation'
+require 'wikipedia/vandalism_detection/features/vulgarism_frequency'
+require 'wikipedia/vandalism_detection/features/vulgarism_impact'
+require 'wikipedia/vandalism_detection/features/weekday'
+require 'wikipedia/vandalism_detection/features/words_increment'

data/lib/wikipedia/vandalism_detection/features/all_wordlists_frequency.rb

@@ -0,0 +1,23 @@
+require 'wikipedia/vandalism_detection/features/frequency_base'
+require 'wikipedia/vandalism_detection/word_lists'
+require 'wikipedia/vandalism_detection/text'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature computes frequency of all wordlists words in the inserted text.
+      class AllWordlistsFrequency < FrequencyBase
+
+        # Returns the percentage of wordlists words in the inserted text.
+        # Returns 0.0 if inserted clean text is of zero length.
+        def calculate(edit)
+          super
+
+          text = Text.new(edit.inserted_words.join("\n")).clean
+          frequency(text, WordLists.all)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/all_wordlists_impact.rb

@@ -0,0 +1,22 @@
+require 'wikipedia/vandalism_detection/features/impact_base'
+require 'wikipedia/vandalism_detection/word_lists'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature computes the percentage by which the edit increases the number of all wordlists words in the text.
+      class AllWordlistsImpact < ImpactBase
+
+        def calculate(edit)
+          super
+
+          old_text = edit.old_revision.text.clean
+          new_text = edit.new_revision.text.clean
+
+          impact(old_text, new_text, WordLists.all)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/anonymity.rb

@@ -0,0 +1,19 @@
+require_relative 'base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature describes whether the contributor of the new revision is an anonymous or
+      # registered Wikipedia user.
+      class Anonymity < Base
+
+        def calculate(edit)
+          super
+
+          edit.new_revision.anonymous_contributor? ? 0 : 1
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/anonymity_previous.rb

@@ -0,0 +1,29 @@
+require_relative 'base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature describes whether the contributor of the old revision is an anonymous or
+      # registered Wikipedia user.
+      class AnonymityPrevious < Base
+
+        def calculate(edit)
+          super
+
+          old_revision = edit.old_revision
+
+          if old_revision.contributor.blank?
+            xml = Wikipedia::api_request({ prop: 'revisions', rvprop: 'user',revids: old_revision.id })
+            contributor = xml.xpath('//rev/@user').text
+            return Features::MISSING_VALUE if contributor.blank?
+
+            old_revision.contributor = contributor
+          end
+
+          old_revision.anonymous_contributor? ? 0 : 1
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/article_size.rb

@@ -0,0 +1,18 @@
+require 'wikipedia/vandalism_detection/features/base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature computes the size of the edit's new revision text (article size).
+      class ArticleSize < Base
+
+        def calculate(edit)
+          super
+
+          edit.new_revision.text.size
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/bad_frequency.rb

@@ -0,0 +1,23 @@
+require 'wikipedia/vandalism_detection/features/frequency_base'
+require 'wikipedia/vandalism_detection/word_lists/bad'
+require 'wikipedia/vandalism_detection/text'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature computes frequency of bad words in the inserted text.
+      class BadFrequency < FrequencyBase
+
+        # Returns the percentage of bad words in the inserted text.
+        # Returns 0.0 if inserted clean text is of zero length.
+        def calculate(edit)
+          super
+
+          text = Text.new(edit.inserted_words.join("\n")).clean
+          frequency(text, WordLists::BAD)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/bad_impact.rb

@@ -0,0 +1,21 @@
+require 'wikipedia/vandalism_detection/features/impact_base'
+require 'wikipedia/vandalism_detection/word_lists/bad'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature computes the percentage by which the edit increases the number of bad words in the text.
+      class BadImpact < ImpactBase
+
+        def calculate(edit)
+          super
+          old_text = edit.old_revision.text.clean
+          new_text = edit.new_revision.text.clean
+
+          impact(old_text, new_text, WordLists::BAD)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/base.rb

@@ -0,0 +1,54 @@
+require 'wikipedia'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      MISSING_VALUE = '?'
+
+      # This class should be the base class for all Wikipedia::Feature classes.
+      class Base
+
+        # Base method for feature calculation.
+        # This method should be overwritten in the concrete Wikipedia::Feature-classes.
+        #
+        # Example:
+        # def calculate(edit)
+        #   super # to handle ArgumentException
+        #
+        #   ... concrete calculation of feature out of edit...
+        # end
+        def calculate(edit)
+          raise ArgumentError.new "parameter should be an Edit" unless edit.kind_of? Edit
+        end
+
+        # Count the apperance of a given single term or multiple terms in the given text
+        # @params terms String
+        # @params options Hash of form {in: String}
+        #
+        # Example of usage:
+        #
+        # feature.count "and", in: text
+        # feature.count ["and", "or"], in: text
+        def count(terms, options = {})
+          terms_is_string = terms.is_a?(String)
+          terms_is_array = terms.is_a?(Array)
+
+          raise ArgumentError, "The second parameter should be a Hash of form {in: text}" unless options[:in]
+          raise ArgumentError, "The first parameter should be an Array or String" unless
+              (terms_is_array || terms_is_string)
+
+          words = options[:in].downcase
+          freq = Hash.new(0)
+          words.gsub(/[\.,'{2,}:\!\?\(\)]/, '').split.each{ |v| freq[v.to_sym] += 1 }
+
+          if terms_is_string
+            freq[terms.downcase.to_sym]
+          else
+            terms.reduce(0) {|r, term| r + freq[term] }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/biased_frequency.rb

@@ -0,0 +1,23 @@
+require 'wikipedia/vandalism_detection/features/frequency_base'
+require 'wikipedia/vandalism_detection/word_lists/biased'
+require 'wikipedia/vandalism_detection/text'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature computes frequency of biased words in the inserted text.
+      class BiasedFrequency < FrequencyBase
+
+        # Returns the percentage of biased words in the inserted text.
+        # Returns 0.0 if inserted clean text is of zero length.
+        def calculate(edit)
+          super
+
+          text = Text.new(edit.inserted_words.join("\n")).clean
+          frequency(text, WordLists::BIASED)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/biased_impact.rb

@@ -0,0 +1,22 @@
+require 'wikipedia/vandalism_detection/features/impact_base'
+require 'wikipedia/vandalism_detection/word_lists/biased'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature computes the percentage by which the edit increases the number of biased words in the text.
+      class BiasedImpact < ImpactBase
+
+        def calculate(edit)
+          super
+
+          old_text = edit.old_revision.text.clean
+          new_text = edit.new_revision.text.clean
+
+          impact(old_text, new_text, WordLists::BIASED)
+        end
+      end
+    end
+  end
+end