wikipedia-vandalism_detection 0.1.0-java

data/lib/wikipedia/vandalism_detection/feature_calculator.rb

@@ -0,0 +1,94 @@
+require 'active_support/core_ext/string'
+require 'active_support/core_ext/array'
+
+require 'wikipedia/vandalism_detection/wikitext_extractor'
+require 'wikipedia/vandalism_detection/features'
+require 'wikipedia/vandalism_detection/edit'
+
+module Wikipedia
+  module VandalismDetection
+    # This class provides methods for calculating a feature set of an edit.
+    # The features that shall be used can be defined in the config/wikipedia-vandalism-detection.yml file
+    # under the 'features:' root attribute like this:
+    #
+    # features:
+    #   - anonymity
+    #   - character sequence
+    #   - ...
+    # etc.
+    class FeatureCalculator
+      def initialize
+        @features = Wikipedia::VandalismDetection.config.features
+        raise FeaturesNotConfiguredError if @features.blank? || @features.empty?
+        @feature_classes = build_feature_classes @features
+      end
+
+      # Calculates the configured festures for the given edit and returns an
+      # array of the computed values.
+      def calculate_features_for(edit)
+        raise ArgumentError, 'Input has to be an Edit.' unless edit.is_a?(Edit)
+
+        features = @feature_classes.map do |feature|
+          begin
+            feature.calculate(edit)
+          rescue WikitextExtractionError
+            $stderr.print %{
+              Edit (#{edit.old_revision.id}, #{edit.new_revision.id}) could not
+              be parsed by the WikitextExtractor and will be discarded.\n""}
+
+            Features::MISSING_VALUE
+          end
+        end
+
+        features
+      end
+
+      # Returns the calculated Numeric feature value for given edit and feature with given name
+      def calculate_feature_for(edit, feature_name)
+        unless edit.is_a?(Edit)
+          raise ArgumentError, 'First parameter has to be an Edit.'
+        end
+
+        unless feature_name.is_a?(String)
+          message = 'Second parameter has to be a feature name String ' \
+                    '(e.g. "anonymity").'
+          raise ArgumentError, message
+        end
+
+        value = Features::MISSING_VALUE
+
+        begin
+          feature = feature_class_from_name(feature_name)
+          value = feature.calculate(edit)
+        rescue WikitextExtractionError
+          $stderr.print %{
+            Edit (#{edit.old_revision.id}, #{edit.new_revision.id}) could not
+            be parsed by the WikitextExtractor and will be discarded.\n""}
+        end
+
+        value
+      end
+
+      # Returns the feature names as defined in
+      # conf/wikipedia-vandalism-detection.yml under 'features:'.
+      def used_features
+        @features
+      end
+
+      private
+
+      # Returns an array of all configured Feature class instances.
+      def build_feature_classes(feature_names)
+        feature_names.map do |name|
+          feature_class_from_name(name)
+        end
+      end
+
+      # Returns the Feature class of the given name
+      def feature_class_from_name(name)
+        camelcased_name = name.split(/[\s-]/).map(&:capitalize!).join('')
+        "Wikipedia::VandalismDetection::Features::#{camelcased_name}".constantize.new
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/all_wordlists_frequency.rb

@@ -0,0 +1,22 @@
+require 'wikipedia/vandalism_detection/features/frequency_base'
+require 'wikipedia/vandalism_detection/word_lists'
+require 'wikipedia/vandalism_detection/text'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes frequency of all wordlists words in the inserted
+      # text.
+      class AllWordlistsFrequency < FrequencyBase
+        # Returns the percentage of wordlists words in the inserted text.
+        # Returns 0.0 if inserted clean text is of zero length.
+        def calculate(edit)
+          super
+
+          text = Text.new(edit.inserted_words.join("\n")).clean
+          frequency(text, WordLists.all)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/all_wordlists_impact.rb

@@ -0,0 +1,21 @@
+require 'wikipedia/vandalism_detection/features/impact_base'
+require 'wikipedia/vandalism_detection/word_lists'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the percentage by which the edit increases the
+      # number of all wordlists words in the text.
+      class AllWordlistsImpact < ImpactBase
+        def calculate(edit)
+          super
+
+          old_text = edit.old_revision.text.clean
+          new_text = edit.new_revision.text.clean
+
+          impact(old_text, new_text, WordLists.all)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/anonymity.rb

@@ -0,0 +1,17 @@
+require_relative 'base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature describes whether the contributor of the new revision is
+      # an anonymous or registered Wikipedia user.
+      class Anonymity < Base
+        def calculate(edit)
+          super
+
+          edit.new_revision.anonymous_contributor? ? 0 : 1
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/anonymity_previous.rb

@@ -0,0 +1,32 @@
+require_relative 'base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature describes whether the contributor of the old revision is
+      # an anonymous or registered Wikipedia user.
+      class AnonymityPrevious < Base
+        def calculate(edit)
+          super
+
+          old_revision = edit.old_revision
+
+          if old_revision.contributor.blank?
+            xml = Wikipedia.api_request(
+              prop: 'revisions',
+              rvprop: 'user',
+              revids: old_revision.id
+            )
+
+            contributor = xml.xpath('//rev/@user').text
+            return Features::MISSING_VALUE if contributor.blank?
+
+            old_revision.contributor = contributor
+          end
+
+          old_revision.anonymous_contributor? ? 0 : 1
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/article_size.rb

@@ -0,0 +1,17 @@
+require 'wikipedia/vandalism_detection/features/base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the size of the edit's new revision text
+      # (article size).
+      class ArticleSize < Base
+        def calculate(edit)
+          super
+
+          edit.new_revision.text.size
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/bad_frequency.rb

@@ -0,0 +1,21 @@
+require 'wikipedia/vandalism_detection/features/frequency_base'
+require 'wikipedia/vandalism_detection/word_lists/bad'
+require 'wikipedia/vandalism_detection/text'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes frequency of bad words in the inserted text.
+      class BadFrequency < FrequencyBase
+        # Returns the percentage of bad words in the inserted text.
+        # Returns 0.0 if inserted clean text is of zero length.
+        def calculate(edit)
+          super
+
+          text = Text.new(edit.inserted_words.join("\n")).clean
+          frequency(text, WordLists::BAD)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/bad_impact.rb

@@ -0,0 +1,20 @@
+require 'wikipedia/vandalism_detection/features/impact_base'
+require 'wikipedia/vandalism_detection/word_lists/bad'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the percentage by which the edit increases the
+      # number of bad words in the text.
+      class BadImpact < ImpactBase
+        def calculate(edit)
+          super
+          old_text = edit.old_revision.text.clean
+          new_text = edit.new_revision.text.clean
+
+          impact(old_text, new_text, WordLists::BAD)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/base.rb

@@ -0,0 +1,61 @@
+require 'wikipedia'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      MISSING_VALUE = '?'.freeze
+
+      # This class should be the base class for all Wikipedia::Feature classes.
+      class Base
+        # Base method for feature calculation.
+        # This method should be overwritten in the concrete
+        # Wikipedia::Feature-classes.
+        #
+        # @example
+        #   def calculate(edit)
+        #     super # to handle ArgumentException
+        #
+        #     ... concrete calculation of feature out of edit...
+        #   end
+        def calculate(edit)
+          return if edit.is_a?(Edit)
+          raise ArgumentError, 'Passed argument has to be an Edit'
+        end
+
+        # Count the apperance of a given single term or multiple terms in the
+        #   given text
+        #
+        # @param terms String
+        # @param options Hash of form { in: String }
+        #
+        # @example
+        #   feature.count "and", in: text
+        #   feature.count ["and", "or"], in: text
+        #
+        # @return Integer
+        def count(terms, options = {})
+          unless options[:in]
+            raise ArgumentError, 'The options hash must include the in: key'
+          end
+
+          unless terms.is_a?(String) || terms.is_a?(Array)
+            raise ArgumentError, 'The 1st arg should be an Array or String'
+          end
+
+          words = options[:in].downcase
+          freq = Hash.new(0)
+
+          words.gsub(/[\.,'{2,}:\!\?\(\)]/, '').split.each do |word|
+            freq[word.to_sym] += 1
+          end
+
+          if terms.is_a?(String)
+            freq[terms.downcase.to_sym]
+          else
+            terms.reduce(0) { |result, term| result + freq[term] }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/biased_frequency.rb

@@ -0,0 +1,21 @@
+require 'wikipedia/vandalism_detection/features/frequency_base'
+require 'wikipedia/vandalism_detection/word_lists/biased'
+require 'wikipedia/vandalism_detection/text'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes frequency of biased words in the inserted text.
+      class BiasedFrequency < FrequencyBase
+        # Returns the percentage of biased words in the inserted text.
+        # Returns 0.0 if inserted clean text is of zero length.
+        def calculate(edit)
+          super
+
+          text = Text.new(edit.inserted_words.join("\n")).clean
+          frequency(text, WordLists::BIASED)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/biased_impact.rb

@@ -0,0 +1,21 @@
+require 'wikipedia/vandalism_detection/features/impact_base'
+require 'wikipedia/vandalism_detection/word_lists/biased'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the percentage by which the edit increases the
+      #  number of biased words in the text.
+      class BiasedImpact < ImpactBase
+        def calculate(edit)
+          super
+
+          old_text = edit.old_revision.text.clean
+          new_text = edit.new_revision.text.clean
+
+          impact(old_text, new_text, WordLists::BIASED)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/blanking.rb

@@ -0,0 +1,26 @@
+require 'wikipedia/vandalism_detection/features/base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature returns whether the edit's is a blanking.
+      # size < 7, based on Mola Velasco 2010 implementation.
+      class Blanking < Base
+        BLANKING_THRESHOLD = 7
+
+        def calculate(edit)
+          super
+
+          old_text_size = edit.old_revision.text.size
+          new_text_size = edit.new_revision.text.size
+
+          text_removed = old_text_size > new_text_size
+          above_threshold = new_text_size < BLANKING_THRESHOLD
+
+          blanking = text_removed && above_threshold
+          blanking ? 1.0 : 0.0
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/character_diversity.rb

@@ -0,0 +1,23 @@
+require 'wikipedia/vandalism_detection/features/base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the character diversity of the edit's new revision inserted text.
+      # I.e. how many unique characters are amongst all inserted?
+      #
+      # Random typing leads to less unique characters relative to full length =>
+      class CharacterDiversity < Base
+        def calculate(edit)
+          super
+
+          inserted_letters = edit.inserted_text.scan(/[^\s]/)
+          all_letters_count = inserted_letters.size
+          unique_count = inserted_letters.uniq.size
+
+          all_letters_count**(1.0 / unique_count)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/character_sequence.rb

@@ -0,0 +1,19 @@
+require 'wikipedia/vandalism_detection/features/base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the length of the longest sequence of the same
+      # character in the inserted text.
+      class CharacterSequence < Base
+        def calculate(edit)
+          super
+
+          sequence_hash = edit.inserted_text.scan(/((.)\2*)/)
+          sequence_hash = sequence_hash.group_by { |seq, _| seq.length }
+          sequence_hash.empty? ? 0 : sequence_hash.max.first
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/comment_bad_frequency.rb

@@ -0,0 +1,21 @@
+require 'wikipedia/vandalism_detection/features/frequency_base'
+require 'wikipedia/vandalism_detection/word_lists/bad'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes frequency of bad words in the comment of the
+      # edit's new revision.
+      class CommentBadFrequency < FrequencyBase
+        # Returns the percentage of bad words in the new revision's comment.
+        # Returns 0.0 if text is of zero length.
+        def calculate(edit)
+          super
+
+          comment = edit.new_revision.comment.clean
+          frequency(comment, WordLists::BAD)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/comment_biased_frequency.rb

@@ -0,0 +1,21 @@
+require 'wikipedia/vandalism_detection/features/frequency_base'
+require 'wikipedia/vandalism_detection/word_lists/biased'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes frequency of biased words in the comment of the
+      # edit's new revision.
+      class CommentBiasedFrequency < FrequencyBase
+        # Returns the percentage of biased words in the new revision's comment.
+        # Returns 0.0 if text is of zero length.
+        def calculate(edit)
+          super
+
+          comment = edit.new_revision.comment.clean
+          frequency(comment, WordLists::BIASED)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/comment_length.rb

@@ -0,0 +1,16 @@
+require 'wikipedia/vandalism_detection/features/base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the byte length of the edit's new revision's
+      # comment.
+      class CommentLength < Base
+        def calculate(edit)
+          super
+          edit.new_revision.comment.clean.bytesize
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/comment_markup_frequency.rb

@@ -0,0 +1,30 @@
+require 'wikipedia/vandalism_detection/features/base'
+require 'wikipedia/vandalism_detection/word_lists/markup'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes frequency of markup words in the comment of the
+      # edit's new revision.
+      class CommentMarkupFrequency < Base
+        MARKUP_REGEX = /(#{WordLists::MARKUP.join('|')})/
+
+        # Returns the percentage of markup words in the new revision's comment.
+        # Returns 0.0 if text is of zero length.
+        def calculate(edit)
+          super
+
+          comment = edit.new_revision.comment
+          all_words_count = comment.split.count
+          markup_words_count = comment.scan(MARKUP_REGEX).count
+
+          if all_words_count > 0
+            markup_words_count.to_f / all_words_count.to_f
+          else
+            0.0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/comment_pronoun_frequency.rb

@@ -0,0 +1,21 @@
+require 'wikipedia/vandalism_detection/features/frequency_base'
+require 'wikipedia/vandalism_detection/word_lists/pronouns'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the frequency of pronouns in the comment of the
+      # new revision.
+      class CommentPronounFrequency < FrequencyBase
+        # Returns the percentage of pronoun words in the new revision's
+        # comment. Returns 0.0 if text is of zero length.
+        def calculate(edit)
+          super
+
+          comment = edit.new_revision.comment.clean
+          frequency(comment, WordLists::PRONOUNS)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/comment_sex_frequency.rb

@@ -0,0 +1,21 @@
+require 'wikipedia/vandalism_detection/features/frequency_base'
+require 'wikipedia/vandalism_detection/word_lists/sex'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes frequency of sex words in the comment of the
+      # edit's new revision.
+      class CommentSexFrequency < FrequencyBase
+        # Returns the percentage of sex words in the new revision's comment.
+        # Returns 0.0 if text is of zero length.
+        def calculate(edit)
+          super
+
+          comment = edit.new_revision.comment.clean
+          frequency(comment, WordLists::SEX)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/comment_vulgarism_frequency.rb

@@ -0,0 +1,21 @@
+require 'wikipedia/vandalism_detection/features/frequency_base'
+require 'wikipedia/vandalism_detection/word_lists/vulgarism'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes frequency of vulgarism words in the comment of the
+      # edit's new revision.
+      class CommentVulgarismFrequency < FrequencyBase
+        # Returns the percentage of vulgarism words in the new revision's
+        # comment.Returns 0.0 if text is of zero length.
+        def calculate(edit)
+          super
+
+          comment = edit.new_revision.comment.clean
+          frequency(comment, WordLists::VULGARISM)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/compressibility.rb

@@ -0,0 +1,31 @@
+require 'wikipedia/vandalism_detection/features/base'
+require 'zlib'
+require 'wikipedia/vandalism_detection/diff'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature describes compressibility ratio of compressed and
+      # uncompressed inserted text.
+      class Compressibility < Base
+        # Calculates the compressibility ratio of the inserted text.
+        # Values above 0.5 are higher compressed and therefor can stand for
+        # nonsense text as:
+        # 'AAAAAAAAAAAAAAAAAAAhhhhhhhhhhhhhhhh!' etc.
+        def calculate(edit)
+          super
+
+          inserted_text = edit.inserted_text
+          uncompressed_size = inserted_text.bytesize.to_f
+          compressed_size = Zlib::Deflate.deflate(inserted_text).bytesize.to_f
+
+          if inserted_text.empty?
+            0.5
+          else
+            uncompressed_size / (compressed_size + uncompressed_size)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/contains_base.rb

@@ -0,0 +1,16 @@
+require 'wikipedia/vandalism_detection/features/base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      class ContainsBase < Base
+        # Returns whether the comment contains the given term.
+        # It returns 0 if term is not included, else 1.
+        def contains(comment, terms)
+          terms = terms.is_a?(Array) ? terms.join('|') : terms
+          comment =~ /#{terms}/i ? 1 : 0
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/copyedit.rb

@@ -0,0 +1,17 @@
+require 'wikipedia/vandalism_detection/features/contains_base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature returns whether the edit's comment includes reverted key
+      # words.
+      class Reverted < ContainsBase
+        def calculate(edit)
+          super
+
+          contains(edit.new_revision.comment, %w[rvt rvv revert])
+        end
+      end
+    end
+  end
+end