wikipedia-vandalism_detection 0.0.1

data/lib/wikipedia/vandalism_detection/features/blanking.rb

@@ -0,0 +1,25 @@
+require 'wikipedia/vandalism_detection/features/base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature returns whether the edit's is a blanking.
+      # size < 7, based on Mola Velasco 2010 implementation.
+      class Blanking < Base
+
+        BLANKING_THRESHOLD = 7
+
+        def calculate(edit)
+          super
+
+          old_text_size = edit.old_revision.text.size
+          new_text_size = edit.new_revision.text.size
+
+          blanking = (old_text_size > new_text_size) && (new_text_size < BLANKING_THRESHOLD)
+          blanking ? 1.0 : 0.0
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/character_diversity.rb

@@ -0,0 +1,25 @@
+require 'wikipedia/vandalism_detection/features/base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature computes the character diversity of the edit's new revision inserted text.
+      # I.e. how many unique characters are amongst all inserted?
+      #
+      # Random typing leads to less unique characters relative to full length =>
+      class CharacterDiversity < Base
+
+        def calculate(edit)
+          super
+
+          inserted_letters = edit.inserted_text.scan(/[^\s]/)
+          all_letters_count = inserted_letters.size
+          unique_count = inserted_letters.uniq.size
+
+          all_letters_count ** (1.0 / unique_count)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/character_sequence.rb

@@ -0,0 +1,19 @@
+require 'wikipedia/vandalism_detection/features/base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature computes the length of the longest sequence of the same character in the inserted text.
+      class CharacterSequence < Base
+
+        def calculate(edit)
+          super
+
+          sequence_hash = edit.inserted_text.scan(/((.)\2*)/).group_by{ |s, c| s.length }
+          sequence_hash.empty? ? 0 : sequence_hash.max.first
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/comment_bad_frequency.rb

@@ -0,0 +1,22 @@
+require 'wikipedia/vandalism_detection/features/frequency_base'
+require 'wikipedia/vandalism_detection/word_lists/bad'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature computes frequency of bad words in the comment of the edit's new revision.
+      class CommentBadFrequency < FrequencyBase
+
+        # Returns the percentage of bad words in the new revision's comment.
+        # Returns 0.0 if text is of zero length.
+        def calculate(edit)
+          super
+
+          comment = edit.new_revision.comment.clean
+          frequency(comment, WordLists::BAD)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/comment_biased_frequency.rb

@@ -0,0 +1,22 @@
+require 'wikipedia/vandalism_detection/features/frequency_base'
+require 'wikipedia/vandalism_detection/word_lists/biased'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature computes frequency of biased words in the comment of the edit's new revision.
+      class CommentBiasedFrequency < FrequencyBase
+
+        # Returns the percentage of biased words in the new revision's comment.
+        # Returns 0.0 if text is of zero length.
+        def calculate(edit)
+          super
+
+          comment = edit.new_revision.comment.clean
+          frequency(comment, WordLists::BIASED)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/comment_length.rb

@@ -0,0 +1,17 @@
+require 'wikipedia/vandalism_detection/features/base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature computes the byte length of the edit's new revision's comment.
+      class CommentLength < Base
+
+        def calculate(edit)
+          super
+          edit.new_revision.comment.clean.bytesize
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/comment_markup_frequency.rb

@@ -0,0 +1,27 @@
+require 'wikipedia/vandalism_detection/features/base'
+require 'wikipedia/vandalism_detection/word_lists/markup'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature computes frequency of markup words in the comment of the edit's new revision.
+      class CommentMarkupFrequency < Base
+
+        # Returns the percentage of markup words in the new revision's comment.
+        # Returns 0.0 if text is of zero length.
+        def calculate(edit)
+          super
+
+          comment = edit.new_revision.comment
+          all_words_count = comment.split.count
+
+          regex = /(#{WordLists::MARKUP.join('|')})/
+          markup_words_count = comment.scan(regex).count
+
+          (all_words_count > 0) ? (markup_words_count.to_f) / (all_words_count.to_f) : 0.0
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/comment_pronoun_frequency.rb

@@ -0,0 +1,22 @@
+require 'wikipedia/vandalism_detection/features/frequency_base'
+require 'wikipedia/vandalism_detection/word_lists/pronouns'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature computes the frequency of pronouns in the comment of the new revision.
+      class CommentPronounFrequency < FrequencyBase
+
+        # Returns the percentage of pronoun words in the new revision's comment.
+        # Returns 0.0 if text is of zero length.
+        def calculate(edit)
+          super
+
+          comment = edit.new_revision.comment.clean
+          frequency(comment, WordLists::PRONOUNS)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/comment_sex_frequency.rb

@@ -0,0 +1,22 @@
+require 'wikipedia/vandalism_detection/features/frequency_base'
+require 'wikipedia/vandalism_detection/word_lists/sex'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature computes frequency of sex words in the comment of the edit's new revision.
+      class CommentSexFrequency < FrequencyBase
+
+        # Returns the percentage of sex words in the new revision's comment.
+        # Returns 0.0 if text is of zero length.
+        def calculate(edit)
+          super
+
+          comment = edit.new_revision.comment.clean
+          frequency(comment, WordLists::SEX)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/comment_vulgarism_frequency.rb

@@ -0,0 +1,22 @@
+require 'wikipedia/vandalism_detection/features/frequency_base'
+require 'wikipedia/vandalism_detection/word_lists/vulgarism'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature computes frequency of vulgarism words in the comment of the edit's new revision.
+      class CommentVulgarismFrequency < FrequencyBase
+
+        # Returns the percentage of vulgarism words in the new revision's comment.
+        # Returns 0.0 if text is of zero length.
+        def calculate(edit)
+          super
+
+          comment = edit.new_revision.comment.clean
+          frequency(comment, WordLists::VULGARISM)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/compressibility.rb

@@ -0,0 +1,27 @@
+require 'wikipedia/vandalism_detection/features/base'
+require 'zlib'
+require 'wikipedia/vandalism_detection/diff'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature describes compressibility ratio of compressed and uncompressed inserted text.
+      class Compressibility < Base
+
+        # Claculates the compressibility ratio of the inserted text.
+        # Values above 0.5 are higher compressed and therefor can stand for nonsense text as:
+        # 'AAAAAAAAAAAAAAAAAAAhhhhhhhhhhhhhhhh!' etc.
+        def calculate(edit)
+          super
+
+          inserted_text = edit.inserted_text
+          uncompressed_size = inserted_text.bytesize.to_f
+          compressed_size = Zlib::Deflate.deflate(inserted_text).bytesize.to_f
+
+          inserted_text.empty? ? 0.5 : (uncompressed_size / ( compressed_size + uncompressed_size))
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/contains_base.rb

@@ -0,0 +1,18 @@
+require 'wikipedia/vandalism_detection/features/base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      class ContainsBase < Base
+
+        # Returns whether the comment contains the given term.
+        # It returns 0 if term is not included, else 1.
+        def contains(comment, terms)
+          terms = terms.is_a?(Array) ? terms.join('|') : terms
+          (comment =~ /#{terms}/i) ? 1 : 0
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/copyedit.rb

@@ -0,0 +1,18 @@
+require 'wikipedia/vandalism_detection/features/contains_base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature returns whether the edit's comment includes reverted key words.
+      class Reverted < ContainsBase
+
+        def calculate(edit)
+          super
+
+          contains(edit.new_revision.comment, ['rvt', 'rvv', 'revert'])
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/digit_ratio.rb

@@ -0,0 +1,24 @@
+require 'wikipedia/vandalism_detection/features/base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature computes the digit to all letters ratio of the edit's new revision inserted text.
+      class DigitRatio < Base
+
+        def calculate(edit)
+          super
+
+          text = edit.inserted_text
+          return 0.0 if text.empty?
+
+          all_letters_count = text.scan(/[[:alnum:]]/).size
+          digit_count = text.scan(/[[:digit:]]/).size
+
+          (1.0 + digit_count) / (1.0 + all_letters_count)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/edits_per_user.rb

@@ -0,0 +1,65 @@
+require 'wikipedia/vandalism_detection/features/base'
+require 'open-uri'
+require 'nokogiri'
+require 'date'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature calculates the number of submitted edits by the same editor (IP or ID) as the edit's editor.
+      class EditsPerUser < Base
+
+        # Returns the number of edits the edit's editor made in the same article.
+        # Attention! This is pretty time consuming (~2sec) due to the url request.
+        def calculate(edit)
+          super
+
+          revision = edit.new_revision
+          page = edit.page
+
+          if page && page.id
+            edits_count_from_page(edit)
+          else
+            edits_count_from_api_request(revision)
+          end
+        end
+
+        protected
+
+        def edits_count_from_page(edit)
+          edit_revision = edit.new_revision
+
+          previous_count = edit.page.edits.reduce(0) do |count, page_edit|
+            page_revision = page_edit.new_revision
+            count += 1 if page_revision.contributor == edit_revision.contributor &&
+                time_diff_in_sec(page_revision.timestamp, edit_revision.timestamp) < 0
+            count
+          end
+        end
+
+        def edits_count_from_api_request(revision)
+          xml = Wikipedia::api_request({list: 'usercontribs', ucuser: revision.contributor, ucprop: 'ids|timestamp'})
+          page_item = xml.xpath("//item[@revid='#{revision.id}']").first
+
+          if page_item
+            page_id = page_item.xpath("@pageid").text
+
+            # count only edits before current
+            count = xml.xpath("//item[@pageid='#{page_id}']").reduce(0) do |count, item|
+              time = item.attr('timestamp')
+              count += 1 if time_diff_in_sec(revision.timestamp, time) > 0
+              count
+            end
+          else
+            0
+          end
+        end
+
+        def time_diff_in_sec(time1, time2)
+          ((DateTime.parse(time1) - DateTime.parse(time2)) * 24 * 60 * 60).to_i
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/emoticons_frequency.rb

@@ -0,0 +1,27 @@
+require 'wikipedia/vandalism_detection/features/base'
+require 'wikipedia/vandalism_detection/word_lists/emoticons'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature computes frequency of emoticon words in the inserted text.
+      class EmoticonsFrequency < Base
+
+        # Returns the percentage of emoticon words in the inserted text.
+        # Returns 0.0 if inserted clean text is of zero length.
+        def calculate(edit)
+          super
+
+          inserted_text = edit.inserted_text
+          regex = /(^|\s)(#{WordLists::EMOTICONS.join('|')})(?=\s|$|\Z|[\.,!?]\s|[\.!?]\Z)/
+
+          emoticons_count = inserted_text.scan(regex).flatten.reject { |c| c.size < 2 }.count
+          total_count = inserted_text.split.count
+
+          (total_count > 0) ? (emoticons_count.to_f) / (total_count.to_f) : 0.0
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/emoticons_impact.rb

@@ -0,0 +1,29 @@
+require 'wikipedia/vandalism_detection/features/base'
+require 'wikipedia/vandalism_detection/word_lists/emoticons'
+require 'wikipedia/vandalism_detection/text'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature computes impact of emoticons words in the inserted text.
+      class EmoticonsImpact < Base
+
+        def calculate(edit)
+          super
+
+          old_text = edit.old_revision.text
+          new_text = edit.new_revision.text
+
+          regex = /(^|\s)(#{WordLists::EMOTICONS.join('|')})(?=\s|$|\Z|[\.,!?]\s|[\.!?]\Z)/
+
+          old_count = old_text.scan(regex).flatten.reject { |c| c.size < 2 }.count.to_f
+          new_count = new_text.scan(regex).flatten.reject { |c| c.size < 2 }.count.to_f
+
+          no_terms_in_both = (old_count == 0 && new_count == 0)
+          no_terms_in_both ? 0.5 : (old_count / (old_count + new_count))
+        end
+      end
+    end
+  end
+end