wikipedia-vandalism_detection 0.1.0-java

data/lib/wikipedia/vandalism_detection/features/digit_ratio.rb

@@ -0,0 +1,23 @@
+require 'wikipedia/vandalism_detection/features/base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the digit to all letters ratio of the edit's new
+      # revision inserted text.
+      class DigitRatio < Base
+        def calculate(edit)
+          super
+
+          text = edit.inserted_text
+          return 0.0 if text.empty?
+
+          all_letters_count = text.scan(/[[:alnum:]]/).size
+          digit_count = text.scan(/[[:digit:]]/).size
+
+          (1.0 + digit_count) / (1.0 + all_letters_count)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/edits_per_user.rb

@@ -0,0 +1,72 @@
+require 'wikipedia/vandalism_detection/features/base'
+require 'open-uri'
+require 'nokogiri'
+require 'date'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature calculates the number of submitted edits by the same editor
+      # (IP or ID) as the edit's editor.
+      class EditsPerUser < Base
+        # Returns the number of edits the edit's editor made in the same
+        # article. Attention: This is pretty time consuming (~2sec) due to the
+        # url request.
+        def calculate(edit)
+          super
+
+          revision = edit.new_revision
+          page = edit.page
+
+          if page && page.id
+            edits_count_from_page(edit)
+          else
+            edits_count_from_api_request(revision)
+          end
+        end
+
+        protected
+
+        def edits_count_from_page(edit)
+          edit_revision = edit.new_revision
+
+          edit.page.edits.reduce(0) do |count, page_edit|
+            page_revision = page_edit.new_revision
+
+            same_user = page_revision.contributor == edit_revision.contributor
+            diff = time_diff(page_revision.timestamp, edit_revision.timestamp)
+
+            count += 1 if same_user && diff < 0
+            count
+          end
+        end
+
+        def edits_count_from_api_request(revision)
+          params = {
+            list: 'usercontribs',
+            ucuser: revision.contributor,
+            ucprop: 'ids|timestamp'
+          }
+
+          xml = Wikipedia.api_request(params)
+
+          page_item = xml.xpath("//item[@revid='#{revision.id}']").first
+          return 0 unless page_item
+
+          page_id = page_item.xpath('@pageid').text
+
+          # count only edits before current
+          xml.xpath("//item[@pageid='#{page_id}']").reduce(0) do |count, item|
+            time = item.attr('timestamp')
+            count += 1 if time_diff(time, revision.timestamp) < 0
+            count
+          end
+        end
+
+        def time_diff(time1, time2)
+          ((DateTime.parse(time1) - DateTime.parse(time2)) * 24 * 60 * 60).to_i
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/emoticons_frequency.rb

@@ -0,0 +1,27 @@
+require 'wikipedia/vandalism_detection/features/base'
+require 'wikipedia/vandalism_detection/word_lists/emoticons'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes frequency of emoticon words in the inserted text.
+      class EmoticonsFrequency < Base
+        EMOJI_REGEX = /(^|\s)(#{WordLists::EMOTICONS.join('|')})(?=\s|$|\Z|[\.,!?]\s|[\.!?]\Z)/
+
+        # Returns the percentage of emoticon words in the inserted text.
+        # Returns 0.0 if inserted clean text is of zero length.
+        def calculate(edit)
+          super
+
+          inserted_text = edit.inserted_text
+          emoticons_count = inserted_text.scan(EMOJI_REGEX).flatten
+            .reject { |c| c.size < 2 }.count
+
+          total_count = inserted_text.split.count
+
+          total_count > 0 ? emoticons_count.to_f / total_count.to_f : 0.0
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/emoticons_impact.rb

@@ -0,0 +1,29 @@
+require 'wikipedia/vandalism_detection/features/base'
+require 'wikipedia/vandalism_detection/word_lists/emoticons'
+require 'wikipedia/vandalism_detection/text'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes impact of emoticons words in the inserted text.
+      class EmoticonsImpact < Base
+        EMOJI_REGEX = /(^|\s)(#{WordLists::EMOTICONS.join('|')})(?=\s|$|\Z|[\.,!?]\s|[\.!?]\Z)/
+
+        def calculate(edit)
+          super
+
+          old_text = edit.old_revision.text
+          new_text = edit.new_revision.text
+
+          old_count = old_text.scan(EMOJI_REGEX).flatten
+            .reject { |c| c.size < 2 }.count.to_f
+          new_count = new_text.scan(EMOJI_REGEX).flatten
+            .reject { |c| c.size < 2 }.count.to_f
+
+          no_terms_in_both = old_count.zero? && new_count.zero?
+          no_terms_in_both ? 0.5 : old_count / (old_count + new_count)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/frequency_base.rb

@@ -0,0 +1,18 @@
+require 'wikipedia/vandalism_detection/features/base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      class FrequencyBase < Base
+        # Returns the ratio of given numbers.
+        # For frequency calculation it returns 0.0 if total_count is zero.
+        def frequency(text, terms)
+          total_count = text.split.count
+          term_count = count terms, in: text
+
+          total_count > 0 ? term_count.to_f / total_count.to_f : 0.0
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/impact_base.rb

@@ -0,0 +1,25 @@
+require 'wikipedia/vandalism_detection/features/base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      class ImpactBase < Base
+        # Returns the ratio of given text's terms count.
+        # For impact calculation it returns 0.5 if number of terms in old text
+        # is zero.
+        def impact(old_text, new_text, terms)
+          old_terms_count = (count terms, in: old_text).to_f
+          new_terms_count = (count terms, in: new_text).to_f
+
+          no_terms_in_both = old_terms_count.zero? && new_terms_count.zero?
+
+          if no_terms_in_both
+            0.5
+          else
+            old_terms_count / (old_terms_count + new_terms_count)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/inserted_character_distribution.rb

@@ -0,0 +1,23 @@
+require 'wikipedia/vandalism_detection/features/base'
+require 'wikipedia/vandalism_detection/algorithms'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the Kullback-Leibler Divergence of the inserted
+      # text's character distribution
+      # relative to the character distribution of the old revision's text.
+      # The smaller the divergence, the higher the similarity of the
+      # distributions and conversely.
+      class InsertedCharacterDistribution < Base
+        include Algorithms
+
+        def calculate(edit)
+          super
+
+          kullback_leibler_divergence(edit.old_revision.text, edit.inserted_text)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/inserted_external_links.rb

@@ -0,0 +1,19 @@
+require 'wikipedia/vandalism_detection/features/base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the number of inserted external links of format
+      # [http://example.com].
+      class InsertedExternalLinks < Base
+        URL_REGEX = %r{\[?(https?|ftp)\s?:\s?\/\/[^\s\/$.?#].[^\s]*]?}i
+
+        def calculate(edit)
+          super
+
+          edit.inserted_text.scan(URL_REGEX).count
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/inserted_internal_links.rb

@@ -0,0 +1,18 @@
+require 'wikipedia/vandalism_detection/features/base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the number of inserted internal links [[link]].
+      class InsertedInternalLinks < Base
+        INTERNAL_LINK_REGEX = /\[{2}([^\[].*?)\]{2}/
+
+        def calculate(edit)
+          super
+
+          edit.inserted_text.scan(INTERNAL_LINK_REGEX).count
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/inserted_size.rb

@@ -0,0 +1,19 @@
+require 'wikipedia/vandalism_detection/features/base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the size of inserted text in the edit's new
+      # revision.
+      class InsertedSize < Base
+        # Returns the size of inserted character in the new revision.
+        def calculate(edit)
+          super
+
+          size = edit.inserted_text.size
+          size < 0 ? 0 : size
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/inserted_words.rb

@@ -0,0 +1,17 @@
+require 'wikipedia/vandalism_detection/features/base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the number of inserted words in the edit's new
+      # revision.
+      class InsertedWords < Base
+        def calculate(edit)
+          super
+
+          edit.inserted_words.count
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/longest_word.rb

@@ -0,0 +1,21 @@
+require 'wikipedia/vandalism_detection/features/base'
+require 'wikipedia/vandalism_detection/text'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the length of the longest word in the inserted
+      # text.
+      class LongestWord < Base
+        def calculate(edit)
+          super
+
+          sequence_hash = Text.new(edit.inserted_words.join("\n"))
+            .clean.split(/[\b\s+,;:]/).group_by(&:length)
+
+          sequence_hash.empty? ? 0 : sequence_hash.max.first
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/markup_frequency.rb

@@ -0,0 +1,29 @@
+require 'wikipedia/vandalism_detection/features/base'
+require 'wikipedia/vandalism_detection/word_lists/markup'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes frequency of bad words in the inserted text.
+      class MarkupFrequency < Base
+        MARKUP_REGEX = /(#{WordLists::MARKUP.join('|')})/
+
+        # Returns the percentage of markup related words in the inserted text.
+        # Returns 0.0 if inserted clean text is of zero length.
+        def calculate(edit)
+          super
+
+          text = edit.inserted_text
+          all_words_count = edit.inserted_words.count
+          markup_words_count = text.scan(MARKUP_REGEX).count
+
+          if all_words_count > 0
+            markup_words_count.to_f / all_words_count.to_f
+          else
+            0.0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/markup_impact.rb

@@ -0,0 +1,30 @@
+require 'wikipedia/vandalism_detection/features/base'
+require 'wikipedia/vandalism_detection/word_lists/markup'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the percentage by which the edit increases the
+      # number of markup words in the text.
+      class MarkupImpact < Base
+        MARKUP_REGEX = /(#{WordLists::MARKUP.join('|')})/
+
+        def calculate(edit)
+          super
+
+          old_text = edit.old_revision.text
+          new_text = edit.new_revision.text
+
+          old_markup_count = old_text.scan(MARKUP_REGEX).count.to_f
+          new_markup_count = new_text.scan(MARKUP_REGEX).count.to_f
+
+          if old_markup_count.zero? && new_markup_count.zero?
+            0.5
+          else
+            old_markup_count / (old_markup_count + new_markup_count)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/non_alphanumeric_ratio.rb

@@ -0,0 +1,23 @@
+require 'wikipedia/vandalism_detection/features/base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the non-alphanumeric to all letters ratio of the
+      # edit's new revision inserted text.
+      class NonAlphanumericRatio < Base
+        def calculate(edit)
+          super
+
+          text = edit.inserted_text
+          return 0.0 if text.empty?
+
+          non_alpha_count = text.scan(/[^a-zA-Z0-9\s]/).size
+          all_letters_count = text.scan(/[^\s]/).size
+
+          (1.0 + non_alpha_count) / (1.0 + all_letters_count)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/personal_life.rb

@@ -0,0 +1,17 @@
+require 'wikipedia/vandalism_detection/features/contains_base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature returns whether the edit's comment includes
+      # 'personal life'.
+      class PersonalLife < ContainsBase
+        def calculate(edit)
+          super
+
+          contains(edit.new_revision.comment, 'personal life')
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/pronoun_frequency.rb

@@ -0,0 +1,21 @@
+require 'wikipedia/vandalism_detection/features/frequency_base'
+require 'wikipedia/vandalism_detection/word_lists/pronouns'
+require 'wikipedia/vandalism_detection/text'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the frequency of pronouns in the insetred text.
+      class PronounFrequency < FrequencyBase
+        # Returns the percentage of pronoun words in the inserted text.
+        # Returns 0.0 if inserted clean text is of zero length.
+        def calculate(edit)
+          super
+
+          text = Text.new(edit.inserted_words.join("\n")).clean
+          frequency(text, WordLists::PRONOUNS)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/pronoun_impact.rb

@@ -0,0 +1,20 @@
+require 'wikipedia/vandalism_detection/features/impact_base'
+require 'wikipedia/vandalism_detection/word_lists/pronouns'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the percentage by which the edit increases the
+      # number of pronouns in the text.
+      class PronounImpact < ImpactBase
+        def calculate(edit)
+          super
+          old_text = edit.old_revision.text.clean
+          new_text = edit.new_revision.text.clean
+
+          impact(old_text, new_text, WordLists::PRONOUNS)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/removed_all_wordlists_frequency.rb

@@ -0,0 +1,22 @@
+require 'wikipedia/vandalism_detection/features/frequency_base'
+require 'wikipedia/vandalism_detection/word_lists'
+require 'wikipedia/vandalism_detection/text'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes frequency of all wordlists words in the removed
+      # text.
+      class RemovedAllWordlistsFrequency < FrequencyBase
+        # Returns the percentage of wordlists words in the removed text.
+        # Returns 0.0 if removed clean text is of zero length.
+        def calculate(edit)
+          super
+
+          text = Text.new(edit.removed_words.join("\n")).clean
+          frequency(text, WordLists.all)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/removed_bad_frequency.rb

@@ -0,0 +1,21 @@
+require 'wikipedia/vandalism_detection/features/frequency_base'
+require 'wikipedia/vandalism_detection/word_lists/sex'
+require 'wikipedia/vandalism_detection/text'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the frequency of bad words in the removed text.
+      class RemovedBadFrequency < FrequencyBase
+        # Returns the percentage of bad words in the removed text.
+        # Returns 0.0 if cleaned removed text is of zero length.
+        def calculate(edit)
+          super
+
+          text = Text.new(edit.removed_words.join("\n")).clean
+          frequency(text, WordLists::BAD)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/removed_biased_frequency.rb

@@ -0,0 +1,21 @@
+require 'wikipedia/vandalism_detection/features/frequency_base'
+require 'wikipedia/vandalism_detection/word_lists/biased'
+require 'wikipedia/vandalism_detection/text'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes frequency of biased words in the removed text.
+      class RemovedBiasedFrequency < FrequencyBase
+        # Returns the percentage of biased words in the removed text.
+        # Returns 0.0 if removed clean text is of zero length.
+        def calculate(edit)
+          super
+
+          text = Text.new(edit.removed_words.join("\n")).clean
+          frequency(text, WordLists::BIASED)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/removed_character_distribution.rb

@@ -0,0 +1,23 @@
+require 'wikipedia/vandalism_detection/features/base'
+require 'wikipedia/vandalism_detection/algorithms'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the Kullback-Leibler Divergence of the removed
+      # text's character distribution relative to the character distribution
+      # of the new revision's text.
+      # The smaller the divergence, the higher the similarity of the
+      # distributions and conversely.
+      class RemovedCharacterDistribution < Base
+        include Algorithms
+
+        def calculate(edit)
+          super
+
+          kullback_leibler_divergence(edit.new_revision.text, edit.removed_text)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/removed_emoticons_frequency.rb

@@ -0,0 +1,28 @@
+require 'wikipedia/vandalism_detection/features/base'
+require 'wikipedia/vandalism_detection/word_lists/emoticons'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the frequency of emoticon words in the removed
+      # text.
+      class RemovedEmoticonsFrequency < Base
+        # Returns the percentage of markup words in the removed text.
+        # Returns 0.0 if cleaned removed text is of zero length.
+        def calculate(edit)
+          super
+
+          removed_text = edit.removed_text
+          emojis = WordLists::EMOTICONS.join('|')
+          regex = /(^|\s)(#{emojis})(?=\s|$|\Z|[\.,!?]\s|[\.!?]\Z)/
+
+          emoticons_count = removed_text.scan(regex).flatten
+            .reject { |c| c.size < 2 }.count
+          total_count = removed_text.split.count
+
+          total_count > 0 ? emoticons_count.to_f / total_count.to_f : 0.0
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/removed_markup_frequency.rb

@@ -0,0 +1,30 @@
+require 'wikipedia/vandalism_detection/features/base'
+require 'wikipedia/vandalism_detection/word_lists/markup'
+require 'wikipedia/vandalism_detection/text'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the frequency of markup words in the removed text.
+      class RemovedMarkupFrequency < Base
+        MARKUP_REGEX = /(#{WordLists::MARKUP.join('|')})/
+
+        # Returns the percentage of markup words in the removed text.
+        # Returns 0.0 if cleaned removed text is of zero length.
+        def calculate(edit)
+          super
+
+          text = edit.removed_text
+          all_words_count = edit.removed_words.count
+          markup_words_count = text.scan(MARKUP_REGEX).count
+
+          if all_words_count > 0
+            markup_words_count.to_f / all_words_count.to_f
+          else
+            0.0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/removed_pronoun_frequency.rb

@@ -0,0 +1,21 @@
+require 'wikipedia/vandalism_detection/features/frequency_base'
+require 'wikipedia/vandalism_detection/word_lists/pronouns'
+require 'wikipedia/vandalism_detection/text'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the frequency of pronouns in the removed text.
+      class RemovedPronounFrequency < FrequencyBase
+        # Returns the percentage of pronoun words in the removed text.
+        # Returns 0.0 if cleaned removed text is of zero length.
+        def calculate(edit)
+          super
+
+          text = Text.new(edit.removed_words.join("\n")).clean
+          frequency(text, WordLists::PRONOUNS)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/removed_sex_frequency.rb

@@ -0,0 +1,21 @@
+require 'wikipedia/vandalism_detection/features/frequency_base'
+require 'wikipedia/vandalism_detection/word_lists/sex'
+require 'wikipedia/vandalism_detection/text'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+      # This feature computes the frequency of sex words in the removed text.
+      class RemovedSexFrequency < FrequencyBase
+        # Returns the percentage of sex words in the removed text.
+        # Returns 0.0 if cleaned removed text is of zero length.
+        def calculate(edit)
+          super
+
+          text = Text.new(edit.removed_words.join("\n")).clean
+          frequency(text, WordLists::SEX)
+        end
+      end
+    end
+  end
+end