wikipedia-vandalism_detection 0.0.1

data/lib/wikipedia/vandalism_detection/features/upper_case_words_ratio.rb

@@ -0,0 +1,31 @@
+# encoding: UTF-8
+
+require 'wikipedia/vandalism_detection/features/base'
+require 'wikipedia/vandalism_detection/text'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature computes the uppercase to all words ratio of the edit's new revision inserted text.
+      class UpperCaseWordsRatio < Base
+
+        def calculate(edit)
+          super
+
+          inserted_alpha_text = edit.inserted_words.delete_if{ |w| w.gsub(/[^A-Za-z]/, '').empty? }.join("\n")
+          words = Text.new(inserted_alpha_text).clean.gsub(/[^\w\s]/, '').split
+
+          return 0.0 if words.empty?
+
+          uppercase_words_count = words.reduce(0) do |count, word|
+            count += 1 if word == word.upcase
+            count
+          end
+
+          (1.0 + uppercase_words_count) / (1.0 + words.count)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/upper_to_lower_case_ratio.rb

@@ -0,0 +1,24 @@
+require 'wikipedia/vandalism_detection/features/base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature computes the uppercase to all letters ratio of the edit's new revision inserted text.
+      class UpperToLowerCaseRatio< Base
+
+        def calculate(edit)
+          super
+
+          text = edit.inserted_text
+          return 0.0 if text.empty?
+
+          uppercase_count = text.scan(/[[:upper:]]/).size
+          lowercase_count = text.scan(/[[:lower:]]/).size
+
+          (1.0 + uppercase_count) / (1.0 + lowercase_count)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/user_reputation.rb

@@ -0,0 +1,38 @@
+require 'wikipedia/vandalism_detection/features/base'
+require 'open-uri'
+require 'nokogiri'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature calculates the average editor's reputation on the current article.
+      class UserReputation < Base
+
+        # Attention! This can be pretty time consuming (up to 2 sec) due to the url request.
+        def calculate(edit)
+          super
+
+          revision = edit.new_revision
+          page_id = edit.page.id || Wikipedia::api_request({ titles: edit.page.title }).xpath("//page/@pageid").first
+
+          text = Wikipedia::wikitrust_request({ pageid: page_id, revid: revision.id })
+          contributions = text.scan(/(\{\{#t:\d+,\d+,#{revision.contributor}\}\})/)
+          # {{#t:trust,revision_id,UserName}}
+
+          trust = 0.0
+
+          unless contributions.empty?
+            sum = contributions.reduce(0.0) do |sum, contribution|
+              sum += contribution[0].split(',').first.split(':').last.to_f
+            end
+
+            trust = sum / contributions.count.to_f
+          end
+
+          trust
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/vulgarism_frequency.rb

@@ -0,0 +1,23 @@
+require 'wikipedia/vandalism_detection/features/frequency_base'
+require 'wikipedia/vandalism_detection/word_lists/vulgarism'
+require 'wikipedia/vandalism_detection/text'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature computes frequency of vulgarism words in the inserted text.
+      class VulgarismFrequency < FrequencyBase
+
+        # Returns the percentage of vulgarism words in the inserted text.
+        # Returns 0.0 if inserted clean text is of zero length.
+        def calculate(edit)
+          super
+
+          text = Text.new(edit.inserted_words.join("\n")).clean
+          frequency(text, WordLists::VULGARISM)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/vulgarism_impact.rb

@@ -0,0 +1,22 @@
+require 'wikipedia/vandalism_detection/features/impact_base'
+require 'wikipedia/vandalism_detection/word_lists/vulgarism'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature computes the percentage by which the edit increases the number of vulgarism words in the text.
+      class VulgarismImpact < ImpactBase
+
+        def calculate(edit)
+          super
+
+          old_text = edit.old_revision.text.clean
+          new_text = edit.new_revision.text.clean
+
+          impact(old_text, new_text, WordLists::VULGARISM)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/weekday.rb

@@ -0,0 +1,21 @@
+require 'wikipedia/vandalism_detection/features/base'
+require 'date'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature calculates the weekday of new revision edit as numeric value.
+      # Monday => 1, Thuesday => 2, etc.
+      class Weekday < Base
+
+        def calculate(edit)
+          super
+
+          timestamp = edit.new_revision.timestamp
+          DateTime.parse(timestamp).wday
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/features/words_increment.rb

@@ -0,0 +1,22 @@
+require 'wikipedia/vandalism_detection/features/base'
+
+module Wikipedia
+  module VandalismDetection
+    module Features
+
+      # This feature computes the increment of the edit's revisions words.
+      class WordsIncrement < Base
+
+        #	computation: |inserted| - |removed|
+        def calculate(edit)
+          super
+
+          inserted_count = edit.inserted_words.count
+          removed_count = edit.removed_words.count
+
+          inserted_count - removed_count
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/instances.rb

@@ -0,0 +1,130 @@
+require 'ruby-band'
+require 'wikipedia/vandalism_detection/configuration'
+require 'weka/classifiers/meta/one_class_classifier'
+
+module Wikipedia
+  module VandalismDetection
+
+    class Instances
+
+      REGULAR_CLASS_INDEX = 0
+      VANDALISM_CLASS_INDEX = 1
+      NOT_KNOWN_INDEX = 2
+
+      CLASS = 'class'
+      VANDALISM = 'vandalism'
+      REGULAR = 'regular'
+      NOT_KNOWN = '?'
+      OUTLIER = Weka::Classifiers::Meta::OneClassClassifier::OUTLIER_LABEL
+
+      VANDALISM_SHORT= 'V'
+      REGULAR_SHORT = 'R'
+
+      OLD_REVISION_ID = 'oldrevisionid'
+      NEW_REVISION_ID = 'newrevisionid'
+
+
+      CLASSES = {
+          REGULAR_CLASS_INDEX => REGULAR,
+          VANDALISM_CLASS_INDEX => VANDALISM,
+          NOT_KNOWN_INDEX => NOT_KNOWN
+      }
+
+      CLASSES_SHORT = {
+          REGULAR_CLASS_INDEX => REGULAR_SHORT,
+          VANDALISM_CLASS_INDEX => VANDALISM_SHORT,
+          NOT_KNOWN_INDEX => NOT_KNOWN
+      }
+
+      # Returns an empty instances dataset of type Java::WekaCore::Instances::Base.
+      # This dataset is used for feature computation and classification for Wikipedia vandalism detection
+      # while training.
+      #
+      # @example
+      #   datset = Wikipedia::VandalismDetection::Instances.empty
+      #   => #<Java::WekaCore::Instances::Base:0xf0f9a00
+      #      @positions=[
+      #        #<Java::WekaCore::Attribute:0x17207a76>,
+      #        #<Java::WekaCore::Attribute:0x5547e4d6>,
+      #        #<Java::WekaCore::Attribute:0x6300c957>,
+      #        ...,
+      #        #<Java::WekaCore::Attribute:0x5a74fae4>]>
+      def self.empty
+        features = Wikipedia::VandalismDetection.configuration.features
+        classes = dataset_classes
+
+        dataset = Core::Type::Instances::Base.new do
+          features.each do |name|
+            numeric :"#{name.gsub(' ', '_')}"
+          end
+
+          nominal :class, classes
+        end
+
+        dataset.class_index = features.count
+        dataset
+      end
+
+      # Returns an empty instances dataset of type Java::WekaCore::Instances::Base.
+      # This dataset is used for feature computation and classification for Wikipedia vandalism detection
+      # while training.
+      #
+      # @example
+      #   datset = Wikipedia::VandalismDetection::Instances.empty
+      #   => #<Java::WekaCore::Instances::Base:0xf0f9a00
+      #      @positions=[
+      #        #<Java::WekaCore::Attribute:0x17207a76>
+      def self.empty_for_feature(name)
+        classes = dataset_classes
+
+        dataset = Core::Type::Instances::Base.new do
+          numeric :"#{name.gsub(' ', '_')}"
+          nominal :class, classes
+        end
+
+        dataset.class_index = 1
+        dataset
+      end
+
+      # Returns an empty instances dataset of type Java::WekaCore::Instances::Base.
+      # This dataset is used for feature computation and classification for Wikipedia vandalism detection
+      # while testing.
+      #
+      # @example
+      #   datset = Wikipedia::VandalismDetection::Instances.empty_for_test
+      #   => #<Java::WekaCore::Instances::Base:0xf0f9a00
+      #      @positions=[
+      #        #<Java::WekaCore::Attribute:0x17207a76>]>
+      def self.empty_for_test_feature(name)
+        dataset = Core::Type::Instances::Base.new do
+          numeric :"#{name.gsub(' ', '_')}"
+          numeric OLD_REVISION_ID.to_sym
+          numeric NEW_REVISION_ID.to_sym
+        end
+
+        dataset
+      end
+
+      # Returns an empty instances dataset of type Java::WekaCore::Instances::Base.
+      # This dataset is used for creating the ground truth classification.
+      def self.empty_for_test_class
+        classes = dataset_classes
+
+        dataset = Core::Type::Instances::Base.new do
+          nominal :class, classes
+        end
+
+        dataset
+      end
+
+      def self.dataset_classes
+        classes = Array.new
+        classes[VANDALISM_CLASS_INDEX] = VANDALISM
+        classes[REGULAR_CLASS_INDEX] = REGULAR
+        classes
+      end
+
+      private_class_method :dataset_classes
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/page.rb

@@ -0,0 +1,88 @@
+require 'nokogiri'
+require_relative 'edit'
+
+module Wikipedia
+  module VandalismDetection
+
+    # Represents a full wikipedia page history.
+    class Page
+
+      START_TAG = '<page>'.freeze
+      END_TAG = '</page>'.freeze
+
+      attr_accessor :id, :title
+      attr_reader :revisions
+
+      def initialize
+        @revisions = {}
+        @edits = []
+        @update_edits = false
+        @update_reverted_edits = false
+      end
+
+      def add_revision(revision)
+        @revisions[revision.id] = revision
+
+        @update_edits = true
+        @update_reverted_edits = true
+      end
+
+      def edits
+        @edits = create_edits_from @revisions if @update_edits
+        @edits
+      end
+
+      def reverted_edits
+        @reverted_edits = create_reverted_edits_from @revisions if @update_reverted_edits
+        @reverted_edits
+      end
+
+      private
+
+      def create_edits_from(revisions)
+        @update_edits = false
+        edits = []
+
+        revisions.each do |_, new_revision|
+          old_revision = revisions[new_revision.parent_id]
+          edits << Edit.new(old_revision, new_revision) unless old_revision.nil?
+        end
+
+        edits.each { |edit| edit.instance_variable_set(:@page, self) }
+        edits
+      end
+
+      # Returns the reverted edits by comparing the text's sha1 hashes of multiple revisions.
+      # If the next but one revision has the same sha1 hash as a base revision and the base revision has another
+      # hash than the one before, the in-between revision is reverted.
+      # The resulting edit holds the base revision as old revision and the reverted as new revision.
+      def create_reverted_edits_from(revisions)
+        @update_reverted_edits = false
+        edits = []
+
+        revisions.each do |current_id, first_revision|
+          second_revision_select = revisions.select { |_, value| value.parent_id == current_id }.first
+          next unless second_revision_select
+
+          second_revision = second_revision_select[1]
+
+          third_revision_select = revisions.select { |_, value| value.parent_id == second_revision.id }.first
+          next unless third_revision_select
+
+          first_sha1 = first_revision.sha1
+          second_sha1 = second_revision.sha1
+          third_sha1 = third_revision_select[1].sha1
+
+          previous_revision_select = revisions.select { |_, value| value.id == first_revision.parent_id }.first
+          previous_sha1 = previous_revision_select ? previous_revision_select[1].sha1 : nil
+
+          if (first_sha1 == third_sha1) && (second_sha1 != previous_sha1)
+            edits << Edit.new(first_revision, second_revision)
+          end
+        end
+
+        edits
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/page_parser.rb

@@ -0,0 +1,52 @@
+require 'nokogiri'
+require_relative 'page'
+require_relative 'revision'
+
+module Wikipedia
+  module VandalismDetection
+    class PageParser
+
+      # Parses an xml string and returns a Wikipedia::VandalismDetection::Page.
+      def parse(xml)
+        @page = Page.new
+        document = Nokogiri::XML(xml, nil, 'UTF-8')
+
+        @page.title = document.xpath('//page/title').inner_text
+        @page.id = document.xpath('//page/id').inner_text
+
+        build_revisions_from document
+
+        @page
+      end
+
+      private
+
+      # Builds and saves the available revisions to the @page variable
+      def build_revisions_from(document)
+        elements = document.xpath('//revision')
+
+        elements.each do |element|
+          revision = Revision.new
+
+          revision.id = element.xpath('id').inner_text
+          revision.timestamp = element.xpath('timestamp').inner_text
+          revision.comment = Text.new(element.xpath('comment').inner_text)
+          revision.text = Text.new(element.xpath('text').inner_text)
+          revision.sha1 = element.xpath('sha1').inner_text
+
+          parent_id_node = element.xpath('parentid')
+          contributor_id_node = element.xpath('contributor/id')
+          contributor_ip_node = element.xpath('contributor/ip')
+          contributor_username_node = element.xpath('contributor/username')
+
+          revision.parent_id = parent_id_node.inner_text unless parent_id_node.empty?
+          revision.contributor = contributor_id_node.inner_text unless contributor_id_node.empty?
+          revision.contributor = contributor_ip_node.inner_text unless contributor_ip_node.empty?
+          revision.contributor_username = contributor_username_node.inner_text unless contributor_username_node.empty?
+
+          @page.add_revision(revision)
+        end
+      end
+    end
+  end
+end