wikipedia-vandalism_detection 0.1.0-java

data/lib/wikipedia/vandalism_detection/instances.rb

@@ -0,0 +1,121 @@
+require 'weka'
+require 'wikipedia/vandalism_detection/configuration'
+require 'weka/classifiers/meta/one_class_classifier'
+
+module Wikipedia
+  module VandalismDetection
+    class Instances
+      REGULAR_CLASS_INDEX = 0
+      VANDALISM_CLASS_INDEX = 1
+      NOT_KNOWN_INDEX = 2
+
+      CLASS = 'class'.freeze
+      VANDALISM = 'vandalism'.freeze
+      REGULAR = 'regular'.freeze
+      NOT_KNOWN = '?'.freeze
+      OUTLIER = Weka::Classifiers::Meta::OneClassClassifier::OUTLIER_LABEL
+
+      VANDALISM_SHORT = 'V'.freeze
+      REGULAR_SHORT = 'R'.freeze
+
+      OLD_REVISION_ID = 'oldrevisionid'.freeze
+      NEW_REVISION_ID = 'newrevisionid'.freeze
+
+      CLASSES = {
+        REGULAR_CLASS_INDEX => REGULAR,
+        VANDALISM_CLASS_INDEX => VANDALISM,
+        NOT_KNOWN_INDEX => NOT_KNOWN
+      }.freeze
+
+      CLASSES_SHORT = {
+        REGULAR_CLASS_INDEX => REGULAR_SHORT,
+        VANDALISM_CLASS_INDEX => VANDALISM_SHORT,
+        NOT_KNOWN_INDEX => NOT_KNOWN
+      }.freeze
+
+      class << self
+        # Returns an empty instances dataset of type Java::WekaCore::Instances.
+        # This dataset is used for feature computation and classification for
+        # Wikipedia vandalism detection while training.
+        #
+        # @example
+        #   datset = Wikipedia::VandalismDetection::Instances.empty
+        #   => #<Java::WekaCore::Instances:0xf0f9a00
+        #      @positions=[
+        #        #<Java::WekaCore::Attribute:0x17207a76>,
+        #        #<Java::WekaCore::Attribute:0x5547e4d6>,
+        #        #<Java::WekaCore::Attribute:0x6300c957>,
+        #        ...,
+        #        #<Java::WekaCore::Attribute:0x5a74fae4>]>
+        def empty
+          features = Wikipedia::VandalismDetection.config.features
+          classes = dataset_classes
+
+          dataset = Weka::Core::Instances.new.with_attributes do
+            features.each do |name|
+              numeric name.tr(' ', '_')
+            end
+
+            nominal :class, values: classes, class_attribute: true
+          end
+
+          dataset
+        end
+
+        # Returns an empty instances dataset of type Java::WekaCore::Instances.
+        # This dataset is used for feature computation and classification for
+        # Wikipedia vandalism detection while training.
+        #
+        # @example
+        #   datset = Wikipedia::VandalismDetection::Instances.empty
+        #   => #<Java::WekaCore::Instances:0xf0f9a00
+        #      @positions=[
+        #        #<Java::WekaCore::Attribute:0x17207a76>
+        def empty_for_feature(name)
+          classes = dataset_classes
+
+          Weka::Core::Instances.new.with_attributes do
+            numeric name.tr(' ', '_')
+            nominal :class, values: classes, class_attribute: true
+          end
+        end
+
+        # Returns an empty instances dataset of type Java::WekaCore::Instances.
+        # This dataset is used for feature computation and classification for
+        # Wikipedia vandalism detection while testing.
+        #
+        # @example
+        #   datset = Wikipedia::VandalismDetection::Instances.empty_for_test
+        #   => #<Java::WekaCore::Instances:0xf0f9a00
+        #      @positions=[
+        #        #<Java::WekaCore::Attribute:0x17207a76>]>
+        def empty_for_test_feature(name)
+          Weka::Core::Instances.new.with_attributes do
+            numeric name.tr(' ', '_')
+            numeric OLD_REVISION_ID
+            numeric NEW_REVISION_ID
+          end
+        end
+
+        # Returns an empty instances dataset of type Java::WekaCore::Instances.
+        # This dataset is used for creating the ground truth classification.
+        def empty_for_test_class
+          classes = dataset_classes
+
+          Weka::Core::Instances.new.with_attributes do
+            nominal :class, values: classes
+          end
+        end
+
+        private
+
+        def dataset_classes
+          classes = []
+          classes[VANDALISM_CLASS_INDEX] = VANDALISM
+          classes[REGULAR_CLASS_INDEX]   = REGULAR
+          classes
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/page.rb

@@ -0,0 +1,101 @@
+require 'nokogiri'
+require_relative 'edit'
+
+module Wikipedia
+  module VandalismDetection
+    # Represents a full wikipedia page history.
+    class Page
+      START_TAG = '<page>'.freeze
+      END_TAG = '</page>'.freeze
+
+      attr_accessor :id, :title
+      attr_reader :revisions
+
+      def initialize
+        @revisions = {}
+        @edits = []
+        @update_edits = false
+        @update_reverted_edits = false
+      end
+
+      def add_revision(revision)
+        @revisions[revision.id] = revision
+
+        @update_edits = true
+        @update_reverted_edits = true
+      end
+
+      def edits
+        @edits = create_edits_from @revisions if @update_edits
+        @edits
+      end
+
+      def reverted_edits
+        if @update_reverted_edits
+          @reverted_edits = create_reverted_edits_from @revisions
+        end
+
+        @reverted_edits
+      end
+
+      private
+
+      def create_edits_from(revisions)
+        @update_edits = false
+        edits = []
+
+        revisions.each do |_, new_revision|
+          old_revision = revisions[new_revision.parent_id]
+          edits << Edit.new(old_revision, new_revision) unless old_revision.nil?
+        end
+
+        edits.each { |edit| edit.instance_variable_set(:@page, self) }
+        edits
+      end
+
+      # Returns the reverted edits by comparing the text's sha1 hashes of
+      # multiple revisions.
+      # If the next but one revision has the same sha1 hash as a base revision
+      # and the base revision has another hash than the one before, the
+      # in-between revision is reverted.
+      # The resulting edit holds the base revision as old revision and the
+      # reverted as new revision.
+      def create_reverted_edits_from(revisions)
+        @update_reverted_edits = false
+        edits = []
+
+        revisions.each do |current_id, first_revision|
+          second_revision_select = revisions
+            .select { |_, value| value.parent_id == current_id }
+            .first
+
+          next unless second_revision_select
+
+          second_revision = second_revision_select[1]
+
+          third_revision_select = revisions
+            .select { |_, value| value.parent_id == second_revision.id }
+            .first
+
+          next unless third_revision_select
+
+          first_sha1 = first_revision.sha1
+          second_sha1 = second_revision.sha1
+          third_sha1 = third_revision_select[1].sha1
+
+          previous_revision_select = revisions
+            .select { |_, value| value.id == first_revision.parent_id }
+            .first
+
+          previous_sha1 = previous_revision_select && previous_revision_select[1].sha1
+
+          if (first_sha1 == third_sha1) && (second_sha1 != previous_sha1)
+            edits << Edit.new(first_revision, second_revision)
+          end
+        end
+
+        edits
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/page_parser.rb

@@ -0,0 +1,63 @@
+require 'nokogiri'
+require_relative 'page'
+require_relative 'revision'
+
+module Wikipedia
+  module VandalismDetection
+    class PageParser
+      # Parses an xml string and returns a Wikipedia::VandalismDetection::Page.
+      def parse(xml)
+        @page = Page.new
+        document = Nokogiri::XML(xml, nil, 'UTF-8')
+
+        @page.title = document.xpath('//page/title').inner_text
+        @page.id = document.xpath('//page/id').inner_text
+
+        build_revisions_from(document)
+
+        @page
+      end
+
+      private
+
+      def node_value(document, xpath)
+        node = document.xpath(xpath.to_s)
+        return if node.empty?
+
+        node.inner_text
+      end
+
+      def node_text(document, xpath)
+        value = node_value(document, xpath)
+        return if value.blank?
+
+        Text.new(value)
+      end
+
+      # Builds and saves the available revisions to the @page variable
+      def build_revisions_from(document)
+        elements = document.xpath('//revision')
+
+        elements.each do |element|
+          revision = Revision.new
+
+          revision.id = node_value(element, :id)
+          revision.timestamp = node_value(element, :timestamp)
+          revision.comment = node_text(element, :comment)
+          revision.text = node_text(element, :text)
+          revision.sha1 = node_value(element, :sha1)
+          revision.parent_id = node_value(element, :parentid)
+          revision.contributor_username = node_value(element, 'contributor/username')
+
+          contributor_id = node_value(element, 'contributor/id')
+          contributor_ip = node_value(element, 'contributor/ip')
+
+          revision.contributor = contributor_id if contributor_id
+          revision.contributor = contributor_ip if contributor_ip
+
+          @page.add_revision(revision)
+        end
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/revision.rb

@@ -0,0 +1,75 @@
+require_relative 'text'
+require 'zlib'
+
+module Wikipedia
+  module VandalismDetection
+    class Revision
+
+      START_TAG = '<revision>'.freeze
+      END_TAG = '</revision>'.freeze
+      REDIRECT_PATTERN = /#REDIRECT\s+\[\[.*?\]\]/
+
+      attr_accessor :id,
+                    :parent_id,
+                    :timestamp,
+                    :contributor_username,
+                    :sha1
+
+      attr_reader :comment,
+                  :contributor_id,
+                  :contributor_ip
+
+      def initialize
+        @text = Zlib::Deflate.deflate('')
+        @comment = Text.new
+      end
+
+      def contributor=(contributor)
+        if ip_v4? contributor
+          @contributor_ip = contributor
+        else
+          @contributor_id = contributor
+        end
+      end
+
+      def contributor
+        @contributor_id || @contributor_ip
+      end
+
+      def anonymous_contributor?
+        !@contributor_ip.nil?
+      end
+
+      def redirect?
+        !!(text =~ REDIRECT_PATTERN)
+      end
+
+      # Compresses text when set
+      def text=(text)
+        text = '' unless text.is_a?(String)
+
+        # remove invalid utf-8 byte sequences
+        text.encode!('UTF-16', 'UTF-8', invalid: :replace, replace: '')
+        text.encode!('UTF-8', 'UTF-16')
+        @text = Zlib::Deflate.deflate(text)
+      end
+
+      # Decompresses text when called
+      def text
+        Text.new(Zlib::Inflate.inflate(@text).force_encoding('utf-8'))
+      end
+
+      def comment=(comment)
+        comment = '' unless comment.is_a?(String)
+        @comment = Text.new(comment)
+      end
+
+      private
+
+      # Returns whether the given value is an IPv4.
+      def ip_v4?(value)
+        !!value.to_s.match(/(\d+)\.(\d+)\.(\d+)\.(\d+)/)
+      end
+    end
+  end
+end

data/lib/wikipedia/vandalism_detection/revision_parser.rb

@@ -0,0 +1,67 @@
+# encoding: utf-8
+
+require 'nokogiri'
+require 'wikipedia/vandalism_detection/revision'
+
+module Wikipedia
+  module VandalismDetection
+    class RevisionParser
+      DEFAULT_PROPERTIES = %i[
+        id
+        parent_id
+        timestamp
+        contributor
+        comment
+        text
+        sha1
+      ].freeze
+
+      # Parses an xml string and returns a Revision.
+      def parse(xml, options = {})
+        document = Nokogiri::XML(xml, nil, 'UTF-8').xpath('//revision')
+        revision = Revision.new
+
+        properties = options[:only] || DEFAULT_PROPERTIES
+
+        revision.id        = node_value(document, properties, :id)
+        revision.timestamp = node_value(document, properties, :timestamp)
+        revision.comment   = node_text(document, properties, :comment)
+        revision.text      = node_text(document, properties, :text)
+        revision.sha1      = node_value(document, properties, :sha1)
+
+        if properties.include?(:contributor)
+          revision.contributor = node_presence(document, 'contributor/id')
+          revision.contributor = node_presence(document, 'contributor/ip')
+          revision.contributor_username = node_presence(document, 'contributor/username')
+        end
+
+        if properties.include?(:parent_id)
+          revision.parent_id = node_presence(document, 'parentid')
+        end
+
+        revision
+      end
+
+      private
+
+      def node_value(document, properties, attribute)
+        return unless properties.include?(attribute)
+        node_presence(document, attribute)
+      end
+
+      def node_text(document, properties, attribute)
+        value = node_value(document, properties, attribute)
+        return if value.blank?
+
+        Text.new(value)
+      end
+
+      def node_presence(document, xpath)
+        node = document.xpath(xpath.to_s)
+        return if node.empty?
+
+        node.inner_text
+      end
+    end
+  end
+end