wukong 3.0.0.pre → 3.0.0.pre2

data/examples/munging/wikipedia/articles/extract_articles-parsed.rb

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+
+# Extracts wikipedia articles from bzipped xml, outputs them in TSV.  Article
+# text is XML encoded, but all newlines and tabs (in fact, all control
+# characters) are converted to XML entities, making it safe to truck around as
+# TSV.
+#
+# Sample Pig LOAD Statement:
+#
+# all_articles = LOAD '$articles' AS
+#   (id:long, namespace:int, title:chararray, revision_timestamp:long, redirect:chararray, text:chararray);
+#
+
+# ## Usage
+#
+# Flattens the wikipedia 'enwiki-latest-pages-articles.xml.gz' into a
+# one-line-per-record heap.
+#
+#    examples/munging/wikipedia/articles/extract_articles-templated.rb --rm --run \
+#      /data/origin/dumps.wikimedia.org/enwiki/20120601/enwiki-20120601-pages-articles.xml
+#      /data/results/wikipedia/full/articles.json.tsv
+#
+
+require 'wukong'
+require 'wukong/streamer/encoding_cleaner'
+require 'crack/xml'
+require 'multi_json'
+require 'oj'
+require_relative '../utils/munging_utils.rb'
+
+module ArticlesExtractor
+  class Mapper < Wukong::Streamer::LineStreamer
+    include Wukong::Streamer::EncodingCleaner
+    include MungingUtils
+
+    def lines
+      @lines ||= []
+    end
+
+    def recordize line
+      lines << line
+      if line =~ /<\/page>/
+        result = Crack::XML::parse(lines.join("\n"))
+        @lines = []
+        return [result]
+      else
+        return nil
+      end
+    end
+
+    def process record
+      if record.has_key? 'mediawiki'
+        record = record['mediawiki']
+      end
+
+      redirect  = record['page']['redirect'] ? record['page']['redirect']['title'] : ''
+      timestamp = Time.iso8601(record['page']['revision']['timestamp']).to_flat
+      raw_text  = record['page']['revision']['text']
+
+      # some few parts per million articles have an empty body -- workaround
+      raw_text = '' if not raw_text.is_a?(String)
+
+      result = [
+        record['page']['id'],
+        record['page']['ns'],
+        scrub_control_chars(record['page']['title']),
+        record['page']['revision']['id'],
+        timestamp,
+        scrub_control_chars(redirect),
+        safe_json_encode(raw_text)
+      ]
+      yield result
+    end
+  end
+end
+
+# Force it to run in a single map task, to avoid writing a custom input format.
+# The job runs in 2 hours, once; much less than the time it'd take me to do so.
+Wukong::Script.new(ArticlesExtractor::Mapper, nil, min_split_size: 1152921504606846976).run

data/examples/munging/wikipedia/articles/extract_articles-templated.rb

@@ -0,0 +1,136 @@
+#!/usr/bin/env ruby
+
+# Extracts wikipedia articles from bzipped xml, outputs them in TSV.  Article
+# text is XML encoded, but all newlines and tabs (in fact, all control
+# characters) are converted to XML entities, making it safe to truck around as
+# TSV.
+
+# ## Schema
+#
+# Sample Pig LOAD Statement:
+#
+# all_articles = LOAD '$articles' AS
+#   (id:long, namespace:int, title:chararray, revision_timestamp:long, redirect:chararray, text:chararray);
+#
+
+# ## Usage
+#
+# Flattens the wikipedia 'enwiki-latest-pages-articles.xml.gz' into a
+# one-line-per-record heap.
+#
+#    examples/munging/wikipedia/articles/extract_articles-templated.rb --rm --run \
+#      /data/origin/dumps.wikimedia.org/enwiki/20120601/enwiki-20120601-pages-articles.xml
+#      /data/results/wikipedia/full/articles.tsv
+#
+
+require 'wukong'
+require 'wukong/streamer/encoding_cleaner'
+require 'crack/xml'
+require 'multi_json'
+require_relative '../utils/munging_utils.rb'
+
+# <page>
+#   <title>Anarchism</title>
+#   <ns>0</ns>
+#   <id>12</id>
+#   <revision>
+#     <id>370845941</id>
+#     <timestamp>2010-06-29T20:14:56Z</timestamp>
+#     <contributor>
+#       <username>Centographer</username>
+#       <id>12640258</id>
+#     </contributor>
+#     <minor />
+#     <comment>clarifying not ordinary anarcho-socialism</comment>
+#     <text xml:space="preserve">
+#       ...snip ...
+#     </text>
+#     <sha1>...</sha1>
+#   </revision>
+# </page>
+#
+module ArticlesExtractor
+  class Mapper < Wukong::Streamer::LineStreamer
+    include Wukong::Streamer::EncodingCleaner
+    include MungingUtils
+
+    def initialize(*)
+      super
+      @lines      = []
+      @state        = :out_of_article
+      @num_lines    = 0
+    end
+
+    # Bolt together all lines between a <page> and a </page> marker.
+    def recordize line
+      @num_lines += 1
+      return if @state == :out_of_article && (ARTICLE_BEG_RE !~ line)
+      @state = :in_article
+      #
+      @lines << line
+      if ARTICLE_END_RE =~ line
+        result   = @lines.join("\n")
+        @lines   = []
+        @state = :out_of_article
+        return   [result]
+      else
+        return nil
+      end
+    end
+
+    def process article
+      info = ARTICLE_RE.match(article)
+      if not info then warn "Bad match line #{@lines}: #{article.to_s[0..2000]}" ; return ; end
+
+      timestamp = [info[:rts_yr], info[:rts_mo], info[:rts_day], info[:rts_hr], info[:rts_min], info[:rts_sec], 'Z'].join
+      text      = Crack::XML::parse("<text>#{info[:text]}</text>")['text'] || ''
+      redirect  = info[:redirect] || ''
+
+      record = [
+        info[:id],
+        info[:ns],
+        scrub_control_chars(info[:title]),
+        info[:revision_id],
+        timestamp,
+        scrub_control_chars(redirect),
+        safe_json_encode(text)
+      ]
+      yield record
+    end
+  end
+
+  ARTICLE_BEG_RE  = %r{\A\s*<page>\z}
+  ARTICLE_END_RE  = %r{\A\s*</page>\z}
+  ARTICLE_RE = %r{\A
+\s*<page>
+\s*  <title>(?<title>[^<]*)</title>
+\s*    <ns>(?<ns>\d+)</ns>
+\s*    <id>(?<id>\d+)</id>
+\s* (?:<redirect\stitle=\"(?<redirect>[^\"]+)\"\s/>)?
+\s* (?:<restrictions>(?<restrictions>[^<]+)</restrictions>)?
+\s*    <revision>
+\s*      <id>(?<revision_id>\d+)</id>
+\s*      <timestamp>(?<rts_yr>\d\d\d\d)-(?<rts_mo>\d\d)-(?<rts_day>\d\d)T(?<rts_hr>\d\d):(?<rts_min>\d\d):(?<rts_sec>\d\d)Z</timestamp>
+\s* (?:
+          <contributor>\s*<username>[^<]+</username>\s*<id>\d+</id>\s*</contributor> |
+          <contributor>\s*<ip>[\d\.]+</ip>\s*</contributor> |
+          <contributor\sdeleted="deleted"\s/>
+    )
+\s*      (?:<minor\s/>)?
+\s*      (?:<comment>[^<]*</comment>|<comment\sdeleted="deleted"\s/>)?
+\s* (?:
+         <text\sxml:space="preserve">
+            (?<text>.*)
+         </text>
+     |
+         <text\sxml:space="preserve"\s/>
+    )
+\s*      (?:<sha1>(?<sha1>[a-z0-9]+)</sha1> | <sha1\s/>)
+\s*  </revision>
+\s*</page>\s*\z}xmo
+
+end
+
+# Force it to run in a single map task, to avoid writing a custom input format.
+# The job runs in 2 hours, once; much less than the time it'd take me to do so.
+Wukong::Script.new(ArticlesExtractor::Mapper, nil, min_split_size: 1152921504606846976).run

data/examples/munging/wikipedia/articles/textualize_articles.rb

@@ -0,0 +1,54 @@
+#!/usr/bin/env ruby
+
+# Generate plain-text versions of articles from the tsv-converted raw article data
+# (output from extract_articles)
+#
+# This strips out template tags, wiki links, and so forth
+#
+# Everything that's left is either actual text, or nicely detached punctuation.
+
+# ## Usage
+#
+# Uses the output of extract_articles-templated.rb:
+#
+#    examples/munging/wikipedia/articles/textualize_articles.rb --rm --run \
+#      /data/results/wikipedia/full/articles.json.tsv      \
+#      /data/results/wikipedia/full/article_texts.json.tsv
+#
+
+require 'wukong'
+require 'multi_json'
+require 'oj'
+require 'strscan'
+require 'find'
+require 'sanitize'
+#
+require_relative '../utils/munging_utils.rb'
+require_relative './wp2txt_article'
+require_relative './wp2txt_utils'
+
+module TextualizeArticles
+
+  class Mapper < Wukong::Streamer::RecordStreamer
+    include MungingUtils
+
+    @@errors   = 0
+    MAX_ERRORS = 1_000
+
+    def process(id, namespace, title, revision_id, timestamp, redirect, raw_text)
+
+      text          = MultiJson.decode(raw_text)
+      article       = Wp2txt::Article.new(text, title)
+      jsonized_text = MultiJson.encode(article.polish)
+
+      yield [id, namespace, title, revision_id, timestamp, redirect, jsonized_text]
+
+    rescue StandardError => err
+      Wukong.bad_record("Bad Record", err, raw_text)
+      raise "Too many errors" if (@@errors += 1) > MAX_ERRORS
+    end
+
+  end
+end
+
+Wukong::Script.new(TextualizeArticles::Mapper, nil).run

data/examples/munging/wikipedia/articles/verify_structure.rb

@@ -0,0 +1,43 @@
+
+# cat /mnt/data/origin/dumps.wikimedia.org/enwiki/20120601/enwiki-20120601-pages-articles.xml  | ruby -ne '$_.chomp!; case $_ when %r{\A(\s*)<redirect title="[^"]+" />\z} then puts %Q{#{$1}<redirect title=\"...\" />} when %r{\A(\s*<[^>]+>)[^<]*(</\w+>)\z} then puts "#{$1}...#{$2}" ; when %r{\A(\s*<[^>]+>)} then puts $1 ; when %r{\A[^<]*(</[\w-]+>)\z} then puts $1 else false end ' > /mnt/data/origin/dumps.wikimedia.org/enwiki/20120601/xml-tags-seen.txt &
+
+# cat /mnt/data/origin/dumps.wikimedia.org/enwiki/20120601/xml-tags-seen.txt | sort -S1G --temp=/mnt{,2,3,4}/tmp | uniq -c  | sort -n | tee xml-tags-census.txt
+
+# cat /mnt/data/origin/dumps.wikimedia.org/enwiki/20120601/xml-tags-seen.txt | sort -S1800M --temp=/mnt{2,3,4}/tmp | uniq -c  | sort -n | tee xml-tags-census.txt &
+
+
+# 609844   <page>
+# 609844     <title>...</title>
+# 609844     <ns>...</ns>
+# 609844     <id>...</id>
+#    714     <restrictions>...</restrictions>
+#
+# 251254     <redirect title="..." />
+# 609844     <revision>
+# 609844       <id>...</id>
+# 609844       <timestamp>...</timestamp>
+#
+# 609843       <contributor>
+# 548236         <username>...</username>
+#  61607         <ip>...</ip>
+# 548236         <id>...</id>
+# 609843       </contributor>
+#
+# 288319       <minor />
+# 529222       <comment>...</comment>
+#    108       <comment>
+#    108 </comment>
+#
+# 224818       <text xml:space="preserve">...</text>
+# 385019       <text xml:space="preserve">
+#      7       <text xml:space="preserve" />
+#
+# 346490       <sha1>...</sha1>
+# 263354       <sha1 />
+#
+# 609843     </revision>
+# 609843   </page>
+#
+# 384998 </text>
+#     20  </text>
+#  643676 </page>

data/examples/munging/wikipedia/articles/wp2txt-LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Yoichiro Hasebe
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/examples/munging/wikipedia/articles/wp2txt_article.rb

@@ -0,0 +1,259 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+# Taken from Yoichiro Yohasebe's [`wp2txt` project](https://github.com/yohasebe/wp2txt)
+# with liberal modifications for our purposes.
+#
+# This software is distributed under the MIT License. Please see the `./wp2txt-LICENSE.txt` file.
+
+require 'strscan'
+require_relative 'wp2txt_utils'
+
+module Wp2txt
+
+  # possible element type, which could be later chosen to print or not to print
+  # :mw_heading
+  # :mw_htable
+  # :mw_quote
+  # :mw_unordered
+  # :mw_ordered
+  # :mw_definition
+  # :mw_pre
+  # :mw_paragraph
+  # :mw_comment
+  # :mw_math
+  # :mw_source
+  # :mw_inputbox
+  # :mw_template
+  # :mw_link
+  # :mw_summary
+  # :mw_blank
+  # :mw_redirect
+
+  # an article contains elements, each of which is [TYPE, string]
+  class Article
+
+    include Wp2txt
+    attr_accessor :elements, :title
+
+    # class varialbes to save resource for generating regexps
+    # those with a trailing number 1 represent opening tag/markup
+    # those with a trailing number 2 represent closing tag/markup
+    # those without a trailing number contain both opening/closing tags/markups
+
+    @@in_template_regex = Regexp.new('^\s*\{\{[^\}]+\}\}\s*$')
+    @@in_link_regex = Regexp.new('^\s*\[.*\]\s*$')
+
+    @@in_inputbox_regex  = Regexp.new('<inputbox>.*?<\/inputbox>')
+    @@in_inputbox_regex1  = Regexp.new('<inputbox>')
+    @@in_inputbox_regex2  = Regexp.new('<\/inputbox>')
+
+    @@in_source_regex  = Regexp.new('<source.*?>.*?<\/source>')
+    @@in_source_regex1  = Regexp.new('<source.*?>')
+    @@in_source_regex2  = Regexp.new('<\/source>')
+
+    @@in_math_regex  = Regexp.new('<math.*?>.*?<\/math>')
+    @@in_math_regex1  = Regexp.new('<math.*?>')
+    @@in_math_regex2  = Regexp.new('<\/math>')
+
+    @@in_heading_regex  = Regexp.new('^=+.*?=+$')
+
+    @@in_html_table_regex = Regexp.new('<table.*?><\/table>')
+    @@in_html_table_regex1 = Regexp.new('<table\b')
+    @@in_html_table_regex2 = Regexp.new('<\/\s*table>')
+
+    @@in_table_regex1 = Regexp.new('^\s*\{\|')
+    @@in_table_regex2 = Regexp.new('^\|\}.*?$')
+
+    @@in_unordered_regex  = Regexp.new('^\*')
+    @@in_ordered_regex    = Regexp.new('^\#')
+    @@in_pre_regex = Regexp.new('^ ')
+    @@in_definition_regex  = Regexp.new('^[\;\:]')
+
+    @@blank_line_regex = Regexp.new('^\s*$')
+
+    @@redirect_regex = Regexp.new('#(?:REDIRECT|転送)\s+\[\[(.+)\]\]', Regexp::IGNORECASE)
+
+    def initialize(text, title = "", strip_tmarker = false)
+      @title = title.strip
+      @strip_tmarker = strip_tmarker
+      parse text
+    end
+
+    def create_element(tp, text)
+      [tp, text]
+    end
+
+    def parse(source)
+      self.class.remove_comments(source)
+      @elements = []
+      mode = nil
+      open_stack  = []
+      close_stack = []
+      source.each_line do |line|
+
+        case mode
+        when :mw_table
+          if @@in_table_regex2 =~ line
+            mode = nil
+          end
+          @elements.last.last << line
+          next
+        when :mw_inputbox
+          if @@in_inputbox_regex2 =~ line
+            mode = nil
+          end
+          @elements.last.last << line
+          next
+        when :mw_source
+          if @@in_source_regex2 =~ line
+            mode = nil
+          end
+          @elements.last.last << line
+          next
+        when :mw_math
+          if @@in_math_regex2 =~ line
+            mode = nil
+          end
+          @elements.last.last << line
+          next
+        when :mw_htable
+          if @@in_html_table_regex2 =~ line
+            mode = nil
+          end
+          @elements.last.last << line
+          next
+        end
+
+        case line
+        when @@blank_line_regex
+          @elements << create_element(:mw_blank, "\n")
+        when @@redirect_regex
+          @elements << create_element(:mw_redirect, line)
+        when @@in_template_regex
+          @elements << create_element(:mw_template, line)
+        when @@in_heading_regex
+          @elements << create_element(:mw_heading, "\n" + line + "\n")
+        when @@in_inputbox_regex
+          @elements << create_element(:mw_inputbox, line)
+        when @@in_inputbox_regex1
+          mode = :mw_inputbox
+          @elements << create_element(:mw_inputbox, line)
+        when @@in_source_regex
+        @elements << create_element(:mw_source, line)
+        when @@in_source_regex1
+          mode = :mw_source
+          @elements << create_element(:mw_source, line)
+        when @@in_math_regex
+          @elements << create_element(:mw_math, line)
+        when @@in_math_regex1
+          mode = :mw_math
+          @elements << create_element(:mw_math, line)
+        when @@in_html_table_regex
+          @elements << create_element(:mw_htable, line)
+        when @@in_html_table_regex1
+          mode = :mw_htable
+          @elements << create_element(:mw_htable, line)
+        when @@in_table_regex1
+          mode = :mw_table
+          @elements << create_element(:mw_table, line)
+        when @@in_unordered_regex
+          line = line.sub(/\A[\*\#\;\:\ ]+/, "") if @strip_tmarker
+          @elements << create_element(:mw_unordered, line)
+        when @@in_ordered_regex
+          line = line.sub(/\A[\*\#\;\:\ ]+/, "") if @strip_tmarker
+          @elements << create_element(:mw_ordered, line)
+        when @@in_pre_regex
+          line = line.sub(/\A\^\ /, "") if @strip_tmarker
+          @elements << create_element(:mw_pre, line)
+        when @@in_definition_regex
+          line = line.sub(/\A[\;\:\ ]+/, "") if @strip_tmarker
+          @elements << create_element(:mw_definition, line)
+        when @@in_link_regex
+          @elements << create_element(:mw_link, line)
+        else
+          @elements << create_element(:mw_paragraph, line)
+        end
+      end
+      @elements
+    end
+
+    def self.remove_comments(text)
+      # remove all comment texts
+      # and insert as many number of new line chars included in
+      # each comment instead
+      text.gsub!(/\<\!\-\-(.*?)\-\-\>/m) do |content|
+        num_of_newlines = content.count("\n")
+        (num_of_newlines == 0) ? "" : ("\n" * num_of_newlines)
+      end
+    end
+
+    EXCLUDE_SECTIONS = {
+      mw_title:      false,
+      mw_heading:    false,
+      mw_paragraph:  false,
+      mw_link:       false,
+      mw_redirect:   false,
+      #
+      mw_pre:        false,
+      mw_quote:      false,
+      mw_unordered:  false,
+      mw_ordered:    false,
+      mw_definition: false,
+      #
+      mw_table:      true,
+      mw_htable:     true,
+      mw_blank:      true,
+      mw_math:       true,
+      mw_source:     true,
+      mw_template:   true,
+    }
+
+    def polish
+      contents = []
+      elements.each do |el_type, element|
+        contents << "+#{el_type.to_s.upcase}+\t" if $DEBUG_MODE
+        next if EXCLUDE_SECTIONS[el_type]
+        #
+        case el_type
+        when :mw_heading            then contents << format_wiki(element)
+        when :mw_paragraph          then contents << format_wiki(element)
+        when :mw_link               then contents << format_wiki(element)
+        when :mw_redirect           then contents << format_wiki(element) << "\n\n"
+        when :mw_pre                then contents << element
+        when :mw_quote              then contents << format_wiki(element)
+        when :mw_unordered          then contents << format_wiki(element)
+        when :mw_ordered            then contents << format_wiki(element)
+        when :mw_definition         then contents << format_wiki(element)
+        when :mw_table, :mw_htable  then contents << format_wiki(element)
+        when :mw_math, :mw_source   then contents << format_wiki(element)
+        when :mw_blank              then contents << format_wiki(element)
+        else
+          warn "Unknown section #{el_type}, content '#{element.to_s.gsub(/[\r\n]+/m,'')[0..200]}'"
+          contents << format_wiki(element)
+        end
+      end
+      text = contents.join
+
+      # Extract text from <b>..</b> and so forth; remove contents of <ref>...</ref> completely
+      text = clean_html(text)
+      # translate some recognizable special characters
+      text = special_chr(text)
+      # re-hang the no-wiki segments
+      unescape_nowiki(text)
+      # strip out templates. Several parts per million of these will fail for
+      # bad structure; I assume that means some parts per thousand will be
+      # mis-estimated. C'est la UGC.
+      text = remove_templates(text) if EXCLUDE_SECTIONS[:mw_template]
+
+      return '' if /\A\s*\z/m =~ text
+      #
+      result = EXCLUDE_SECTIONS[:mw_title] ? "" : "# #{format_wiki(title)}\n\n"
+      result << text
+      result.gsub!(/\n\n\n+/m){"\n\n"}
+      result << "\n"
+
+      result
+    end
+  end
+end