wukong 3.0.0.pre → 3.0.0.pre2

data/examples/munging/wikipedia/articles/wp2txt_utils.rb

@@ -0,0 +1,452 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+# Taken from Yoichiro Yohasebe's [`wp2txt` project](https://github.com/yohasebe/wp2txt)
+# with liberal modifications for our purposes.
+#
+# This software is distributed under the MIT License. Please see the `./wp2txt-LICENSE.txt` file.
+
+require 'strscan'
+require 'find'
+require 'sanitize'
+
+module Wp2txt
+
+  def format_wiki(original_text, has_retried = false)
+    begin
+      text = original_text + ""
+
+      text = chrref_to_utf(text)
+      text = escape_nowiki(text)
+
+      text = process_interwiki_links(text)
+      text = process_external_links(text)
+
+      text = remove_directive(text)
+      text = remove_emphasis(text)
+
+      text = mndash(text)
+
+      text = remove_hr(text)
+
+      return text
+
+      text = special_chr(text)
+
+      unescape_nowiki(text)
+    rescue # detect invalid byte sequence in UTF-8
+      if has_retried
+        puts "invalid byte sequence detected"
+        puts "******************************"
+        File.open("error_log.txt", "w") do |f|
+          f.write original_text
+        end
+        exit
+      else
+        fixed_text = original_text.encode("UTF-16", :invalid => :replace, :replace => '').encode("UTF-8")
+        return format_wiki(fixed_text, true)
+      end
+    end
+  end
+
+  #################### parser for nested structure ####################
+
+  def process_nested_structure(scanner, left, right, &block)
+    buffer = ""
+    while str = scanner.scan_until(/(#{Regexp.escape(left)}|#{Regexp.escape(right)})/m)
+      # begin
+      case scanner[1]
+      when left
+        buffer << str
+        has_left = true
+      when right
+        if has_left
+          buffer = buffer[0...-(left.size)]
+          contents = block.call(str[0...-(left.size)])
+          buffer << contents
+          break
+        else
+          buffer << str
+        end
+      end
+    end
+    buffer << scanner.rest
+
+    if buffer == scanner.string
+      return scanner.string
+    else
+      scanner.string = buffer
+      return process_nested_structure(scanner, left, right, &block) || ""
+    end
+  end
+
+  def remove_templates(str, only_not_inline = true)
+    scanner = StringScanner.new(str)
+    result = process_nested_structure(scanner, "{{", "}}") do |contents|
+      # if contents.index("\n")
+      #   "\n"
+      # else
+      #   "[tpl]#{contents}[/tpl]"
+      # end
+      ''
+    end
+  rescue SystemStackError => err
+    Wukong.bad_record("Poorly nested templates", err, str)
+    return str.gsub!(/{{.*}}/m, "**BAD TEMPLATE**")
+  end
+
+
+  #################### methods used from format_wiki ####################
+
+  def escape_nowiki(str)
+    if @nowikis
+      @nowikis.clear
+    else
+      @nowikis = {}
+    end
+    str.gsub(/<nowiki>(.*?)<\/nowiki>/m) do
+      nowiki = $1
+      nowiki_id = nowiki.object_id
+      @nowikis[nowiki_id] = nowiki
+      "<nowiki nowikiid=\"#{nowiki_id}\">"
+    end
+  end
+
+  def unescape_nowiki(str)
+    str.gsub(/<nowiki nowikiid=\"(\d+)\">/) do
+      obj_id = $1.to_i
+      @nowikis[obj_id]
+    end
+  end
+
+  def process_interwiki_links(str)
+    scanner = StringScanner.new(str)
+    result = process_nested_structure(scanner, "[[", "]]") do |contents|
+      str_new = ""
+      parts = contents.split("|")
+      case parts.size
+      when 1
+        parts.first || ""
+      else
+        parts.shift
+        parts.join("|")
+      end
+    end
+    result
+  rescue SystemStackError => err
+    Wukong.bad_record("Poorly nested internal links", err, str)
+    return str.gsub!(/\[\[.*\]\]/m, "**BAD INTERWIKI LINKS**")
+  end
+
+  def process_external_links(str)
+    scanner = StringScanner.new(str)
+    result = process_nested_structure(scanner, "[", "]") do |contents|
+      parts = contents.split(" ", 2)
+      case parts.size
+      when 1
+        parts.first || ""
+      else
+        parts.last || ""
+      end
+    end
+    result
+  rescue SystemStackError => err
+    Wukong.bad_record("Poorly nested external links", err, str)
+    return str.gsub!(/\[.*\]/m, "**BAD EXTERNAL LINKS**")
+  end
+
+  def special_chr(str)
+    unless @sp_hash
+      html = ['&nbsp;', '&lt;', '&gt;', '&amp;', '&quot;']\
+      .zip([' ', '<', '>', '&', '"'])
+
+      umraut_accent = ['&Agrave;', '&Aacute;', '&Acirc;', '&Atilde;', '&Auml;',
+      '&Aring;', '&AElig;', '&Ccedil;', '&Egrave;', '&Eacute;', '&Ecirc;',
+      '&Euml;', '&Igrave;', '&Iacute;', '&Icirc;', '&Iuml;', '&Ntilde;',
+      '&Ograve;', '&Oacute;', '&Ocirc;', '&Otilde;', '&Ouml;', '&Oslash;',
+      '&Ugrave;', '&Uacute;', '&Ucirc;', '&Uuml;', '&szlig;', '&agrave;',
+      '&aacute;', '&acirc;', '&atilde;', '&auml;', '&aring;', '&aelig;',
+      '&ccedil;', '&egrave;', '&eacute;', '&ecirc;', '&euml;', '&igrave;',
+      '&iacute;', '&icirc;', '&iuml;', '&ntilde;', '&ograve;', '&oacute;',
+      '&ocirc;', '&oelig;', '&otilde;', '&ouml;', '&oslash;', '&ugrave;',
+      '&uacute;', '&ucirc;', '&uuml;', '&yuml;']\
+      .zip(['À', 'Á', 'Â', 'Ã', 'Ä', 'Å', 'Æ', 'Ç', 'È', 'É', 'Ê', 'Ë', 'Ì', 'Í',
+      'Î', 'Ï', 'Ñ', 'Ò', 'Ó', 'Ô', 'Õ', 'Ö', 'Ø', 'Ù', 'Ú', 'Û', 'Ü', 'ß', 'à',
+      'á', 'â', 'ã', 'ä', 'å', 'æ', 'ç', 'è', 'é', 'ê', 'ë', 'ì', 'í', 'î', 'ï',
+      'ñ', 'ò', 'ó', 'ô','œ', 'õ', 'ö', 'ø', 'ù', 'ú', 'û', 'ü', 'ÿ'])
+
+      punctuation = ['&iquest;', '&iexcl;', '&laquo;', '&raquo;', '&sect;',
+      '&para;', '&dagger;', '&Dagger;', '&bull;', '&ndash;', '&mdash;']\
+      .zip(['¿', '¡', '«', '»', '§', '¶', '†', '‡', '•', '–', '—'])
+
+      commercial = ['&trade;', '&copy;', '&reg;', '&cent;', '&euro;', '&yen;',
+      '&pound;', '&curren;'].zip(['™', '©', '®', '¢', '€', '¥', '£', '¤'])
+
+      greek_chr = ['&alpha;', '&beta;', '&gamma;', '&delta;', '&epsilon;',
+      '&zeta;', '&eta;', '&theta;', '&iota;', '&kappa;', '&lambda;', '&mu;',
+      '&nu;', '&xi;', '&omicron;', '&pi;', '&rho;', '&sigma;', '&sigmaf;',
+      '&tau;', '&upsilon;', '&phi;', '&chi;', '&psi;', '&omega;', '&Gamma;',
+      '&Delta;', '&Theta;', '&Lambda;', '&Xi;', '&Pi;', '&Sigma;', '&Phi;',
+      '&Psi;', '&Omega;']\
+      .zip(['α', 'β', 'γ', 'δ', 'ε', 'ζ', 'η', 'θ', 'ι', 'κ', 'λ',
+      'μ', 'ν', 'ξ', 'ο', 'π', 'ρ', 'σ', 'ς', 'τ', 'υ', 'φ', 'χ',
+      'ψ', 'ω', 'Γ', 'Δ', 'Θ', 'Λ', 'Ξ', 'Π', 'Σ', 'Φ', 'Ψ', 'Ω'])
+
+      math_chr1 = ['&int;', '&sum;', '&prod;', '&radic;', '&minus;', '&plusmn;',
+      '&infin;', '&asymp;', '&prop;', '&equiv;', '&ne;', '&le;', '&ge;',
+      '&times;', '&middot;', '&divide;', '&part;', '&prime;', '&Prime;',
+      '&nabla;', '&permil;', '&deg;', '&there4;', '&oslash;', '&isin;', '&cap;',
+      '&cup;', '&sub;', '&sup;', '&sube;', '&supe;', '&not;', '&and;', '&or;',
+      '&exist;', '&forall;', '&rArr;', '&hArr;', '&rarr;', '&harr;', '&uarr;']\
+      .zip(['∫', '∑', '∏', '√', '−', '±', '∞', '≈', '∝', '≡', '≠', '≤',
+      '≥', '×', '·', '÷', '∂', '′', '″', '∇', '‰', '°', '∴', 'ø', '∈',
+      '∩', '∪', '⊂', '⊃', '⊆', '⊇', '¬', '∧', '∨', '∃', '∀', '⇒',
+      '⇔', '→', '↔', '↑'])
+
+      math_chr2 = ['&alefsym;', '&notin;'].zip(['ℵ', '∉'])
+
+      others = ['&uml;', '&ordf;',
+      '&macr;', '&acute;', '&micro;', '&cedil;', '&ordm;', '&lsquo;', '&rsquo;',
+      '&ldquo;', '&sbquo;', '&rdquo;', '&bdquo;', '&spades;', '&clubs;', '&loz;',
+      '&hearts;', '&larr;', '&diams;', '&lsaquo;', '&rsaquo;', '&darr;']\
+      .zip(['¨', 'ª', '¯', '´', 'µ', '¸', 'º', '‘', '’', '“', '‚', '”',
+      '„', '♠', '♣', '◊', '♥', '←', '♦', '‹', '›', '↓'] )
+
+      spc_array = html + umraut_accent + punctuation + commercial + greek_chr +
+                  math_chr1 + math_chr2 + others
+      @sp_hash  = Hash[*spc_array.flatten]
+      @sp_regex = Regexp.new("(" + @sp_hash.keys.join("|") + ")")
+    end
+    #str.gsub!("&amp;"){'&'}
+    str.gsub!(@sp_regex) do
+      @sp_hash[$1]
+    end
+    return str
+  end
+
+  def remove_tag(str, tagset = ['<', '>'])
+    if tagset == ['<', '>']
+      return remove_html_tag(str)
+    end
+    tagsets = Regexp.quote(tagset.uniq.join(""))
+    regex = /#{Regexp.escape(tagset[0])}[^#{tagsets}]*#{Regexp.escape(tagset[1])}/
+    newstr = str.gsub(regex, "")
+    # newstr = newstr.gsub(/<\!\-\-.*?\-\->/, "")
+    return newstr
+  end
+
+  def remove_html_tag(str)
+    str = ::Sanitize.clean(str)
+  end
+
+  def clean_html(text)
+    text.gsub!(%r{<(\w+)\s[^>]*?(/?)>}, '<\1\2>' )
+    text = ::Sanitize.clean(text, remove_contents: ['ref'])
+  end
+
+  def remove_emphasis(str)
+    str.gsub(/(''+)(.+?)\1/) do
+      $2
+    end
+  end
+
+  def chrref_to_utf(num_str)
+    begin
+      utf_str = num_str.gsub(/&#(x?)([0-9a-fA-F]+);/) do
+        if $1 == 'x'
+          ch = $2.to_i(16)
+        else
+          ch = $2.to_i
+        end
+        hi = ch>>8
+        lo = ch&0xff
+        u = "\377\376" << lo.chr << hi.chr
+        u.encode("UTF-8", "UTF-16")
+      end
+    rescue StandardError
+      return num_str
+    end
+    return utf_str
+  end
+
+  def remove_directive(str)
+    remove_tag(str, ['__', '__'])
+  end
+
+  def mndash(str)
+    str = str.gsub(/\{(mdash|ndash|–)\}/, "–")
+  end
+
+  def remove_hr(page)
+    page = page.gsub(/^\s*\-+\s*$/, "")
+  end
+
+  def make_reference(str)
+    str.gsub!(%r{<br ?\/>}m,             "\n")
+    str.gsub!(%r{<ref[^>]*\/>}m,         '')
+    str.gsub!(%r{<ref[^>]*>.*?<\/ref>}m, '')
+    str
+  end
+
+  def format_ref(page)
+    page = page.gsub(/\[ref\](.*?)\[\/ref\]/m) do
+      ref = $1.dup
+      ref.gsub(/(?:[\r\n]+|<br ?\/>)/, " ")
+    end
+  end
+
+  #################### methods currently unused ####################
+
+  def process_template(str)
+    scanner = StringScanner.new(str)
+    result = process_nested_structure(scanner, "{{", "}}") do |contents|
+      parts = contents.split("|")
+      case parts.size
+      when 0
+        ""
+      when 1
+        parts.first || ""
+      else
+        if parts.last.split("=").size > 1
+          parts.first || ""
+        else
+          parts.last || ""
+        end
+      end
+    end
+    result
+  rescue SystemStackError => err
+    Wukong.bad_record("Poorly nested templates", err, str)
+    return str.gsub!(/\[\[.*\]\]/m, "**BAD TEMPLATES**")
+  end
+
+  def remove_table(str)
+    new_str = str.gsub(/\{\|[^\{\|\}]*?\|\}/m, "")
+    if str != new_str
+      new_str = remove_table(new_str)
+    end
+    new_str = remove_table(new_str) unless str == new_str
+    return new_str
+  end
+
+  def remove_clade(page)
+    new_page = page.gsub(/\{\{(?:C|c)lade[^\{\}]*\}\}/m, "")
+    new_page = remove_clade(new_page) unless page == new_page
+    new_page
+  end
+
+  def remove_inline_template(str)
+    str.gsub(/\{\{(.*?)\}\}/) do
+       key = $1
+       if /\A[^\|]+\z/ =~ key
+         result = key
+       else
+         info = key.split("|")
+         type_code = info.first
+         case type_code
+         when /\Alang*/i, /\AIPA/i, /\AIEP/i, /\ASEP/i, /\Aindent/i, /\Aaudio/i, /\Asmall/i,
+              /\Admoz/i, /\Apron/i, /\Aunicode/i, /\Anote label/i, /\Anowrap/i,
+              /\AArabDIN/i, /\Atrans/i, /\ANihongo/i, /\APolytonic/i
+           out = info[-1]
+         else
+           out = "{" + info.collect{|i|i.chomp}.join("|") + "}"
+         end
+         result = out
+       end
+     end
+  end
+
+  #################### file related utilities ####################
+
+  # collect filenames recursively
+  def collect_files(str, regex = nil)
+    regex ||= //
+    text_array = Array.new
+    Find.find(str) do |f|
+      text_array << f if regex =~ f
+    end
+    text_array.sort
+  end
+
+  # modify a file using block/yield mechanism
+  def file_mod(file_path, backup = false, &block)
+    File.open(file_path, "r") do |fr|
+      str = fr.read
+      newstr = yield(str)
+      str = newstr unless newstr == nil
+      File.open("temp", "w") do |tf|
+        tf.write(str)
+      end
+    end
+
+    File.rename(file_path, file_path + ".bak")
+    File.rename("temp", file_path)
+    File.unlink(file_path + ".bak") unless backup
+  end
+
+  # modify files under a directry (recursive)
+  def batch_file_mod(dir_path, &block)
+    if FileTest.directory?(dir_path)
+      collect_files(dir_path).each do |file|
+        yield file if FileTest.file?(file)
+      end
+    else
+      yield dir_path if FileTest.file?(dir_path)
+    end
+  end
+
+  # take care of difference of separators among environments
+  def correct_separator(input)
+    if input.is_a?(String)
+      ret_str = String.new
+      if RUBY_PLATFORM.index("win32")
+        ret_str = input.gsub("/", "\\")
+      else
+        ret_str = input.gsub("\\", "/")
+      end
+      return ret_str
+    elsif input.is_a?(Array)
+      ret_array = Array.new
+      input.each do |item|
+        ret_array << correct_separator(item)
+      end
+      return ret_array
+    end
+  end
+
+  def rename(files)
+    # num of digits necessary to name the last file generated
+    maxwidth = 0
+
+    files.each do |f|
+      width = f.slice(/\-(\d+)\z/, 1).to_s.length.to_i
+      maxwidth = width if maxwidth < width
+    end
+
+    files.each do |f|
+      newname= f.sub(/\-(\d+)\z/) do
+        "-" + sprintf("%0#{maxwidth}d", $1.to_i)
+      end
+      File.rename(f, newname + ".txt")
+    end
+  end
+
+  # convert int of seconds to string in the format 00:00:00
+  def sec_to_str(int)
+    unless int
+      str = "--:--:--"
+      return str
+    end
+    h = int / 3600
+    m = (int - h * 3600) / 60
+    s = int % 60
+    str = sprintf("%02d:%02d:%02d", h, m, s)
+    return str
+  end
+
+  def decimal_format(i)
+    str = i.to_s.reverse
+    return str.scan(/.?.?./).join(',').reverse
+  end
+
+end

data/examples/munging/wikipedia/dbpedia/dbpedia_common.rb

@@ -0,0 +1,4 @@
+
+require 'wukong'
+require 'multi_json'
+require_relative '../utils/munging_utils.rb'