wp2txt 0.8.0 → 0.9.1

data/lib/wp2txt.rb

@@ -4,6 +4,7 @@
 $: << File.join(File.dirname(__FILE__))
 
 require "nokogiri"
+require "parallel"
 
 require 'pp'
 require "wp2txt/article"
@@ -24,7 +25,7 @@ module Wp2txt
 
     include Wp2txt
 
-    def initialize(parent, input_file, output_dir = ".", tfile_size = 10, convert = true, strip_tmarker = false)
+    def initialize(parent, input_file, output_dir = ".", tfile_size = 10, num_threads = 1, convert = true, strip_tmarker = false)
       @parent = parent
       @fp = nil
       
@@ -33,6 +34,8 @@ module Wp2txt
       @tfile_size = tfile_size
       @convert = convert
       @strip_tmarker = strip_tmarker
+      num_cores_available = Etc.nprocessors
+      @num_threads = num_threads <= num_cores_available ? num_threads : num_cores_available
     end
     
     def file_size(file) 
@@ -97,6 +100,7 @@ module Wp2txt
       if /.bz2$/ =~ @input_file
         unless NO_BZ2
           file = Bzip2::Reader.new File.open(@input_file, "r:UTF-8")
+          @parent.msg("WP2TXT is spawming #{@num_threads} threads to process data \n", 0)         
           @parent.msg("Preparing ... This may take several minutes or more ", 0)         
           @infile_size = file_size(file)
           @parent.msg("... Done.", 1)
@@ -108,6 +112,7 @@ module Wp2txt
           else
             file = IO.popen("bzip2 -c -d #{@input_file}") 
           end
+          @parent.msg("WP2TXT is spawming #{@num_threads} threads to process data \n", 0)         
           @parent.msg("Preparing ... This may take several minutes or more ", 0)         
           @infile_size = file_size(file)
           @parent.msg("... Done.", 1)
@@ -232,53 +237,71 @@ module Wp2txt
       end_flag = false
       terminal_round = false
       output_text = ""
+      pages = []
+      data_empty = false
 
-      while page = get_page
-        xmlns = '<mediawiki xmlns="http://www.mediawiki.org/xml/export-0.5/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.mediawiki.org/xml/export-0.5/ http://www.mediawiki.org/xml/export-0.5.xsd" version="0.5" xml:lang="en">' + "\n"
-        xml = xmlns + page + "</mediawiki>"
+      begin
+        page = get_page
+        if page
+          pages << page
+        else
+          data_empty = true
+        end
+        if data_empty || pages.size == @num_threads
+          # pages_text = Parallel.map_with_index(pages, in_threads: @num_threads) do |page, n|
+          pages_text = Parallel.map_with_index(pages, in_threads: @num_threads) do |page, n|
+            page_text = {:order => n, :data => nil}
+            xmlns = '<mediawiki xmlns="http://www.mediawiki.org/xml/export-0.5/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.mediawiki.org/xml/export-0.5/ http://www.mediawiki.org/xml/export-0.5.xsd" version="0.5" xml:lang="en">' + "\n"
+            xml = xmlns + page + "</mediawiki>"
 
-        input = Nokogiri::XML(xml, nil, 'UTF-8')
-        page = input.xpath("//xmlns:text").first
-        pp_title = page.parent.parent.at_css "title"
-        title = pp_title.content
-        next if /\:/ =~ title
-        text = page.content
+            input = Nokogiri::XML(xml, nil, 'UTF-8')
+            page = input.xpath("//xmlns:text").first
+            pp_title = page.parent.parent.at_css "title"
+            title = pp_title.content
+            unless  /\:/ =~ title
+              text = page.content
+              text.gsub!(/\<\!\-\-(.*?)\-\-\>/m) do |content|
+                num_of_newlines = content.count("\n")
+                if num_of_newlines == 0
+                  ""
+                else
+                  "\n" * num_of_newlines
+                end
+              end
+              article = Article.new(text, title, @strip_tmarker)
+              page_text[:data] = block.call(article)
+            end
+            page_text
+          end
+          pages.clear
+          pages_text = pages_text.sort_by{|v| v[:order]}.map{|v| v[:data]}.compact
+          pages_text.each do |page_text|
+            output_text << page_text
+            @count ||= 0; @count += 1;
+            @total_size = output_text.bytesize
+            # flagged when data exceeds the size of output file
+            end_flag = true if @total_size > (@tfile_size * 1024 * 1024)
+          end
 
-        text.gsub!(/\<\!\-\-(.*?)\-\-\>/m) do |content|
-          num_of_newlines = content.count("\n")
-          if num_of_newlines == 0
-            ""
-          else
-            "\n" * num_of_newlines
+          #close the present file, then open a new one
+          if end_flag
+            cleanup!(output_text)
+            @fp.puts(output_text)
+            output_text = ""
+            @total_size = 0
+            end_flag = false
+            @fp.close
+            @file_index += 1
+            outfilename = File.join(@output_dir, @outfile_base + @file_index.to_s)
+            @outfiles << outfilename
+            @fp = File.open(outfilename, "w")
+            next
           end
         end
-        
-        @count ||= 0;@count += 1;
-
-        article = Article.new(text, title, @strip_tmarker)
-        output_text += block.call(article)
-        @total_size = output_text.bytesize
-
-        # flagged when data exceeds the size of output file
-        end_flag = true if @total_size > (@tfile_size * 1024 * 1024)
+      end while !data_empty 
 
-        #close the present file, then open a new one
-        if end_flag
-          output_text.strip!
-          @fp.puts(output_text)
-          output_text = ""
-          @total_size = 0
-          end_flag = false
-          @fp.close
-          @file_index += 1
-          outfilename = File.join(@output_dir, @outfile_base + @file_index.to_s)
-          @outfiles << outfilename
-          @fp = File.open(outfilename, "w")
-          next
-        end
-      end
       if output_text != ""
-        output_text.strip!
+        cleanup!(output_text)
         @fp.puts(output_text) 
       end
       notify_parent(true)
@@ -320,4 +343,5 @@ module Wp2txt
       @parent.msg("Processing finished", 1)
     end 
   end
-end
+end
+

data/lib/wp2txt/article.rb

@@ -37,6 +37,10 @@ module Wp2txt
     def initialize(text, title = "", strip_tmarker = false)
       @title = title.strip
       @strip_tmarker = strip_tmarker
+      convert_characters!(text)    
+      make_reference!(text)
+      remove_ref!(text)
+      
       parse text
     end
     
@@ -58,6 +62,22 @@ module Wp2txt
         end
 
         case mode
+        when :mw_ml_template
+          scanner = StringScanner.new(line)
+          str= process_nested_structure(scanner, "{{", "}}") {""}
+          if $ml_template_end_regex =~ str
+            mode = nil
+          end
+          @elements.last.last << line
+          next
+        when :mw_ml_link
+          scanner = StringScanner.new(line)
+          str= process_nested_structure(scanner, "[[", "]]") {""}
+          if $ml_link_end_regex =~ str
+            mode = nil
+          end
+          @elements.last.last << line
+          next
         when :mw_table
           if $in_table_regex2 =~ line
             mode = nil
@@ -91,17 +111,27 @@ module Wp2txt
         end
 
         case line
+        when $isolated_template_regex
+          @elements << create_element(:mw_isolated_template, line)
+        when $isolated_tag_regex
+          @elements << create_element(:mw_isolated_tag, line)
         when $blank_line_regex
           @elements << create_element(:mw_blank, "\n")      
         when $redirect_regex
           @elements << create_element(:mw_redirect, line)
-        when $in_template_regex
-          @elements << create_element(:mw_template, line)
+        # when $in_template_regex
+        #   @elements << create_element(:mw_template, line)
         when $in_heading_regex
           line = line.sub($heading_onset_regex){$1}.sub($heading_coda_regex){$1}          
           @elements << create_element(:mw_heading, "\n" + line + "\n")
         when $in_inputbox_regex
           @elements << create_element(:mw_inputbox, line)
+        when $ml_template_onset_regex 
+          @elements << create_element(:mw_ml_template, line)
+          mode = :mw_ml_template
+        when $ml_link_onset_regex 
+          @elements << create_element(:mw_ml_link, line)
+          mode = :mw_ml_link
         when $in_inputbox_regex1
           mode = :mw_inputbox
           @elements << create_element(:mw_inputbox, line)
@@ -138,7 +168,7 @@ module Wp2txt
         when $in_link_regex
           @elements << create_element(:mw_link, line)
         else 
-          @elements << create_element(:mw_paragraph, line)
+          @elements << create_element(:mw_paragraph, "\n" + line)
         end
       end
       @elements

data/lib/wp2txt/utils.rb

@@ -16,43 +16,36 @@ $html_decoder = HTMLEntities.new
 $entities = ['&nbsp;', '&lt;', '&gt;', '&amp;', '&quot;'].zip([' ', '<', '>', '&', '"'])
 $html_hash  = Hash[*$entities.flatten]
 $html_regex = Regexp.new("(" + $html_hash.keys.join("|") + ")")
-
-$in_template_regex = Regexp.new('^\s*\{\{[^\}]+\}\}\s*$')
+$ml_template_onset_regex = Regexp.new('^\{\{[^\}]*$')
+$ml_template_end_regex   = Regexp.new('\}\}\s*$')
+$ml_link_onset_regex = Regexp.new('^\[\[[^\]]*$')
+$ml_linkend_regex   = Regexp.new('\]\]\s*$')
+$isolated_template_regex = Regexp.new('^\s*\{\{.+\}\}\s*$')
+$isolated_tag_regex = Regexp.new('^\s*\<[^\<\>]+\>.+\<[^\<\>]+\>\s*$')
 $in_link_regex = Regexp.new('^\s*\[.*\]\s*$')
-    
 $in_inputbox_regex  = Regexp.new('<inputbox>.*?<\/inputbox>')
 $in_inputbox_regex1  = Regexp.new('<inputbox>')
 $in_inputbox_regex2  = Regexp.new('<\/inputbox>')
-
 $in_source_regex  = Regexp.new('<source.*?>.*?<\/source>')
 $in_source_regex1  = Regexp.new('<source.*?>')
 $in_source_regex2  = Regexp.new('<\/source>')
-
 $in_math_regex  = Regexp.new('<math.*?>.*?<\/math>')
 $in_math_regex1  = Regexp.new('<math.*?>')
 $in_math_regex2  = Regexp.new('<\/math>')
-
 $in_heading_regex  = Regexp.new('^=+.*?=+$')
-
 $in_html_table_regex = Regexp.new('<table.*?><\/table>')
 $in_html_table_regex1 = Regexp.new('<table\b')
 $in_html_table_regex2 = Regexp.new('<\/\s*table>')
-
 $in_table_regex1 = Regexp.new('^\s*\{\|')
 $in_table_regex2 = Regexp.new('^\|\}.*?$')
-
 $in_unordered_regex  = Regexp.new('^\*')
 $in_ordered_regex    = Regexp.new('^\#')
 $in_pre_regex = Regexp.new('^ ')
 $in_definition_regex  = Regexp.new('^[\;\:]')    
-
 $blank_line_regex = Regexp.new('^\s*$')
-
 $redirect_regex = Regexp.new('#(?:REDIRECT|転送)\s+\[\[(.+)\]\]', Regexp::IGNORECASE)
-
 $remove_tag_regex = Regexp.new("\<[^\<\>]*\>")
 $remove_directives_regex = Regexp.new("\_\_[^\_]*\_\_")
-
 $remove_emphasis_regex = Regexp.new('(' + Regexp.escape("''") + '+)(.+?)\1')
 $chrref_to_utf_regex = Regexp.new('&#(x?)([0-9a-fA-F]+);')
 $mndash_regex = Regexp.new('\{(mdash|ndash|–)\}')
@@ -75,6 +68,7 @@ $category_regex = Regexp.new('[\{\[\|\b](?:' + $category_patterns + ')\:(.*?)[\}
 $escape_nowiki_regex = Regexp.new('<nowiki>(.*?)<\/nowiki>', Regexp::MULTILINE)
 $unescape_nowiki_regex = Regexp.new('<nowiki\-(\d+?)>')
 
+$remove_isolated_regex = Regexp.new('^\s*\{\{(.*?)\}\}\s*$')
 $remove_inline_regex = Regexp.new('\{\{(.*?)\}\}')
 $type_code_regex = Regexp.new('\A(?:lang*|\AIPA|IEP|SEP|indent|audio|small|dmoz|pron|unicode|note label|nowrap|ArabDIN|trans|Nihongo|Polytonic)', Regexp::IGNORECASE)
 
@@ -111,26 +105,26 @@ module Wp2txt
   
   def format_wiki!(text, has_retried = false)
     escape_nowiki!(text)
-
     process_interwiki_links!(text)
     process_external_links!(text)
-
     unescape_nowiki!(text)      
-  end
-  
-  def format_article!(text)
+    #####
     remove_directive!(text)
     remove_emphasis!(text)
     mndash!(text)
-    make_reference!(text)
-    format_ref!(text)
     remove_hr!(text)
     remove_tag!(text)
-    convert_characters!(text)    
-    correct_inline_template!(text) unless $leave_template
-    remove_templates!(text) unless $leave_template
+    correct_inline_template!(text) unless $leave_inline_template
+    remove_templates!(text) unless $leave_inline_template
+    # remove_table!(text) unless $leave_table
   end
   
+  def cleanup!(text)
+    text.gsub!(/\[ref\]\s*\[\/ref\]/m){""}
+    text.gsub!(/\n\n\n+/m){"\n\n"}
+    text.strip!
+    text << "\n\n"
+  end
   #################### parser for nested structure ####################
    
   def process_nested_structure(scanner, left, right, &block)
@@ -236,6 +230,10 @@ module Wp2txt
     result = process_nested_structure(scanner, "{{", "}}") do |contents|
       ""
     end
+    scanner = StringScanner.new(result)
+    result = process_nested_structure(scanner, "{", "}") do |contents|
+      ""
+    end
     str.replace(result)
   end
   
@@ -309,36 +307,33 @@ module Wp2txt
     str.gsub!($make_reference_regex_d){"[/ref]"}
   end
 
-  def format_ref!(page)
-    ###### do nothing for now
-    # page.gsub!($format_ref_regex) do
-    # end
-  end
-
   def correct_inline_template!(str)
-    str.gsub!($remove_inline_regex) do
-      key = $1
-      if $onset_bar_regex =~ key
-        result = key
-      elsif
-        info = key.split("|")
-        type_code = info.first
-        case type_code
-        when $type_code_regex
-          out = info[-1]
-        else
-          if $leave_template
-            out = "{" + info.collect{|i|i.chomp}.join("|") + "}"
-          else
-            out = ""
+    scanner = StringScanner.new(str)
+    result = process_nested_structure(scanner, "{{", "}}") do |contents|
+      parts = contents.split("|")
+      # type_code = parts.first
+      # case type_code
+      # when $type_code_regex
+      #   out = parts[-1]
+      # else
+      #   case parts.size
+      #   when 0
+      #     out = ""
+      #   when 1
+      #     out = parts.first || ""
+      #   else
+          # while parts.size > 2 && parts.last.split("=").size > 1
+          while parts.size > 1 && parts.last.split("=").size > 1
+            parts.pop
           end
-        end
-        out
-      else
-        ""
-      end
+          out = parts.last || ""
+      #   end
+      # end
+      out.strip
     end
+    str.replace result
   end
+
   
 #################### file related utilities ####################
 

data/lib/wp2txt/version.rb

@@ -1,3 +1,3 @@
 module Wp2txt
-  VERSION = "0.8.0"
+  VERSION = "0.9.1"
 end

data/spec/utils_spec.rb

@@ -182,22 +182,34 @@ describe "Wp2txt" do
     end
   end
   
-  # describe "process_template" do
-  #   it "removes brackets and leaving some text" do
-  #     str_before = "{{}}"
-  #     str_after = ""
-  #     expect(process_template(str_before)).to eq str_after
-  #     str_before = "{{lang|en|Japan}}"
-  #     str_after  = "Japan"
-  #     expect(process_template(str_before)).to eq str_after
-  #     str_before = "{{a|b=c|d=f}}"
-  #     str_after  = "a"
-  #     expect(process_template(str_before)).to eq str_after
-  #     str_before = "{{a|b|{{c|d|e}}}}"
-  #     str_after  = "e"
-  #     expect(process_template(str_before)).to eq str_after
-  #   end
-  # end
+  describe "correct_inline_template!" do
+    it "removes brackets and leaving some text" do
+      # str_before = "{{}}"
+      # str_after = ""
+      # correct_inline_template!(str_before)
+      # expect(str_before).to eq str_after
+      str_before = "{{MedalCountry | {{JPN}} }}"
+      str_after  = "JPN"
+      correct_inline_template!(str_before)
+      expect(str_before).to eq str_after
+      str_before = "{{lang|en|Japan}}"
+      str_after  = "Japan"
+      correct_inline_template!(str_before)
+      expect(str_before).to eq str_after
+      str_before = "{{a|b=c|d=f}}"
+      str_after  = "a"
+      correct_inline_template!(str_before)
+      expect(str_before).to eq str_after
+      str_before = "{{a|b|{{c|d|e}}}}"
+      str_after  = "e"
+      correct_inline_template!(str_before)
+      expect(str_before).to eq str_after
+      str_before = "{{要出典範囲|日本人に多く見受けられる|date=2013年8月|title=日本人特有なのか、本当に多いのかを示す必要がある}}"
+      str_after = "日本人に多く見受けられる"
+      correct_inline_template!(str_before)
+      expect(str_before).to eq str_after
+    end
+  end
   
   #   describe "expand_template" do
   #     it "gets data corresponding to a given template using mediawiki api" do