wp2txt 0.7.8 → 0.9.2

data/lib/wp2txt/article.rb

@@ -37,6 +37,10 @@ module Wp2txt
     def initialize(text, title = "", strip_tmarker = false)
       @title = title.strip
       @strip_tmarker = strip_tmarker
+      convert_characters!(text)    
+      make_reference!(text)
+      remove_ref!(text)
+      
       parse text
     end
     
@@ -58,6 +62,22 @@ module Wp2txt
         end
 
         case mode
+        when :mw_ml_template
+          scanner = StringScanner.new(line)
+          str= process_nested_structure(scanner, "{{", "}}") {""}
+          if $ml_template_end_regex =~ str
+            mode = nil
+          end
+          @elements.last.last << line
+          next
+        when :mw_ml_link
+          scanner = StringScanner.new(line)
+          str= process_nested_structure(scanner, "[[", "]]") {""}
+          if $ml_link_end_regex =~ str
+            mode = nil
+          end
+          @elements.last.last << line
+          next
         when :mw_table
           if $in_table_regex2 =~ line
             mode = nil
@@ -91,19 +111,29 @@ module Wp2txt
         end
 
         case line
+        when $isolated_template_regex
+          @elements << create_element(:mw_isolated_template, line)
+        when $isolated_tag_regex
+          @elements << create_element(:mw_isolated_tag, line)
         when $blank_line_regex
           @elements << create_element(:mw_blank, "\n")      
         when $redirect_regex
           @elements << create_element(:mw_redirect, line)
-        when $in_template_regex
-          @elements << create_element(:mw_template, line)
+        # when $in_template_regex
+        #   @elements << create_element(:mw_template, line)
         when $in_heading_regex
           line = line.sub($heading_onset_regex){$1}.sub($heading_coda_regex){$1}          
           @elements << create_element(:mw_heading, "\n" + line + "\n")
         when $in_inputbox_regex
           @elements << create_element(:mw_inputbox, line)
+        when $ml_template_onset_regex 
+          @elements << create_element(:mw_ml_template, line)
+          mode = :mw_ml_template
+        when $ml_link_onset_regex 
+          @elements << create_element(:mw_ml_link, line)
+          mode = :mw_ml_link
         when $in_inputbox_regex1
-          mode = :mw_inputbox 
+          mode = :mw_inputbox
           @elements << create_element(:mw_inputbox, line)
         when $in_source_regex
         @elements << create_element(:mw_source, line)
@@ -138,7 +168,7 @@ module Wp2txt
         when $in_link_regex
           @elements << create_element(:mw_link, line)
         else 
-          @elements << create_element(:mw_paragraph, line)
+          @elements << create_element(:mw_paragraph, "\n" + line)
         end
       end
       @elements

data/lib/wp2txt/utils.rb

@@ -16,43 +16,36 @@ $html_decoder = HTMLEntities.new
 $entities = ['&nbsp;', '&lt;', '&gt;', '&amp;', '&quot;'].zip([' ', '<', '>', '&', '"'])
 $html_hash  = Hash[*$entities.flatten]
 $html_regex = Regexp.new("(" + $html_hash.keys.join("|") + ")")
-
-$in_template_regex = Regexp.new('^\s*\{\{[^\}]+\}\}\s*$')
+$ml_template_onset_regex = Regexp.new('^\{\{[^\}]*$')
+$ml_template_end_regex   = Regexp.new('\}\}\s*$')
+$ml_link_onset_regex = Regexp.new('^\[\[[^\]]*$')
+$ml_linkend_regex   = Regexp.new('\]\]\s*$')
+$isolated_template_regex = Regexp.new('^\s*\{\{.+\}\}\s*$')
+$isolated_tag_regex = Regexp.new('^\s*\<[^\<\>]+\>.+\<[^\<\>]+\>\s*$')
 $in_link_regex = Regexp.new('^\s*\[.*\]\s*$')
-    
 $in_inputbox_regex  = Regexp.new('<inputbox>.*?<\/inputbox>')
 $in_inputbox_regex1  = Regexp.new('<inputbox>')
 $in_inputbox_regex2  = Regexp.new('<\/inputbox>')
-
 $in_source_regex  = Regexp.new('<source.*?>.*?<\/source>')
 $in_source_regex1  = Regexp.new('<source.*?>')
 $in_source_regex2  = Regexp.new('<\/source>')
-
 $in_math_regex  = Regexp.new('<math.*?>.*?<\/math>')
 $in_math_regex1  = Regexp.new('<math.*?>')
 $in_math_regex2  = Regexp.new('<\/math>')
-
 $in_heading_regex  = Regexp.new('^=+.*?=+$')
-
 $in_html_table_regex = Regexp.new('<table.*?><\/table>')
 $in_html_table_regex1 = Regexp.new('<table\b')
 $in_html_table_regex2 = Regexp.new('<\/\s*table>')
-
 $in_table_regex1 = Regexp.new('^\s*\{\|')
 $in_table_regex2 = Regexp.new('^\|\}.*?$')
-
 $in_unordered_regex  = Regexp.new('^\*')
 $in_ordered_regex    = Regexp.new('^\#')
 $in_pre_regex = Regexp.new('^ ')
 $in_definition_regex  = Regexp.new('^[\;\:]')    
-
 $blank_line_regex = Regexp.new('^\s*$')
-
 $redirect_regex = Regexp.new('#(?:REDIRECT|転送)\s+\[\[(.+)\]\]', Regexp::IGNORECASE)
-
 $remove_tag_regex = Regexp.new("\<[^\<\>]*\>")
 $remove_directives_regex = Regexp.new("\_\_[^\_]*\_\_")
-
 $remove_emphasis_regex = Regexp.new('(' + Regexp.escape("''") + '+)(.+?)\1')
 $chrref_to_utf_regex = Regexp.new('&#(x?)([0-9a-fA-F]+);')
 $mndash_regex = Regexp.new('\{(mdash|ndash|–)\}')
@@ -75,6 +68,7 @@ $category_regex = Regexp.new('[\{\[\|\b](?:' + $category_patterns + ')\:(.*?)[\}
 $escape_nowiki_regex = Regexp.new('<nowiki>(.*?)<\/nowiki>', Regexp::MULTILINE)
 $unescape_nowiki_regex = Regexp.new('<nowiki\-(\d+?)>')
 
+$remove_isolated_regex = Regexp.new('^\s*\{\{(.*?)\}\}\s*$')
 $remove_inline_regex = Regexp.new('\{\{(.*?)\}\}')
 $type_code_regex = Regexp.new('\A(?:lang*|\AIPA|IEP|SEP|indent|audio|small|dmoz|pron|unicode|note label|nowrap|ArabDIN|trans|Nihongo|Polytonic)', Regexp::IGNORECASE)
 
@@ -111,27 +105,26 @@ module Wp2txt
   
   def format_wiki!(text, has_retried = false)
     escape_nowiki!(text)
-
     process_interwiki_links!(text)
     process_external_links!(text)
-
     unescape_nowiki!(text)      
-  end
-  
-  def format_article!(text)
+    #####
     remove_directive!(text)
     remove_emphasis!(text)
     mndash!(text)
-    make_reference!(text)
-    format_ref!(text)
     remove_hr!(text)
     remove_tag!(text)
-    convert_characters!(text)    
-    correct_inline_template!(text) unless $leave_template
-    remove_templates!(text) unless $leave_template
-    remove_table!(text) unless $leave_table
+    correct_inline_template!(text) unless $leave_inline_template
+    remove_templates!(text) unless $leave_inline_template
+    # remove_table!(text) unless $leave_table
   end
   
+  def cleanup!(text)
+    text.gsub!(/\[ref\]\s*\[\/ref\]/m){""}
+    text.gsub!(/\n\n\n+/m){"\n\n"}
+    text.strip!
+    text << "\n\n"
+  end
   #################### parser for nested structure ####################
    
   def process_nested_structure(scanner, left, right, &block)
@@ -237,6 +230,10 @@ module Wp2txt
     result = process_nested_structure(scanner, "{{", "}}") do |contents|
       ""
     end
+    scanner = StringScanner.new(result)
+    result = process_nested_structure(scanner, "{", "}") do |contents|
+      ""
+    end
     str.replace(result)
   end
   
@@ -295,48 +292,48 @@ module Wp2txt
     str.gsub!($mndash_regex, "–")
   end
 
-  def remove_hr!(page)
-    page.gsub!($remove_hr_regex, "")
+  def remove_hr!(str)
+    str.gsub!($remove_hr_regex, "")
   end
 
+  def remove_ref!(str)
+    str.gsub!($format_ref_regex){""}
+  end
+  
   def make_reference!(str)
     str.gsub!($make_reference_regex_a){"\n"}
     str.gsub!($make_reference_regex_b){""}
     str.gsub!($make_reference_regex_c){"[ref]"}
     str.gsub!($make_reference_regex_d){"[/ref]"}
-    str.gsub!($format_ref_regex){""} unless $leave_ref
-  end
-
-  def format_ref!(page)
-    ###### do nothing for now
-    # page.gsub!($format_ref_regex) do
-    # end
   end
 
   def correct_inline_template!(str)
-    str.gsub!($remove_inline_regex) do
-      key = $1
-      if $onset_bar_regex =~ key
-        result = key
-      elsif
-        info = key.split("|")
-        type_code = info.first
-        case type_code
-        when $type_code_regex
-          out = info[-1]
-        else
-          if $leave_template
-            out = "{" + info.collect{|i|i.chomp}.join("|") + "}"
-          else
-            out = ""
+    scanner = StringScanner.new(str)
+    result = process_nested_structure(scanner, "{{", "}}") do |contents|
+      parts = contents.split("|")
+      # type_code = parts.first
+      # case type_code
+      # when $type_code_regex
+      #   out = parts[-1]
+      # else
+      #   case parts.size
+      #   when 0
+      #     out = ""
+      #   when 1
+      #     out = parts.first || ""
+      #   else
+          # while parts.size > 2 && parts.last.split("=").size > 1
+          while parts.size > 1 && parts.last.split("=").size > 1
+            parts.pop
           end
-        end
-        out
-      else
-        ""
-      end
+          out = parts.last || ""
+      #   end
+      # end
+      out.strip
     end
+    str.replace result
   end
+
   
 #################### file related utilities ####################
 

data/lib/wp2txt/version.rb

@@ -1,3 +1,3 @@
 module Wp2txt
-  VERSION = "0.7.8"
+  VERSION = "0.9.2"
 end

data/lib/wp2txt.rb

@@ -3,13 +3,10 @@
 
 $: << File.join(File.dirname(__FILE__))
 
-# require "rubygems"
-# require "bundler/setup"
-
 require "nokogiri"
-# require "oga"
-# require "ox"
+require "parallel"
 
+require 'etc'
 require 'pp'
 require "wp2txt/article"
 require "wp2txt/utils"
@@ -29,7 +26,7 @@ module Wp2txt
 
     include Wp2txt
 
-    def initialize(parent, input_file, output_dir = ".", tfile_size = 10, convert = true, strip_tmarker = false)
+    def initialize(parent, input_file, output_dir = ".", tfile_size = 10, num_threads = 1, convert = true, strip_tmarker = false)
       @parent = parent
       @fp = nil
       
@@ -38,6 +35,8 @@ module Wp2txt
       @tfile_size = tfile_size
       @convert = convert
       @strip_tmarker = strip_tmarker
+      num_cores_available = Etc.nprocessors
+      @num_threads = num_threads <= num_cores_available ? num_threads : num_cores_available
     end
     
     def file_size(file) 
@@ -102,6 +101,7 @@ module Wp2txt
       if /.bz2$/ =~ @input_file
         unless NO_BZ2
           file = Bzip2::Reader.new File.open(@input_file, "r:UTF-8")
+          @parent.msg("WP2TXT is spawming #{@num_threads} threads to process data \n", 0)         
           @parent.msg("Preparing ... This may take several minutes or more ", 0)         
           @infile_size = file_size(file)
           @parent.msg("... Done.", 1)
@@ -113,6 +113,7 @@ module Wp2txt
           else
             file = IO.popen("bzip2 -c -d #{@input_file}") 
           end
+          @parent.msg("WP2TXT is spawming #{@num_threads} threads to process data \n", 0)         
           @parent.msg("Preparing ... This may take several minutes or more ", 0)         
           @infile_size = file_size(file)
           @parent.msg("... Done.", 1)
@@ -237,81 +238,73 @@ module Wp2txt
       end_flag = false
       terminal_round = false
       output_text = ""
+      pages = []
+      data_empty = false
 
-      while page = get_page
-        xmlns = '<mediawiki xmlns="http://www.mediawiki.org/xml/export-0.5/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.mediawiki.org/xml/export-0.5/ http://www.mediawiki.org/xml/export-0.5.xsd" version="0.5" xml:lang="en">' + "\n"
-        xml = xmlns + page + "</mediawiki>"
-
-        input = Nokogiri::XML(xml, nil, 'UTF-8')
-        page = input.xpath("//xmlns:text").first
-        pp_title = page.parent.parent.at_css "title"
-        title = pp_title.content
-        next if /\:/ =~ title
-        text = page.content
-
-        # input = Oga.parse_xml(xml)
-        # page = input.xpath("//xmlns:text").first
-        # title = page.parent.parent.xpath("//xmlns:title").first.text
-        # next if /\:/ =~ title
-        # text = page.text
+      begin
+        page = get_page
+        if page
+          pages << page
+        else
+          data_empty = true
+        end
+        if data_empty || pages.size == @num_threads
+          # pages_text = Parallel.map_with_index(pages, in_threads: @num_threads) do |page, n|
+          pages_text = Parallel.map_with_index(pages, in_threads: @num_threads) do |page, n|
+            page_text = {:order => n, :data => nil}
+            xmlns = '<mediawiki xmlns="http://www.mediawiki.org/xml/export-0.5/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.mediawiki.org/xml/export-0.5/ http://www.mediawiki.org/xml/export-0.5.xsd" version="0.5" xml:lang="en">' + "\n"
+            xml = xmlns + page + "</mediawiki>"
 
-        # input = Ox.load(xml, :encoding => "UTF-8")
-        # title = ""
-        # text  = ""
-        # input.nodes.first.nodes.each do |n|
-        #   if n.name == "title"
-        #     title = n.nodes.first
-        #     if /\:/ =~ title
-        #       title = ""
-        #       break
-        #     end
-        #   elsif n.name == "revision"
-        #     n.nodes.each do |o|
-        #       if o.name == "text"
-        #         text = o.nodes.first
-        #         break
-        #       end
-        #     end
-        #   end
-        # end
-        # next if title == "" || text == ""
+            input = Nokogiri::XML(xml, nil, 'UTF-8')
+            page = input.xpath("//xmlns:text").first
+            pp_title = page.parent.parent.at_css "title"
+            title = pp_title.content
+            unless  /\:/ =~ title
+              text = page.content
+              text.gsub!(/\<\!\-\-(.*?)\-\-\>/m) do |content|
+                num_of_newlines = content.count("\n")
+                if num_of_newlines == 0
+                  ""
+                else
+                  "\n" * num_of_newlines
+                end
+              end
+              article = Article.new(text, title, @strip_tmarker)
+              page_text[:data] = block.call(article)
+            end
+            page_text
+          end
+          pages.clear
+          pages_text = pages_text.sort_by{|v| v[:order]}.map{|v| v[:data]}.compact
+          pages_text.each do |page_text|
+            output_text << page_text
+            @count ||= 0; @count += 1;
+            @total_size = output_text.bytesize
+            # flagged when data exceeds the size of output file
+            end_flag = true if @total_size > (@tfile_size * 1024 * 1024)
+          end
 
-        # remove all comment texts
-        # and insert as many number of new line chars included in 
-        # each comment instead
-        text.gsub!(/\<\!\-\-(.*?)\-\-\>/m) do |content|
-          num_of_newlines = content.count("\n")
-          if num_of_newlines == 0
-            ""
-          else
-            "\n" * num_of_newlines
+          #close the present file, then open a new one
+          if end_flag
+            cleanup!(output_text)
+            @fp.puts(output_text)
+            output_text = ""
+            @total_size = 0
+            end_flag = false
+            @fp.close
+            @file_index += 1
+            outfilename = File.join(@output_dir, @outfile_base + @file_index.to_s)
+            @outfiles << outfilename
+            @fp = File.open(outfilename, "w")
+            next
           end
         end
-        
-        @count ||= 0;@count += 1;
-
-        article = Article.new(text, title, @strip_tmarker)
-        output_text += block.call(article)
-        @total_size = output_text.bytesize
+      end while !data_empty 
 
-        # flagged when data exceeds the size of output file
-        end_flag = true if @total_size > (@tfile_size * 1024 * 1024)
-
-        #close the present file, then open a new one
-        if end_flag
-          @fp.puts(output_text)
-          output_text = ""
-          @total_size = 0
-          end_flag = false
-          @fp.close
-          @file_index += 1
-          outfilename = File.join(@output_dir, @outfile_base + @file_index.to_s)
-          @outfiles << outfilename
-          @fp = File.open(outfilename, "w")
-          next
-        end
+      if output_text != ""
+        cleanup!(output_text)
+        @fp.puts(output_text) 
       end
-      @fp.puts(output_text) if output_text != ""
       notify_parent(true)
       @parent.after
       @fp.close    
@@ -351,4 +344,5 @@ module Wp2txt
       @parent.msg("Processing finished", 1)
     end 
   end
-end
+end
+

data/spec/utils_spec.rb

@@ -182,22 +182,34 @@ describe "Wp2txt" do
     end
   end
   
-  # describe "process_template" do
-  #   it "removes brackets and leaving some text" do
-  #     str_before = "{{}}"
-  #     str_after = ""
-  #     expect(process_template(str_before)).to eq str_after
-  #     str_before = "{{lang|en|Japan}}"
-  #     str_after  = "Japan"
-  #     expect(process_template(str_before)).to eq str_after
-  #     str_before = "{{a|b=c|d=f}}"
-  #     str_after  = "a"
-  #     expect(process_template(str_before)).to eq str_after
-  #     str_before = "{{a|b|{{c|d|e}}}}"
-  #     str_after  = "e"
-  #     expect(process_template(str_before)).to eq str_after
-  #   end
-  # end
+  describe "correct_inline_template!" do
+    it "removes brackets and leaving some text" do
+      # str_before = "{{}}"
+      # str_after = ""
+      # correct_inline_template!(str_before)
+      # expect(str_before).to eq str_after
+      str_before = "{{MedalCountry | {{JPN}} }}"
+      str_after  = "JPN"
+      correct_inline_template!(str_before)
+      expect(str_before).to eq str_after
+      str_before = "{{lang|en|Japan}}"
+      str_after  = "Japan"
+      correct_inline_template!(str_before)
+      expect(str_before).to eq str_after
+      str_before = "{{a|b=c|d=f}}"
+      str_after  = "a"
+      correct_inline_template!(str_before)
+      expect(str_before).to eq str_after
+      str_before = "{{a|b|{{c|d|e}}}}"
+      str_after  = "e"
+      correct_inline_template!(str_before)
+      expect(str_before).to eq str_after
+      str_before = "{{要出典範囲|日本人に多く見受けられる|date=2013年8月|title=日本人特有なのか、本当に多いのかを示す必要がある}}"
+      str_after = "日本人に多く見受けられる"
+      correct_inline_template!(str_before)
+      expect(str_before).to eq str_after
+    end
+  end
   
   #   describe "expand_template" do
   #     it "gets data corresponding to a given template using mediawiki api" do

data/wp2txt.gemspec

@@ -23,6 +23,7 @@ Gem::Specification.new do |s|
   # s.add_development_dependency "rake"
 
   s.add_dependency "nokogiri"
+  s.add_dependency "parallel"
   s.add_dependency "htmlentities"
-  s.add_dependency "trollop"
+  s.add_dependency "optimist"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wp2txt
 version: !ruby/object:Gem::Version
-  version: 0.7.8
+  version: 0.9.2
 platform: ruby
 authors:
 - Yoichiro Hasebe
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-12-01 00:00:00.000000000 Z
+date: 2022-07-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: htmlentities
   requirement: !ruby/object:Gem::Requirement
@@ -39,7 +53,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: trollop
+  name: optimist
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -69,8 +83,10 @@ files:
 - Rakefile
 - bin/benchmark.rb
 - bin/wp2txt
-- data/testdata.bz2
-- error_log.txt
+- data/output_samples/testdata_en.txt
+- data/output_samples/testdata_ja.txt
+- data/testdata_en.bz2
+- data/testdata_ja.bz2
 - lib/wp2txt.rb
 - lib/wp2txt/article.rb
 - lib/wp2txt/mw_api.rb
@@ -83,7 +99,7 @@ files:
 homepage: http://github.com/yohasebe/wp2txt
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -98,9 +114,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: wp2txt
-rubygems_version: 2.4.2
-signing_key: 
+rubygems_version: 3.3.3
+signing_key:
 specification_version: 4
 summary: Wikipedia dump to text converter
 test_files:

data/error_log.txt

@@ -1 +0,0 @@
-[[アンパサンド]]