wp2txt 0.9.2 → 0.9.5

data/lib/wp2txt/article.rb

@@ -37,10 +37,11 @@ module Wp2txt
     def initialize(text, title = "", strip_tmarker = false)
       @title = title.strip
       @strip_tmarker = strip_tmarker
-      convert_characters!(text)    
+      convert_characters!(text)
+      remove_html!(text)
+      remove_complex!(text)
       make_reference!(text)
       remove_ref!(text)
-      
       parse text
     end
     

data/lib/wp2txt/utils.rb

@@ -108,7 +108,7 @@ module Wp2txt
     process_interwiki_links!(text)
     process_external_links!(text)
     unescape_nowiki!(text)      
-    #####
+
     remove_directive!(text)
     remove_emphasis!(text)
     mndash!(text)
@@ -116,11 +116,15 @@ module Wp2txt
     remove_tag!(text)
     correct_inline_template!(text) unless $leave_inline_template
     remove_templates!(text) unless $leave_inline_template
-    # remove_table!(text) unless $leave_table
+    remove_table!(text) unless $leave_table
   end
   
   def cleanup!(text)
     text.gsub!(/\[ref\]\s*\[\/ref\]/m){""}
+    text.gsub!(/^File:.+$/){""}
+    text.gsub!(/^\|.*$/){""}
+    text.gsub!(/^{{.*$/){""}
+    text.gsub!(/^}}.*$/){""}
     text.gsub!(/\n\n\n+/m){"\n\n"}
     text.strip!
     text << "\n\n"
@@ -128,8 +132,9 @@ module Wp2txt
   #################### parser for nested structure ####################
    
   def process_nested_structure(scanner, left, right, &block)
+    test = false
     buffer = ""
-    begin
+    # begin
     if left == "[" && right == "]"
       regex = $single_square_bracket_regex
     elsif left == "[[" && right == "]]"
@@ -141,7 +146,7 @@ module Wp2txt
     elsif left == "{|" && right == "|}"
       regex = $curly_square_bracket_regex
     else
-      regex = Regexp.new('(#{Regexp.escape(left)}|#{Regexp.escape(right)})', Regexp::MULTILINE)
+      regex = Regexp.new("(#{Regexp.escape(left)}|#{Regexp.escape(right)})")
     end
     while str = scanner.scan_until(regex)
       case scanner[1]
@@ -167,9 +172,9 @@ module Wp2txt
       scanner.string = buffer
       return process_nested_structure(scanner, left, right, &block) || ""
     end
-    rescue => e
-      return scanner.string
-    end
+    # rescue => e
+    #   return scanner.string
+    # end
   end  
 
   #################### methods used from format_wiki ####################
@@ -234,6 +239,10 @@ module Wp2txt
     result = process_nested_structure(scanner, "{", "}") do |contents|
       ""
     end
+    scanner = StringScanner.new(result)
+    result = process_nested_structure(scanner, "{{", "}}") do |contents|
+      ""
+    end
     str.replace(result)
   end
   
@@ -299,6 +308,24 @@ module Wp2txt
   def remove_ref!(str)
     str.gsub!($format_ref_regex){""}
   end
+
+  def remove_html!(str)
+    ["div", "gallery", "timeline"].each do |tag|
+      scanner = StringScanner.new(str)
+      result = process_nested_structure(scanner, "<#{tag}", "#{tag}>") do |contents|
+        ""
+      end
+      str.replace(result)
+    end
+  end
+
+  def remove_complex!(str)
+    str.gsub!(/(?:'')?\[https?\:[^\[\]]+?\s([^\]]++)?\](?:'')?/){$1}
+    str.gsub!(/(?:'')?\[https?\:[^\[\]]++\](?:'')?\s?/){""}
+    str.gsub!(/\<\<([^<>]++)\>\>\s?/){"《#{$1}》"}
+    str.gsub!(/\{\{(?:Infobox|efn|Sfn|div col|no col|bar box|formatnum\:|Refnest\||Refnest\||Col\||See also\||R\|)((?:[^{}]++|\{\{\g<1>\}\})++)\}\}/im){""}
+    str.gsub!(/\[\[(?:File|ファイル)\:((?:[^\[\]]++|\[\[\g<1>\]\])++)\]\]/im){""}
+  end
   
   def make_reference!(str)
     str.gsub!($make_reference_regex_a){"\n"}
@@ -311,30 +338,28 @@ module Wp2txt
     scanner = StringScanner.new(str)
     result = process_nested_structure(scanner, "{{", "}}") do |contents|
       parts = contents.split("|")
-      # type_code = parts.first
-      # case type_code
-      # when $type_code_regex
-      #   out = parts[-1]
-      # else
-      #   case parts.size
-      #   when 0
-      #     out = ""
-      #   when 1
-      #     out = parts.first || ""
-      #   else
-          # while parts.size > 2 && parts.last.split("=").size > 1
-          while parts.size > 1 && parts.last.split("=").size > 1
-            parts.pop
-          end
-          out = parts.last || ""
-      #   end
-      # end
+      if /\A(?:lang|fontsize)\z/i =~ parts[0]
+        parts.shift
+      elsif /\Alang=/i =~ parts[1]
+        parts.shift
+      end
+
+      if parts.size == 1
+        out = parts[0]
+      else
+        keyval = parts[1].split("=")
+        if keyval.size > 1
+          out = keyval[1]
+        else
+          out = parts[1] || ""
+        end
+      end
+
       out.strip
     end
     str.replace result
   end
 
-  
 #################### file related utilities ####################
 
   # collect filenames recursively
@@ -427,5 +452,4 @@ module Wp2txt
     str = i.to_s.reverse
     return str.scan(/.?.?./).join(',').reverse
   end
-    
 end

data/lib/wp2txt/version.rb

@@ -1,3 +1,3 @@
 module Wp2txt
-  VERSION = "0.9.2"
+  VERSION = "0.9.5"
 end

data/lib/wp2txt.rb

@@ -101,7 +101,7 @@ module Wp2txt
       if /.bz2$/ =~ @input_file
         unless NO_BZ2
           file = Bzip2::Reader.new File.open(@input_file, "r:UTF-8")
-          @parent.msg("WP2TXT is spawming #{@num_threads} threads to process data \n", 0)         
+          @parent.msg("WP2TXT is spawning #{@num_threads} threads to process data \n", 0)         
           @parent.msg("Preparing ... This may take several minutes or more ", 0)         
           @infile_size = file_size(file)
           @parent.msg("... Done.", 1)
@@ -113,7 +113,7 @@ module Wp2txt
           else
             file = IO.popen("bzip2 -c -d #{@input_file}") 
           end
-          @parent.msg("WP2TXT is spawming #{@num_threads} threads to process data \n", 0)         
+          @parent.msg("WP2TXT is spawning #{@num_threads} threads to process data \n", 0)         
           @parent.msg("Preparing ... This may take several minutes or more ", 0)         
           @infile_size = file_size(file)
           @parent.msg("... Done.", 1)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wp2txt
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.9.5
 platform: ruby
 authors:
 - Yoichiro Hasebe
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-07-25 00:00:00.000000000 Z
+date: 2022-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -84,7 +84,11 @@ files:
 - bin/benchmark.rb
 - bin/wp2txt
 - data/output_samples/testdata_en.txt
+- data/output_samples/testdata_en_categories.txt
+- data/output_samples/testdata_en_summary.txt
 - data/output_samples/testdata_ja.txt
+- data/output_samples/testdata_ja_categories.txt
+- data/output_samples/testdata_ja_summary.txt
 - data/testdata_en.bz2
 - data/testdata_ja.bz2
 - lib/wp2txt.rb
@@ -114,7 +118,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.3
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: Wikipedia dump to text converter