wp2txt 0.9.3 → 0.9.5.1

data/lib/wp2txt/article.rb

@@ -37,10 +37,11 @@ module Wp2txt
     def initialize(text, title = "", strip_tmarker = false)
       @title = title.strip
       @strip_tmarker = strip_tmarker
-      convert_characters!(text)    
+      convert_characters!(text)
+      remove_html!(text)
+      remove_complex!(text)
       make_reference!(text)
       remove_ref!(text)
-      
       parse text
     end
     

data/lib/wp2txt/utils.rb

@@ -108,7 +108,7 @@ module Wp2txt
     process_interwiki_links!(text)
     process_external_links!(text)
     unescape_nowiki!(text)      
-    #####
+
     remove_directive!(text)
     remove_emphasis!(text)
     mndash!(text)
@@ -116,11 +116,15 @@ module Wp2txt
     remove_tag!(text)
     correct_inline_template!(text) unless $leave_inline_template
     remove_templates!(text) unless $leave_inline_template
-    # remove_table!(text) unless $leave_table
+    remove_table!(text) unless $leave_table
   end
   
   def cleanup!(text)
     text.gsub!(/\[ref\]\s*\[\/ref\]/m){""}
+    text.gsub!(/^File:.+$/){""}
+    text.gsub!(/^\|.*$/){""}
+    text.gsub!(/^{{.*$/){""}
+    text.gsub!(/^}}.*$/){""}
     text.gsub!(/\n\n\n+/m){"\n\n"}
     text.strip!
     text << "\n\n"
@@ -128,45 +132,46 @@ module Wp2txt
   #################### parser for nested structure ####################
    
   def process_nested_structure(scanner, left, right, &block)
+    test = false
     buffer = ""
     begin
-    if left == "[" && right == "]"
-      regex = $single_square_bracket_regex
-    elsif left == "[[" && right == "]]"
-      regex = $double_square_bracket_regex
-    elsif left == "{" && right == "}"
-      regex = $single_curly_bracket_regex
-    elsif left == "{{" && right == "}}"
-      regex = $double_curly_bracket_regex
-    elsif left == "{|" && right == "|}"
-      regex = $curly_square_bracket_regex
-    else
-      regex = Regexp.new('(#{Regexp.escape(left)}|#{Regexp.escape(right)})', Regexp::MULTILINE)
-    end
-    while str = scanner.scan_until(regex)
-      case scanner[1]
-      when left
-        buffer << str
-        has_left = true
-      when right
-        if has_left
-          buffer = buffer[0...-(left.size)]
-          contents = block.call(str[0...-(left.size)])
-          buffer << contents
-          break
-        else
+      if left == "[" && right == "]"
+        regex = $single_square_bracket_regex
+      elsif left == "[[" && right == "]]"
+        regex = $double_square_bracket_regex
+      elsif left == "{" && right == "}"
+        regex = $single_curly_bracket_regex
+      elsif left == "{{" && right == "}}"
+        regex = $double_curly_bracket_regex
+      elsif left == "{|" && right == "|}"
+        regex = $curly_square_bracket_regex
+      else
+        regex = Regexp.new("(#{Regexp.escape(left)}|#{Regexp.escape(right)})")
+      end
+      while str = scanner.scan_until(regex)
+        case scanner[1]
+        when left
           buffer << str
+          has_left = true
+        when right
+          if has_left
+            buffer = buffer[0...-(left.size)]
+            contents = block.call(str[0...-(left.size)])
+            buffer << contents
+            break
+          else
+            buffer << str
+          end
         end
       end
-    end
-    buffer << scanner.rest
+      buffer << scanner.rest
 
-    if buffer == scanner.string
-      return buffer
-    else
-      scanner.string = buffer
-      return process_nested_structure(scanner, left, right, &block) || ""
-    end
+      if buffer == scanner.string
+        return buffer
+      else
+        scanner.string = buffer
+        return process_nested_structure(scanner, left, right, &block) || ""
+      end
     rescue => e
       return scanner.string
     end
@@ -234,6 +239,10 @@ module Wp2txt
     result = process_nested_structure(scanner, "{", "}") do |contents|
       ""
     end
+    scanner = StringScanner.new(result)
+    result = process_nested_structure(scanner, "{{", "}}") do |contents|
+      ""
+    end
     str.replace(result)
   end
   
@@ -299,6 +308,24 @@ module Wp2txt
   def remove_ref!(str)
     str.gsub!($format_ref_regex){""}
   end
+
+  def remove_html!(str)
+    ["div", "gallery", "timeline"].each do |tag|
+      scanner = StringScanner.new(str)
+      result = process_nested_structure(scanner, "<#{tag}", "#{tag}>") do |contents|
+        ""
+      end
+      str.replace(result)
+    end
+  end
+
+  def remove_complex!(str)
+    str.gsub!(/(?:'')?\[https?\:[^\[\]]+?\s([^\]]++)?\](?:'')?/){$1}
+    str.gsub!(/(?:'')?\[https?\:[^\[\]]++\](?:'')?\s?/){""}
+    str.gsub!(/\<\<([^<>]++)\>\>\s?/){"《#{$1}》"}
+    str.gsub!(/\{\{(?:Infobox|efn|Sfn|div col|no col|bar box|formatnum\:|Refnest\||Refnest\||Col\||See also\||R\|)((?:[^{}]++|\{\{\g<1>\}\})++)\}\}/im){""}
+    str.gsub!(/\[\[(?:File|ファイル)\:((?:[^\[\]]++|\[\[\g<1>\]\])++)\]\]/im){""}
+  end
   
   def make_reference!(str)
     str.gsub!($make_reference_regex_a){"\n"}
@@ -311,30 +338,32 @@ module Wp2txt
     scanner = StringScanner.new(str)
     result = process_nested_structure(scanner, "{{", "}}") do |contents|
       parts = contents.split("|")
-      # type_code = parts.first
-      # case type_code
-      # when $type_code_regex
-      #   out = parts[-1]
-      # else
-      #   case parts.size
-      #   when 0
-      #     out = ""
-      #   when 1
-      #     out = parts.first || ""
-      #   else
-          # while parts.size > 2 && parts.last.split("=").size > 1
-          while parts.size > 1 && parts.last.split("=").size > 1
-            parts.pop
+      if /\A(?:lang|fontsize)\z/i =~ parts[0]
+        parts.shift
+      elsif /\Alang=/i =~ parts[1]
+        parts.shift
+      end
+
+      if parts.size == 1
+        out = parts[0]
+      else
+        begin
+          keyval = parts[1].split("=")
+          if keyval.size > 1
+            out = keyval[1]
+          else
+            out = parts[1] || ""
           end
-          out = parts.last || ""
-      #   end
-      # end
+        rescue
+          out = parts[1] || ""
+        end
+      end
+
       out.strip
     end
     str.replace result
   end
 
-  
 #################### file related utilities ####################
 
   # collect filenames recursively
@@ -427,5 +456,4 @@ module Wp2txt
     str = i.to_s.reverse
     return str.scan(/.?.?./).join(',').reverse
   end
-    
 end

data/lib/wp2txt/version.rb

@@ -1,3 +1,3 @@
 module Wp2txt
-  VERSION = "0.9.3"
+  VERSION = "0.9.5.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wp2txt
 version: !ruby/object:Gem::Version
-  version: 0.9.3
+  version: 0.9.5.1
 platform: ruby
 authors:
 - Yoichiro Hasebe
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-07-29 00:00:00.000000000 Z
+date: 2022-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -85,8 +85,10 @@ files:
 - bin/wp2txt
 - data/output_samples/testdata_en.txt
 - data/output_samples/testdata_en_categories.txt
+- data/output_samples/testdata_en_summary.txt
 - data/output_samples/testdata_ja.txt
 - data/output_samples/testdata_ja_categories.txt
+- data/output_samples/testdata_ja_summary.txt
 - data/testdata_en.bz2
 - data/testdata_ja.bz2
 - lib/wp2txt.rb
@@ -116,7 +118,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.3
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: Wikipedia dump to text converter