wp2txt 0.7.8 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ee8448d2dc341c9f26a613522c0b9a225b62a7df
-  data.tar.gz: 036aa5184a19b4351c65af605f2ebc23b9e73398
+  metadata.gz: d0610b7e28e04c4cd9c3a1401c88e15f6ddb16ec
+  data.tar.gz: b866915631fdc956395c005735b089ddff7956e5
 SHA512:
-  metadata.gz: 05dd0bd2462bc72f030c0bd03233e359d1febdb4b30ad1309f4baf35ab6241684d164269ae1bae527163da787188d915ccb7ab460d83cd83732fbf9627d7ada1
-  data.tar.gz: 2bc83d1854656a4b3a83e6a2e1b9cfe86c86163d27a64582f994fc997b8104e4ab28d8d28881c054e323fd69934c53b63909cd7458a8d2ed0243c95702f8a14e
+  metadata.gz: e9fbef3de5ed866de0b3c7fadd96bdf0ff501b71c2d9f6f282eed538194bdfff8d9659cf53aedf95062c9aadf2ec90393075158ef9c8ae78f3e53ce84119f764
+  data.tar.gz: 36ad316986d94a6be89ccb591dec510dc4695bede188448fde0745702faad8039d0df4a84d2f0730dd11749a63d58e6b63b51b2bce585d8f8ccb3ff02553c3c8

data/README.md

@@ -2,7 +2,7 @@
 
 Wikipedia dump file to text converter
 
-**Important** This is a project *work in progress* and it could be slow, unstable, and even destructive! Please use it with caution 
+**Important: This is a project *work in progress* and it could be slow, unstable, and even destructive! Please use it with caution!**
 
 ### About ###
 

data/bin/wp2txt

@@ -4,7 +4,7 @@
 $: << File.join(File.dirname(__FILE__))
 $: << File.join(File.dirname(__FILE__), '..', 'lib')
 
-$DEBUG_MODE = false
+DEBUG_MODE = true
 SHAREDIR = File.join(File.dirname(__FILE__), '..', 'share')
 DOCDIR = File.join(File.dirname(__FILE__), '..', 'doc')
 
@@ -76,16 +76,18 @@ wpconv.extract_text do |article|
     when :mw_heading
       next if !config[:heading]
       format_wiki!(e.last)
+      format_article!(e.last)
       line = e.last
       line << "+HEADING+" if $DEBUG_MODE
     when :mw_paragraph
       # next if !config[:paragraph]
       format_wiki!(e.last)
-      line = e.last
+      format_article!(e.last)
+      line = e.last + "\n"
       line << "+PARAGRAPH+" if $DEBUG_MODE
     when :mw_table, :mw_htable
       next if !config[:table]
-      format_wiki!(e.last)
+      # format_wiki!(e.last)
       line = e.last
       line << "+TABLE+" if $DEBUG_MODE
     when :mw_pre
@@ -94,23 +96,23 @@ wpconv.extract_text do |article|
       line << "+PRE+" if $DEBUG_MODE
     when :mw_quote
       # next if !config[:quote]
-      format_wiki!(e.last)
+      # format_wiki!(e.last)
       line = e.last
       line << "+QUOTE+" if $DEBUG_MODE
     when :mw_unordered, :mw_ordered, :mw_definition
       next if !config[:list]
-      format_wiki!(e.last)
+      # format_wiki!(e.last)
       line = e.last
       line << "+LIST+" if $DEBUG_MODE
     when :mw_redirect
       next if !config[:redirect]
-      format_wiki!(e.last)
+      # format_wiki!(e.last)
       line = e.last
       line << "+REDIRECT+" if $DEBUG_MODE
       line << "\n\n"
     else
       if $DEBUG_MODE
-        format_wiki!(e.last)
+        # format_wiki!(e.last)
         line = e.last
         line << "+OTHER+"
       else
@@ -119,8 +121,9 @@ wpconv.extract_text do |article|
     end
     contents << line    
   end
-  format_article!(contents)
   convert_characters!(contents)
+  remove_table!(contents) unless $leave_table
+  remove_ref!(contents) unless $leave_ref
     
   ##### cleanup #####
   if /\A\s*\z/m =~ contents
@@ -129,7 +132,6 @@ wpconv.extract_text do |article|
     result = config[:title] ? title + "\n" << contents : contents
   end
   result.gsub!(/\[ref\]\s*\[\/ref\]/m){""}
-  result.gsub!(/^[\s\W]+$/)
   result.gsub!(/\n\n\n+/m){"\n\n"}
   result << "\n"  
 end

data/lib/wp2txt.rb

@@ -3,12 +3,7 @@
 
 $: << File.join(File.dirname(__FILE__))
 
-# require "rubygems"
-# require "bundler/setup"
-
 require "nokogiri"
-# require "oga"
-# require "ox"
 
 require 'pp'
 require "wp2txt/article"
@@ -249,36 +244,6 @@ module Wp2txt
         next if /\:/ =~ title
         text = page.content
 
-        # input = Oga.parse_xml(xml)
-        # page = input.xpath("//xmlns:text").first
-        # title = page.parent.parent.xpath("//xmlns:title").first.text
-        # next if /\:/ =~ title
-        # text = page.text
-
-        # input = Ox.load(xml, :encoding => "UTF-8")
-        # title = ""
-        # text  = ""
-        # input.nodes.first.nodes.each do |n|
-        #   if n.name == "title"
-        #     title = n.nodes.first
-        #     if /\:/ =~ title
-        #       title = ""
-        #       break
-        #     end
-        #   elsif n.name == "revision"
-        #     n.nodes.each do |o|
-        #       if o.name == "text"
-        #         text = o.nodes.first
-        #         break
-        #       end
-        #     end
-        #   end
-        # end
-        # next if title == "" || text == ""
-
-        # remove all comment texts
-        # and insert as many number of new line chars included in 
-        # each comment instead
         text.gsub!(/\<\!\-\-(.*?)\-\-\>/m) do |content|
           num_of_newlines = content.count("\n")
           if num_of_newlines == 0
@@ -299,6 +264,7 @@ module Wp2txt
 
         #close the present file, then open a new one
         if end_flag
+          output_text.strip!
           @fp.puts(output_text)
           output_text = ""
           @total_size = 0
@@ -311,7 +277,10 @@ module Wp2txt
           next
         end
       end
-      @fp.puts(output_text) if output_text != ""
+      if output_text != ""
+        output_text.strip!
+        @fp.puts(output_text) 
+      end
       notify_parent(true)
       @parent.after
       @fp.close    

data/lib/wp2txt/article.rb

@@ -103,7 +103,7 @@ module Wp2txt
         when $in_inputbox_regex
           @elements << create_element(:mw_inputbox, line)
         when $in_inputbox_regex1
-          mode = :mw_inputbox 
+          mode = :mw_inputbox
           @elements << create_element(:mw_inputbox, line)
         when $in_source_regex
         @elements << create_element(:mw_source, line)

data/lib/wp2txt/utils.rb

@@ -129,7 +129,6 @@ module Wp2txt
     convert_characters!(text)    
     correct_inline_template!(text) unless $leave_template
     remove_templates!(text) unless $leave_template
-    remove_table!(text) unless $leave_table
   end
   
   #################### parser for nested structure ####################
@@ -295,16 +294,19 @@ module Wp2txt
     str.gsub!($mndash_regex, "–")
   end
 
-  def remove_hr!(page)
-    page.gsub!($remove_hr_regex, "")
+  def remove_hr!(str)
+    str.gsub!($remove_hr_regex, "")
   end
 
+  def remove_ref!(str)
+    str.gsub!($format_ref_regex){""}
+  end
+  
   def make_reference!(str)
     str.gsub!($make_reference_regex_a){"\n"}
     str.gsub!($make_reference_regex_b){""}
     str.gsub!($make_reference_regex_c){"[ref]"}
     str.gsub!($make_reference_regex_d){"[/ref]"}
-    str.gsub!($format_ref_regex){""} unless $leave_ref
   end
 
   def format_ref!(page)

data/lib/wp2txt/version.rb

@@ -1,3 +1,3 @@
 module Wp2txt
-  VERSION = "0.7.8"
+  VERSION = "0.8.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wp2txt
 version: !ruby/object:Gem::Version
-  version: 0.7.8
+  version: 0.8.0
 platform: ruby
 authors:
 - Yoichiro Hasebe
@@ -70,7 +70,6 @@ files:
 - bin/benchmark.rb
 - bin/wp2txt
 - data/testdata.bz2
-- error_log.txt
 - lib/wp2txt.rb
 - lib/wp2txt/article.rb
 - lib/wp2txt/mw_api.rb

data/error_log.txt

@@ -1 +0,0 @@
-[[アンパサンド]]