wp2txt 0.9.5 → 1.0.1

data/lib/wp2txt.rb

@@ -4,42 +4,25 @@
 $: << File.join(File.dirname(__FILE__))
 
 require "nokogiri"
-require "parallel"
-
-require 'etc'
-require 'pp'
 require "wp2txt/article"
 require "wp2txt/utils"
-require "wp2txt/progressbar"
-# require "wp2txt/mw_api"
-
-begin
-  require "bzip2-ruby"
-  NO_BZ2 = false  
-rescue LoadError
-  # in case bzip2-ruby gem is not available
-  NO_BZ2 = true
-end
 
 module Wp2txt
-  class Runner
-
+  class Splitter
     include Wp2txt
-
-    def initialize(parent, input_file, output_dir = ".", tfile_size = 10, num_threads = 1, convert = true, strip_tmarker = false)
-      @parent = parent
+    def initialize(input_file, output_dir = ".", tfile_size = 10, bz2_gem = false)
       @fp = nil
-      
       @input_file = input_file
       @output_dir = output_dir
       @tfile_size = tfile_size
-      @convert = convert
-      @strip_tmarker = strip_tmarker
-      num_cores_available = Etc.nprocessors
-      @num_threads = num_threads <= num_cores_available ? num_threads : num_cores_available
+      if bz2_gem
+        require "bzip2-ruby"
+      end
+      @bz2_gem = bz2_gem
+      prepare
     end
-    
-    def file_size(file) 
+
+    def file_size(file)
       origin = Time.now
       size = 0;  unit = 10485760; star = 0; before = Time.now.to_f
       error_count = 10
@@ -49,7 +32,7 @@ module Wp2txt
         rescue => e
           a = nil
         end
-        break unless a    
+        break unless a
 
         present = Time.now.to_f
         size += a.size
@@ -57,88 +40,62 @@ module Wp2txt
           star = 0 if star > 10
           star += 1
           before = present
-        end    
+        end
       end
       time_elapsed = Time.now - origin
       size
     end
-    
-    # control the display of command line progressbar (or gui which is not available for now)
-    def notify_parent(last = false)
-      @last_time ||= Time.now.to_f
-      @elapsed_sum ||= 0
-      time_now = Time.now.to_f
-      elapsed_from_last = (time_now - @last_time).to_i
-
-      if elapsed_from_last > 0.3 || last
 
-        @last_time = time_now        
-        @elapsed_sum += elapsed_from_last
-        gvalue = (@size_read.to_f / @infile_size.to_f * 100 * 100).to_i
-        elt_str = sec_to_str(@elapsed_sum)
-        if last
-          eta_str = "00:00:00"
-        else
-          lines_persec = @size_read / @elapsed_sum if @elapsed_sum > 0
-          eta_sec = (@infile_size - @size_read) / lines_persec
-          eta_str = sec_to_str(eta_sec)
-        end
-        @parent.prg_update(gvalue, elt_str, eta_str)
+    # check if a given command exists: return the path if it does, return false if not
+    def command_exist?(command)
+      basename = File.basename(command)
+      path = ""
+      print "Checking #{basename}: "
+      if open("| which #{command} 2>/dev/null"){ |f| path = f.gets.strip }
+        puts "detected [#{path}]"
+        return path.strip
+      elsif open("| which #{basename} 2>/dev/null"){ |f| path = f.gets.strip }
+        puts "detected [#{path}]"
+        return path.strip
+      else
+        puts "not found"
+        return false
       end
     end
 
-    # check the size of input file (bz2 or plain xml) when uncompressed
+    # check the size of input file (bz2 or plain xml) when decompressed
     def prepare
-
       # if output_dir is not specified, output in the same directory
       # as the imput file
       if !@output_dir && @input_file
         @output_dir = File.dirname(@input_file)
       end
 
-      # if input file is bz2 compressed, use bz2-ruby if available,
-      # use command line bzip2 program otherwise.
       if /.bz2$/ =~ @input_file
-        unless NO_BZ2
-          file = Bzip2::Reader.new File.open(@input_file, "r:UTF-8")
-          @parent.msg("WP2TXT is spawning #{@num_threads} threads to process data \n", 0)         
-          @parent.msg("Preparing ... This may take several minutes or more ", 0)         
-          @infile_size = file_size(file)
-          @parent.msg("... Done.", 1)
-          file.close
+        if @bz2_gem
           file = Bzip2::Reader.new File.open(@input_file, "r:UTF-8")
+        elsif RUBY_PLATFORM.index("win32")
+          file = IO.popen("bunzip2.exe -c #{@input_file}")
         else
-          if RUBY_PLATFORM.index("win32")
-            file = IO.popen("bunzip2.exe -c #{@input_file}")
-          else
-            file = IO.popen("bzip2 -c -d #{@input_file}") 
-          end
-          @parent.msg("WP2TXT is spawning #{@num_threads} threads to process data \n", 0)         
-          @parent.msg("Preparing ... This may take several minutes or more ", 0)         
-          @infile_size = file_size(file)
-          @parent.msg("... Done.", 1)
-          file.close  # try to reopen since rewind method is unavailable
-          if RUBY_PLATFORM.index("win32")
-            file = IO.popen("bunzip2.exe -c #{@input_file}")
-          else
-            file = IO.popen("bzip2 -c -d #{@input_file}") 
+          if bzpath = command_exist?("lbzip2") ||
+                      command_exist?("pbzip2") ||
+                      command_exist?("bzip2")
+            file = IO.popen("#{bzpath} -c -d #{@input_file}")
           end
-        end 
+        end
       else # meaning that it is a text file
         @infile_size = File.stat(@input_file).size
         file = open(@input_file)
       end
 
       #create basename of output file
-      @outfile_base = File.basename(@input_file, ".*") + "-"            
+      @outfile_base = File.basename(@input_file, ".*") + "-"
       @total_size = 0
       @file_index = 1
       outfilename = File.join(@output_dir, @outfile_base + @file_index.to_s)
       @outfiles = []
       @outfiles << outfilename
-      @fp = File.open(outfilename, "w")    
-      @parent.before
-      @parent.data_set(@input_file, 100 * 100)
+      @fp = File.open(outfilename, "w")
       @file_pointer = file
       return true
     end
@@ -156,7 +113,110 @@ module Wp2txt
         # temp_buf is filled with text split by "\n"
         temp_buf = []
         ss = StringScanner.new(new_lines)
-        while ss.scan(/.*?\n/m)             
+        while ss.scan(/.*?\n/m)
+          temp_buf << ss[0]
+        end
+        temp_buf << ss.rest unless ss.eos?
+
+        new_first_line = temp_buf.shift
+        if new_first_line[-1, 1] == "\n" # new_first_line.index("\n")
+          @buffer.last <<  new_first_line
+          @buffer << ""
+        else
+          @buffer.last << new_first_line
+        end
+        @buffer += temp_buf unless temp_buf.empty?
+        if @buffer.last[-1, 1] == "\n" # @buffer.last.index("\n")
+          @buffer << ""
+        end
+        break if @buffer.size > 1
+      end
+      return true
+    end
+
+    def get_newline
+      @buffer ||= [""]
+      if @buffer.size == 1
+        return nil unless fill_buffer
+      end
+      if @buffer.empty?
+        return nil
+      else
+        new_line = @buffer.shift
+        return new_line
+      end
+    end
+
+    def split_file
+      output_text = ""
+      end_flag = false
+      while text = get_newline
+        @count ||= 0;@count += 1;
+        @size_read ||=0
+        @size_read += text.bytesize
+        @total_size += text.bytesize
+        output_text << text
+        end_flag = true if @total_size > (@tfile_size * 1024 * 1024)
+        # never close the file until the end of the page even if end_flag is on
+        if end_flag && /<\/page/ =~ text
+          @fp.puts(output_text)
+          output_text = ""
+          @total_size = 0
+          end_flag = false
+          @fp.close
+          @file_index += 1
+          outfilename = File.join(@output_dir, @outfile_base + @file_index.to_s)
+          @outfiles << outfilename
+          @fp = File.open(outfilename, "w")
+          next
+        end
+      end
+      @fp.puts(output_text) if output_text != ""
+      @fp.close
+
+      if File.size(outfilename) == 0
+        File.delete(outfilename)
+        @outfiles.delete(outfilename)
+      end
+
+      rename(@outfiles, "xml")
+    end
+  end
+
+  class Runner
+    include Wp2txt
+
+    def initialize(input_file, output_dir = ".", strip_tmarker = false, del_interfile = true)
+      @fp = nil
+      @input_file = input_file
+      @output_dir = output_dir
+      @strip_tmarker = strip_tmarker
+      @del_interfile = del_interfile
+      prepare
+    end
+
+    def prepare
+      @infile_size = File.stat(@input_file).size
+      file = open(@input_file)
+      @file_pointer = file
+      @outfile_base = File.basename(@input_file, ".*")
+      @total_size = 0
+      return true
+    end
+
+    def fill_buffer
+      while true do
+        begin
+          new_lines = @file_pointer.read(10485760)
+        rescue => e
+          return nil
+        end
+        return nil unless new_lines
+
+        # temp_buf is filled with text split by "\n"
+        temp_buf = []
+        ss = StringScanner.new(new_lines)
+        while ss.scan(/.*?\n/m)
           temp_buf << ss[0]
         end
         temp_buf << ss.rest unless ss.eos?
@@ -178,25 +238,22 @@ module Wp2txt
     end
 
     def get_newline
-      @buffer ||= [""]   
+      @buffer ||= [""]
       if @buffer.size == 1
         return nil unless fill_buffer
       end
       if @buffer.empty?
         return nil
-      else 
+      else
         new_line = @buffer.shift
         return new_line
-      end  
+      end
     end
 
     def get_page
       inside_page = false
       page = ""
       while line = get_newline
-        notify_parent        
-        @size_read ||=0; @size_read += line.bytesize
-        
         if /<page>/ =~ line #
           page << line
           inside_page = true
@@ -215,22 +272,7 @@ module Wp2txt
       end
     end
 
-    # call this method to do the job
     def extract_text(&block)
-      prepare            
-      if @convert
-        if block
-          extract_and_convert(&block)
-        else
-          extract_and_convert
-        end
-      else
-        # output the original xml only split to files of the specified size
-        extract
-      end
-    end
-    
-    def extract_and_convert(&block)
       in_text = false
       in_message = false
       result_text = ""
@@ -241,17 +283,15 @@ module Wp2txt
       pages = []
       data_empty = false
 
-      begin
+      while !data_empty
         page = get_page
         if page
           pages << page
         else
           data_empty = true
         end
-        if data_empty || pages.size == @num_threads
-          # pages_text = Parallel.map_with_index(pages, in_threads: @num_threads) do |page, n|
-          pages_text = Parallel.map_with_index(pages, in_threads: @num_threads) do |page, n|
-            page_text = {:order => n, :data => nil}
+        if data_empty
+          pages.each do |page|
             xmlns = '<mediawiki xmlns="http://www.mediawiki.org/xml/export-0.5/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.mediawiki.org/xml/export-0.5/ http://www.mediawiki.org/xml/export-0.5.xsd" version="0.5" xml:lang="en">' + "\n"
             xml = xmlns + page + "</mediawiki>"
 
@@ -270,79 +310,22 @@ module Wp2txt
                 end
               end
               article = Article.new(text, title, @strip_tmarker)
-              page_text[:data] = block.call(article)
+              page_text = block.call(article)
+              output_text << page_text
             end
-            page_text
-          end
-          pages.clear
-          pages_text = pages_text.sort_by{|v| v[:order]}.map{|v| v[:data]}.compact
-          pages_text.each do |page_text|
-            output_text << page_text
-            @count ||= 0; @count += 1;
-            @total_size = output_text.bytesize
-            # flagged when data exceeds the size of output file
-            end_flag = true if @total_size > (@tfile_size * 1024 * 1024)
           end
 
-          #close the present file, then open a new one
-          if end_flag
-            cleanup!(output_text)
+          cleanup!(output_text)
+          if output_text.size > 0
+            outfilename = File.join(@output_dir, @outfile_base + ".txt")
+            @fp = File.open(outfilename, "w")
             @fp.puts(output_text)
-            output_text = ""
-            @total_size = 0
-            end_flag = false
             @fp.close
-            @file_index += 1
-            outfilename = File.join(@output_dir, @outfile_base + @file_index.to_s)
-            @outfiles << outfilename
-            @fp = File.open(outfilename, "w")
-            next
           end
-        end
-      end while !data_empty 
-
-      if output_text != ""
-        cleanup!(output_text)
-        @fp.puts(output_text) 
-      end
-      notify_parent(true)
-      @parent.after
-      @fp.close    
-      rename(@outfiles)    
-      @parent.msg("Processing finished", 1)
-    end
-
-    def extract
-      output_text = ""
-      end_flag = false
-      while text = get_newline
-        @count ||= 0;@count += 1;
-        @size_read ||=0;@size_read += text.bytesize
-        @total_size += text.bytesize
-        output_text << text
-        end_flag = true if @total_size > (@tfile_size * 1024 * 1024)
-        notify_parent
-        # never close the file until the end of the page even if end_flag is on
-        if end_flag && /<\/page/ =~ text 
-          @fp.puts(output_text)
+          File.delete(@input_file) if @del_interfile
           output_text = ""
-          @total_size = 0
-          end_flag = false
-          @fp.close
-          @file_index += 1
-          outfilename = File.join(@output_dir, @outfile_base + @file_index.to_s)
-          @outfiles << outfilename
-          @fp = File.open(outfilename, "w")
-          next
         end
       end
-      @fp.puts(output_text) if output_text != ""
-      notify_parent(true)
-      @parent.after
-      @fp.close    
-      rename(@outfiles)    
-      @parent.msg("Processing finished", 1)
-    end 
+    end
   end
 end
-

data/spec/utils_spec.rb

@@ -184,10 +184,6 @@ describe "Wp2txt" do
   
   describe "correct_inline_template!" do
     it "removes brackets and leaving some text" do
-      # str_before = "{{}}"
-      # str_after = ""
-      # correct_inline_template!(str_before)
-      # expect(str_before).to eq str_after
       str_before = "{{MedalCountry | {{JPN}} }}"
       str_after  = "JPN"
       correct_inline_template!(str_before)
@@ -197,11 +193,11 @@ describe "Wp2txt" do
       correct_inline_template!(str_before)
       expect(str_before).to eq str_after
       str_before = "{{a|b=c|d=f}}"
-      str_after  = "a"
+      str_after  = "c"
       correct_inline_template!(str_before)
       expect(str_before).to eq str_after
       str_before = "{{a|b|{{c|d|e}}}}"
-      str_after  = "e"
+      str_after  = "b"
       correct_inline_template!(str_before)
       expect(str_before).to eq str_after
       str_before = "{{要出典範囲|日本人に多く見受けられる|date=2013年8月|title=日本人特有なのか、本当に多いのかを示す必要がある}}"
@@ -210,18 +206,4 @@ describe "Wp2txt" do
       expect(str_before).to eq str_after
     end
   end
-  
-  #   describe "expand_template" do
-  #     it "gets data corresponding to a given template using mediawiki api" do
-  #       uri = "http://en.wiktionary.org/w/api.php"
-  #       template = "{{en-verb}}"
-  #       word = "kick"
-  #       expanded = expand_template(uri, template, word)
-  #       html =<<EOD
-  # <span class=\"infl-inline\"><b class=\"Latn \" lang=\"en\">kick</b> (''third-person singular simple present'' <span class=\"form-of third-person-singular-form-of\">'''<span class=\"Latn \" lang=\"en\">[[kicks#English|kicks]]</span>'''</span>, ''present participle'' <span class=\"form-of present-participle-form-of\">'''<span class=\"Latn \" lang=\"en\">[[kicking#English|kicking]]</span>'''</span>, ''simple past and past participle'' <span class=\"form-of simple-past-and-participle-form-of\"> '''<span class=\"Latn \" lang=\"en\">[[kicked#English|kicked]]</span>'''</span>)</span>[[Category:English verbs|kick]]
-  # EOD
-  #       html.strip!
-  #       expanded.should == html
-  #     end
-  #   end
-end
+end

data/wp2txt.gemspec

@@ -7,13 +7,14 @@ Gem::Specification.new do |s|
   s.version     = Wp2txt::VERSION
   s.authors     = ["Yoichiro Hasebe"]
   s.email       = ["yohasebe@gmail.com"]
-  s.homepage    = "http://github.com/yohasebe/wp2txt"
-  s.summary     = %q{Wikipedia dump to text converter}
-  s.description = %q{WP2TXT extracts plain text data from Wikipedia dump file (encoded in XML/compressed with Bzip2) stripping all the MediaWiki markups and other metadata.}
+  s.homepage    = "https://github.com/yohasebe/wp2txt"
+  s.summary     = %q{A command-line toolkit to extract text content and category data from Wikipedia dump files}
+  s.description = %q{WP2TXT extracts text and category data from Wikipedia dump files (encoded in XML / compressed with Bzip2), removing MediaWiki markup and other metadata.}
 
   s.rubyforge_project = "wp2txt"
 
   s.files         = `git ls-files`.split("\n")
+  s.files -= ["data/*", "image/*"]
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
@@ -23,7 +24,10 @@ Gem::Specification.new do |s|
   # s.add_development_dependency "rake"
 
   s.add_dependency "nokogiri"
+  s.add_dependency "ruby-progressbar"
   s.add_dependency "parallel"
   s.add_dependency "htmlentities"
   s.add_dependency "optimist"
+  s.add_dependency "pastel"
+  s.add_dependency "tty-spinner"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wp2txt
 version: !ruby/object:Gem::Version
-  version: 0.9.5
+  version: 1.0.1
 platform: ruby
 authors:
 - Yoichiro Hasebe
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-08-02 00:00:00.000000000 Z
+date: 2022-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
@@ -66,12 +80,39 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: WP2TXT extracts plain text data from Wikipedia dump file (encoded in
-  XML/compressed with Bzip2) stripping all the MediaWiki markups and other metadata.
+- !ruby/object:Gem::Dependency
+  name: pastel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: tty-spinner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: WP2TXT extracts text and category data from Wikipedia dump files (encoded
+  in XML / compressed with Bzip2), removing MediaWiki markup and other metadata.
 email:
 - yohasebe@gmail.com
 executables:
-- benchmark.rb
 - wp2txt
 extensions: []
 extra_rdoc_files: []
@@ -81,26 +122,26 @@ files:
 - LICENSE
 - README.md
 - Rakefile
-- bin/benchmark.rb
 - bin/wp2txt
 - data/output_samples/testdata_en.txt
-- data/output_samples/testdata_en_categories.txt
+- data/output_samples/testdata_en_category.txt
 - data/output_samples/testdata_en_summary.txt
 - data/output_samples/testdata_ja.txt
-- data/output_samples/testdata_ja_categories.txt
+- data/output_samples/testdata_ja_category.txt
 - data/output_samples/testdata_ja_summary.txt
 - data/testdata_en.bz2
 - data/testdata_ja.bz2
+- image/screenshot.png
+- image/wp2txt-logo.svg
+- image/wp2txt.svg
 - lib/wp2txt.rb
 - lib/wp2txt/article.rb
-- lib/wp2txt/mw_api.rb
-- lib/wp2txt/progressbar.rb
 - lib/wp2txt/utils.rb
 - lib/wp2txt/version.rb
 - spec/spec_helper.rb
 - spec/utils_spec.rb
 - wp2txt.gemspec
-homepage: http://github.com/yohasebe/wp2txt
+homepage: https://github.com/yohasebe/wp2txt
 licenses: []
 metadata: {}
 post_install_message:
@@ -121,7 +162,8 @@ requirements: []
 rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
-summary: Wikipedia dump to text converter
+summary: A command-line toolkit to extract text content and category data from Wikipedia
+  dump files
 test_files:
 - spec/spec_helper.rb
 - spec/utils_spec.rb