wp2txt 0.9.5.1 → 1.0.0

data/lib/wp2txt/utils.rb

@@ -77,6 +77,22 @@ $double_square_bracket_regex = Regexp.new("(#{Regexp.escape('[[')}|#{Regexp.esca
 $single_curly_bracket_regex = Regexp.new("(#{Regexp.escape('{')}|#{Regexp.escape('}')})", Regexp::MULTILINE)
 $double_curly_bracket_regex = Regexp.new("(#{Regexp.escape('{{')}|#{Regexp.escape('}}')})", Regexp::MULTILINE)
 $curly_square_bracket_regex = Regexp.new("(#{Regexp.escape('{|')}|#{Regexp.escape('|}')})", Regexp::MULTILINE)
+
+$complex_regex_01 = Regexp.new('\<\<([^<>]++)\>\>\s?')
+$complex_regex_02 = Regexp.new('\[\[File\:((?:[^\[\]]++|\[\[\g<1>\]\])++)\]\]', Regexp::MULTILINE | Regexp::IGNORECASE)
+$complex_regex_03 = Regexp.new('^\[\[((?:[^\[\]]++|\[\[\g<1>\]\])++)^\]\]', Regexp::MULTILINE)
+$complex_regex_04 = Regexp.new('\{\{(?:infobox|efn|sfn|unreliable source|refn|reflist|col(?:umns)?\-list|div col|no col|bar box|formatnum\:|col\||see also\||r\||#)((?:[^{}]++|\{\{\g<1>\}\})++)\}\}', Regexp::MULTILINE | Regexp::IGNORECASE)
+$complex_regex_05 = Regexp.new('\{\{[^{}]+?\n\|((?:[^{}]++|\{\{\g<1>\}\})++)\}\}', Regexp::MULTILINE | Regexp::IGNORECASE)
+
+$cleanup_regex_01 = Regexp.new('\[ref\]\s*\[\/ref\]', Regexp::MULTILINE)
+$cleanup_regex_02 = Regexp.new('^File:.+$')
+$cleanup_regex_03 = Regexp.new('^\|.*$')
+$cleanup_regex_04 = Regexp.new('\{\{.*$')
+$cleanup_regex_05 = Regexp.new('^.*\}\}')
+$cleanup_regex_06 = Regexp.new('\{\|.*$')
+$cleanup_regex_07 = Regexp.new('^.*\|\}')
+$cleanup_regex_08 = Regexp.new('\n\n\n+', Regexp::MULTILINE)
+
 ###################################################
 
 module Wp2txt
@@ -104,11 +120,12 @@ module Wp2txt
   end
   
   def format_wiki!(text, has_retried = false)
+    remove_complex!(text)
+
     escape_nowiki!(text)
     process_interwiki_links!(text)
     process_external_links!(text)
     unescape_nowiki!(text)      
-
     remove_directive!(text)
     remove_emphasis!(text)
     mndash!(text)
@@ -120,15 +137,18 @@ module Wp2txt
   end
   
   def cleanup!(text)
-    text.gsub!(/\[ref\]\s*\[\/ref\]/m){""}
-    text.gsub!(/^File:.+$/){""}
-    text.gsub!(/^\|.*$/){""}
-    text.gsub!(/^{{.*$/){""}
-    text.gsub!(/^}}.*$/){""}
-    text.gsub!(/\n\n\n+/m){"\n\n"}
+    text.gsub!($cleanup_regex_01){""}
+    text.gsub!($cleanup_regex_02){""}
+    text.gsub!($cleanup_regex_03){""}
+    text.gsub!($cleanup_regex_04){""}
+    text.gsub!($cleanup_regex_05){""}
+    text.gsub!($cleanup_regex_06){""}
+    text.gsub!($cleanup_regex_07){""}
+    text.gsub!($cleanup_regex_08){"\n\n"}
     text.strip!
     text << "\n\n"
   end
+
   #################### parser for nested structure ####################
    
   def process_nested_structure(scanner, left, right, &block)
@@ -217,12 +237,16 @@ module Wp2txt
   def process_external_links!(str)
     scanner = StringScanner.new(str)
     result = process_nested_structure(scanner, "[", "]") do |contents|
-      parts = contents.split(" ", 2)
-      case parts.size
-      when 1
-        parts.first || ""
+      if /\A\s.+\s\z/ =~ contents
+        " (#{contents.strip}) "
       else
-        parts.last || ""
+        parts = contents.split(" ", 2)
+        case parts.size
+        when 1
+          parts.first || ""
+        else
+          parts.last || ""
+        end
       end
     end
     str.replace(result)
@@ -239,10 +263,6 @@ module Wp2txt
     result = process_nested_structure(scanner, "{", "}") do |contents|
       ""
     end
-    scanner = StringScanner.new(result)
-    result = process_nested_structure(scanner, "{{", "}}") do |contents|
-      ""
-    end
     str.replace(result)
   end
   
@@ -310,7 +330,8 @@ module Wp2txt
   end
 
   def remove_html!(str)
-    ["div", "gallery", "timeline"].each do |tag|
+    str.gsub!(/<[^<>]+\/>/){""}
+    ["div", "gallery", "timeline", "noinclude"].each do |tag|
       scanner = StringScanner.new(str)
       result = process_nested_structure(scanner, "<#{tag}", "#{tag}>") do |contents|
         ""
@@ -320,11 +341,11 @@ module Wp2txt
   end
 
   def remove_complex!(str)
-    str.gsub!(/(?:'')?\[https?\:[^\[\]]+?\s([^\]]++)?\](?:'')?/){$1}
-    str.gsub!(/(?:'')?\[https?\:[^\[\]]++\](?:'')?\s?/){""}
-    str.gsub!(/\<\<([^<>]++)\>\>\s?/){"《#{$1}》"}
-    str.gsub!(/\{\{(?:Infobox|efn|Sfn|div col|no col|bar box|formatnum\:|Refnest\||Refnest\||Col\||See also\||R\|)((?:[^{}]++|\{\{\g<1>\}\})++)\}\}/im){""}
-    str.gsub!(/\[\[(?:File|ファイル)\:((?:[^\[\]]++|\[\[\g<1>\]\])++)\]\]/im){""}
+    str.gsub!($complex_regex_01){"《#{$1}》"}
+    str.gsub!($complex_regex_02){""}
+    str.gsub!($complex_regex_03){""}
+    str.gsub!($complex_regex_04){""}
+    str.gsub!($complex_regex_05){""}
   end
   
   def make_reference!(str)
@@ -340,6 +361,8 @@ module Wp2txt
       parts = contents.split("|")
       if /\A(?:lang|fontsize)\z/i =~ parts[0]
         parts.shift
+      elsif /\Alang\-/i =~ parts[0]
+        parts.shift
       elsif /\Alang=/i =~ parts[1]
         parts.shift
       end
@@ -422,7 +445,7 @@ module Wp2txt
     end
   end
 
-  def rename(files)    
+  def rename(files, ext = "txt")    
     # num of digits necessary to name the last file generated
     maxwidth = 0  
 
@@ -435,8 +458,9 @@ module Wp2txt
       newname= f.sub(/\-(\d+)\z/) do
         "-" + sprintf("%0#{maxwidth}d", $1.to_i)
       end
-      File.rename(f, newname + ".txt")
+      File.rename(f, newname + ".#{ext}")
     end
+    return true
   end
 
   # convert int of seconds to string in the format 00:00:00

data/lib/wp2txt/version.rb

@@ -1,3 +1,3 @@
 module Wp2txt
-  VERSION = "0.9.5.1"
+  VERSION = "1.0.0"
 end

data/lib/wp2txt.rb

@@ -4,14 +4,8 @@
 $: << File.join(File.dirname(__FILE__))
 
 require "nokogiri"
-require "parallel"
-
-require 'etc'
-require 'pp'
 require "wp2txt/article"
 require "wp2txt/utils"
-require "wp2txt/progressbar"
-# require "wp2txt/mw_api"
 
 begin
   require "bzip2-ruby"
@@ -22,21 +16,14 @@ rescue LoadError
 end
 
 module Wp2txt
-  class Runner
-
+  class Splitter
     include Wp2txt
-
-    def initialize(parent, input_file, output_dir = ".", tfile_size = 10, num_threads = 1, convert = true, strip_tmarker = false)
-      @parent = parent
+    def initialize(input_file, output_dir = ".", tfile_size = 10)
       @fp = nil
-      
       @input_file = input_file
       @output_dir = output_dir
       @tfile_size = tfile_size
-      @convert = convert
-      @strip_tmarker = strip_tmarker
-      num_cores_available = Etc.nprocessors
-      @num_threads = num_threads <= num_cores_available ? num_threads : num_cores_available
+      prepare            
     end
     
     def file_size(file) 
@@ -62,34 +49,9 @@ module Wp2txt
       time_elapsed = Time.now - origin
       size
     end
-    
-    # control the display of command line progressbar (or gui which is not available for now)
-    def notify_parent(last = false)
-      @last_time ||= Time.now.to_f
-      @elapsed_sum ||= 0
-      time_now = Time.now.to_f
-      elapsed_from_last = (time_now - @last_time).to_i
-
-      if elapsed_from_last > 0.3 || last
 
-        @last_time = time_now        
-        @elapsed_sum += elapsed_from_last
-        gvalue = (@size_read.to_f / @infile_size.to_f * 100 * 100).to_i
-        elt_str = sec_to_str(@elapsed_sum)
-        if last
-          eta_str = "00:00:00"
-        else
-          lines_persec = @size_read / @elapsed_sum if @elapsed_sum > 0
-          eta_sec = (@infile_size - @size_read) / lines_persec
-          eta_str = sec_to_str(eta_sec)
-        end
-        @parent.prg_update(gvalue, elt_str, eta_str)
-      end
-    end
-
-    # check the size of input file (bz2 or plain xml) when uncompressed
+    # check the size of input file (bz2 or plain xml) when decompressed
     def prepare
-
       # if output_dir is not specified, output in the same directory
       # as the imput file
       if !@output_dir && @input_file
@@ -101,28 +63,12 @@ module Wp2txt
       if /.bz2$/ =~ @input_file
         unless NO_BZ2
           file = Bzip2::Reader.new File.open(@input_file, "r:UTF-8")
-          @parent.msg("WP2TXT is spawning #{@num_threads} threads to process data \n", 0)         
-          @parent.msg("Preparing ... This may take several minutes or more ", 0)         
-          @infile_size = file_size(file)
-          @parent.msg("... Done.", 1)
-          file.close
-          file = Bzip2::Reader.new File.open(@input_file, "r:UTF-8")
         else
           if RUBY_PLATFORM.index("win32")
             file = IO.popen("bunzip2.exe -c #{@input_file}")
           else
             file = IO.popen("bzip2 -c -d #{@input_file}") 
           end
-          @parent.msg("WP2TXT is spawning #{@num_threads} threads to process data \n", 0)         
-          @parent.msg("Preparing ... This may take several minutes or more ", 0)         
-          @infile_size = file_size(file)
-          @parent.msg("... Done.", 1)
-          file.close  # try to reopen since rewind method is unavailable
-          if RUBY_PLATFORM.index("win32")
-            file = IO.popen("bunzip2.exe -c #{@input_file}")
-          else
-            file = IO.popen("bzip2 -c -d #{@input_file}") 
-          end
         end 
       else # meaning that it is a text file
         @infile_size = File.stat(@input_file).size
@@ -137,8 +83,6 @@ module Wp2txt
       @outfiles = []
       @outfiles << outfilename
       @fp = File.open(outfilename, "w")    
-      @parent.before
-      @parent.data_set(@input_file, 100 * 100)
       @file_pointer = file
       return true
     end
@@ -190,13 +134,113 @@ module Wp2txt
       end  
     end
 
+    def split_file
+      output_text = ""
+      end_flag = false
+      while text = get_newline
+        @count ||= 0;@count += 1;
+        @size_read ||=0
+        @size_read += text.bytesize
+        @total_size += text.bytesize
+        output_text << text
+        end_flag = true if @total_size > (@tfile_size * 1024 * 1024)
+        # never close the file until the end of the page even if end_flag is on
+        if end_flag && /<\/page/ =~ text 
+          @fp.puts(output_text)
+          output_text = ""
+          @total_size = 0
+          end_flag = false
+          @fp.close
+          @file_index += 1
+          outfilename = File.join(@output_dir, @outfile_base + @file_index.to_s)
+          @outfiles << outfilename
+          @fp = File.open(outfilename, "w")
+          next
+        end
+      end
+      @fp.puts(output_text) if output_text != ""
+      @fp.close    
+
+      if File.size(outfilename) == 0
+        File.delete(outfilename) 
+        @outfiles.delete(outfilename)
+      end
+
+      rename(@outfiles, "xml")    
+    end 
+  end
+
+  class Runner
+    include Wp2txt
+
+    def initialize(input_file, output_dir = ".", strip_tmarker = false, del_interfile = true)
+      @fp = nil
+      @input_file = input_file
+      @output_dir = output_dir
+      @strip_tmarker = strip_tmarker
+      @del_interfile = del_interfile
+      prepare
+    end
+    
+    def prepare
+      @infile_size = File.stat(@input_file).size
+      file = open(@input_file)
+      @file_pointer = file
+      @outfile_base = File.basename(@input_file, ".*")
+      @total_size = 0
+      return true
+    end
+
+    def fill_buffer
+      while true do
+        begin
+          new_lines = @file_pointer.read(10485760)
+        rescue => e
+          return nil
+        end
+        return nil unless new_lines
+
+        # temp_buf is filled with text split by "\n"
+        temp_buf = []
+        ss = StringScanner.new(new_lines)
+        while ss.scan(/.*?\n/m)             
+          temp_buf << ss[0]
+        end
+        temp_buf << ss.rest unless ss.eos?
+
+        new_first_line = temp_buf.shift
+        if new_first_line[-1, 1] == "\n" # new_first_line.index("\n")
+          @buffer.last <<  new_first_line
+          @buffer << ""
+        else
+          @buffer.last << new_first_line
+        end
+        @buffer += temp_buf unless temp_buf.empty?
+        if @buffer.last[-1, 1] == "\n" # @buffer.last.index("\n")
+          @buffer << ""
+        end
+        break if @buffer.size > 1
+      end
+      return true
+    end
+
+    def get_newline
+      @buffer ||= [""]   
+      if @buffer.size == 1
+        return nil unless fill_buffer
+      end
+      if @buffer.empty?
+        return nil
+      else 
+        new_line = @buffer.shift
+        return new_line
+      end  
+    end
+
     def get_page
       inside_page = false
       page = ""
       while line = get_newline
-        notify_parent        
-        @size_read ||=0; @size_read += line.bytesize
-        
         if /<page>/ =~ line #
           page << line
           inside_page = true
@@ -215,22 +259,7 @@ module Wp2txt
       end
     end
 
-    # call this method to do the job
     def extract_text(&block)
-      prepare            
-      if @convert
-        if block
-          extract_and_convert(&block)
-        else
-          extract_and_convert
-        end
-      else
-        # output the original xml only split to files of the specified size
-        extract
-      end
-    end
-    
-    def extract_and_convert(&block)
       in_text = false
       in_message = false
       result_text = ""
@@ -241,17 +270,15 @@ module Wp2txt
       pages = []
       data_empty = false
 
-      begin
+      while !data_empty 
         page = get_page
         if page
           pages << page
         else
           data_empty = true
         end
-        if data_empty || pages.size == @num_threads
-          # pages_text = Parallel.map_with_index(pages, in_threads: @num_threads) do |page, n|
-          pages_text = Parallel.map_with_index(pages, in_threads: @num_threads) do |page, n|
-            page_text = {:order => n, :data => nil}
+        if data_empty
+          pages.each do |page|
             xmlns = '<mediawiki xmlns="http://www.mediawiki.org/xml/export-0.5/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.mediawiki.org/xml/export-0.5/ http://www.mediawiki.org/xml/export-0.5.xsd" version="0.5" xml:lang="en">' + "\n"
             xml = xmlns + page + "</mediawiki>"
 
@@ -270,79 +297,22 @@ module Wp2txt
                 end
               end
               article = Article.new(text, title, @strip_tmarker)
-              page_text[:data] = block.call(article)
+              page_text = block.call(article)
+              output_text << page_text
             end
-            page_text
-          end
-          pages.clear
-          pages_text = pages_text.sort_by{|v| v[:order]}.map{|v| v[:data]}.compact
-          pages_text.each do |page_text|
-            output_text << page_text
-            @count ||= 0; @count += 1;
-            @total_size = output_text.bytesize
-            # flagged when data exceeds the size of output file
-            end_flag = true if @total_size > (@tfile_size * 1024 * 1024)
           end
 
-          #close the present file, then open a new one
-          if end_flag
-            cleanup!(output_text)
+          cleanup!(output_text)
+          if output_text.size > 0
+            outfilename = File.join(@output_dir, @outfile_base + ".txt")
+            @fp = File.open(outfilename, "w")
             @fp.puts(output_text)
-            output_text = ""
-            @total_size = 0
-            end_flag = false
             @fp.close
-            @file_index += 1
-            outfilename = File.join(@output_dir, @outfile_base + @file_index.to_s)
-            @outfiles << outfilename
-            @fp = File.open(outfilename, "w")
-            next
           end
-        end
-      end while !data_empty 
-
-      if output_text != ""
-        cleanup!(output_text)
-        @fp.puts(output_text) 
-      end
-      notify_parent(true)
-      @parent.after
-      @fp.close    
-      rename(@outfiles)    
-      @parent.msg("Processing finished", 1)
-    end
-
-    def extract
-      output_text = ""
-      end_flag = false
-      while text = get_newline
-        @count ||= 0;@count += 1;
-        @size_read ||=0;@size_read += text.bytesize
-        @total_size += text.bytesize
-        output_text << text
-        end_flag = true if @total_size > (@tfile_size * 1024 * 1024)
-        notify_parent
-        # never close the file until the end of the page even if end_flag is on
-        if end_flag && /<\/page/ =~ text 
-          @fp.puts(output_text)
+          File.delete(@input_file) if @del_interfile
           output_text = ""
-          @total_size = 0
-          end_flag = false
-          @fp.close
-          @file_index += 1
-          outfilename = File.join(@output_dir, @outfile_base + @file_index.to_s)
-          @outfiles << outfilename
-          @fp = File.open(outfilename, "w")
-          next
         end
       end
-      @fp.puts(output_text) if output_text != ""
-      notify_parent(true)
-      @parent.after
-      @fp.close    
-      rename(@outfiles)    
-      @parent.msg("Processing finished", 1)
-    end 
+    end
   end
 end
-

data/spec/utils_spec.rb

@@ -184,10 +184,6 @@ describe "Wp2txt" do
   
   describe "correct_inline_template!" do
     it "removes brackets and leaving some text" do
-      # str_before = "{{}}"
-      # str_after = ""
-      # correct_inline_template!(str_before)
-      # expect(str_before).to eq str_after
       str_before = "{{MedalCountry | {{JPN}} }}"
       str_after  = "JPN"
       correct_inline_template!(str_before)
@@ -197,11 +193,11 @@ describe "Wp2txt" do
       correct_inline_template!(str_before)
       expect(str_before).to eq str_after
       str_before = "{{a|b=c|d=f}}"
-      str_after  = "a"
+      str_after  = "c"
       correct_inline_template!(str_before)
       expect(str_before).to eq str_after
       str_before = "{{a|b|{{c|d|e}}}}"
-      str_after  = "e"
+      str_after  = "b"
       correct_inline_template!(str_before)
       expect(str_before).to eq str_after
       str_before = "{{要出典範囲|日本人に多く見受けられる|date=2013年8月|title=日本人特有なのか、本当に多いのかを示す必要がある}}"
@@ -210,18 +206,4 @@ describe "Wp2txt" do
       expect(str_before).to eq str_after
     end
   end
-  
-  #   describe "expand_template" do
-  #     it "gets data corresponding to a given template using mediawiki api" do
-  #       uri = "http://en.wiktionary.org/w/api.php"
-  #       template = "{{en-verb}}"
-  #       word = "kick"
-  #       expanded = expand_template(uri, template, word)
-  #       html =<<EOD
-  # <span class=\"infl-inline\"><b class=\"Latn \" lang=\"en\">kick</b> (''third-person singular simple present'' <span class=\"form-of third-person-singular-form-of\">'''<span class=\"Latn \" lang=\"en\">[[kicks#English|kicks]]</span>'''</span>, ''present participle'' <span class=\"form-of present-participle-form-of\">'''<span class=\"Latn \" lang=\"en\">[[kicking#English|kicking]]</span>'''</span>, ''simple past and past participle'' <span class=\"form-of simple-past-and-participle-form-of\"> '''<span class=\"Latn \" lang=\"en\">[[kicked#English|kicked]]</span>'''</span>)</span>[[Category:English verbs|kick]]
-  # EOD
-  #       html.strip!
-  #       expanded.should == html
-  #     end
-  #   end
-end
+end

data/wp2txt.gemspec

@@ -14,6 +14,7 @@ Gem::Specification.new do |s|
   s.rubyforge_project = "wp2txt"
 
   s.files         = `git ls-files`.split("\n")
+  s.files -= ["data/*", "image/*"]
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
@@ -23,7 +24,10 @@ Gem::Specification.new do |s|
   # s.add_development_dependency "rake"
 
   s.add_dependency "nokogiri"
+  s.add_dependency "ruby-progressbar"
   s.add_dependency "parallel"
   s.add_dependency "htmlentities"
   s.add_dependency "optimist"
+  s.add_dependency "pastel"
+  s.add_dependency "tty-spinner"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wp2txt
 version: !ruby/object:Gem::Version
-  version: 0.9.5.1
+  version: 1.0.0
 platform: ruby
 authors:
 - Yoichiro Hasebe
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-08-02 00:00:00.000000000 Z
+date: 2022-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
@@ -66,12 +80,39 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: pastel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: tty-spinner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: WP2TXT extracts plain text data from Wikipedia dump file (encoded in
   XML/compressed with Bzip2) stripping all the MediaWiki markups and other metadata.
 email:
 - yohasebe@gmail.com
 executables:
-- benchmark.rb
 - wp2txt
 extensions: []
 extra_rdoc_files: []
@@ -81,20 +122,20 @@ files:
 - LICENSE
 - README.md
 - Rakefile
-- bin/benchmark.rb
 - bin/wp2txt
 - data/output_samples/testdata_en.txt
-- data/output_samples/testdata_en_categories.txt
+- data/output_samples/testdata_en_category.txt
 - data/output_samples/testdata_en_summary.txt
 - data/output_samples/testdata_ja.txt
-- data/output_samples/testdata_ja_categories.txt
+- data/output_samples/testdata_ja_category.txt
 - data/output_samples/testdata_ja_summary.txt
 - data/testdata_en.bz2
 - data/testdata_ja.bz2
+- image/screenshot.png
+- image/wp2txt-logo.svg
+- image/wp2txt.svg
 - lib/wp2txt.rb
 - lib/wp2txt/article.rb
-- lib/wp2txt/mw_api.rb
-- lib/wp2txt/progressbar.rb
 - lib/wp2txt/utils.rb
 - lib/wp2txt/version.rb
 - spec/spec_helper.rb
@@ -118,7 +159,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.3.3
 signing_key:
 specification_version: 4
 summary: Wikipedia dump to text converter