wp2txt 1.0.1 → 1.1.0

data/spec/utils_spec.rb

@@ -1,10 +1,9 @@
-#!/usr/bin/env ruby
-# -*- coding: utf-8 -*-
+# frozen_string_literal: true
 
-require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
-require 'wp2txt'
-require 'wp2txt/article'
-require 'wp2txt/utils'
+require_relative "spec_helper"
+require_relative "../lib/wp2txt"
+require_relative "../lib/wp2txt/article"
+require_relative "../lib/wp2txt/utils"
 
 describe "Wp2txt" do
   it "contains mediawiki-format related functions:" do
@@ -17,14 +16,14 @@ describe "Wp2txt" do
 
   describe "process_nested_structure" do
     it "parse nested structure replacing str in the format specified" do
-      str_before = "[[ab[[cde[[alfa]]]]fg]]"
-      str_after  = "<<ab<<cde<<alfa>>>>fg>>"
-      scanner = StringScanner.new(str_before)
-      str_processed = process_nested_structure(scanner, "[[", "]]") do |content|
+      str_before1 = "[[ab[[cde[[alfa]]]]fg]]"
+      str_after1  = "<<ab<<cde<<alfa>>>>fg>>"
+      scanner1 = StringScanner.new(str_before1)
+      str_processed = process_nested_structure(scanner1, "[[", "]]") do |content|
         "<<" + content + ">>"
       end
-      expect(str_processed).to eq str_after
-      
+      expect(str_processed).to eq str_after1
+
       str_before = "#* {{quote-book|1503|year_published=1836|chapter=19 Henry VII. c. 5: Coin||A Collection of Statutes Connected with the General Administration of the Law|page=158|url=http://books.google.com/books?id=QtYuAAAAIAAJ
       |passage={{...}} every of them, being gold, whole and weight, shall '''go''' and be current in payment throughout this his realm for the sum that they were coined for.}}"
       str_after = "#* <<quote-book|1503|year_published=1836|chapter=19 Henry VII. c. 5: Coin||A Collection of Statutes Connected with the General Administration of the Law|page=158|url=http://books.google.com/books?id=QtYuAAAAIAAJ
@@ -33,177 +32,155 @@ describe "Wp2txt" do
       str_processed = process_nested_structure(scanner, "{{", "}}") do |content|
         "<<" + content + ">>"
       end
-      #str_processed.should == str_after
       expect(str_processed).to eq str_after
-      
     end
   end
-  
-  describe "special_chr!" do
+
+  describe "special_chr" do
     it "replaces character references with real characters" do
       str_before = "&nbsp; &lt; &gt; &amp; &quot;"
       str_after  = "  < > & \""
-      special_chr!(str_before)
-      expect(str_before).to eq str_after
-    end    
+      expect(special_chr(str_before)).to eq str_after
+    end
   end
-  
-  describe "chrref_to_utf!" do
+
+  describe "chrref_to_utf" do
     it "replaces character references with real characters" do
       str_before = "&#x266A;"
       str_after  = "♪"
-      chrref_to_utf!(str_before)
-      expect(str_before).to eq str_after
+      expect(chrref_to_utf(str_before)).to eq str_after
     end
   end
-  
-  describe "mndash!" do
+
+  describe "mndash" do
     it "replaces {mdash}, {ndash}, or {–} with '–'" do
       str_before = "{mdash} {ndash} {–}"
       str_after  = "– – –"
-      mndash!(str_before)
-      expect(str_before).to eq str_after
+      expect(mndash(str_before)).to eq str_after
     end
   end
-  
+
   describe "make_reference" do
     it "replaces <ref> tag with [ref]" do
       str_before = "<ref> ... </ref>"
       str_after  = "[ref] ... [/ref]"
-      make_reference!(str_before)
-      expect(str_before).to eq str_after
-    end    
+      expect(make_reference(str_before)).to eq str_after
+    end
   end
-  
-  describe "remove_table!" do
+
+  describe "remove_table" do
     it "removes table formated parts" do
       str_before = "{| ... \n{| ... \n ...|}\n ...|}"
       str_after  = ""
-      remove_table!(str_before)
-      expect(str_before).to eq str_after
-    end    
-  end
-
-  # describe "remove_clade" do
-  #   it "removes clade formated parts" do
-  #     str_before = "\{\{clade ... \n ... \n ... \n\}\}"
-  #     str_after  = ""
-  #     expect(remove_clade(str_before)).to eq str_after
-  #   end
-  # end
-  
-  describe "remove_hr!" do
+      expect(remove_table(str_before)).to eq str_after
+    end
+  end
+
+  describe "remove_hr" do
     it "removes horizontal lines" do
       str_before = "\n----\n--\n--\n"
       str_after  = "\n\n"
-      remove_hr!(str_before)
-      expect(str_before).to eq str_after
-    end    
+      expect(remove_hr(str_before)).to eq str_after
+    end
   end
 
-  describe "remove_inbetween!" do
+  describe "remove_inbetween" do
     it "removes tags and its contents" do
-      str_before = "<tag>abc</tag>"
-      str_after  = "abc"
-      remove_tag!(str_before)
-      expect(str_before).to eq str_after
-      str_before = "[tag]def[/tag]"
-      str_after  = "def"
-      remove_inbetween!(str_before, ['[', ']'])
-      expect(str_before).to eq str_after
-    end    
-  end
-  
-  describe "remove_directive!" do
+      str_before1 = "<tag>abc</tag>"
+      str_after1  = "abc"
+      expect(remove_tag(str_before1)).to eq str_after1
+
+      str_before2 = "[tag]def[/tag]"
+      str_after2  = "def"
+      expect(remove_inbetween(str_before2, ["[", "]"])).to eq str_after2
+    end
+  end
+
+  describe "remove_directive" do
     it "removes directive" do
       str_before = "__abc__\n __def__"
       str_after  = "\n "
-      remove_directive!(str_before)
-      expect(str_before).to eq str_after
-    end    
+      expect(remove_directive(str_before)).to eq str_after
+    end
   end
 
-  describe "remove_emphasis!" do
+  describe "remove_emphasis" do
     it "removes directive" do
       str_before = "''abc''\n'''def'''"
       str_after  = "abc\ndef"
-      remove_emphasis!(str_before)
-      expect(str_before).to eq str_after
-    end    
+      expect(remove_emphasis(str_before)).to eq str_after
+    end
   end
-  
-  describe "escape_nowiki!" do
+
+  describe "escape_nowiki" do
     it "replaces <nowiki>...</nowiki> with <nowiki-object_id>" do
       str_before = "<nowiki>[[abc]]</nowiki>def<nowiki>[[ghi]]</nowiki>"
       str_after  = Regexp.new("<nowiki-\\d+>def<nowiki-\\d+>")
-      escape_nowiki!(str_before)
-      expect(str_before).to match str_after
+      expect(escape_nowiki(str_before)).to match str_after
     end
   end
 
-  describe "unescape_nowiki!" do
+  describe "unescape_nowiki" do
     it "replaces <nowiki-object_id> with string stored elsewhere" do
-      @nowikis = {123 => "[[abc]]", 124 => "[[ghi]]"}
+      @nowikis = { 123 => "[[abc]]", 124 => "[[ghi]]" }
       str_before = "<nowiki-123>def<nowiki-124>"
       str_after  = "[[abc]]def[[ghi]]"
-      unescape_nowiki!(str_before)
-      expect(str_before).to eq str_after
+      expect(unescape_nowiki(str_before)).to eq str_after
     end
   end
-  
-  describe "process_interwiki_links!" do
+
+  describe "process_interwiki_links" do
     it "formats text link and remove brackets" do
-      a = "[[a b]]"
-      b = "[[a b|c]]"
-      c = "[[a|b|c]]"
-      d = "[[硬口蓋鼻音|[ɲ], /J/]]"
-      process_interwiki_links!(a)
-      process_interwiki_links!(b)
-      process_interwiki_links!(c)
-      process_interwiki_links!(d)
-      expect(a).to eq "a b"
-      expect(b).to eq "c"
-      expect(c).to eq "b|c"
-      expect(d).to eq "[ɲ], /J/"
+      a1 = "[[a b]]"
+      b1 = "[[a b|c]]"
+      c1 = "[[a|b|c]]"
+      d1 = "[[硬口蓋鼻音|[ɲ], /J/]]"
+      a2 = process_interwiki_links(a1)
+      b2 = process_interwiki_links(b1)
+      c2 = process_interwiki_links(c1)
+      d2 = process_interwiki_links(d1)
+      expect(a2).to eq "a b"
+      expect(b2).to eq "c"
+      expect(c2).to eq "b|c"
+      expect(d2).to eq "[ɲ], /J/"
     end
   end
 
-  describe "process_external_links!" do
+  describe "process_external_links" do
     it "formats text link and remove brackets" do
-      a = "[http://yohasebe.com yohasebe.com]"
-      b = "[http://yohasebe.com]"
-      c = "* Turkish: {{t+|tr|köken bilimi}}]], {{t+|tr|etimoloji}}"
-      process_external_links!(a)
-      process_external_links!(b)
-      process_external_links!(c)
-      expect(a).to eq "yohasebe.com"      
-      expect(b).to eq "http://yohasebe.com"
-      expect(c).to eq "* Turkish: {{t+|tr|köken bilimi}}]], {{t+|tr|etimoloji}}"
+      a1 = "[http://yohasebe.com yohasebe.com]"
+      b1 = "[http://yohasebe.com]"
+      c1 = "* Turkish: {{t+|tr|köken bilimi}}]], {{t+|tr|etimoloji}}"
+      a2 = process_external_links(a1)
+      b2 = process_external_links(b1)
+      c2 = process_external_links(c1)
+      expect(a2).to eq "yohasebe.com"
+      expect(b2).to eq "http://yohasebe.com"
+      expect(c2).to eq "* Turkish: {{t+|tr|köken bilimi}}]], {{t+|tr|etimoloji}}"
     end
   end
-  
-  describe "correct_inline_template!" do
+
+  describe "correct_inline_template" do
     it "removes brackets and leaving some text" do
-      str_before = "{{MedalCountry | {{JPN}} }}"
-      str_after  = "JPN"
-      correct_inline_template!(str_before)
-      expect(str_before).to eq str_after
-      str_before = "{{lang|en|Japan}}"
-      str_after  = "Japan"
-      correct_inline_template!(str_before)
-      expect(str_before).to eq str_after
-      str_before = "{{a|b=c|d=f}}"
-      str_after  = "c"
-      correct_inline_template!(str_before)
-      expect(str_before).to eq str_after
-      str_before = "{{a|b|{{c|d|e}}}}"
-      str_after  = "b"
-      correct_inline_template!(str_before)
-      expect(str_before).to eq str_after
-      str_before = "{{要出典範囲|日本人に多く見受けられる|date=2013年8月|title=日本人特有なのか、本当に多いのかを示す必要がある}}"
-      str_after = "日本人に多く見受けられる"
-      correct_inline_template!(str_before)
-      expect(str_before).to eq str_after
+      str_before1 = "{{MedalCountry | {{JPN}} }}"
+      str_after1  = "JPN"
+      expect(correct_inline_template(str_before1)).to eq str_after1
+
+      str_before2 = "{{lang|en|Japan}}"
+      str_after2  = "Japan"
+      expect(correct_inline_template(str_before2)).to eq str_after2
+
+      str_before3 = "{{a|b=c|d=f}}"
+      str_after3  = "c"
+      expect(correct_inline_template(str_before3)).to eq str_after3
+
+      str_before4 = "{{a|b|{{c|d|e}}}}"
+      str_after4  = "b"
+      expect(correct_inline_template(str_before4)).to eq str_after4
+
+      str_before5 = "{{要出典範囲|日本人に多く見受けられる|date=2013年8月|title=日本人特有なのか、本当に多いのかを示す必要がある}}"
+      str_after5 = "日本人に多く見受けられる"
+      expect(correct_inline_template(str_before5)).to eq str_after5
     end
   end
 end

data/wp2txt.gemspec

@@ -1,6 +1,6 @@
- # -*- coding: utf-8 -*-
-$:.push File.expand_path("../lib", __FILE__)
-require "wp2txt/version"
+# frozen_string_literal: true
+
+require_relative "lib/wp2txt/version"
 
 Gem::Specification.new do |s|
   s.name        = "wp2txt"
@@ -8,26 +8,24 @@ Gem::Specification.new do |s|
   s.authors     = ["Yoichiro Hasebe"]
   s.email       = ["yohasebe@gmail.com"]
   s.homepage    = "https://github.com/yohasebe/wp2txt"
-  s.summary     = %q{A command-line toolkit to extract text content and category data from Wikipedia dump files}
-  s.description = %q{WP2TXT extracts text and category data from Wikipedia dump files (encoded in XML / compressed with Bzip2), removing MediaWiki markup and other metadata.}
-
+  s.summary     = "A command-line toolkit to extract text content and category data from Wikipedia dump files"
+  s.description = "WP2TXT extracts text and category data from Wikipedia dump files (encoded in XML / compressed with Bzip2), removing MediaWiki markup and other metadata."
   s.rubyforge_project = "wp2txt"
-
-  s.files         = `git ls-files`.split("\n")
+  s.license = "MIT"
+  s.required_ruby_version = Gem::Requirement.new(">= 2.6")
+  s.files = `git ls-files`.split("\n")
   s.files -= ["data/*", "image/*"]
-  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
-  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
   s.require_paths = ["lib"]
-
-  # s.add_development_dependency "bundler"
-  # s.add_development_dependency "rspec"
-  # s.add_development_dependency "rake"
-
-  s.add_dependency "nokogiri"
-  s.add_dependency "ruby-progressbar"
-  s.add_dependency "parallel"
+  s.add_development_dependency "bundler"
+  s.add_development_dependency "rake"
+  s.add_development_dependency "rspec"
   s.add_dependency "htmlentities"
+  s.add_dependency "nokogiri"
   s.add_dependency "optimist"
+  s.add_dependency "parallel"
   s.add_dependency "pastel"
+  s.add_dependency "ruby-progressbar"
   s.add_dependency "tty-spinner"
 end

metadata

@@ -1,23 +1,23 @@
 --- !ruby/object:Gem::Specification
 name: wp2txt
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors:
 - Yoichiro Hasebe
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-08-11 00:00:00.000000000 Z
+date: 2023-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -25,13 +25,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: ruby-progressbar
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -39,13 +39,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: parallel
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: optimist
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pastel
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +122,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: tty-spinner
   requirement: !ruby/object:Gem::Requirement
@@ -117,7 +159,12 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".dockerignore"
+- ".github/workflows/ci.yml"
 - ".gitignore"
+- ".rubocop.yml"
+- ".solargraph.yml"
+- Dockerfile
 - Gemfile
 - LICENSE
 - README.md
@@ -136,13 +183,15 @@ files:
 - image/wp2txt.svg
 - lib/wp2txt.rb
 - lib/wp2txt/article.rb
+- lib/wp2txt/regex.rb
 - lib/wp2txt/utils.rb
 - lib/wp2txt/version.rb
 - spec/spec_helper.rb
 - spec/utils_spec.rb
 - wp2txt.gemspec
 homepage: https://github.com/yohasebe/wp2txt
-licenses: []
+licenses:
+- MIT
 metadata: {}
 post_install_message:
 rdoc_options: []
@@ -152,14 +201,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.4.1
 signing_key:
 specification_version: 4
 summary: A command-line toolkit to extract text content and category data from Wikipedia