wp2txt 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d33a41cf46688679a14eb8c3eb16f6ed33ce9175c7f5b566c9f87998ba2c8401
-  data.tar.gz: 7371e0f7b06b2f0846f01d66f461c7e106778adc6e686919302f0f29b1f80a9e
+  metadata.gz: 62f1e8d6ab1932f3ae3c34fb71930b7e73500c832481dcea6288742c38850a79
+  data.tar.gz: f0ff0a5488b635b828338d41029c5ad191a0c88282c0fa294a9facf2d93c055b
 SHA512:
-  metadata.gz: cab8d9c27989387acc6dbbe052029d2205508ce10e38b8eedc111c822328d8eba551d603020684cbb3844a87b747f261a5959f711267acd96a3b97ccef4f6834
-  data.tar.gz: 4de59be37d57ef3d14ae2304660e8dde069bdf645a7cff862026562b26327984f1be13840e9d6ec1f25110222367f71c84a0286b649d71fec0c13805c6b0a647
+  metadata.gz: 7bca85758e88d53dcd33fe43e83a251624f89329a2cb55ffb97b41141bcf8fe5ace7c48e3b8e49f5aa42f84724247cfe4ad376238a949e9154876d4d07469afe
+  data.tar.gz: de59399d5163afed2947e0802abf2e0365894d566c8a1f11823bc901d4948346e7af47d6fba558387f5af7e1301a6725a51a322ac1cd4810264dc3003e0729e2

data/.dockerignore

@@ -0,0 +1,8 @@
+.git
+.github
+image
+pkg
+spec
+.dockerignore
+.gitignore
+Gemfile.lock

data/.github/workflows/ci.yml

@@ -0,0 +1,36 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+  schedule:
+    - cron: '42 5 * * *'
+
+jobs:
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '3.1' ]
+
+    runs-on: ubuntu-latest
+    name: Ruby ${{matrix.ruby}}
+    container: ruby:${{matrix.ruby}}
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Show Ruby Version
+      run: ruby -v
+
+    - name: Install dependencies
+      run: bundle install
+
+    - name: Install rspec
+      run: gem install rspec
+
+    - name: Run tests
+      run: rspec
+
+

data/.gitignore

@@ -18,4 +18,4 @@ tmp
 .DS_Store
 *.bak
 *.~
-
+tags

data/.rubocop.yml

@@ -0,0 +1,80 @@
+AllCops:
+  NewCops: disable
+  SuggestExtensions: false
+  TargetRubyVersion: 2.6
+
+Documentation:
+  Enabled: false
+
+Naming/AccessorMethodName:
+  Enabled: false
+
+Naming/VariableNumber:
+  Enabled: false
+
+Naming/FileName:
+  Enabled: false
+
+Security/MarshalLoad:
+  Enabled: false
+
+Security/Open:
+  Enabled: false
+
+Layout/EndOfLine:
+   Enabled: False
+
+Style/FormatStringToken:
+   Enabled: false
+
+Style/ClassVars:
+  Enabled: false
+
+Style/OptionalBooleanParameter:
+  Enabled: false
+
+Style/StringConcatenation:
+  Enabled: false
+
+Style/PerlBackrefs:
+  Enabled: false
+
+Style/StringLiterals:
+  Enabled: false
+
+Style/StringLiteralsInInterpolation:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Style/WordArray:
+  Enabled: false
+
+Style/EvalWithLocation:
+  Enabled: false
+
+Layout/LineLength:
+  Max: 400
+
+Metrics/MethodLength:
+  Max: 200
+
+Metrics/BlockLength:
+  Max: 200
+
+Metrics/AbcSize:
+  Max: 200
+
+Metrics/PerceivedComplexity:
+  Max: 60
+
+Metrics/ClassLength:
+  Max: 800
+
+Metrics/CyclomaticComplexity:
+  Max: 60
+
+Metrics/ParameterLists:
+  Max: 8
+
+Metrics/ModuleLength:
+  Max: 600

data/.solargraph.yml

@@ -0,0 +1,22 @@
+---
+include:
+- "**/*.rb"
+exclude:
+- spec/**/*
+- test/**/*
+- vendor/**/*
+- ".bundle/**/*"
+require: []
+domains: []
+reporters:
+- rubocop
+# - require_not_found
+formatter:
+  rubocop:
+    cops: safe
+    except: []
+    only: []
+    extra_args: []
+require_paths: []
+plugins: []
+max_files: 5000

data/Dockerfile

@@ -0,0 +1,20 @@
+FROM ruby:3.1.3-alpine3.17
+
+WORKDIR /wp2txt
+COPY . ./
+RUN rm -Rf wp2txt/Gemfile.lock
+
+RUN apk update && \
+    apk upgrade && \
+    apk add --no-cache linux-headers libxml2-dev make gcc libc-dev bash && \
+    apk add --no-cache -t .build-packages --no-cache build-base curl-dev wget gcompat && \
+    bundle install -j4
+
+RUN wget https://fossies.org/linux/privat/lbzip2-2.5.tar.gz -O lbzip2.tar.gz && \
+    tar -xvf lbzip2.tar.gz && cd lbzip2-2.5 && \
+    bash configure && make && make install && \
+    cd .. && rm -rf lbzip2*
+
+WORKDIR /
+ENV PATH $PATH:/wp2txt/bin
+CMD ["bash"]

data/Gemfile

@@ -1,4 +1,11 @@
+# frozen_string_literal: true
+
 source "http://rubygems.org"
 
-# Specify your gem's dependencies in wp2txt.gemspec
-gemspec
+gem "htmlentities"
+gem "nokogiri"
+gem "optimist"
+gem "parallel"
+gem "pastel"
+gem "ruby-progressbar"
+gem "tty-spinner"

data/README.md

@@ -6,20 +6,30 @@ A command-line toolkit to extract text content and category data from Wikipedia
 
 WP2TXT extracts text and category data from Wikipedia dump files (encoded in XML / compressed with Bzip2), removing MediaWiki markup and other metadata.
 
-**UPDATE (August 2022)**
+## Changelog
 
-1. A new option `--category-only` has been added. When this option is enabled, only the title and category information of the article is extracted.
-2. A new option `--summary-only` has been added. If this option is enabled, only the title, category information, and opening paragraphs of the article will be extracted.
-3. Text conversion with the current version of WP2TXT is *more than 2x times faster* than the previous version due to parallel processing of multiple files (the rate of speedup depends on the CPU cores used for processing).
+**December 2022**
+
+- Docker images available via Docker Hub
+
+**November 2022**
+
+- Code added to suppress "Invalid byte sequence error" when an ilegal UTF-8 character is input.
+
+**August 2022**
+
+- A new option `--category-only` has been added. When this option is enabled, only the title and category information of the article is extracted.
+- A new option `--summary-only` has been added. If this option is enabled, only the title, category information, and opening paragraphs of the article will be extracted.
+- Text conversion with the current version of WP2TXT is *more than 2x times faster* than the previous version due to parallel processing of multiple files (the rate of speedup depends on the CPU cores used for processing).
 
 ## Screenshot
 
-<img src='https://raw.githubusercontent.com/yohasebe/wp2txt/master/image/screenshot.png' width="700" />
+<img src='https://raw.githubusercontent.com/yohasebe/wp2txt/master/image/screenshot.png' width="800" />
 
-**Environment** 
+**Environment**
 
 - WP2TXT 1.0.1
-- MacBook Pro (2021 Apple M1 Pro) 
+- MacBook Pro (2021 Apple M1 Pro)
 - enwiki-20220720-pages-articles.xml.bz2 (19.98 GB)
 
 In the above environment, the process (decompression, splitting, extraction, and conversion) to obtain the plain text data of the English Wikipedia takes less than 1.5 hours.
@@ -32,9 +42,28 @@ In the above environment, the process (decompression, splitting, extraction, and
 - Allows extracting category information of the article
 - Allows extracting opening paragraphs of the article
 
-## Preparation
+## Setting Up
+
+### WP2TXT on Docker
+
+1. Install [Docker Desktop](https://www.docker.com/products/docker-desktop/) (Mac/Windows/Linux)
+2. Execute `docker` command in a terminal:
+
+```shell
+docker run -it -v /Users/me/localdata:/data yohasebe/wp2txt
+```
+
+- Make sure to Replace `/Users/me/localdata` with the full path to the data directory in your local computer
+
+3. The Docker image will begin downloading and a bash prompt will appear when finished.
+4. The `wp2txt` command will be avalable anywhare in the Docker container. Use the `/data` directory as the location of the input dump files and the output text files.
+
+**IMPORTANT:**
+
+- Configure Docker Desktop resource settings (number of cores, amount of memory, etc.) to get the best performance possible.
+- When running the `wp2txt` command inside a Docker container, be sure to set the output directory to somewhere in the mounted local directory specified by the `docker run` command.
 
-### For MacOS / Linux/ WSL2
+### WP2TXT on MacOS and Linux
 
 WP2TXT requires that one of the following commands be installed on the system in order to decompress `bz2` files:
 
@@ -48,7 +77,7 @@ If you are using MacOS with Homebrew installed, you can install `lbzip2` with th
 
     $ brew install lbzip2
 
-### For Windows
+### WP2TXT on Windows
 
 Install [Bzip2 for Windows](http://gnuwin32.sourceforge.net/packages/bzip2.htm) and set the path so that WP2TXT can use the bunzip2.exe command. Alternatively, you can extract the Wikipedia dump file in your own way and process the resulting XML file with WP2TXT.
 
@@ -184,11 +213,11 @@ The author will appreciate your mentioning one of these in your research.
 Or use this BibTeX entry:
 
 ```
-@misc{WP2TXT_2022,
+@misc{wp2txt_2022,
   author = {Yoichiro Hasebe},
   title = {WP2TXT: A command-line toolkit to extract text content and category data from Wikipedia dump files},
-  url = {https://github.com/yohasebe/wp2txt}
-  year = {2022},
+  url = {https://github.com/yohasebe/wp2txt},
+  year = {2022}
 }
 ```
 

data/Rakefile

@@ -1,9 +1,30 @@
+# frozen_string_literal: true
+
 require "bundler/gem_tasks"
-require 'rspec/core'
-require 'rspec/core/rake_task'
+require "rspec/core"
+require "rspec/core/rake_task"
+require_relative "./lib/wp2txt/version"
+
+class String
+  def strip_heredoc
+    gsub(/^#{scan(/^[ \t]*(?=\S)/).min}/, "")
+  end
+end
 
 RSpec::Core::RakeTask.new(:spec) do |spec|
-  spec.pattern = FileList['spec/**/*_spec.rb']
+  spec.pattern = FileList["spec/**/*_spec.rb"]
 end
 
-task :default => :spec
+task default: :spec
+
+desc "Push Docker images"
+task :push do
+  sh <<-SCRIPT.strip_heredoc, { verbose: false }
+    /bin/bash -xeu <<'BASH'
+      # docker buildx create --name mybuilder
+      # docker buildx use mybuilder
+      # docker buildx inspect --bootstrap
+      docker buildx build --platform linux/amd64,linux/arm64 -t yohasebe/wp2txt:#{Wp2txt::VERSION} -t yohasebe/wp2txt:latest . --push
+    BASH
+  SCRIPT
+end