wp2txt 1.0.2 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bb540f4f17f7825786d110245c235ac556e3e64cedb17efae3e0591887425801
-  data.tar.gz: 479c357f7ba117ae10d9a5a04d24ce3aca2e54d942a156b02eb932c1aab55c8b
+  metadata.gz: 0bcba2c84286504ae628176aad55dbdea05889dfaa7f471cf080ae933691cffc
+  data.tar.gz: f83c63d7c6e91270da1da2aed54ab6e5c352c5695340ccd6378fdd20c43fc332
 SHA512:
-  metadata.gz: 940d47d2c8bce06029fe76e3b3744563d089e26e297e5224b36e65d815295da57117eae84cbb43abeddf2f2c052e2a987d668cba52c7af6148e935b571b6d403
-  data.tar.gz: 8ce76523a3bf181ac7a5da11f088dd14cfb1e1d7ac0d5239832db52968d183db16a3ece6074513b634eebe0e5ca28ceea945eaef6542ecb1933266caf4e89a3c
+  metadata.gz: ae0eae028a98d4299a0e93278220b991e53e13deb80b88cce2971cd889d769808305e0d3aa8ee4e73af0cc55f07f27c0cc6a9f0d440e4693a410ba7d0a6333ba
+  data.tar.gz: 25bff247bf80b4a0b5ff785ed51b60e3e21e6a3ca5e0bfeace9961df5060832ae047c3fa606a3267f9f770c253fd816176372c4d2f7ba73999d5317ea59933e6

data/.dockerignore

@@ -0,0 +1,8 @@
+.git
+.github
+image
+pkg
+spec
+.dockerignore
+.gitignore
+Gemfile.lock

data/.github/workflows/ci.yml

@@ -0,0 +1,36 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+  schedule:
+    - cron: '42 5 * * *'
+
+jobs:
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '3.1' ]
+
+    runs-on: ubuntu-latest
+    name: Ruby ${{matrix.ruby}}
+    container: ruby:${{matrix.ruby}}
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Show Ruby Version
+      run: ruby -v
+
+    - name: Install dependencies
+      run: bundle install
+
+    - name: Install rspec
+      run: gem install rspec
+
+    - name: Run tests
+      run: rspec
+
+

data/.gitignore

@@ -18,4 +18,4 @@ tmp
 .DS_Store
 *.bak
 *.~
-
+tags

data/.rubocop.yml

@@ -0,0 +1,80 @@
+AllCops:
+  NewCops: disable
+  SuggestExtensions: false
+  TargetRubyVersion: 2.6
+
+Documentation:
+  Enabled: false
+
+Naming/AccessorMethodName:
+  Enabled: false
+
+Naming/VariableNumber:
+  Enabled: false
+
+Naming/FileName:
+  Enabled: false
+
+Security/MarshalLoad:
+  Enabled: false
+
+Security/Open:
+  Enabled: false
+
+Layout/EndOfLine:
+   Enabled: False
+
+Style/FormatStringToken:
+   Enabled: false
+
+Style/ClassVars:
+  Enabled: false
+
+Style/OptionalBooleanParameter:
+  Enabled: false
+
+Style/StringConcatenation:
+  Enabled: false
+
+Style/PerlBackrefs:
+  Enabled: false
+
+Style/StringLiterals:
+  Enabled: false
+
+Style/StringLiteralsInInterpolation:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Style/WordArray:
+  Enabled: false
+
+Style/EvalWithLocation:
+  Enabled: false
+
+Layout/LineLength:
+  Max: 400
+
+Metrics/MethodLength:
+  Max: 200
+
+Metrics/BlockLength:
+  Max: 200
+
+Metrics/AbcSize:
+  Max: 200
+
+Metrics/PerceivedComplexity:
+  Max: 60
+
+Metrics/ClassLength:
+  Max: 800
+
+Metrics/CyclomaticComplexity:
+  Max: 60
+
+Metrics/ParameterLists:
+  Max: 8
+
+Metrics/ModuleLength:
+  Max: 600

data/.solargraph.yml

@@ -0,0 +1,22 @@
+---
+include:
+- "**/*.rb"
+exclude:
+- spec/**/*
+- test/**/*
+- vendor/**/*
+- ".bundle/**/*"
+require: []
+domains: []
+reporters:
+- rubocop
+# - require_not_found
+formatter:
+  rubocop:
+    cops: safe
+    except: []
+    only: []
+    extra_args: []
+require_paths: []
+plugins: []
+max_files: 5000

data/Dockerfile

@@ -0,0 +1,20 @@
+FROM ruby:3.1.3-alpine3.17
+
+WORKDIR /wp2txt
+COPY . ./
+RUN rm -Rf wp2txt/Gemfile.lock
+
+RUN apk update && \
+    apk upgrade && \
+    apk add --no-cache linux-headers libxml2-dev make gcc libc-dev bash && \
+    apk add --no-cache -t .build-packages --no-cache build-base curl-dev wget gcompat && \
+    bundle install -j4
+
+RUN wget https://fossies.org/linux/privat/lbzip2-2.5.tar.gz -O lbzip2.tar.gz && \
+    tar -xvf lbzip2.tar.gz && cd lbzip2-2.5 && \
+    bash configure && make && make install && \
+    cd .. && rm -rf lbzip2*
+
+WORKDIR /
+ENV PATH $PATH:/wp2txt/bin
+CMD ["bash"]

data/Gemfile

@@ -1,4 +1,11 @@
+# frozen_string_literal: true
+
 source "http://rubygems.org"
 
-# Specify your gem's dependencies in wp2txt.gemspec
-gemspec
+gem "htmlentities"
+gem "nokogiri"
+gem "optimist"
+gem "parallel"
+gem "pastel"
+gem "ruby-progressbar"
+gem "tty-spinner"

data/README.md

@@ -8,6 +8,15 @@ WP2TXT extracts text and category data from Wikipedia dump files (encoded in XML
 
 ## Changelog
 
+**January 2023**
+
+- Bug related to command line arguments fixed
+- Code cleanup introducing Rubocop
+
+**December 2022**
+
+- Docker images available via Docker Hub
+
 **November 2022**
 
 - Code added to suppress "Invalid byte sequence error" when an ilegal UTF-8 character is input.
@@ -38,9 +47,28 @@ In the above environment, the process (decompression, splitting, extraction, and
 - Allows extracting category information of the article
 - Allows extracting opening paragraphs of the article
 
-## Preparation
+## Setting Up
+
+### WP2TXT on Docker
+
+1. Install [Docker Desktop](https://www.docker.com/products/docker-desktop/) (Mac/Windows/Linux)
+2. Execute `docker` command in a terminal:
+
+```shell
+docker run -it -v /Users/me/localdata:/data yohasebe/wp2txt
+```
+
+- Make sure to Replace `/Users/me/localdata` with the full path to the data directory in your local computer
+
+3. The Docker image will begin downloading and a bash prompt will appear when finished.
+4. The `wp2txt` command will be avalable anywhare in the Docker container. Use the `/data` directory as the location of the input dump files and the output text files.
+
+**IMPORTANT:**
+
+- Configure Docker Desktop resource settings (number of cores, amount of memory, etc.) to get the best performance possible.
+- When running the `wp2txt` command inside a Docker container, be sure to set the output directory to somewhere in the mounted local directory specified by the `docker run` command.
 
-### For MacOS and Linux
+### WP2TXT on MacOS and Linux
 
 WP2TXT requires that one of the following commands be installed on the system in order to decompress `bz2` files:
 
@@ -54,7 +82,7 @@ If you are using MacOS with Homebrew installed, you can install `lbzip2` with th
 
     $ brew install lbzip2
 
-### For Windows
+### WP2TXT on Windows
 
 Install [Bzip2 for Windows](http://gnuwin32.sourceforge.net/packages/bzip2.htm) and set the path so that WP2TXT can use the bunzip2.exe command. Alternatively, you can extract the Wikipedia dump file in your own way and process the resulting XML file with WP2TXT.
 
@@ -70,7 +98,7 @@ Download the latest Wikipedia dump file for the desired language at a URL such a
 
     https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-pages-articles.xml.bz2
 
-Here, `enwiki` refers to the English Wikipedia. To get the Japanese Wikipedia dump file, for instance, change this to jawiki (Japanese). In doing so, note that there are two instances of `enwiki` in the URL above.
+Here, `enwiki` refers to the English Wikipedia. To get the Japanese Wikipedia dump file, for instance, change this to `jawiki` (Japanese). In doing so, note that there are two instances of `enwiki` in the URL above.
 
 Alternatively, you can also select Wikipedia dump files created on a specific date from [here](http://dumps.wikimedia.org/backup-index.html). Make sure to download a file named in the following format:
 
@@ -190,11 +218,11 @@ The author will appreciate your mentioning one of these in your research.
 Or use this BibTeX entry:
 
 ```
-@misc{wp2txt_2022,
+@misc{wp2txt_2023,
   author = {Yoichiro Hasebe},
   title = {WP2TXT: A command-line toolkit to extract text content and category data from Wikipedia dump files},
   url = {https://github.com/yohasebe/wp2txt},
-  year = {2022}
+  year = {2023}
 }
 ```
 

data/Rakefile

@@ -1,9 +1,30 @@
+# frozen_string_literal: true
+
 require "bundler/gem_tasks"
-require 'rspec/core'
-require 'rspec/core/rake_task'
+require "rspec/core"
+require "rspec/core/rake_task"
+require_relative "./lib/wp2txt/version"
+
+class String
+  def strip_heredoc
+    gsub(/^#{scan(/^[ \t]*(?=\S)/).min}/, "")
+  end
+end
 
 RSpec::Core::RakeTask.new(:spec) do |spec|
-  spec.pattern = FileList['spec/**/*_spec.rb']
+  spec.pattern = FileList["spec/**/*_spec.rb"]
 end
 
-task :default => :spec
+task default: :spec
+
+desc "Push Docker images"
+task :push do
+  sh <<-SCRIPT.strip_heredoc, { verbose: false }
+    /bin/bash -xeu <<'BASH'
+      # docker buildx create --name mybuilder
+      # docker buildx use mybuilder
+      # docker buildx inspect --bootstrap
+      docker buildx build --platform linux/amd64,linux/arm64 -t yohasebe/wp2txt:#{Wp2txt::VERSION} -t yohasebe/wp2txt:latest . --push
+    BASH
+  SCRIPT
+end