wp2txt 1.1.3 → 2.1.0

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: wp2txt
 version: !ruby/object:Gem::Version
-  version: 1.1.3
+  version: 2.1.0
 platform: ruby
 authors:
 - Yoichiro Hasebe
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-05-13 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +51,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: htmlentities
   requirement: !ruby/object:Gem::Requirement
@@ -123,7 +150,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: ruby-progressbar
+  name: tty-progressbar
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -150,6 +177,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: WP2TXT extracts text and category data from Wikipedia dump files (encoded
   in XML / compressed with Bzip2), removing MediaWiki markup and other metadata.
 email:
@@ -162,38 +203,95 @@ files:
 - ".dockerignore"
 - ".github/workflows/ci.yml"
 - ".gitignore"
-- ".rubocop.yml"
 - ".solargraph.yml"
+- CHANGELOG.md
+- DEVELOPMENT.md
+- DEVELOPMENT_ja.md
 - Dockerfile
 - Gemfile
 - LICENSE
 - README.md
+- README_ja.md
 - Rakefile
 - bin/wp2txt
-- data/output_samples/testdata_en.txt
-- data/output_samples/testdata_en_category.txt
-- data/output_samples/testdata_en_summary.txt
-- data/output_samples/testdata_ja.txt
-- data/output_samples/testdata_ja_category.txt
-- data/output_samples/testdata_ja_summary.txt
-- data/testdata_en.bz2
-- data/testdata_ja.bz2
-- image/screenshot.png
 - image/wp2txt-logo.svg
 - image/wp2txt.svg
 - lib/wp2txt.rb
 - lib/wp2txt/article.rb
+- lib/wp2txt/bz2_validator.rb
+- lib/wp2txt/category_cache.rb
+- lib/wp2txt/cli.rb
+- lib/wp2txt/cli_ui.rb
+- lib/wp2txt/config.rb
+- lib/wp2txt/constants.rb
+- lib/wp2txt/data/html_entities.json
+- lib/wp2txt/data/language_metadata.json
+- lib/wp2txt/data/language_tiers.json
+- lib/wp2txt/data/mediawiki_aliases.json
+- lib/wp2txt/data/template_aliases.json
+- lib/wp2txt/data/wikipedia_entities.json
+- lib/wp2txt/extractor.rb
+- lib/wp2txt/file_utils.rb
+- lib/wp2txt/formatter.rb
+- lib/wp2txt/global_data_cache.rb
+- lib/wp2txt/index_cache.rb
+- lib/wp2txt/magic_words.rb
+- lib/wp2txt/memory_monitor.rb
+- lib/wp2txt/multistream.rb
+- lib/wp2txt/output_writer.rb
+- lib/wp2txt/parser_functions.rb
+- lib/wp2txt/ractor_worker.rb
 - lib/wp2txt/regex.rb
+- lib/wp2txt/section_extractor.rb
+- lib/wp2txt/stream_processor.rb
+- lib/wp2txt/template_expander.rb
+- lib/wp2txt/text_processing.rb
 - lib/wp2txt/utils.rb
 - lib/wp2txt/version.rb
+- scripts/benchmark_regex.rb
+- scripts/fetch_html_entities.rb
+- scripts/fetch_language_metadata.rb
+- scripts/fetch_mediawiki_data.rb
+- scripts/fetch_template_data.rb
+- scripts/profile_memory.rb
+- spec/article_spec.rb
+- spec/auto_download_spec.rb
+- spec/bz2_validator_spec.rb
+- spec/category_cache_spec.rb
+- spec/category_fetcher_spec.rb
+- spec/cleanup_spec.rb
+- spec/cli_options_spec.rb
+- spec/cli_spec.rb
+- spec/config_spec.rb
+- spec/constants_spec.rb
+- spec/file_utils_spec.rb
+- spec/fixtures/samples.rb
+- spec/formatter_sections_spec.rb
+- spec/global_data_cache_spec.rb
+- spec/index_cache_spec.rb
+- spec/integration_spec.rb
+- spec/magic_words_spec.rb
+- spec/markers_spec.rb
+- spec/memory_monitor_spec.rb
+- spec/multistream_spec.rb
+- spec/output_writer_spec.rb
+- spec/parser_functions_spec.rb
+- spec/ractor_worker_spec.rb
+- spec/regex_spec.rb
+- spec/section_extractor_spec.rb
 - spec/spec_helper.rb
+- spec/stream_processor_spec.rb
+- spec/template_data_spec.rb
+- spec/template_expander_spec.rb
+- spec/template_processing_spec.rb
+- spec/text_processing_spec.rb
 - spec/utils_spec.rb
+- spec/wp2txt_spec.rb
 - wp2txt.gemspec
 homepage: https://github.com/yohasebe/wp2txt
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -201,18 +299,48 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.6'
+      version: '3.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.12
-signing_key:
+rubygems_version: 4.0.3
 specification_version: 4
 summary: A command-line toolkit to extract text content and category data from Wikipedia
   dump files
 test_files:
+- spec/article_spec.rb
+- spec/auto_download_spec.rb
+- spec/bz2_validator_spec.rb
+- spec/category_cache_spec.rb
+- spec/category_fetcher_spec.rb
+- spec/cleanup_spec.rb
+- spec/cli_options_spec.rb
+- spec/cli_spec.rb
+- spec/config_spec.rb
+- spec/constants_spec.rb
+- spec/file_utils_spec.rb
+- spec/fixtures/samples.rb
+- spec/formatter_sections_spec.rb
+- spec/global_data_cache_spec.rb
+- spec/index_cache_spec.rb
+- spec/integration_spec.rb
+- spec/magic_words_spec.rb
+- spec/markers_spec.rb
+- spec/memory_monitor_spec.rb
+- spec/multistream_spec.rb
+- spec/output_writer_spec.rb
+- spec/parser_functions_spec.rb
+- spec/ractor_worker_spec.rb
+- spec/regex_spec.rb
+- spec/section_extractor_spec.rb
 - spec/spec_helper.rb
+- spec/stream_processor_spec.rb
+- spec/template_data_spec.rb
+- spec/template_expander_spec.rb
+- spec/template_processing_spec.rb
+- spec/text_processing_spec.rb
 - spec/utils_spec.rb
+- spec/wp2txt_spec.rb

data/.rubocop.yml

@@ -1,80 +0,0 @@
-AllCops:
-  NewCops: disable
-  SuggestExtensions: false
-  TargetRubyVersion: 2.6
-
-Documentation:
-  Enabled: false
-
-Naming/AccessorMethodName:
-  Enabled: false
-
-Naming/VariableNumber:
-  Enabled: false
-
-Naming/FileName:
-  Enabled: false
-
-Security/MarshalLoad:
-  Enabled: false
-
-Security/Open:
-  Enabled: false
-
-Layout/EndOfLine:
-   Enabled: False
-
-Style/FormatStringToken:
-   Enabled: false
-
-Style/ClassVars:
-  Enabled: false
-
-Style/OptionalBooleanParameter:
-  Enabled: false
-
-Style/StringConcatenation:
-  Enabled: false
-
-Style/PerlBackrefs:
-  Enabled: false
-
-Style/StringLiterals:
-  Enabled: false
-
-Style/StringLiteralsInInterpolation:
-  Enabled: true
-  EnforcedStyle: double_quotes
-
-Style/WordArray:
-  Enabled: false
-
-Style/EvalWithLocation:
-  Enabled: false
-
-Layout/LineLength:
-  Max: 400
-
-Metrics/MethodLength:
-  Max: 200
-
-Metrics/BlockLength:
-  Max: 200
-
-Metrics/AbcSize:
-  Max: 200
-
-Metrics/PerceivedComplexity:
-  Max: 60
-
-Metrics/ClassLength:
-  Max: 800
-
-Metrics/CyclomaticComplexity:
-  Max: 60
-
-Metrics/ParameterLists:
-  Max: 8
-
-Metrics/ModuleLength:
-  Max: 600