httpspell 1.2.1 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dcf823c527b930c3048fd15e17d0106af6aee7c9375b685d884c8690e285b484
-  data.tar.gz: 123fcd42841af054627d69dfa3d4a655e0c97bb5eeeec96fd4b2aa46be9eedd3
+  metadata.gz: e6245dacfbe0f2b1f7a95e95118a058e592f0fcbe88407b804e756bd41691054
+  data.tar.gz: 731ddea385cdca5f40c14c10289583b1544b9055bd82015ce2c5beb3b9495b52
 SHA512:
-  metadata.gz: a03721c821750a533eacb9f2e53148d2afb0489a18ee278ad3b9973424ea043973c5989f3a052764fb3611177bb3561ddfc287bca80f9351abcd63ea9f0638cb
-  data.tar.gz: 9ee1bef8ed9d0b1bdd2e0a2aeb74a28f764cd21c422430802a11cf055d119aca683b64fb26ab4b3ac92f7ead13f920bff7b368fc6c4e081bdfc1bb9acd8d51bf
+  metadata.gz: 2295d29b287812f6f1330d4cc17a5eae9ab3a14e99bb9ab4d4215761d3146a3f27b8ddcb66331999156c68a1ac58e2ac33b47d7043b7a2771e56faca03eb7b62
+  data.tar.gz: 6cbab0f82ae37c684a6bf4ad60e30a7b5a9921e5d90956bd4baa088e4190c9e8b2b8359e41c457dc6d594c962e43b13e3483046b0208f5bb5273d0928e74950f

data/.github/dependabot.yml

@@ -0,0 +1,14 @@
+version: 2
+updates:
+- package-ecosystem: bundler
+  directory: "/"
+  schedule:
+    interval: daily
+    time: "09:00"
+  open-pull-requests-limit: 10
+  ignore:
+  - dependency-name: rubocop
+    versions:
+    - 1.10.0
+    - 1.11.0
+    - 1.9.0

data/.gitignore

@@ -1 +1,3 @@
+.rake_tasks
 pkg
+spec/dictionaries

data/.mergify.yml

@@ -0,0 +1,8 @@
+pull_request_rules:
+  - name: automatic merge for Dependabot pull requests
+    conditions:
+      - author~=^dependabot(|-preview)\[bot\]$
+      - check-success=Travis CI - Pull Request
+    actions:
+      merge:
+        method: merge

data/.rubocop.yml

@@ -1,5 +1,6 @@
 AllCops:
-  TargetRubyVersion: 2.5.1
+  NewCops: enable
+  TargetRubyVersion: 3.3
   Include:
     - '**/Gemfile'
     - '**/Rakefile'
@@ -8,23 +9,17 @@ AllCops:
   Exclude:
     - vendor/**/*
     - db/migrations/**/*
-
   DisplayCopNames:
     Enabled: true
-
   DisplayStyleGuide:
     Enabled: true
-
 Naming/FileName:
   Exclude:
-  - Guardfile
-
+    - Guardfile
 Metrics/BlockLength:
   Exclude:
     - spec/**/*
-
-Metrics/LineLength:
+Layout/LineLength:
   Max: 160
-
 Style/Documentation:
   Enabled: false

data/.ruby-version

@@ -0,0 +1 @@
+ruby-3.3.1

data/Gemfile

@@ -2,3 +2,20 @@
 
 source 'https://rubygems.org'
 gemspec
+
+group :development do
+  gem 'aruba'
+  gem 'bundler'
+  gem 'guard'
+  gem 'guard-bundler'
+  gem 'guard-rspec'
+  gem 'httpx'
+  gem 'pry'
+  gem 'pry-byebug'
+  gem 'rake'
+  gem 'rspec'
+  gem 'rubocop'
+  gem 'rubocop-rake'
+  gem 'rubocop-rspec'
+  gem 'stub_server'
+end

data/Gemfile.lock

@@ -1,61 +1,66 @@
 PATH
   remote: .
   specs:
-    httpspell (1.2.1)
+    httpspell (1.4.0)
       addressable
       nokogiri
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.5.2)
-      public_suffix (>= 2.0.2, < 4.0)
-    aruba (0.14.5)
-      childprocess (>= 0.6.3, < 0.10.0)
-      contracts (~> 0.9)
-      cucumber (>= 1.3.19)
-      ffi (~> 1.9.10)
-      rspec-expectations (>= 2.99)
-      thor (~> 0.19)
-    ast (2.4.0)
-    backports (3.11.3)
-    builder (3.2.3)
-    byebug (10.0.2)
-    childprocess (0.9.0)
-      ffi (~> 1.0, >= 1.0.11)
-    coderay (1.1.2)
-    contracts (0.16.0)
-    cucumber (3.1.0)
-      builder (>= 2.1.2)
-      cucumber-core (~> 3.1.0)
-      cucumber-expressions (~> 5.0.4)
-      cucumber-wire (~> 0.0.1)
-      diff-lcs (~> 1.3)
-      gherkin (~> 5.0)
-      multi_json (>= 1.7.5, < 2.0)
-      multi_test (>= 0.1.2)
-    cucumber-core (3.1.0)
-      backports (>= 3.8.0)
-      cucumber-tag_expressions (~> 1.1.0)
-      gherkin (>= 5.0.0)
-    cucumber-expressions (5.0.18)
-    cucumber-tag_expressions (1.1.1)
-    cucumber-wire (0.0.1)
-    diff-lcs (1.3)
-    ffi (1.9.23)
-    formatador (0.2.5)
-    gherkin (5.1.0)
-    guard (2.14.2)
+    addressable (2.8.6)
+      public_suffix (>= 2.0.2, < 6.0)
+    aruba (2.2.0)
+      bundler (>= 1.17, < 3.0)
+      contracts (>= 0.16.0, < 0.18.0)
+      cucumber (>= 8.0, < 10.0)
+      rspec-expectations (~> 3.4)
+      thor (~> 1.0)
+    ast (2.4.2)
+    bigdecimal (3.1.8)
+    builder (3.2.4)
+    byebug (11.1.3)
+    coderay (1.1.3)
+    contracts (0.17)
+    cucumber (9.2.0)
+      builder (~> 3.2)
+      cucumber-ci-environment (> 9, < 11)
+      cucumber-core (> 13, < 14)
+      cucumber-cucumber-expressions (~> 17.0)
+      cucumber-gherkin (> 24, < 28)
+      cucumber-html-formatter (> 20.3, < 22)
+      cucumber-messages (> 19, < 25)
+      diff-lcs (~> 1.5)
+      mini_mime (~> 1.1)
+      multi_test (~> 1.1)
+      sys-uname (~> 1.2)
+    cucumber-ci-environment (10.0.1)
+    cucumber-core (13.0.2)
+      cucumber-gherkin (>= 27, < 28)
+      cucumber-messages (>= 20, < 23)
+      cucumber-tag-expressions (> 5, < 7)
+    cucumber-cucumber-expressions (17.1.0)
+      bigdecimal
+    cucumber-gherkin (27.0.0)
+      cucumber-messages (>= 19.1.4, < 23)
+    cucumber-html-formatter (21.3.1)
+      cucumber-messages (> 19, < 25)
+    cucumber-messages (22.0.0)
+    cucumber-tag-expressions (6.1.0)
+    diff-lcs (1.5.1)
+    ffi (1.16.3)
+    formatador (1.1.0)
+    guard (2.18.1)
       formatador (>= 0.2.4)
       listen (>= 2.7, < 4.0)
       lumberjack (>= 1.0.12, < 2.0)
       nenv (~> 0.1)
       notiffany (~> 0.0)
-      pry (>= 0.9.12)
+      pry (>= 0.13.0)
       shellany (~> 0.0)
       thor (>= 0.18.1)
-    guard-bundler (2.1.0)
-      bundler (~> 1.0)
+    guard-bundler (3.0.1)
+      bundler (>= 2.1, < 3)
       guard (~> 2.2)
       guard-compat (~> 1.1)
     guard-compat (1.2.1)
@@ -63,68 +68,108 @@ GEM
       guard (~> 2.1)
       guard-compat (~> 1.1)
       rspec (>= 2.99.0, < 4.0)
-    listen (3.1.5)
-      rb-fsevent (~> 0.9, >= 0.9.4)
-      rb-inotify (~> 0.9, >= 0.9.7)
-      ruby_dep (~> 1.2)
-    lumberjack (1.0.13)
-    method_source (0.9.0)
-    mini_portile2 (2.3.0)
-    multi_json (1.13.1)
-    multi_test (0.1.2)
+    http-2-next (1.0.3)
+    httpx (1.2.5)
+      http-2-next (>= 1.0.3)
+    json (2.7.2)
+    language_server-protocol (3.17.0.3)
+    listen (3.9.0)
+      rb-fsevent (~> 0.10, >= 0.10.3)
+      rb-inotify (~> 0.9, >= 0.9.10)
+    lumberjack (1.2.10)
+    method_source (1.1.0)
+    mini_mime (1.1.5)
+    multi_test (1.1.0)
     nenv (0.3.0)
-    nokogiri (1.8.2)
-      mini_portile2 (~> 2.3.0)
-    notiffany (0.1.1)
+    nokogiri (1.16.5-arm64-darwin)
+      racc (~> 1.4)
+    nokogiri (1.16.5-x86_64-darwin)
+      racc (~> 1.4)
+    nokogiri (1.16.5-x86_64-linux)
+      racc (~> 1.4)
+    notiffany (0.1.3)
       nenv (~> 0.1)
       shellany (~> 0.0)
-    parallel (1.12.1)
-    parser (2.5.1.0)
-      ast (~> 2.4.0)
-    powerpack (0.1.1)
-    pry (0.11.3)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-    pry-byebug (3.6.0)
-      byebug (~> 10.0)
-      pry (~> 0.10)
-    public_suffix (3.0.2)
-    rack (2.0.5)
-    rainbow (3.0.0)
-    rake (12.3.1)
-    rb-fsevent (0.10.3)
-    rb-inotify (0.9.10)
-      ffi (>= 0.5.0, < 2)
-    rspec (3.7.0)
-      rspec-core (~> 3.7.0)
-      rspec-expectations (~> 3.7.0)
-      rspec-mocks (~> 3.7.0)
-    rspec-core (3.7.1)
-      rspec-support (~> 3.7.0)
-    rspec-expectations (3.7.0)
+    parallel (1.24.0)
+    parser (3.3.1.0)
+      ast (~> 2.4.1)
+      racc
+    pry (0.14.2)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    pry-byebug (3.10.1)
+      byebug (~> 11.0)
+      pry (>= 0.13, < 0.15)
+    public_suffix (5.0.5)
+    racc (1.8.0)
+    rack (3.0.11)
+    rackup (0.2.3)
+      rack (>= 3.0.0.beta1)
+      webrick
+    rainbow (3.1.1)
+    rake (13.2.1)
+    rb-fsevent (0.11.2)
+    rb-inotify (0.11.1)
+      ffi (~> 1.0)
+    regexp_parser (2.9.2)
+    rexml (3.2.8)
+      strscan (>= 3.0.9)
+    rspec (3.13.0)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.0)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-mocks (3.7.0)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-support (3.7.1)
-    rubocop (0.56.0)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.1)
+    rubocop (1.64.0)
+      json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 2.5)
-      powerpack (~> 0.1)
+      parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (~> 1.0, >= 1.0.1)
-    ruby-progressbar (1.9.0)
-    ruby_dep (1.5.0)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.31.3)
+      parser (>= 3.3.1.0)
+    rubocop-capybara (2.20.0)
+      rubocop (~> 1.41)
+    rubocop-factory_bot (2.25.1)
+      rubocop (~> 1.41)
+    rubocop-rake (0.6.0)
+      rubocop (~> 1.0)
+    rubocop-rspec (2.29.2)
+      rubocop (~> 1.40)
+      rubocop-capybara (~> 2.17)
+      rubocop-factory_bot (~> 2.22)
+      rubocop-rspec_rails (~> 2.28)
+    rubocop-rspec_rails (2.28.3)
+      rubocop (~> 1.40)
+    ruby-progressbar (1.13.0)
     shellany (0.0.1)
-    stub_server (0.3.0)
-      rack
-    thor (0.20.0)
-    unicode-display_width (1.3.3)
+    strscan (3.1.0)
+    stub_server (0.7.0)
+      rackup (~> 0.2.2)
+      webrick
+    sys-uname (1.2.3)
+      ffi (~> 1.1)
+    thor (1.3.1)
+    unicode-display_width (2.5.0)
+    webrick (1.8.1)
 
 PLATFORMS
-  ruby
+  arm64-darwin-22
+  arm64-darwin-23
+  x86_64-darwin-21
+  x86_64-linux
 
 DEPENDENCIES
   aruba
@@ -133,12 +178,15 @@ DEPENDENCIES
   guard-bundler
   guard-rspec
   httpspell!
+  httpx
   pry
   pry-byebug
   rake
   rspec
   rubocop
+  rubocop-rake
+  rubocop-rspec
   stub_server
 
 BUNDLED WITH
-   1.16.1
+   2.5.9

data/Guardfile

@@ -0,0 +1,25 @@
+guard :bundler do
+  require 'guard/bundler'
+  require 'guard/bundler/verify'
+  helper = Guard::Bundler::Verify.new
+
+  files = ['Gemfile']
+  files += Dir['*.gemspec'] if files.any? { |f| helper.uses_gemspec?(f) }
+
+  # Assume files are symlinked from somewhere
+  files.each { |file| watch(helper.real_path(file)) }
+end
+
+guard :rspec, cmd: "bundle exec rspec" do
+  require "guard/rspec/dsl"
+  dsl = Guard::RSpec::Dsl.new(self)
+
+  rspec = dsl.rspec
+  watch(rspec.spec_helper) { rspec.spec_dir }
+  watch(rspec.spec_support) { rspec.spec_dir }
+  watch(rspec.spec_files)
+
+  # Ruby files
+  ruby = dsl.ruby
+  dsl.watch_spec_files_for(ruby.lib_files)
+end

data/README.markdown

@@ -1,7 +1,5 @@
 # `httpspell`
 
-[![Build Status](https://travis-ci.org/suhlig/httpspell.svg?branch=master)](https://travis-ci.org/suhlig/httpspell)
-
 This is a spellchecker that recursively fetches HTML pages, converts them to plain text (using [pandoc](http://pandoc.org/)), and spellchecks them with [hunspell](https://hunspell.github.io/). Unknown words will be printed to `stdout`, which makes the tool a good candidate for CI pipelines where you might want to take action when a spelling error is found on a web page.
 
 Words that are not in the dictionary for the given language (inferred from the `lang` attribute of the HTML document's root element) can be added to a personal dictionary, which will mark the word as correctly spelled.
@@ -46,3 +44,26 @@ If you produce content with kramdown (e.g. using Jekyll), setting `spellcheck='f
 ```
 {: spellcheck="false"}
 ```
+
+# Dictionaries
+
+Hunspell uses the system dictionary paths; on the Mac this is `~/Library/Spelling/`. Get some dictionaries as explained in the [hunspell](https://github.com/hunspell/hunspell) project:
+
+```command
+$ wget -O ~/Library/Spelling/en_US.aff https://cgit.freedesktop.org/libreoffice/dictionaries/plain/en/en_US.aff
+$ wget -O ~/Library/Spelling/en_US.dic https://cgit.freedesktop.org/libreoffice/dictionaries/plain/en/en_US.dic
+```
+
+German:
+
+```command
+$ wget -O ~/Library/Spelling/de_DE.dic https://cgit.freedesktop.org/libreoffice/dictionaries/plain/de/de_DE_frami.dic
+$ wget -O ~/Library/Spelling/de_DE.aff https://cgit.freedesktop.org/libreoffice/dictionaries/plain/de/de_DE_frami.aff
+```
+
+Italian (for integration tests):
+
+```command
+$ wget -O ~/Library/Spelling/it_IT.dic https://cgit.freedesktop.org/libreoffice/dictionaries/plain/it_IT/it_IT.dic
+$ wget -O ~/Library/Spelling/it_IT.aff https://cgit.freedesktop.org/libreoffice/dictionaries/plain/it_IT/it_IT.aff
+```

data/Rakefile

@@ -10,7 +10,7 @@ task default: ['spec:all']
 
 namespace :spec do
   desc 'Run all specs'
-  task all: %i[rubocop:auto_correct unit system]
+  task all: %i[rubocop:autocorrect unit system]
 
   %w[unit system].each do |type|
     desc "Run #{type} tests"

data/TODO.markdown

@@ -0,0 +1,5 @@
+* Bail out if lang cannot be inferred and is not given on cmdline
+* exe/httpspell:    # TODO: --recursive, defaults to false
+* exe/httpspell:    # TODO wget has some additional options for recursive behavior that should be reviewed
+* lib/httpspell/spider.rb:          # TODO Print _which_ entry of the blacklist matches
+* lib/httpspell/spider.rb:        # TODO Ignore same page links (some anchor)

data/exe/httpspell

@@ -59,27 +59,38 @@ if ARGV.size != 1
   exit 1
 end
 
-spell_checker = HttpSpell::SpellChecker.new(personal_dictionary_path, tracing: tracing)
+def check(doc, lang, personal_dictionary_path, verbose)
+  unknown_words = HttpSpell::SpellChecker.new(personal_dictionary_path, verbose:).check(doc, lang)
+
+  if unknown_words.empty?
+    warn 'No unknown words.' if verbose
+  else
+    warn "#{unknown_words.size} unknown words:" if verbose
+    puts unknown_words
+    true
+  end
+end
+
 has_unknown_words = false
 
-spider_success = HttpSpell::Spider.new(ARGV.first, whitelist: whitelist, blacklist: blacklist, tracing: tracing).start do |url, doc|
-  lang = force_language || doc.root['lang'] || ENV['LANGUAGE']
+spider_success = HttpSpell::Spider.new(ARGV.first, whitelist:, blacklist:, verbose:, tracing:).start do |url, doc|
+  lang = force_language || doc.root['lang'] || ENV.fetch('LANGUAGE', nil)
+  warn "Checking #{url} as #{lang}" if verbose
 
-  # Remove sections that are not to be spellchecked
+  # Remove elements that are not to be spellchecked
   doc.css('pre').each(&:unlink)
   doc.css('code').each(&:unlink)
   doc.css('[spellcheck=false]').each(&:unlink)
 
-  # TODO: Find sections with a lang attribute and handle them separately
-  unknown_words = spell_checker.check(doc.to_s, lang)
-
-  if unknown_words.empty?
-    warn "No unknown words (language is #{lang}) at #{url}." if verbose
-  else
-    warn "#{unknown_words.size} unknown words (language is #{lang}) at #{url}:" if verbose
-    puts unknown_words
-    has_unknown_words = true
+  # Handle elements with a different lang attribute separately
+  doc.css(%([lang]:not([lang="#{lang}"]))).each do |element|
+    warn "Handling #{element.name} with lang #{element['lang']}:" if verbose
+    has_unknown_words |= check(element.to_s, element['lang'], personal_dictionary_path, verbose)
+    element.unlink
   end
+
+  # Everything else
+  has_unknown_words |= check(doc.to_s, lang, personal_dictionary_path, verbose)
 end
 
 exit 2 unless spider_success

data/httpspell.gemspec

@@ -4,7 +4,6 @@ lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'httpspell/version'
 
-# rubocop:disable Metrics/BlockLength
 Gem::Specification.new do |spec|
   spec.name          = 'httpspell'
   spec.version       = HttpSpell::VERSION
@@ -13,8 +12,8 @@ Gem::Specification.new do |spec|
 
   spec.summary       = 'HTTP spellchecker'
   spec.description   = %(httpspell is a spellchecker that recursively fetches
-    HTML pages, converts them to plain text using pandoc, and
-    spellchecks them with hunspell.)
+  HTML pages, converts them to plain text using pandoc, and
+  spellchecks them with hunspell.)
   spec.license       = 'MIT'
 
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
@@ -26,17 +25,5 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'addressable'
   spec.add_dependency 'nokogiri'
-
-  spec.add_development_dependency 'aruba'
-  spec.add_development_dependency 'bundler'
-  spec.add_development_dependency 'guard'
-  spec.add_development_dependency 'guard-bundler'
-  spec.add_development_dependency 'guard-rspec'
-  spec.add_development_dependency 'pry'
-  spec.add_development_dependency 'pry-byebug'
-  spec.add_development_dependency 'rake'
-  spec.add_development_dependency 'rspec'
-  spec.add_development_dependency 'rubocop'
-  spec.add_development_dependency 'stub_server'
+  spec.metadata['rubygems_mfa_required'] = 'true'
 end
-# rubocop:enable Metrics/BlockLength

data/lib/httpspell/spellchecker.rb

@@ -1,8 +1,8 @@
 module HttpSpell
   class SpellChecker
-    def initialize(personal_dictionary_path = nil, tracing: false)
+    def initialize(personal_dictionary_path = nil, verbose: false)
       @personal_dictionary_arg = "-p #{personal_dictionary_path}" if personal_dictionary_path
-      @tracing = tracing
+      @verbose = verbose
     end
 
     def check(doc, lang)
@@ -11,7 +11,7 @@ module HttpSpell
         "hunspell -d #{translate(lang)} #{@personal_dictionary_arg} -i UTF-8 -l",
       ]
 
-      if @tracing
+      if @verbose
         warn "Piping the HTML document into the following chain of commands:"
         warn commands
       end

data/lib/httpspell/spider.rb

@@ -8,12 +8,13 @@ module HttpSpell
   class Spider
     attr_reader :todo, :done
 
-    def initialize(starting_point, whitelist: nil, blacklist: [], tracing: false)
+    def initialize(starting_point, whitelist: nil, blacklist: [], verbose: false, tracing: false)
       @todo = []
       @done = []
       todo << Addressable::URI.parse(starting_point)
       @whitelist = whitelist || [/^#{starting_point}/]
       @blacklist = blacklist
+      @verbose = verbose
       @tracing = tracing
     end
 
@@ -46,10 +47,10 @@ module HttpSpell
     private
 
     def links(uri)
-      response = URI(uri).read # We are using open-uri, which follows redirects and also provides the content-type.
+      response = http_get(uri)
 
-      if response.content_type != 'text/html'
-        warn "Skipping #{uri} because it is not HTML" if @tracing
+      if response.respond_to?(:content_type) && response.content_type != 'text/html'
+        warn "Skipping #{uri} because it is not HTML" if @verbose
         return []
       end
 
@@ -60,13 +61,13 @@ module HttpSpell
         link = uri.join(link) if link.relative?
 
         if @whitelist.none? { |re| re.match?(link.to_s) }
-          warn "Skipping #{link} because it is not on the whitelist #{@whitelist}" if @tracing
+          warn "Skipping #{link} because it is not on the whitelist #{@whitelist}" if @verbose
           next
         end
 
         if @blacklist.any? { |re| re.match?(link.to_s) }
           # TODO Print _which_ entry of the blacklist matches
-          warn "Skipping #{link} because it is on the blacklist #{@blacklist}" if @tracing
+          warn "Skipping #{link} because it is on the blacklist #{@blacklist}" if @verbose
           next
         end
 
@@ -79,8 +80,21 @@ module HttpSpell
 
       yield uri, doc if block_given?
 
-      warn "Adding #{links.size} links from #{uri}" if @tracing
+      warn "Adding #{links.size} links from #{uri}" if @verbose
       links
     end
+
+    # https://twin.github.io/improving-open-uri/
+    def http_get(uri)
+      tries = 10
+
+      begin
+        URI.open(uri, redirect: false)
+      rescue OpenURI::HTTPRedirect => redirect
+        uri = redirect.uri
+        retry if (tries -= 1) > 0
+        raise
+      end
+    end
   end
 end

data/lib/httpspell/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HttpSpell
-  VERSION = '1.2.1'
+  VERSION = '1.4.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: httpspell
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.4.0
 platform: ruby
 authors:
 - Steffen Uhlig
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2018-06-01 00:00:00.000000000 Z
+date: 2024-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -38,164 +38,10 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: aruba
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: guard
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: guard-bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: guard-rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: pry
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: pry-byebug
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rubocop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: stub_server
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: |-
   httpspell is a spellchecker that recursively fetches
-      HTML pages, converts them to plain text using pandoc, and
-      spellchecks them with hunspell.
+    HTML pages, converts them to plain text using pandoc, and
+    spellchecks them with hunspell.
 email:
 - steffen@familie-uhlig.net
 executables:
@@ -203,24 +49,29 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/dependabot.yml"
 - ".gitignore"
+- ".mergify.yml"
 - ".rspec"
 - ".rubocop.yml"
-- ".travis.yml"
+- ".ruby-version"
 - Gemfile
 - Gemfile.lock
+- Guardfile
 - README.markdown
 - Rakefile
+- TODO.markdown
 - exe/httpspell
 - httpspell.gemspec
 - lib/httpspell/spellchecker.rb
 - lib/httpspell/spider.rb
 - lib/httpspell/version.rb
-homepage: 
+homepage:
 licenses:
 - MIT
-metadata: {}
-post_install_message: 
+metadata:
+  rubygems_mfa_required: 'true'
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -235,9 +86,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.5.9
+signing_key:
 specification_version: 4
 summary: HTTP spellchecker
 test_files: []

data/.travis.yml

@@ -1,6 +0,0 @@
-language: ruby
-rvm:
-- 2.5.1
-before_install:
-- sudo apt-get -qq update
-- sudo apt-get install -y pandoc hunspell hunspell-en-us