httpspell 1.4.1 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dc09324c003c7b14e08fa255b7a31c0a9aeb143df033da9aea300619a47268ba
-  data.tar.gz: 6890352a3cef38e243e2506398d58736c8179c2e0443a2b6ff341165e724dba0
+  metadata.gz: d66cfcc88c0bc7e0e237033b8c76f1aaccc40f9aba3f68766d45204a2b133401
+  data.tar.gz: 6c488170f95d0f33fdcbc5c55f2416d654f2b4558214943f21d19e1220f2ad96
 SHA512:
-  metadata.gz: 826bb8e875b2f1584dd5c052ab9777e616e1da0d6844263589b027c3eabfb07955155e0c43b8b1b8dc253d720eba952e80330c38035fff53fc1943420dea7454
-  data.tar.gz: 7a4e3c9aaa586d4fbdc41971424cd5f064793ff18cba8d8606a452b3cee36070af44aa2f78ab307c71a613404cc1e490af1f56eca11068675183625f5360790e
+  metadata.gz: f493b3411cd162e4a714203b05277f26810d71a0c23e52d69c36297c19e03db7b97692c1e4928c8f8fb0b9bc9a59f04b9ae4113c5cdd309c1edcf2a493d68687
+  data.tar.gz: ed44c8adf0dcd63330e8e7f837d9f515fd6131e58ae02808a01ed430ef42e5ee258a8c8d5a6fb9552c6b2c0dc460966453d3f1cda6e359739a18b6a01a25cdeb

data/Gemfile.lock

@@ -1,15 +1,12 @@
 PATH
   remote: .
   specs:
-    httpspell (1.4.1)
-      addressable
+    httpspell (1.5.0)
       nokogiri
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.8.6)
-      public_suffix (>= 2.0.2, < 6.0)
     aruba (2.2.0)
       bundler (>= 1.17, < 3.0)
       contracts (>= 0.16.0, < 0.18.0)
@@ -91,7 +88,7 @@ GEM
       nenv (~> 0.1)
       shellany (~> 0.0)
     parallel (1.24.0)
-    parser (3.3.1.0)
+    parser (3.3.2.0)
       ast (~> 2.4.1)
       racc
     pry (0.14.2)
@@ -100,7 +97,6 @@ GEM
     pry-byebug (3.10.1)
       byebug (~> 11.0)
       pry (>= 0.13, < 0.15)
-    public_suffix (5.0.5)
     racc (1.8.0)
     rack (3.0.11)
     rackup (0.2.3)
@@ -127,7 +123,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.1)
-    rubocop (1.64.0)
+    rubocop (1.64.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)

data/TODO.markdown

@@ -1,4 +1,4 @@
 * Bail out if lang cannot be inferred and is not given on cmdline
 * exe/httpspell:    # TODO: --recursive, defaults to false
 * exe/httpspell:    # TODO wget has some additional options for recursive behavior that should be reviewed
-* lib/httpspell/spider.rb:          # TODO Print _which_ entry of the blacklist matches
+* lib/httpspell/spider.rb:          # TODO Print _which_ entry of the exclude list matches

data/exe/httpspell

@@ -10,8 +10,8 @@ personal_dictionary_path = nil
 force_language = nil
 tracing = nil
 verbose = nil
-whitelist = nil
-blacklist = []
+included = nil
+excluded = []
 
 begin
   OptionParser.new do |parser|
@@ -29,9 +29,9 @@ begin
       force_language = l
     end
 
-    parser.on('-w', '--whitelist=EXPRESSION', 'when recursively retrieving URLs, allow only those matching the given regular EXPRESSION') do |w|
-      whitelist ||= []
-      whitelist << Regexp.new(w)
+    parser.on('-i', '--include=EXPRESSION', 'when recursively retrieving URLs, allow only those matching the given regular EXPRESSION') do |w|
+      included ||= []
+      included << Regexp.new(w)
     end
 
     parser.on('-t', '--trace', 'enable error tracing') do
@@ -42,15 +42,15 @@ begin
       verbose = true
     end
 
-    parser.on('-b', '--blacklist=EXPRESSION', 'blacklist (ignore) URLs matching the given regular EXPRESSION') do |b|
-      blacklist << Regexp.new(b)
+    parser.on('-e', '--exclude=EXPRESSION', 'exclude URLs matching the given regular EXPRESSION') do |b|
+      excluded << Regexp.new(b)
     end
 
     # TODO: --recursive, defaults to false
     # TODO wget has some additional options for recursive behavior that should be reviewed
   end.parse!
 rescue StandardError
-  warn "Error - #{$ERROR_INFO}"
+  warn "Error: #{$ERROR_INFO}"
   exit 1
 end
 
@@ -59,13 +59,14 @@ if ARGV.size != 1
   exit 1
 end
 
-def check(doc, lang, personal_dictionary_path, verbose)
+def check(url, doc, lang, personal_dictionary_path, verbose)
   unknown_words = HttpSpell::SpellChecker.new(personal_dictionary_path, verbose:).check(doc, lang)
 
   if unknown_words.empty?
-    warn 'No unknown words.' if verbose
+    warn "#{url} (lang=#{lang}): No unknown words" if verbose
+    false
   else
-    warn "#{unknown_words.size} unknown words:" if verbose
+    warn "#{url} (lang=#{lang}): #{unknown_words.size} unknown words:" if verbose
     puts unknown_words
     true
   end
@@ -73,24 +74,23 @@ end
 
 has_unknown_words = false
 
-spider_success = HttpSpell::Spider.new(ARGV.first, whitelist:, blacklist:, verbose:, tracing:).start do |url, doc|
+spider_success = HttpSpell::Spider.new(ARGV.first, included:, excluded:, verbose:, tracing:).start do |url, doc|
   lang = force_language || doc.root['lang'] || ENV.fetch('LANGUAGE', nil)
-  warn "Checking #{url} as #{lang}" if verbose
 
   # Remove elements that are not to be spellchecked
   doc.css('pre').each(&:unlink)
   doc.css('code').each(&:unlink)
+  doc.css('iframe').each(&:unlink)
   doc.css('[spellcheck=false]').each(&:unlink)
 
   # Handle elements with a different lang attribute separately
   doc.css(%([lang]:not([lang="#{lang}"]))).each do |element|
-    warn "Handling #{element.name} with lang #{element['lang']}:" if verbose
-    has_unknown_words |= check(element.to_s, element['lang'], personal_dictionary_path, verbose)
+    has_unknown_words |= check("#{url} => #{element.name} with", element.to_s, element['lang'], personal_dictionary_path, verbose)
     element.unlink
   end
 
   # Everything else
-  has_unknown_words |= check(doc.to_s, lang, personal_dictionary_path, verbose)
+  has_unknown_words |= check("#{url} => document with", doc.to_s, lang, personal_dictionary_path, verbose)
 end
 
 exit 2 unless spider_success

data/httpspell.gemspec

@@ -23,7 +23,6 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'addressable'
   spec.add_dependency 'nokogiri'
   spec.metadata['rubygems_mfa_required'] = 'true'
 end

data/lib/http_spell/spellchecker.rb

@@ -13,11 +13,6 @@ module HttpSpell
         "hunspell -d #{translate(lang)} #{@personal_dictionary_arg} -i UTF-8 -l",
       ]
 
-      if @verbose
-        warn 'Piping the HTML document into the following chain of commands:'
-        warn commands
-      end
-
       Open3.pipeline_rw(*commands) do |stdin, stdout, _wait_thrs|
         stdin.puts(doc)
         stdin.close

data/lib/http_spell/spider.rb

@@ -1,21 +1,21 @@
 # frozen_string_literal: true
 
 require 'nokogiri'
+require 'uri'
 require 'open-uri'
 require 'open3'
-require 'addressable/uri'
 require 'English'
 
 module HttpSpell
   class Spider
     attr_reader :todo, :done
 
-    def initialize(starting_point, whitelist: nil, blacklist: [], verbose: false, tracing: false)
+    def initialize(starting_point, included: nil, excluded: [], verbose: false, tracing: false)
       @todo = []
       @done = []
-      todo << Addressable::URI.parse(starting_point)
-      @whitelist = whitelist || [/^#{starting_point}/]
-      @blacklist = blacklist
+      todo << URI(starting_point)
+      @included = included || [/^#{starting_point}/]
+      @excluded = excluded
       @verbose = verbose
       @tracing = tracing
     end
@@ -35,7 +35,12 @@ module HttpSpell
           end
 
           done.append(url)
-          todo.concat(extracted - done - todo).uniq!
+          new_links = (extracted - done - todo).uniq
+
+          if new_links.any?
+            warn "Adding #{new_links.size} new links found at #{url}" if @verbose
+            todo.concat(extracted - done - todo).uniq!
+          end
         rescue StandardError
           warn "Skipping #{url} because of #{$ERROR_INFO.message}"
           warn $ERROR_INFO.backtrace if @tracing
@@ -52,46 +57,43 @@ module HttpSpell
       response = http_get(uri)
 
       if response.respond_to?(:content_type) && response.content_type != 'text/html'
-        warn "Skipping #{uri} because it is not HTML" if @verbose
+        warn "Skipping #{response.base_uri} because it is not HTML" if @verbose
         return []
       end
 
       doc = Nokogiri::HTML(response)
 
       links = doc.css('a[href]').map do |e|
-        link = Addressable::URI.parse(e['href'])
-        link = uri.join(link) if link.relative?
+        next if e['href'].start_with?('#') # Ignore fragment on the same page; we always check the whole page
+
+        link = URI.join(response.base_uri, e['href'])
+        link.fragment = nil # Ignore fragment in links to other pages, too
 
-        if @whitelist.none? { |re| re.match?(link.to_s) }
-          warn "Skipping #{link} because it is not on the whitelist #{@whitelist}" if @verbose
+        if @included.none? { |re| re.match?(link.to_s) }
+          warn "Skipping #{link} because it is not on the included #{@included}" if @verbose
           next
         end
 
-        if @blacklist.any? { |re| re.match?(link.to_s) }
-          # TODO: Print _which_ entry of the blacklist matches
-          warn "Skipping #{link} because it is on the blacklist #{@blacklist}" if @verbose
+        if @excluded.any? { |re| re.match?(link.to_s) }
+          # TODO: Print _which_ entry of the excluded matches
+          warn "Skipping #{link} because it is on the excluded #{@excluded}" if @verbose
           next
         end
 
-        # Ignore fragment; we always check the whole page
-        link.fragment = nil
-
         link
       rescue StandardError
-        warn $ERROR_INFO.message
+        warn "Error: #{$ERROR_INFO}"
         warn $ERROR_INFO.backtrace if @tracing
       end.compact
 
-      yield uri, doc if block_given?
+      yield response.base_uri, doc if block_given?
 
-      warn "Adding #{links.size} links from #{uri}" if @verbose
       links
     end
 
     # https://twin.github.io/improving-open-uri/
     def http_get(uri)
       tries = 10
-
       begin
         URI.parse(uri).open(redirect: false)
       rescue OpenURI::HTTPRedirect => e

data/lib/http_spell/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HttpSpell
-  VERSION = '1.4.1'
+  VERSION = '1.5.0'
 end

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: httpspell
 version: !ruby/object:Gem::Version
-  version: 1.4.1
+  version: 1.5.0
 platform: ruby
 authors:
 - Steffen Uhlig
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-05-30 00:00:00.000000000 Z
+date: 2024-05-31 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: addressable
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement