httpspell 1.4.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6245dacfbe0f2b1f7a95e95118a058e592f0fcbe88407b804e756bd41691054
-  data.tar.gz: 731ddea385cdca5f40c14c10289583b1544b9055bd82015ce2c5beb3b9495b52
+  metadata.gz: d66cfcc88c0bc7e0e237033b8c76f1aaccc40f9aba3f68766d45204a2b133401
+  data.tar.gz: 6c488170f95d0f33fdcbc5c55f2416d654f2b4558214943f21d19e1220f2ad96
 SHA512:
-  metadata.gz: 2295d29b287812f6f1330d4cc17a5eae9ab3a14e99bb9ab4d4215761d3146a3f27b8ddcb66331999156c68a1ac58e2ac33b47d7043b7a2771e56faca03eb7b62
-  data.tar.gz: 6cbab0f82ae37c684a6bf4ad60e30a7b5a9921e5d90956bd4baa088e4190c9e8b2b8359e41c457dc6d594c962e43b13e3483046b0208f5bb5273d0928e74950f
+  metadata.gz: f493b3411cd162e4a714203b05277f26810d71a0c23e52d69c36297c19e03db7b97692c1e4928c8f8fb0b9bc9a59f04b9ae4113c5cdd309c1edcf2a493d68687
+  data.tar.gz: ed44c8adf0dcd63330e8e7f837d9f515fd6131e58ae02808a01ed430ef42e5ee258a8c8d5a6fb9552c6b2c0dc460966453d3f1cda6e359739a18b6a01a25cdeb

data/.rubocop.yml

@@ -1,7 +1,11 @@
+require:
+  - rubocop-rake
+  - rubocop-rspec
 AllCops:
   NewCops: enable
   TargetRubyVersion: 3.3
   Include:
+    - '**/*.rb'
     - '**/Gemfile'
     - '**/Rakefile'
     - '**/config.ru'
@@ -23,3 +27,19 @@ Layout/LineLength:
   Max: 160
 Style/Documentation:
   Enabled: false
+Metrics/AbcSize:
+  Enabled: false
+Metrics/MethodLength:
+  Enabled: false
+Metrics/CyclomaticComplexity:
+  Enabled: false
+Style/TrailingCommaInArrayLiteral:
+  Enabled: false
+RSpec/ExampleWording:
+  Enabled: false
+RSpec/InstanceVariable:
+  AssignmentOnly: true
+RSpec/ExampleLength:
+  Max: 10
+Metrics/PerceivedComplexity:
+  Max: 16

data/Gemfile.lock

@@ -1,15 +1,12 @@
 PATH
   remote: .
   specs:
-    httpspell (1.4.0)
-      addressable
+    httpspell (1.5.0)
       nokogiri
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.8.6)
-      public_suffix (>= 2.0.2, < 6.0)
     aruba (2.2.0)
       bundler (>= 1.17, < 3.0)
       contracts (>= 0.16.0, < 0.18.0)
@@ -91,7 +88,7 @@ GEM
       nenv (~> 0.1)
       shellany (~> 0.0)
     parallel (1.24.0)
-    parser (3.3.1.0)
+    parser (3.3.2.0)
       ast (~> 2.4.1)
       racc
     pry (0.14.2)
@@ -100,7 +97,6 @@ GEM
     pry-byebug (3.10.1)
       byebug (~> 11.0)
       pry (>= 0.13, < 0.15)
-    public_suffix (5.0.5)
     racc (1.8.0)
     rack (3.0.11)
     rackup (0.2.3)
@@ -127,7 +123,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.1)
-    rubocop (1.64.0)
+    rubocop (1.64.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)

data/TODO.markdown

@@ -1,5 +1,4 @@
 * Bail out if lang cannot be inferred and is not given on cmdline
 * exe/httpspell:    # TODO: --recursive, defaults to false
 * exe/httpspell:    # TODO wget has some additional options for recursive behavior that should be reviewed
-* lib/httpspell/spider.rb:          # TODO Print _which_ entry of the blacklist matches
-* lib/httpspell/spider.rb:        # TODO Ignore same page links (some anchor)
+* lib/httpspell/spider.rb:          # TODO Print _which_ entry of the exclude list matches

data/exe/httpspell

@@ -2,16 +2,16 @@
 # frozen_string_literal: true
 
 require 'optparse'
-require 'httpspell/spider'
-require 'httpspell/spellchecker'
-require 'httpspell/version'
+require 'http_spell/spider'
+require 'http_spell/spellchecker'
+require 'http_spell/version'
 
 personal_dictionary_path = nil
 force_language = nil
 tracing = nil
 verbose = nil
-whitelist = nil
-blacklist = []
+included = nil
+excluded = []
 
 begin
   OptionParser.new do |parser|
@@ -29,9 +29,9 @@ begin
       force_language = l
     end
 
-    parser.on('-w', '--whitelist=EXPRESSION', 'when recursively retrieving URLs, allow only those matching the given regular EXPRESSION') do |w|
-      whitelist ||= []
-      whitelist << Regexp.new(w)
+    parser.on('-i', '--include=EXPRESSION', 'when recursively retrieving URLs, allow only those matching the given regular EXPRESSION') do |w|
+      included ||= []
+      included << Regexp.new(w)
     end
 
     parser.on('-t', '--trace', 'enable error tracing') do
@@ -42,15 +42,15 @@ begin
       verbose = true
     end
 
-    parser.on('-b', '--blacklist=EXPRESSION', 'blacklist (ignore) URLs matching the given regular EXPRESSION') do |b|
-      blacklist << Regexp.new(b)
+    parser.on('-e', '--exclude=EXPRESSION', 'exclude URLs matching the given regular EXPRESSION') do |b|
+      excluded << Regexp.new(b)
     end
 
     # TODO: --recursive, defaults to false
     # TODO wget has some additional options for recursive behavior that should be reviewed
   end.parse!
 rescue StandardError
-  warn "Error - #{$ERROR_INFO}"
+  warn "Error: #{$ERROR_INFO}"
   exit 1
 end
 
@@ -59,13 +59,14 @@ if ARGV.size != 1
   exit 1
 end
 
-def check(doc, lang, personal_dictionary_path, verbose)
+def check(url, doc, lang, personal_dictionary_path, verbose)
   unknown_words = HttpSpell::SpellChecker.new(personal_dictionary_path, verbose:).check(doc, lang)
 
   if unknown_words.empty?
-    warn 'No unknown words.' if verbose
+    warn "#{url} (lang=#{lang}): No unknown words" if verbose
+    false
   else
-    warn "#{unknown_words.size} unknown words:" if verbose
+    warn "#{url} (lang=#{lang}): #{unknown_words.size} unknown words:" if verbose
     puts unknown_words
     true
   end
@@ -73,24 +74,23 @@ end
 
 has_unknown_words = false
 
-spider_success = HttpSpell::Spider.new(ARGV.first, whitelist:, blacklist:, verbose:, tracing:).start do |url, doc|
+spider_success = HttpSpell::Spider.new(ARGV.first, included:, excluded:, verbose:, tracing:).start do |url, doc|
   lang = force_language || doc.root['lang'] || ENV.fetch('LANGUAGE', nil)
-  warn "Checking #{url} as #{lang}" if verbose
 
   # Remove elements that are not to be spellchecked
   doc.css('pre').each(&:unlink)
   doc.css('code').each(&:unlink)
+  doc.css('iframe').each(&:unlink)
   doc.css('[spellcheck=false]').each(&:unlink)
 
   # Handle elements with a different lang attribute separately
   doc.css(%([lang]:not([lang="#{lang}"]))).each do |element|
-    warn "Handling #{element.name} with lang #{element['lang']}:" if verbose
-    has_unknown_words |= check(element.to_s, element['lang'], personal_dictionary_path, verbose)
+    has_unknown_words |= check("#{url} => #{element.name} with", element.to_s, element['lang'], personal_dictionary_path, verbose)
     element.unlink
   end
 
   # Everything else
-  has_unknown_words |= check(doc.to_s, lang, personal_dictionary_path, verbose)
+  has_unknown_words |= check("#{url} => document with", doc.to_s, lang, personal_dictionary_path, verbose)
 end
 
 exit 2 unless spider_success

data/httpspell.gemspec

@@ -2,7 +2,7 @@
 
 lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'httpspell/version'
+require 'http_spell/version'
 
 Gem::Specification.new do |spec|
   spec.name          = 'httpspell'
@@ -23,7 +23,6 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'addressable'
   spec.add_dependency 'nokogiri'
   spec.metadata['rubygems_mfa_required'] = 'true'
 end

data/lib/{httpspell → http_spell}/spellchecker.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module HttpSpell
   class SpellChecker
     def initialize(personal_dictionary_path = nil, verbose: false)
@@ -11,11 +13,6 @@ module HttpSpell
         "hunspell -d #{translate(lang)} #{@personal_dictionary_arg} -i UTF-8 -l",
       ]
 
-      if @verbose
-        warn "Piping the HTML document into the following chain of commands:"
-        warn commands
-      end
-
       Open3.pipeline_rw(*commands) do |stdin, stdout, _wait_thrs|
         stdin.puts(doc)
         stdin.close

data/lib/http_spell/spider.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+require 'uri'
+require 'open-uri'
+require 'open3'
+require 'English'
+
+module HttpSpell
+  class Spider
+    attr_reader :todo, :done
+
+    def initialize(starting_point, included: nil, excluded: [], verbose: false, tracing: false)
+      @todo = []
+      @done = []
+      todo << URI(starting_point)
+      @included = included || [/^#{starting_point}/]
+      @excluded = excluded
+      @verbose = verbose
+      @tracing = tracing
+    end
+
+    def start
+      success = true
+
+      while todo.any?
+        url = todo.pop
+
+        begin
+          extracted = links(url) do |u, d|
+            yield u, d if block_given?
+          rescue StandardError
+            warn "Callback error for #{url}: #{$ERROR_INFO}"
+            warn $ERROR_INFO.backtrace if @tracing
+          end
+
+          done.append(url)
+          new_links = (extracted - done - todo).uniq
+
+          if new_links.any?
+            warn "Adding #{new_links.size} new links found at #{url}" if @verbose
+            todo.concat(extracted - done - todo).uniq!
+          end
+        rescue StandardError
+          warn "Skipping #{url} because of #{$ERROR_INFO.message}"
+          warn $ERROR_INFO.backtrace if @tracing
+          success = false
+        end
+      end
+
+      success
+    end
+
+    private
+
+    def links(uri)
+      response = http_get(uri)
+
+      if response.respond_to?(:content_type) && response.content_type != 'text/html'
+        warn "Skipping #{response.base_uri} because it is not HTML" if @verbose
+        return []
+      end
+
+      doc = Nokogiri::HTML(response)
+
+      links = doc.css('a[href]').map do |e|
+        next if e['href'].start_with?('#') # Ignore fragment on the same page; we always check the whole page
+
+        link = URI.join(response.base_uri, e['href'])
+        link.fragment = nil # Ignore fragment in links to other pages, too
+
+        if @included.none? { |re| re.match?(link.to_s) }
+          warn "Skipping #{link} because it is not on the included #{@included}" if @verbose
+          next
+        end
+
+        if @excluded.any? { |re| re.match?(link.to_s) }
+          # TODO: Print _which_ entry of the excluded matches
+          warn "Skipping #{link} because it is on the excluded #{@excluded}" if @verbose
+          next
+        end
+
+        link
+      rescue StandardError
+        warn "Error: #{$ERROR_INFO}"
+        warn $ERROR_INFO.backtrace if @tracing
+      end.compact
+
+      yield response.base_uri, doc if block_given?
+
+      links
+    end
+
+    # https://twin.github.io/improving-open-uri/
+    def http_get(uri)
+      tries = 10
+      begin
+        URI.parse(uri).open(redirect: false)
+      rescue OpenURI::HTTPRedirect => e
+        uri = e.uri
+        retry if (tries -= 1).positive?
+        raise
+      end
+    end
+  end
+end

data/lib/{httpspell → http_spell}/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HttpSpell
-  VERSION = '1.4.0'
+  VERSION = '1.5.0'
 end

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: httpspell
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Steffen Uhlig
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-05-29 00:00:00.000000000 Z
+date: 2024-05-31 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: addressable
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -63,9 +49,9 @@ files:
 - TODO.markdown
 - exe/httpspell
 - httpspell.gemspec
-- lib/httpspell/spellchecker.rb
-- lib/httpspell/spider.rb
-- lib/httpspell/version.rb
+- lib/http_spell/spellchecker.rb
+- lib/http_spell/spider.rb
+- lib/http_spell/version.rb
 homepage:
 licenses:
 - MIT

data/lib/httpspell/spider.rb

@@ -1,100 +0,0 @@
-require 'nokogiri'
-require 'open-uri'
-require 'open3'
-require 'addressable/uri'
-require 'English'
-
-module HttpSpell
-  class Spider
-    attr_reader :todo, :done
-
-    def initialize(starting_point, whitelist: nil, blacklist: [], verbose: false, tracing: false)
-      @todo = []
-      @done = []
-      todo << Addressable::URI.parse(starting_point)
-      @whitelist = whitelist || [/^#{starting_point}/]
-      @blacklist = blacklist
-      @verbose = verbose
-      @tracing = tracing
-    end
-
-    def start
-      success = true
-
-      while todo.any?
-        url = todo.pop
-
-        begin
-          extracted = links(url) do |u, d|
-            yield u, d if block_given?
-          rescue
-            warn "Callback error for #{url}: #{$ERROR_INFO}"
-            warn $ERROR_INFO.backtrace if @tracing
-          end
-
-          done.append(url)
-          todo.concat(extracted - done - todo)
-        rescue StandardError
-          warn "Skipping #{url} because of #{$ERROR_INFO.message}"
-          warn $ERROR_INFO.backtrace if @tracing
-          success = false
-        end
-      end
-
-      return success
-    end
-
-    private
-
-    def links(uri)
-      response = http_get(uri)
-
-      if response.respond_to?(:content_type) && response.content_type != 'text/html'
-        warn "Skipping #{uri} because it is not HTML" if @verbose
-        return []
-      end
-
-      doc = Nokogiri::HTML(response)
-
-      links = doc.css('a[href]').map do |e|
-        link = Addressable::URI.parse(e['href'])
-        link = uri.join(link) if link.relative?
-
-        if @whitelist.none? { |re| re.match?(link.to_s) }
-          warn "Skipping #{link} because it is not on the whitelist #{@whitelist}" if @verbose
-          next
-        end
-
-        if @blacklist.any? { |re| re.match?(link.to_s) }
-          # TODO Print _which_ entry of the blacklist matches
-          warn "Skipping #{link} because it is on the blacklist #{@blacklist}" if @verbose
-          next
-        end
-
-        # TODO Ignore same page links (some anchor)
-        link
-      rescue StandardError
-        warn $ERROR_INFO.message
-        warn $ERROR_INFO.backtrace if @tracing
-      end.compact
-
-      yield uri, doc if block_given?
-
-      warn "Adding #{links.size} links from #{uri}" if @verbose
-      links
-    end
-
-    # https://twin.github.io/improving-open-uri/
-    def http_get(uri)
-      tries = 10
-
-      begin
-        URI.open(uri, redirect: false)
-      rescue OpenURI::HTTPRedirect => redirect
-        uri = redirect.uri
-        retry if (tries -= 1) > 0
-        raise
-      end
-    end
-  end
-end