httpspell 1.2.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dcf823c527b930c3048fd15e17d0106af6aee7c9375b685d884c8690e285b484
-  data.tar.gz: 123fcd42841af054627d69dfa3d4a655e0c97bb5eeeec96fd4b2aa46be9eedd3
+  metadata.gz: ec39e7efff90e1b266f19714e6b6ce91a1a714e149be0df491e0e04cf5bef564
+  data.tar.gz: 58be9917fc0e13ad4653e6687b19b018d30be56b0c0c9d62da8e39ef169264e6
 SHA512:
-  metadata.gz: a03721c821750a533eacb9f2e53148d2afb0489a18ee278ad3b9973424ea043973c5989f3a052764fb3611177bb3561ddfc287bca80f9351abcd63ea9f0638cb
-  data.tar.gz: 9ee1bef8ed9d0b1bdd2e0a2aeb74a28f764cd21c422430802a11cf055d119aca683b64fb26ab4b3ac92f7ead13f920bff7b368fc6c4e081bdfc1bb9acd8d51bf
+  metadata.gz: 045cc45fc3bd73e5cb4694fec14dec5afad13ab8402cdc90173b621f21a8a11c0b02a375404e0dc332c7e6ec29a571ce3e6d5b152283ad6bb9323161f583579d
+  data.tar.gz: de4f50971f4d12a29c829b47a14f54d84c005c9ff7d421fcfcd672fdf065eb7db5ad4c27a83933e4d9de4bf4417d3cfbdf881a312416a0c6b0a95018637592f7

data/Gemfile.lock

@@ -1,51 +1,51 @@
 PATH
   remote: .
   specs:
-    httpspell (1.2.1)
+    httpspell (1.3.0)
       addressable
       nokogiri
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.5.2)
+    addressable (2.6.0)
       public_suffix (>= 2.0.2, < 4.0)
-    aruba (0.14.5)
-      childprocess (>= 0.6.3, < 0.10.0)
+    aruba (0.14.9)
+      childprocess (>= 0.6.3, < 1.1.0)
       contracts (~> 0.9)
       cucumber (>= 1.3.19)
-      ffi (~> 1.9.10)
+      ffi (~> 1.9)
       rspec-expectations (>= 2.99)
       thor (~> 0.19)
     ast (2.4.0)
-    backports (3.11.3)
+    backports (3.13.0)
     builder (3.2.3)
-    byebug (10.0.2)
-    childprocess (0.9.0)
-      ffi (~> 1.0, >= 1.0.11)
+    byebug (11.0.1)
+    childprocess (1.0.1)
+      rake (< 13.0)
     coderay (1.1.2)
     contracts (0.16.0)
-    cucumber (3.1.0)
+    cucumber (3.1.2)
       builder (>= 2.1.2)
-      cucumber-core (~> 3.1.0)
-      cucumber-expressions (~> 5.0.4)
+      cucumber-core (~> 3.2.0)
+      cucumber-expressions (~> 6.0.1)
       cucumber-wire (~> 0.0.1)
       diff-lcs (~> 1.3)
-      gherkin (~> 5.0)
+      gherkin (~> 5.1.0)
       multi_json (>= 1.7.5, < 2.0)
       multi_test (>= 0.1.2)
-    cucumber-core (3.1.0)
+    cucumber-core (3.2.1)
       backports (>= 3.8.0)
       cucumber-tag_expressions (~> 1.1.0)
-      gherkin (>= 5.0.0)
-    cucumber-expressions (5.0.18)
+      gherkin (~> 5.0)
+    cucumber-expressions (6.0.1)
     cucumber-tag_expressions (1.1.1)
     cucumber-wire (0.0.1)
     diff-lcs (1.3)
-    ffi (1.9.23)
+    ffi (1.10.0)
     formatador (0.2.5)
     gherkin (5.1.0)
-    guard (2.14.2)
+    guard (2.15.0)
       formatador (>= 0.2.4)
       listen (>= 2.7, < 4.0)
       lumberjack (>= 1.0.12, < 2.0)
@@ -54,8 +54,8 @@ GEM
       pry (>= 0.9.12)
       shellany (~> 0.0)
       thor (>= 0.18.1)
-    guard-bundler (2.1.0)
-      bundler (~> 1.0)
+    guard-bundler (2.2.1)
+      bundler (>= 1.3.0, < 3)
       guard (~> 2.2)
       guard-compat (~> 1.1)
     guard-compat (1.2.1)
@@ -63,65 +63,67 @@ GEM
       guard (~> 2.1)
       guard-compat (~> 1.1)
       rspec (>= 2.99.0, < 4.0)
+    jaro_winkler (1.5.2)
     listen (3.1.5)
       rb-fsevent (~> 0.9, >= 0.9.4)
       rb-inotify (~> 0.9, >= 0.9.7)
       ruby_dep (~> 1.2)
     lumberjack (1.0.13)
-    method_source (0.9.0)
-    mini_portile2 (2.3.0)
+    method_source (0.9.2)
+    mini_portile2 (2.4.0)
     multi_json (1.13.1)
     multi_test (0.1.2)
     nenv (0.3.0)
-    nokogiri (1.8.2)
-      mini_portile2 (~> 2.3.0)
+    nokogiri (1.10.2)
+      mini_portile2 (~> 2.4.0)
     notiffany (0.1.1)
       nenv (~> 0.1)
       shellany (~> 0.0)
-    parallel (1.12.1)
-    parser (2.5.1.0)
+    parallel (1.17.0)
+    parser (2.6.2.1)
       ast (~> 2.4.0)
-    powerpack (0.1.1)
-    pry (0.11.3)
+    pry (0.12.2)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
-    pry-byebug (3.6.0)
-      byebug (~> 10.0)
+    pry-byebug (3.7.0)
+      byebug (~> 11.0)
       pry (~> 0.10)
-    public_suffix (3.0.2)
-    rack (2.0.5)
+    psych (3.1.0)
+    public_suffix (3.0.3)
+    rack (2.0.7)
     rainbow (3.0.0)
-    rake (12.3.1)
+    rake (12.3.2)
     rb-fsevent (0.10.3)
-    rb-inotify (0.9.10)
-      ffi (>= 0.5.0, < 2)
-    rspec (3.7.0)
-      rspec-core (~> 3.7.0)
-      rspec-expectations (~> 3.7.0)
-      rspec-mocks (~> 3.7.0)
-    rspec-core (3.7.1)
-      rspec-support (~> 3.7.0)
-    rspec-expectations (3.7.0)
+    rb-inotify (0.10.0)
+      ffi (~> 1.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-mocks (3.7.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-support (3.7.1)
-    rubocop (0.56.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    rubocop (0.67.2)
+      jaro_winkler (~> 1.5.1)
       parallel (~> 1.10)
-      parser (>= 2.5)
-      powerpack (~> 0.1)
+      parser (>= 2.5, != 2.5.1.1)
+      psych (>= 3.1.0)
       rainbow (>= 2.2.2, < 4.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (~> 1.0, >= 1.0.1)
-    ruby-progressbar (1.9.0)
+      unicode-display_width (>= 1.4.0, < 1.6)
+    ruby-progressbar (1.10.0)
     ruby_dep (1.5.0)
     shellany (0.0.1)
-    stub_server (0.3.0)
+    stub_server (0.4.0)
       rack
-    thor (0.20.0)
-    unicode-display_width (1.3.3)
+    thor (0.20.3)
+    unicode-display_width (1.5.0)
 
 PLATFORMS
   ruby
@@ -141,4 +143,4 @@ DEPENDENCIES
   stub_server
 
 BUNDLED WITH
-   1.16.1
+   1.17.2

data/TODO.markdown

@@ -0,0 +1,6 @@
+* Bail out if lang cannot be inferred and is not given on cmdline
+* exe/httpspell:    # TODO: --recursive, defaults to false
+* exe/httpspell:    # TODO wget has some additional options for recursive behavior that should be reviewed
+* exe/httpspell:  # TODO: Find sections with a lang attribute and handle them separately
+* lib/httpspell/spider.rb:          # TODO Print _which_ entry of the blacklist matches
+* lib/httpspell/spider.rb:        # TODO Ignore same page links (some anchor)

data/lib/httpspell/spider.rb

@@ -46,7 +46,7 @@ module HttpSpell
     private
 
     def links(uri)
-      response = URI(uri).read # We are using open-uri, which follows redirects and also provides the content-type.
+      response = http_get(URI(uri))
 
       if response.content_type != 'text/html'
         warn "Skipping #{uri} because it is not HTML" if @tracing
@@ -82,5 +82,18 @@ module HttpSpell
       warn "Adding #{links.size} links from #{uri}" if @tracing
       links
     end
+
+    # https://twin.github.io/improving-open-uri/
+    def http_get(uri)
+      tries = 10
+
+      begin
+        uri.open(redirect: false)
+      rescue OpenURI::HTTPRedirect => redirect
+        uri = redirect.uri
+        retry if (tries -= 1) > 0
+        raise
+      end
+    end
   end
 end

data/lib/httpspell/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HttpSpell
-  VERSION = '1.2.1'
+  VERSION = '1.3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: httpspell
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.3.0
 platform: ruby
 authors:
 - Steffen Uhlig
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-06-01 00:00:00.000000000 Z
+date: 2019-04-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -211,6 +211,7 @@ files:
 - Gemfile.lock
 - README.markdown
 - Rakefile
+- TODO.markdown
 - exe/httpspell
 - httpspell.gemspec
 - lib/httpspell/spellchecker.rb
@@ -235,8 +236,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.1
 signing_key: 
 specification_version: 4
 summary: HTTP spellchecker