httpspell 1.3.0 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (19) hide show
  1. checksums.yaml +4 -4
  2. data/.github/dependabot.yml +14 -0
  3. data/.gitignore +2 -0
  4. data/.mergify.yml +8 -0
  5. data/.rubocop.yml +24 -9
  6. data/.ruby-version +1 -0
  7. data/Gemfile +17 -0
  8. data/Gemfile.lock +140 -94
  9. data/Guardfile +25 -0
  10. data/README.markdown +23 -2
  11. data/Rakefile +1 -1
  12. data/TODO.markdown +0 -2
  13. data/exe/httpspell +27 -16
  14. data/httpspell.gemspec +4 -17
  15. data/lib/{httpspell → http_spell}/spellchecker.rb +6 -4
  16. data/lib/{httpspell → http_spell}/spider.rb +21 -16
  17. data/lib/{httpspell → http_spell}/version.rb +1 -1
  18. metadata +18 -168
  19. data/.travis.yml +0 -6
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: ec39e7efff90e1b266f19714e6b6ce91a1a714e149be0df491e0e04cf5bef564
4
- data.tar.gz: 58be9917fc0e13ad4653e6687b19b018d30be56b0c0c9d62da8e39ef169264e6
3
+ metadata.gz: dc09324c003c7b14e08fa255b7a31c0a9aeb143df033da9aea300619a47268ba
4
+ data.tar.gz: 6890352a3cef38e243e2506398d58736c8179c2e0443a2b6ff341165e724dba0
5
5
  SHA512:
6
- metadata.gz: 045cc45fc3bd73e5cb4694fec14dec5afad13ab8402cdc90173b621f21a8a11c0b02a375404e0dc332c7e6ec29a571ce3e6d5b152283ad6bb9323161f583579d
7
- data.tar.gz: de4f50971f4d12a29c829b47a14f54d84c005c9ff7d421fcfcd672fdf065eb7db5ad4c27a83933e4d9de4bf4417d3cfbdf881a312416a0c6b0a95018637592f7
6
+ metadata.gz: 826bb8e875b2f1584dd5c052ab9777e616e1da0d6844263589b027c3eabfb07955155e0c43b8b1b8dc253d720eba952e80330c38035fff53fc1943420dea7454
7
+ data.tar.gz: 7a4e3c9aaa586d4fbdc41971424cd5f064793ff18cba8d8606a452b3cee36070af44aa2f78ab307c71a613404cc1e490af1f56eca11068675183625f5360790e
data/.github/dependabot.yml ADDED
@@ -0,0 +1,14 @@
1
+ version: 2
2
+ updates:
3
+ - package-ecosystem: bundler
4
+ directory: "/"
5
+ schedule:
6
+ interval: daily
7
+ time: "09:00"
8
+ open-pull-requests-limit: 10
9
+ ignore:
10
+ - dependency-name: rubocop
11
+ versions:
12
+ - 1.10.0
13
+ - 1.11.0
14
+ - 1.9.0
data/.gitignore CHANGED
@@ -1 +1,3 @@
1
+ .rake_tasks
1
2
  pkg
3
+ spec/dictionaries
data/.mergify.yml ADDED
@@ -0,0 +1,8 @@
1
+ pull_request_rules:
2
+ - name: automatic merge for Dependabot pull requests
3
+ conditions:
4
+ - author~=^dependabot(|-preview)\[bot\]$
5
+ - check-success=Travis CI - Pull Request
6
+ actions:
7
+ merge:
8
+ method: merge
data/.rubocop.yml CHANGED
@@ -1,6 +1,11 @@
1
+ require:
2
+ - rubocop-rake
3
+ - rubocop-rspec
1
4
  AllCops:
2
- TargetRubyVersion: 2.5.1
5
+ NewCops: enable
6
+ TargetRubyVersion: 3.3
3
7
  Include:
8
+ - '**/*.rb'
4
9
  - '**/Gemfile'
5
10
  - '**/Rakefile'
6
11
  - '**/config.ru'
@@ -8,23 +13,33 @@ AllCops:
8
13
  Exclude:
9
14
  - vendor/**/*
10
15
  - db/migrations/**/*
11
-
12
16
  DisplayCopNames:
13
17
  Enabled: true
14
-
15
18
  DisplayStyleGuide:
16
19
  Enabled: true
17
-
18
20
  Naming/FileName:
19
21
  Exclude:
20
- - Guardfile
21
-
22
+ - Guardfile
22
23
  Metrics/BlockLength:
23
24
  Exclude:
24
25
  - spec/**/*
25
-
26
- Metrics/LineLength:
26
+ Layout/LineLength:
27
27
  Max: 160
28
-
29
28
  Style/Documentation:
30
29
  Enabled: false
30
+ Metrics/AbcSize:
31
+ Enabled: false
32
+ Metrics/MethodLength:
33
+ Enabled: false
34
+ Metrics/CyclomaticComplexity:
35
+ Enabled: false
36
+ Style/TrailingCommaInArrayLiteral:
37
+ Enabled: false
38
+ RSpec/ExampleWording:
39
+ Enabled: false
40
+ RSpec/InstanceVariable:
41
+ AssignmentOnly: true
42
+ RSpec/ExampleLength:
43
+ Max: 10
44
+ Metrics/PerceivedComplexity:
45
+ Max: 16
data/.ruby-version ADDED
@@ -0,0 +1 @@
1
+ ruby-3.3.1
data/Gemfile CHANGED
@@ -2,3 +2,20 @@
2
2
 
3
3
  source 'https://rubygems.org'
4
4
  gemspec
5
+
6
+ group :development do
7
+ gem 'aruba'
8
+ gem 'bundler'
9
+ gem 'guard'
10
+ gem 'guard-bundler'
11
+ gem 'guard-rspec'
12
+ gem 'httpx'
13
+ gem 'pry'
14
+ gem 'pry-byebug'
15
+ gem 'rake'
16
+ gem 'rspec'
17
+ gem 'rubocop'
18
+ gem 'rubocop-rake'
19
+ gem 'rubocop-rspec'
20
+ gem 'stub_server'
21
+ end
data/Gemfile.lock CHANGED
@@ -1,61 +1,66 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- httpspell (1.3.0)
4
+ httpspell (1.4.1)
5
5
  addressable
6
6
  nokogiri
7
7
 
8
8
  GEM
9
9
  remote: https://rubygems.org/
10
10
  specs:
11
- addressable (2.6.0)
12
- public_suffix (>= 2.0.2, < 4.0)
13
- aruba (0.14.9)
14
- childprocess (>= 0.6.3, < 1.1.0)
15
- contracts (~> 0.9)
16
- cucumber (>= 1.3.19)
17
- ffi (~> 1.9)
18
- rspec-expectations (>= 2.99)
19
- thor (~> 0.19)
20
- ast (2.4.0)
21
- backports (3.13.0)
22
- builder (3.2.3)
23
- byebug (11.0.1)
24
- childprocess (1.0.1)
25
- rake (< 13.0)
26
- coderay (1.1.2)
27
- contracts (0.16.0)
28
- cucumber (3.1.2)
29
- builder (>= 2.1.2)
30
- cucumber-core (~> 3.2.0)
31
- cucumber-expressions (~> 6.0.1)
32
- cucumber-wire (~> 0.0.1)
33
- diff-lcs (~> 1.3)
34
- gherkin (~> 5.1.0)
35
- multi_json (>= 1.7.5, < 2.0)
36
- multi_test (>= 0.1.2)
37
- cucumber-core (3.2.1)
38
- backports (>= 3.8.0)
39
- cucumber-tag_expressions (~> 1.1.0)
40
- gherkin (~> 5.0)
41
- cucumber-expressions (6.0.1)
42
- cucumber-tag_expressions (1.1.1)
43
- cucumber-wire (0.0.1)
44
- diff-lcs (1.3)
45
- ffi (1.10.0)
46
- formatador (0.2.5)
47
- gherkin (5.1.0)
48
- guard (2.15.0)
11
+ addressable (2.8.6)
12
+ public_suffix (>= 2.0.2, < 6.0)
13
+ aruba (2.2.0)
14
+ bundler (>= 1.17, < 3.0)
15
+ contracts (>= 0.16.0, < 0.18.0)
16
+ cucumber (>= 8.0, < 10.0)
17
+ rspec-expectations (~> 3.4)
18
+ thor (~> 1.0)
19
+ ast (2.4.2)
20
+ bigdecimal (3.1.8)
21
+ builder (3.2.4)
22
+ byebug (11.1.3)
23
+ coderay (1.1.3)
24
+ contracts (0.17)
25
+ cucumber (9.2.0)
26
+ builder (~> 3.2)
27
+ cucumber-ci-environment (> 9, < 11)
28
+ cucumber-core (> 13, < 14)
29
+ cucumber-cucumber-expressions (~> 17.0)
30
+ cucumber-gherkin (> 24, < 28)
31
+ cucumber-html-formatter (> 20.3, < 22)
32
+ cucumber-messages (> 19, < 25)
33
+ diff-lcs (~> 1.5)
34
+ mini_mime (~> 1.1)
35
+ multi_test (~> 1.1)
36
+ sys-uname (~> 1.2)
37
+ cucumber-ci-environment (10.0.1)
38
+ cucumber-core (13.0.2)
39
+ cucumber-gherkin (>= 27, < 28)
40
+ cucumber-messages (>= 20, < 23)
41
+ cucumber-tag-expressions (> 5, < 7)
42
+ cucumber-cucumber-expressions (17.1.0)
43
+ bigdecimal
44
+ cucumber-gherkin (27.0.0)
45
+ cucumber-messages (>= 19.1.4, < 23)
46
+ cucumber-html-formatter (21.3.1)
47
+ cucumber-messages (> 19, < 25)
48
+ cucumber-messages (22.0.0)
49
+ cucumber-tag-expressions (6.1.0)
50
+ diff-lcs (1.5.1)
51
+ ffi (1.16.3)
52
+ formatador (1.1.0)
53
+ guard (2.18.1)
49
54
  formatador (>= 0.2.4)
50
55
  listen (>= 2.7, < 4.0)
51
56
  lumberjack (>= 1.0.12, < 2.0)
52
57
  nenv (~> 0.1)
53
58
  notiffany (~> 0.0)
54
- pry (>= 0.9.12)
59
+ pry (>= 0.13.0)
55
60
  shellany (~> 0.0)
56
61
  thor (>= 0.18.1)
57
- guard-bundler (2.2.1)
58
- bundler (>= 1.3.0, < 3)
62
+ guard-bundler (3.0.1)
63
+ bundler (>= 2.1, < 3)
59
64
  guard (~> 2.2)
60
65
  guard-compat (~> 1.1)
61
66
  guard-compat (1.2.1)
@@ -63,70 +68,108 @@ GEM
63
68
  guard (~> 2.1)
64
69
  guard-compat (~> 1.1)
65
70
  rspec (>= 2.99.0, < 4.0)
66
- jaro_winkler (1.5.2)
67
- listen (3.1.5)
68
- rb-fsevent (~> 0.9, >= 0.9.4)
69
- rb-inotify (~> 0.9, >= 0.9.7)
70
- ruby_dep (~> 1.2)
71
- lumberjack (1.0.13)
72
- method_source (0.9.2)
73
- mini_portile2 (2.4.0)
74
- multi_json (1.13.1)
75
- multi_test (0.1.2)
71
+ http-2-next (1.0.3)
72
+ httpx (1.2.5)
73
+ http-2-next (>= 1.0.3)
74
+ json (2.7.2)
75
+ language_server-protocol (3.17.0.3)
76
+ listen (3.9.0)
77
+ rb-fsevent (~> 0.10, >= 0.10.3)
78
+ rb-inotify (~> 0.9, >= 0.9.10)
79
+ lumberjack (1.2.10)
80
+ method_source (1.1.0)
81
+ mini_mime (1.1.5)
82
+ multi_test (1.1.0)
76
83
  nenv (0.3.0)
77
- nokogiri (1.10.2)
78
- mini_portile2 (~> 2.4.0)
79
- notiffany (0.1.1)
84
+ nokogiri (1.16.5-arm64-darwin)
85
+ racc (~> 1.4)
86
+ nokogiri (1.16.5-x86_64-darwin)
87
+ racc (~> 1.4)
88
+ nokogiri (1.16.5-x86_64-linux)
89
+ racc (~> 1.4)
90
+ notiffany (0.1.3)
80
91
  nenv (~> 0.1)
81
92
  shellany (~> 0.0)
82
- parallel (1.17.0)
83
- parser (2.6.2.1)
84
- ast (~> 2.4.0)
85
- pry (0.12.2)
86
- coderay (~> 1.1.0)
87
- method_source (~> 0.9.0)
88
- pry-byebug (3.7.0)
93
+ parallel (1.24.0)
94
+ parser (3.3.1.0)
95
+ ast (~> 2.4.1)
96
+ racc
97
+ pry (0.14.2)
98
+ coderay (~> 1.1)
99
+ method_source (~> 1.0)
100
+ pry-byebug (3.10.1)
89
101
  byebug (~> 11.0)
90
- pry (~> 0.10)
91
- psych (3.1.0)
92
- public_suffix (3.0.3)
93
- rack (2.0.7)
94
- rainbow (3.0.0)
95
- rake (12.3.2)
96
- rb-fsevent (0.10.3)
97
- rb-inotify (0.10.0)
102
+ pry (>= 0.13, < 0.15)
103
+ public_suffix (5.0.5)
104
+ racc (1.8.0)
105
+ rack (3.0.11)
106
+ rackup (0.2.3)
107
+ rack (>= 3.0.0.beta1)
108
+ webrick
109
+ rainbow (3.1.1)
110
+ rake (13.2.1)
111
+ rb-fsevent (0.11.2)
112
+ rb-inotify (0.11.1)
98
113
  ffi (~> 1.0)
99
- rspec (3.8.0)
100
- rspec-core (~> 3.8.0)
101
- rspec-expectations (~> 3.8.0)
102
- rspec-mocks (~> 3.8.0)
103
- rspec-core (3.8.0)
104
- rspec-support (~> 3.8.0)
105
- rspec-expectations (3.8.2)
114
+ regexp_parser (2.9.2)
115
+ rexml (3.2.8)
116
+ strscan (>= 3.0.9)
117
+ rspec (3.13.0)
118
+ rspec-core (~> 3.13.0)
119
+ rspec-expectations (~> 3.13.0)
120
+ rspec-mocks (~> 3.13.0)
121
+ rspec-core (3.13.0)
122
+ rspec-support (~> 3.13.0)
123
+ rspec-expectations (3.13.0)
106
124
  diff-lcs (>= 1.2.0, < 2.0)
107
- rspec-support (~> 3.8.0)
108
- rspec-mocks (3.8.0)
125
+ rspec-support (~> 3.13.0)
126
+ rspec-mocks (3.13.1)
109
127
  diff-lcs (>= 1.2.0, < 2.0)
110
- rspec-support (~> 3.8.0)
111
- rspec-support (3.8.0)
112
- rubocop (0.67.2)
113
- jaro_winkler (~> 1.5.1)
128
+ rspec-support (~> 3.13.0)
129
+ rspec-support (3.13.1)
130
+ rubocop (1.64.0)
131
+ json (~> 2.3)
132
+ language_server-protocol (>= 3.17.0)
114
133
  parallel (~> 1.10)
115
- parser (>= 2.5, != 2.5.1.1)
116
- psych (>= 3.1.0)
134
+ parser (>= 3.3.0.2)
117
135
  rainbow (>= 2.2.2, < 4.0)
136
+ regexp_parser (>= 1.8, < 3.0)
137
+ rexml (>= 3.2.5, < 4.0)
138
+ rubocop-ast (>= 1.31.1, < 2.0)
118
139
  ruby-progressbar (~> 1.7)
119
- unicode-display_width (>= 1.4.0, < 1.6)
120
- ruby-progressbar (1.10.0)
121
- ruby_dep (1.5.0)
140
+ unicode-display_width (>= 2.4.0, < 3.0)
141
+ rubocop-ast (1.31.3)
142
+ parser (>= 3.3.1.0)
143
+ rubocop-capybara (2.20.0)
144
+ rubocop (~> 1.41)
145
+ rubocop-factory_bot (2.25.1)
146
+ rubocop (~> 1.41)
147
+ rubocop-rake (0.6.0)
148
+ rubocop (~> 1.0)
149
+ rubocop-rspec (2.29.2)
150
+ rubocop (~> 1.40)
151
+ rubocop-capybara (~> 2.17)
152
+ rubocop-factory_bot (~> 2.22)
153
+ rubocop-rspec_rails (~> 2.28)
154
+ rubocop-rspec_rails (2.28.3)
155
+ rubocop (~> 1.40)
156
+ ruby-progressbar (1.13.0)
122
157
  shellany (0.0.1)
123
- stub_server (0.4.0)
124
- rack
125
- thor (0.20.3)
126
- unicode-display_width (1.5.0)
158
+ strscan (3.1.0)
159
+ stub_server (0.7.0)
160
+ rackup (~> 0.2.2)
161
+ webrick
162
+ sys-uname (1.2.3)
163
+ ffi (~> 1.1)
164
+ thor (1.3.1)
165
+ unicode-display_width (2.5.0)
166
+ webrick (1.8.1)
127
167
 
128
168
  PLATFORMS
129
- ruby
169
+ arm64-darwin-22
170
+ arm64-darwin-23
171
+ x86_64-darwin-21
172
+ x86_64-linux
130
173
 
131
174
  DEPENDENCIES
132
175
  aruba
@@ -135,12 +178,15 @@ DEPENDENCIES
135
178
  guard-bundler
136
179
  guard-rspec
137
180
  httpspell!
181
+ httpx
138
182
  pry
139
183
  pry-byebug
140
184
  rake
141
185
  rspec
142
186
  rubocop
187
+ rubocop-rake
188
+ rubocop-rspec
143
189
  stub_server
144
190
 
145
191
  BUNDLED WITH
146
- 1.17.2
192
+ 2.5.9
data/Guardfile ADDED
@@ -0,0 +1,25 @@
1
+ guard :bundler do
2
+ require 'guard/bundler'
3
+ require 'guard/bundler/verify'
4
+ helper = Guard::Bundler::Verify.new
5
+
6
+ files = ['Gemfile']
7
+ files += Dir['*.gemspec'] if files.any? { |f| helper.uses_gemspec?(f) }
8
+
9
+ # Assume files are symlinked from somewhere
10
+ files.each { |file| watch(helper.real_path(file)) }
11
+ end
12
+
13
+ guard :rspec, cmd: "bundle exec rspec" do
14
+ require "guard/rspec/dsl"
15
+ dsl = Guard::RSpec::Dsl.new(self)
16
+
17
+ rspec = dsl.rspec
18
+ watch(rspec.spec_helper) { rspec.spec_dir }
19
+ watch(rspec.spec_support) { rspec.spec_dir }
20
+ watch(rspec.spec_files)
21
+
22
+ # Ruby files
23
+ ruby = dsl.ruby
24
+ dsl.watch_spec_files_for(ruby.lib_files)
25
+ end
data/README.markdown CHANGED
@@ -1,7 +1,5 @@
1
1
  # `httpspell`
2
2
 
3
- [![Build Status](https://travis-ci.org/suhlig/httpspell.svg?branch=master)](https://travis-ci.org/suhlig/httpspell)
4
-
5
3
  This is a spellchecker that recursively fetches HTML pages, converts them to plain text (using [pandoc](http://pandoc.org/)), and spellchecks them with [hunspell](https://hunspell.github.io/). Unknown words will be printed to `stdout`, which makes the tool a good candidate for CI pipelines where you might want to take action when a spelling error is found on a web page.
6
4
 
7
5
  Words that are not in the dictionary for the given language (inferred from the `lang` attribute of the HTML document's root element) can be added to a personal dictionary, which will mark the word as correctly spelled.
@@ -46,3 +44,26 @@ If you produce content with kramdown (e.g. using Jekyll), setting `spellcheck='f
46
44
  ```
47
45
  {: spellcheck="false"}
48
46
  ```
47
+
48
+ # Dictionaries
49
+
50
+ Hunspell uses the system dictionary paths; on the Mac this is `~/Library/Spelling/`. Get some dictionaries as explained in the [hunspell](https://github.com/hunspell/hunspell) project:
51
+
52
+ ```command
53
+ $ wget -O ~/Library/Spelling/en_US.aff https://cgit.freedesktop.org/libreoffice/dictionaries/plain/en/en_US.aff
54
+ $ wget -O ~/Library/Spelling/en_US.dic https://cgit.freedesktop.org/libreoffice/dictionaries/plain/en/en_US.dic
55
+ ```
56
+
57
+ German:
58
+
59
+ ```command
60
+ $ wget -O ~/Library/Spelling/de_DE.dic https://cgit.freedesktop.org/libreoffice/dictionaries/plain/de/de_DE_frami.dic
61
+ $ wget -O ~/Library/Spelling/de_DE.aff https://cgit.freedesktop.org/libreoffice/dictionaries/plain/de/de_DE_frami.aff
62
+ ```
63
+
64
+ Italian (for integration tests):
65
+
66
+ ```command
67
+ $ wget -O ~/Library/Spelling/it_IT.dic https://cgit.freedesktop.org/libreoffice/dictionaries/plain/it_IT/it_IT.dic
68
+ $ wget -O ~/Library/Spelling/it_IT.aff https://cgit.freedesktop.org/libreoffice/dictionaries/plain/it_IT/it_IT.aff
69
+ ```
data/Rakefile CHANGED
@@ -10,7 +10,7 @@ task default: ['spec:all']
10
10
 
11
11
  namespace :spec do
12
12
  desc 'Run all specs'
13
- task all: %i[rubocop:auto_correct unit system]
13
+ task all: %i[rubocop:autocorrect unit system]
14
14
 
15
15
  %w[unit system].each do |type|
16
16
  desc "Run #{type} tests"
data/TODO.markdown CHANGED
@@ -1,6 +1,4 @@
1
1
  * Bail out if lang cannot be inferred and is not given on cmdline
2
2
  * exe/httpspell: # TODO: --recursive, defaults to false
3
3
  * exe/httpspell: # TODO wget has some additional options for recursive behavior that should be reviewed
4
- * exe/httpspell: # TODO: Find sections with a lang attribute and handle them separately
5
4
  * lib/httpspell/spider.rb: # TODO Print _which_ entry of the blacklist matches
6
- * lib/httpspell/spider.rb: # TODO Ignore same page links (some anchor)
data/exe/httpspell CHANGED
@@ -2,9 +2,9 @@
2
2
  # frozen_string_literal: true
3
3
 
4
4
  require 'optparse'
5
- require 'httpspell/spider'
6
- require 'httpspell/spellchecker'
7
- require 'httpspell/version'
5
+ require 'http_spell/spider'
6
+ require 'http_spell/spellchecker'
7
+ require 'http_spell/version'
8
8
 
9
9
  personal_dictionary_path = nil
10
10
  force_language = nil
@@ -59,27 +59,38 @@ if ARGV.size != 1
59
59
  exit 1
60
60
  end
61
61
 
62
- spell_checker = HttpSpell::SpellChecker.new(personal_dictionary_path, tracing: tracing)
62
+ def check(doc, lang, personal_dictionary_path, verbose)
63
+ unknown_words = HttpSpell::SpellChecker.new(personal_dictionary_path, verbose:).check(doc, lang)
64
+
65
+ if unknown_words.empty?
66
+ warn 'No unknown words.' if verbose
67
+ else
68
+ warn "#{unknown_words.size} unknown words:" if verbose
69
+ puts unknown_words
70
+ true
71
+ end
72
+ end
73
+
63
74
  has_unknown_words = false
64
75
 
65
- spider_success = HttpSpell::Spider.new(ARGV.first, whitelist: whitelist, blacklist: blacklist, tracing: tracing).start do |url, doc|
66
- lang = force_language || doc.root['lang'] || ENV['LANGUAGE']
76
+ spider_success = HttpSpell::Spider.new(ARGV.first, whitelist:, blacklist:, verbose:, tracing:).start do |url, doc|
77
+ lang = force_language || doc.root['lang'] || ENV.fetch('LANGUAGE', nil)
78
+ warn "Checking #{url} as #{lang}" if verbose
67
79
 
68
- # Remove sections that are not to be spellchecked
80
+ # Remove elements that are not to be spellchecked
69
81
  doc.css('pre').each(&:unlink)
70
82
  doc.css('code').each(&:unlink)
71
83
  doc.css('[spellcheck=false]').each(&:unlink)
72
84
 
73
- # TODO: Find sections with a lang attribute and handle them separately
74
- unknown_words = spell_checker.check(doc.to_s, lang)
75
-
76
- if unknown_words.empty?
77
- warn "No unknown words (language is #{lang}) at #{url}." if verbose
78
- else
79
- warn "#{unknown_words.size} unknown words (language is #{lang}) at #{url}:" if verbose
80
- puts unknown_words
81
- has_unknown_words = true
85
+ # Handle elements with a different lang attribute separately
86
+ doc.css(%([lang]:not([lang="#{lang}"]))).each do |element|
87
+ warn "Handling #{element.name} with lang #{element['lang']}:" if verbose
88
+ has_unknown_words |= check(element.to_s, element['lang'], personal_dictionary_path, verbose)
89
+ element.unlink
82
90
  end
91
+
92
+ # Everything else
93
+ has_unknown_words |= check(doc.to_s, lang, personal_dictionary_path, verbose)
83
94
  end
84
95
 
85
96
  exit 2 unless spider_success
data/httpspell.gemspec CHANGED
@@ -2,9 +2,8 @@
2
2
 
3
3
  lib = File.expand_path('lib', __dir__)
4
4
  $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
- require 'httpspell/version'
5
+ require 'http_spell/version'
6
6
 
7
- # rubocop:disable Metrics/BlockLength
8
7
  Gem::Specification.new do |spec|
9
8
  spec.name = 'httpspell'
10
9
  spec.version = HttpSpell::VERSION
@@ -13,8 +12,8 @@ Gem::Specification.new do |spec|
13
12
 
14
13
  spec.summary = 'HTTP spellchecker'
15
14
  spec.description = %(httpspell is a spellchecker that recursively fetches
16
- HTML pages, converts them to plain text using pandoc, and
17
- spellchecks them with hunspell.)
15
+ HTML pages, converts them to plain text using pandoc, and
16
+ spellchecks them with hunspell.)
18
17
  spec.license = 'MIT'
19
18
 
20
19
  spec.files = `git ls-files -z`.split("\x0").reject do |f|
@@ -26,17 +25,5 @@ Gem::Specification.new do |spec|
26
25
 
27
26
  spec.add_dependency 'addressable'
28
27
  spec.add_dependency 'nokogiri'
29
-
30
- spec.add_development_dependency 'aruba'
31
- spec.add_development_dependency 'bundler'
32
- spec.add_development_dependency 'guard'
33
- spec.add_development_dependency 'guard-bundler'
34
- spec.add_development_dependency 'guard-rspec'
35
- spec.add_development_dependency 'pry'
36
- spec.add_development_dependency 'pry-byebug'
37
- spec.add_development_dependency 'rake'
38
- spec.add_development_dependency 'rspec'
39
- spec.add_development_dependency 'rubocop'
40
- spec.add_development_dependency 'stub_server'
28
+ spec.metadata['rubygems_mfa_required'] = 'true'
41
29
  end
42
- # rubocop:enable Metrics/BlockLength
data/lib/{httpspell → http_spell}/spellchecker.rb RENAMED
@@ -1,8 +1,10 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module HttpSpell
2
4
  class SpellChecker
3
- def initialize(personal_dictionary_path = nil, tracing: false)
5
+ def initialize(personal_dictionary_path = nil, verbose: false)
4
6
  @personal_dictionary_arg = "-p #{personal_dictionary_path}" if personal_dictionary_path
5
- @tracing = tracing
7
+ @verbose = verbose
6
8
  end
7
9
 
8
10
  def check(doc, lang)
@@ -11,8 +13,8 @@ module HttpSpell
11
13
  "hunspell -d #{translate(lang)} #{@personal_dictionary_arg} -i UTF-8 -l",
12
14
  ]
13
15
 
14
- if @tracing
15
- warn "Piping the HTML document into the following chain of commands:"
16
+ if @verbose
17
+ warn 'Piping the HTML document into the following chain of commands:'
16
18
  warn commands
17
19
  end
18
20
 
data/lib/{httpspell → http_spell}/spider.rb RENAMED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'nokogiri'
2
4
  require 'open-uri'
3
5
  require 'open3'
@@ -8,12 +10,13 @@ module HttpSpell
8
10
  class Spider
9
11
  attr_reader :todo, :done
10
12
 
11
- def initialize(starting_point, whitelist: nil, blacklist: [], tracing: false)
13
+ def initialize(starting_point, whitelist: nil, blacklist: [], verbose: false, tracing: false)
12
14
  @todo = []
13
15
  @done = []
14
16
  todo << Addressable::URI.parse(starting_point)
15
17
  @whitelist = whitelist || [/^#{starting_point}/]
16
18
  @blacklist = blacklist
19
+ @verbose = verbose
17
20
  @tracing = tracing
18
21
  end
19
22
 
@@ -26,13 +29,13 @@ module HttpSpell
26
29
  begin
27
30
  extracted = links(url) do |u, d|
28
31
  yield u, d if block_given?
29
- rescue
32
+ rescue StandardError
30
33
  warn "Callback error for #{url}: #{$ERROR_INFO}"
31
34
  warn $ERROR_INFO.backtrace if @tracing
32
35
  end
33
36
 
34
37
  done.append(url)
35
- todo.concat(extracted - done - todo)
38
+ todo.concat(extracted - done - todo).uniq!
36
39
  rescue StandardError
37
40
  warn "Skipping #{url} because of #{$ERROR_INFO.message}"
38
41
  warn $ERROR_INFO.backtrace if @tracing
@@ -40,16 +43,16 @@ module HttpSpell
40
43
  end
41
44
  end
42
45
 
43
- return success
46
+ success
44
47
  end
45
48
 
46
49
  private
47
50
 
48
51
  def links(uri)
49
- response = http_get(URI(uri))
52
+ response = http_get(uri)
50
53
 
51
- if response.content_type != 'text/html'
52
- warn "Skipping #{uri} because it is not HTML" if @tracing
54
+ if response.respond_to?(:content_type) && response.content_type != 'text/html'
55
+ warn "Skipping #{uri} because it is not HTML" if @verbose
53
56
  return []
54
57
  end
55
58
 
@@ -60,17 +63,19 @@ module HttpSpell
60
63
  link = uri.join(link) if link.relative?
61
64
 
62
65
  if @whitelist.none? { |re| re.match?(link.to_s) }
63
- warn "Skipping #{link} because it is not on the whitelist #{@whitelist}" if @tracing
66
+ warn "Skipping #{link} because it is not on the whitelist #{@whitelist}" if @verbose
64
67
  next
65
68
  end
66
69
 
67
70
  if @blacklist.any? { |re| re.match?(link.to_s) }
68
- # TODO Print _which_ entry of the blacklist matches
69
- warn "Skipping #{link} because it is on the blacklist #{@blacklist}" if @tracing
71
+ # TODO: Print _which_ entry of the blacklist matches
72
+ warn "Skipping #{link} because it is on the blacklist #{@blacklist}" if @verbose
70
73
  next
71
74
  end
72
75
 
73
- # TODO Ignore same page links (some anchor)
76
+ # Ignore fragment; we always check the whole page
77
+ link.fragment = nil
78
+
74
79
  link
75
80
  rescue StandardError
76
81
  warn $ERROR_INFO.message
@@ -79,7 +84,7 @@ module HttpSpell
79
84
 
80
85
  yield uri, doc if block_given?
81
86
 
82
- warn "Adding #{links.size} links from #{uri}" if @tracing
87
+ warn "Adding #{links.size} links from #{uri}" if @verbose
83
88
  links
84
89
  end
85
90
 
@@ -88,10 +93,10 @@ module HttpSpell
88
93
  tries = 10
89
94
 
90
95
  begin
91
- uri.open(redirect: false)
92
- rescue OpenURI::HTTPRedirect => redirect
93
- uri = redirect.uri
94
- retry if (tries -= 1) > 0
96
+ URI.parse(uri).open(redirect: false)
97
+ rescue OpenURI::HTTPRedirect => e
98
+ uri = e.uri
99
+ retry if (tries -= 1).positive?
95
100
  raise
96
101
  end
97
102
  end
data/lib/{httpspell → http_spell}/version.rb RENAMED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module HttpSpell
4
- VERSION = '1.3.0'
4
+ VERSION = '1.4.1'
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: httpspell
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.3.0
4
+ version: 1.4.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Steffen Uhlig
8
- autorequire:
8
+ autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2019-04-12 00:00:00.000000000 Z
11
+ date: 2024-05-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: addressable
@@ -38,164 +38,10 @@ dependencies:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
- - !ruby/object:Gem::Dependency
42
- name: aruba
43
- requirement: !ruby/object:Gem::Requirement
44
- requirements:
45
- - - ">="
46
- - !ruby/object:Gem::Version
47
- version: '0'
48
- type: :development
49
- prerelease: false
50
- version_requirements: !ruby/object:Gem::Requirement
51
- requirements:
52
- - - ">="
53
- - !ruby/object:Gem::Version
54
- version: '0'
55
- - !ruby/object:Gem::Dependency
56
- name: bundler
57
- requirement: !ruby/object:Gem::Requirement
58
- requirements:
59
- - - ">="
60
- - !ruby/object:Gem::Version
61
- version: '0'
62
- type: :development
63
- prerelease: false
64
- version_requirements: !ruby/object:Gem::Requirement
65
- requirements:
66
- - - ">="
67
- - !ruby/object:Gem::Version
68
- version: '0'
69
- - !ruby/object:Gem::Dependency
70
- name: guard
71
- requirement: !ruby/object:Gem::Requirement
72
- requirements:
73
- - - ">="
74
- - !ruby/object:Gem::Version
75
- version: '0'
76
- type: :development
77
- prerelease: false
78
- version_requirements: !ruby/object:Gem::Requirement
79
- requirements:
80
- - - ">="
81
- - !ruby/object:Gem::Version
82
- version: '0'
83
- - !ruby/object:Gem::Dependency
84
- name: guard-bundler
85
- requirement: !ruby/object:Gem::Requirement
86
- requirements:
87
- - - ">="
88
- - !ruby/object:Gem::Version
89
- version: '0'
90
- type: :development
91
- prerelease: false
92
- version_requirements: !ruby/object:Gem::Requirement
93
- requirements:
94
- - - ">="
95
- - !ruby/object:Gem::Version
96
- version: '0'
97
- - !ruby/object:Gem::Dependency
98
- name: guard-rspec
99
- requirement: !ruby/object:Gem::Requirement
100
- requirements:
101
- - - ">="
102
- - !ruby/object:Gem::Version
103
- version: '0'
104
- type: :development
105
- prerelease: false
106
- version_requirements: !ruby/object:Gem::Requirement
107
- requirements:
108
- - - ">="
109
- - !ruby/object:Gem::Version
110
- version: '0'
111
- - !ruby/object:Gem::Dependency
112
- name: pry
113
- requirement: !ruby/object:Gem::Requirement
114
- requirements:
115
- - - ">="
116
- - !ruby/object:Gem::Version
117
- version: '0'
118
- type: :development
119
- prerelease: false
120
- version_requirements: !ruby/object:Gem::Requirement
121
- requirements:
122
- - - ">="
123
- - !ruby/object:Gem::Version
124
- version: '0'
125
- - !ruby/object:Gem::Dependency
126
- name: pry-byebug
127
- requirement: !ruby/object:Gem::Requirement
128
- requirements:
129
- - - ">="
130
- - !ruby/object:Gem::Version
131
- version: '0'
132
- type: :development
133
- prerelease: false
134
- version_requirements: !ruby/object:Gem::Requirement
135
- requirements:
136
- - - ">="
137
- - !ruby/object:Gem::Version
138
- version: '0'
139
- - !ruby/object:Gem::Dependency
140
- name: rake
141
- requirement: !ruby/object:Gem::Requirement
142
- requirements:
143
- - - ">="
144
- - !ruby/object:Gem::Version
145
- version: '0'
146
- type: :development
147
- prerelease: false
148
- version_requirements: !ruby/object:Gem::Requirement
149
- requirements:
150
- - - ">="
151
- - !ruby/object:Gem::Version
152
- version: '0'
153
- - !ruby/object:Gem::Dependency
154
- name: rspec
155
- requirement: !ruby/object:Gem::Requirement
156
- requirements:
157
- - - ">="
158
- - !ruby/object:Gem::Version
159
- version: '0'
160
- type: :development
161
- prerelease: false
162
- version_requirements: !ruby/object:Gem::Requirement
163
- requirements:
164
- - - ">="
165
- - !ruby/object:Gem::Version
166
- version: '0'
167
- - !ruby/object:Gem::Dependency
168
- name: rubocop
169
- requirement: !ruby/object:Gem::Requirement
170
- requirements:
171
- - - ">="
172
- - !ruby/object:Gem::Version
173
- version: '0'
174
- type: :development
175
- prerelease: false
176
- version_requirements: !ruby/object:Gem::Requirement
177
- requirements:
178
- - - ">="
179
- - !ruby/object:Gem::Version
180
- version: '0'
181
- - !ruby/object:Gem::Dependency
182
- name: stub_server
183
- requirement: !ruby/object:Gem::Requirement
184
- requirements:
185
- - - ">="
186
- - !ruby/object:Gem::Version
187
- version: '0'
188
- type: :development
189
- prerelease: false
190
- version_requirements: !ruby/object:Gem::Requirement
191
- requirements:
192
- - - ">="
193
- - !ruby/object:Gem::Version
194
- version: '0'
195
41
  description: |-
196
42
  httpspell is a spellchecker that recursively fetches
197
- HTML pages, converts them to plain text using pandoc, and
198
- spellchecks them with hunspell.
43
+ HTML pages, converts them to plain text using pandoc, and
44
+ spellchecks them with hunspell.
199
45
  email:
200
46
  - steffen@familie-uhlig.net
201
47
  executables:
@@ -203,25 +49,29 @@ executables:
203
49
  extensions: []
204
50
  extra_rdoc_files: []
205
51
  files:
52
+ - ".github/dependabot.yml"
206
53
  - ".gitignore"
54
+ - ".mergify.yml"
207
55
  - ".rspec"
208
56
  - ".rubocop.yml"
209
- - ".travis.yml"
57
+ - ".ruby-version"
210
58
  - Gemfile
211
59
  - Gemfile.lock
60
+ - Guardfile
212
61
  - README.markdown
213
62
  - Rakefile
214
63
  - TODO.markdown
215
64
  - exe/httpspell
216
65
  - httpspell.gemspec
217
- - lib/httpspell/spellchecker.rb
218
- - lib/httpspell/spider.rb
219
- - lib/httpspell/version.rb
220
- homepage:
66
+ - lib/http_spell/spellchecker.rb
67
+ - lib/http_spell/spider.rb
68
+ - lib/http_spell/version.rb
69
+ homepage:
221
70
  licenses:
222
71
  - MIT
223
- metadata: {}
224
- post_install_message:
72
+ metadata:
73
+ rubygems_mfa_required: 'true'
74
+ post_install_message:
225
75
  rdoc_options: []
226
76
  require_paths:
227
77
  - lib
@@ -236,8 +86,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
236
86
  - !ruby/object:Gem::Version
237
87
  version: '0'
238
88
  requirements: []
239
- rubygems_version: 3.0.1
240
- signing_key:
89
+ rubygems_version: 3.5.9
90
+ signing_key:
241
91
  specification_version: 4
242
92
  summary: HTTP spellchecker
243
93
  test_files: []
data/.travis.yml DELETED
@@ -1,6 +0,0 @@
1
- language: ruby
2
- rvm:
3
- - 2.5.1
4
- before_install:
5
- - sudo apt-get -qq update
6
- - sudo apt-get install -y pandoc hunspell hunspell-en-us