httpspell 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 0fe73f8f1ff3740d6e3ae3af685d3554879a2819fc9d7803c994a21dd3694d91
+  data.tar.gz: 8354f5c3bdc325a073310aa534a6171164d3dfbe7a1c4154f77737f20108eb91
+SHA512:
+  metadata.gz: '073693d2520238d10012e4c02057c4966ab8af80f1c9db868e5ae2a4b95e4ae59a7d0989c162f62649aa0d2194290da0bca1ac5e1186f8ff3569cca581d571ae'
+  data.tar.gz: 8eb778ffa3bcc1f56e8362d160117f695ec5f3ca146592219f4ef43a160ea28b96c67c5a6edeba52fbdc6dc3413b9f4967243fc60431a81861506d2c46435b7b

data/.gitignore

@@ -0,0 +1 @@
+pkg

data/.rubocop.yml

@@ -0,0 +1,30 @@
+AllCops:
+  TargetRubyVersion: 2.5.1
+  Include:
+    - '**/Gemfile'
+    - '**/Rakefile'
+    - '**/config.ru'
+    - '**/*.rake'
+  Exclude:
+    - vendor/**/*
+    - db/migrations/**/*
+
+  DisplayCopNames:
+    Enabled: true
+
+  DisplayStyleGuide:
+    Enabled: true
+
+Naming/FileName:
+  Exclude:
+  - Guardfile
+
+Metrics/BlockLength:
+  Exclude:
+    - spec/**/*
+
+Metrics/LineLength:
+  Max: 160
+
+Style/Documentation:
+  Enabled: false

data/Gemfile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+gemspec

data/Gemfile.lock

@@ -0,0 +1,108 @@
+PATH
+  remote: .
+  specs:
+    httpspell (1.0.0)
+      addressable
+      nokogiri
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
+    ast (2.4.0)
+    byebug (10.0.2)
+    coderay (1.1.2)
+    diff-lcs (1.3)
+    ffi (1.9.23)
+    formatador (0.2.5)
+    guard (2.14.2)
+      formatador (>= 0.2.4)
+      listen (>= 2.7, < 4.0)
+      lumberjack (>= 1.0.12, < 2.0)
+      nenv (~> 0.1)
+      notiffany (~> 0.0)
+      pry (>= 0.9.12)
+      shellany (~> 0.0)
+      thor (>= 0.18.1)
+    guard-bundler (2.1.0)
+      bundler (~> 1.0)
+      guard (~> 2.2)
+      guard-compat (~> 1.1)
+    guard-compat (1.2.1)
+    guard-rspec (4.7.3)
+      guard (~> 2.1)
+      guard-compat (~> 1.1)
+      rspec (>= 2.99.0, < 4.0)
+    listen (3.1.5)
+      rb-fsevent (~> 0.9, >= 0.9.4)
+      rb-inotify (~> 0.9, >= 0.9.7)
+      ruby_dep (~> 1.2)
+    lumberjack (1.0.13)
+    method_source (0.9.0)
+    mini_portile2 (2.3.0)
+    nenv (0.3.0)
+    nokogiri (1.8.2)
+      mini_portile2 (~> 2.3.0)
+    notiffany (0.1.1)
+      nenv (~> 0.1)
+      shellany (~> 0.0)
+    parallel (1.12.1)
+    parser (2.5.1.0)
+      ast (~> 2.4.0)
+    powerpack (0.1.1)
+    pry (0.11.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    pry-byebug (3.6.0)
+      byebug (~> 10.0)
+      pry (~> 0.10)
+    public_suffix (3.0.2)
+    rainbow (3.0.0)
+    rake (12.3.1)
+    rb-fsevent (0.10.3)
+    rb-inotify (0.9.10)
+      ffi (>= 0.5.0, < 2)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.1)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.1)
+    rubocop (0.56.0)
+      parallel (~> 1.10)
+      parser (>= 2.5)
+      powerpack (~> 0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.0, >= 1.0.1)
+    ruby-progressbar (1.9.0)
+    ruby_dep (1.5.0)
+    shellany (0.0.1)
+    thor (0.20.0)
+    unicode-display_width (1.3.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler
+  guard
+  guard-bundler
+  guard-rspec
+  httpspell!
+  pry
+  pry-byebug
+  rake
+  rspec
+  rubocop
+
+BUNDLED WITH
+   1.16.1

data/README.markdown

@@ -0,0 +1,21 @@
+# `httpspell`
+
+This is a spellchecker that recursively fetches HTML pages, converts them to plain text (using [pandoc](http://pandoc.org/)), and spellchecks them with [hunspell](https://hunspell.github.io/). Unknown words will be printed to `stdout`, which makes the tool a good candidate for CI pipelines where you might want to take action when a spelling error is found on a web page.
+
+Words that are not in the dictionary for the given language (inferred from the `lang` attribute of the HTML document's root element) can be added to a personal dictionary, which will mark the word as correctly spelled.
+
+# What is *not* checked
+
+* When spidering a site, `httpspell` will skip all responses with a `content-type` header other than `text/html`.
+* Before converting, `httpspell` removes the following nodes from the HTML DOM as they are not a good target for spellchecking:
+  - `code`
+  - `pre`
+  - Elements with `spellcheck='false'` (this is how HTML5 allows tagging elements as a target for spellchecking)
+
+# Misc
+
+If you produce content with kramdown (e.g. using Jekyll), setting `spellcheck='false'` for an element is a simple as adding this line *after* the element (e.g. heading):
+
+```
+{: spellcheck="false"}
+```

data/Rakefile

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'rspec/core/rake_task'
+require 'bundler/gem_tasks'
+require 'rubocop/rake_task'
+
+RuboCop::RakeTask.new
+
+task default: ['spec:all']
+
+namespace :spec do
+  desc 'Run all specs'
+  task all: ['rubocop:auto_correct', :unit]
+
+  RSpec::Core::RakeTask.new(:unit) do |t|
+    t.pattern = 'spec/unit/**/*_spec.rb'
+  end
+end

data/exe/httpspell

@@ -0,0 +1,51 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'optparse'
+require 'httpspell/spider'
+require 'httpspell/spellchecker'
+
+personal_dictionary_path = nil
+
+begin
+  OptionParser.new do |parser|
+    parser.banner.prepend <<~BANNER
+      Spellchecks a website via HTTP.
+
+    BANNER
+
+    parser.on('-p', '--personal-dictionary=FILE', 'path to the personal dictionary file') do |p|
+      personal_dictionary_path = p
+    end
+
+    # TODO: --recursive, defaults to false
+    # TODO wget has some additional options for recursive behavior that should be reviewed
+  end.parse!
+rescue StandardError
+  warn "Error - #{$ERROR_INFO}"
+  exit 1
+end
+
+if ARGV.size != 1
+  warn "Expected exactly one argument, but received #{ARGV.size}."
+  exit 1
+end
+
+spell_checker = HttpSpell::SpellChecker.new(personal_dictionary_path)
+
+HttpSpell::Spider.new(ARGV.first).start do |url, doc|
+  lang = doc.root['lang'] || 'de-DE'
+
+  # Remove sections that are not to be spellchecked
+  doc.css('pre').each(&:unlink)
+  doc.css('code').each(&:unlink)
+  doc.css('[spellcheck=false]').each(&:unlink)
+
+  # TODO: Find sections with a lang attribute and handle them separately
+  unknown_words = spell_checker.check(doc.to_s, lang)
+
+  unless unknown_words.empty?
+    warn "#{unknown_words.size} unknown words at #{url}:"
+    puts unknown_words
+  end
+end

data/httpspell.gemspec

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'httpspell/version'
+
+# rubocop:disable Metrics/BlockLength
+Gem::Specification.new do |spec|
+  spec.name          = 'httpspell'
+  spec.version       = HttpSpell::VERSION
+  spec.authors       = ['Steffen Uhlig']
+  spec.email         = ['steffen@familie-uhlig.net']
+
+  spec.summary       = 'HTTP spellchecker'
+  spec.description   = %(httpspell is a spellchecker that recursively fetches
+    HTML pages, converts them to plain text using pandoc, and
+    spellchecks them with hunspell.)
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'addressable'
+  spec.add_dependency 'nokogiri'
+
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'guard'
+  spec.add_development_dependency 'guard-bundler'
+  spec.add_development_dependency 'guard-rspec'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'pry-byebug'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rubocop'
+end
+# rubocop:enable Metrics/BlockLength

data/lib/httpspell/spellchecker.rb

@@ -0,0 +1,25 @@
+module HttpSpell
+  class SpellChecker
+    def initialize(personal_dictionary_path = nil)
+      @personal_dictionary_arg = "-p #{personal_dictionary_path}" if personal_dictionary_path
+    end
+
+    def check(doc, lang)
+      Open3.pipeline_rw('pandoc --from html --to plain', "hunspell -d #{translate(lang)} #{@personal_dictionary_arg} -i UTF-8 -l") do |stdin, stdout, _wait_thrs|
+        stdin.puts(doc)
+        stdin.close
+        stdout.read.split.uniq
+      end
+    end
+
+    private
+
+    # The W3C [recommends](https://www.w3.org/International/questions/qa-html-language-declarations)
+    # to specify language using identifiers as per [RFC 5646](https://tools.ietf.org/html/rfc5646)
+    # which uses dashes. Hunspell, however, uses underscores. This method translates RFC-style identifiers
+    # to hunspell-style.
+    def translate(lang)
+      lang.tr('-', '_')
+    end
+  end
+end

data/lib/httpspell/spider.rb

@@ -0,0 +1,62 @@
+require 'nokogiri'
+require 'open-uri'
+require 'open3'
+require 'addressable/uri'
+require 'English'
+
+module HttpSpell
+  # rubocop:disable Metrics/AbcSize
+  # rubocop:disable Metrics/MethodLength
+  class Spider
+    attr_reader :todo, :done
+
+    def initialize(starting_point, base_url = starting_point)
+      @todo = []
+      @done = []
+      todo << Addressable::URI.parse(starting_point)
+      @base_url = Addressable::URI.parse(base_url)
+    end
+
+    def start
+      while todo.any?
+        url = todo.pop
+
+        begin
+          extracted = links(url) do |u, d|
+            yield u, d if block_given?
+          rescue
+            warn "Callback error for #{url}: #{$ERROR_INFO}"
+          end
+
+          done.append(url)
+          todo.concat(extracted - done - todo)
+        rescue StandardError
+          warn "Could not fetch #{url}: #{$ERROR_INFO}"
+        end
+      end
+    end
+
+    private
+
+    def links(uri)
+      # We are using open-uri, which follows redirects and also provides the content-type.
+      response = URI(uri).read
+      return [] unless response.content_type == 'text/html'
+      doc = Nokogiri::HTML(response)
+
+      links = doc.css('a[href]').map do |e|
+        link = Addressable::URI.parse(e['href'])
+        link = uri.join(link) if link.relative?
+        next unless link.to_s.start_with?(@base_url.to_s)
+        link
+      rescue StandardError
+        warn $ERROR_INFO
+      end.compact
+
+      yield uri, doc if block_given?
+      links
+    end
+  end
+  # rubocop:enable Metrics/AbcSize
+  # rubocop:enable Metrics/MethodLength
+end

data/lib/httpspell/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module HttpSpell
+  VERSION = '1.0.0'
+end

metadata

@@ -0,0 +1,213 @@
+--- !ruby/object:Gem::Specification
+name: httpspell
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Steffen Uhlig
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-05-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: |-
+  httpspell is a spellchecker that recursively fetches
+      HTML pages, converts them to plain text using pandoc, and
+      spellchecks them with hunspell.
+email:
+- steffen@familie-uhlig.net
+executables:
+- httpspell
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rubocop.yml"
+- Gemfile
+- Gemfile.lock
+- README.markdown
+- Rakefile
+- exe/httpspell
+- httpspell.gemspec
+- lib/httpspell/spellchecker.rb
+- lib/httpspell/spider.rb
+- lib/httpspell/version.rb
+homepage: 
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: HTTP spellchecker
+test_files: []