ghostwriter 0.1.0.placeholder → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: ebcc9c22952ebb3ffc67cf0e0a6fccc26a7ef03b
-  data.tar.gz: 66ebb78d46f9bff22308d8223cdfeb9b322528b0
+SHA256:
+  metadata.gz: 80d5aced9b18684b3640c28ce1e86c8e9859942f57fefbd26dd1a1f3e7791eaf
+  data.tar.gz: 0f71619c0a7e247cf163f074ca7c8bb54fa8c93daaa5649df8f853fd0ccab1da
 SHA512:
-  metadata.gz: d09deb19bc041c49fb2d87e8b4220b8bd6488deb55d48cc4ca5c72b9ad4eaa6dc94cc41200effdd683fab4727133a9b0a13bd3d7619db3d66215535c4268a531
-  data.tar.gz: 9cacc0548840d48cf2b049bd2d65457c5b248ff0cb4e79b6b7fa2e916dd35fdb69f0a84bfdb7c759099e761c3d8e74d0d0a09dc60b5ee842440412f54a213f94
+  metadata.gz: f9760753d4ffc30bee200a33347cd9aeda0b4593304f07ff9ce53c4ca1f971d51b50644feb763ffc51ec5635202e971704592f45d0cbe021693a7e790e39c7e9
+  data.tar.gz: 203df0d639d25f35a73dcdde0f12c037e8f508e086c34cec5e2da7325a8b0e173db10590da50550a50e853f74f0753ce231b36646619fc3c1f166bdd986e7186

data/.rubocop.yml

@@ -0,0 +1 @@
+inherit_from: ../.rubocop.yml

data/.ruby-version

@@ -0,0 +1 @@
+ruby-2.7.1

data/.travis.yml

@@ -1,4 +1,4 @@
 language: ruby
 rvm:
   - 2.2.1
-before_install: gem install bundler -v 1.11.2
+before_install: gem install bundler -v 1.10.6

data/CODE_OF_CONDUCT.md

@@ -1,49 +1,13 @@
 # Contributor Code of Conduct
 
-As contributors and maintainers of this project, and in the interest of
-fostering an open and welcoming community, we pledge to respect all people who
-contribute through reporting issues, posting feature requests, updating
-documentation, submitting pull requests or patches, and other activities.
+As contributors and maintainers of this project, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities.
 
-We are committed to making participation in this project a harassment-free
-experience for everyone, regardless of level of experience, gender, gender
-identity and expression, sexual orientation, disability, personal appearance,
-body size, race, ethnicity, age, religion, or nationality.
+We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, or religion.
 
-Examples of unacceptable behavior by participants include:
+Examples of unacceptable behavior by participants include the use of sexual language or imagery, derogatory comments or personal attacks, trolling, public or private harassment, insults, or other unprofessional conduct.
 
-* The use of sexualized language or imagery
-* Personal attacks
-* Trolling or insulting/derogatory comments
-* Public or private harassment
-* Publishing other's private information, such as physical or electronic
-  addresses, without explicit permission
-* Other unethical or unprofessional conduct
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. Project maintainers who do not follow the Code of Conduct may be removed from the project team.
 
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
-permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive, or harmful.
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers.
 
-By adopting this Code of Conduct, project maintainers commit themselves to
-fairly and consistently applying these principles to every aspect of managing
-this project. Project maintainers who do not follow or enforce the Code of
-Conduct may be permanently removed from the project team.
-
-This code of conduct applies both within project spaces and in public spaces
-when an individual is representing the project or its community.
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting a project maintainer at robin@tenjin.ca. All
-complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. Maintainers are
-obligated to maintain confidentiality with regard to the reporter of an
-incident.
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage],
-version 1.3.0, available at
-[http://contributor-covenant.org/version/1/3/0/][version]
-
-[homepage]: http://contributor-covenant.org
-[version]: http://contributor-covenant.org/version/1/3/0/
+This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.0.0, available at [http://contributor-covenant.org/version/1/0/0/](http://contributor-covenant.org/version/1/0/0/)

data/Gemfile

@@ -1,4 +1,6 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in ghostwriter.gemspec
+# Specify gem dependencies in ghostwriter.gemspec
 gemspec

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2016 Robin Miller
+Copyright (c) 2015 Robin Miller
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -1,8 +1,15 @@
 # Ghostwriter
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/ghostwriter`. To experiment with that code, run `bin/console` for an interactive prompt.
+Ghostwriter rewrites HTML as plain text while preserving as much legibility and functionality as possible.
 
-TODO: Delete this and the text above, and describe your gem
+It's sort of like a reverse-markdown or a very, very simple screen reader.
+
+## But Why, Though?
+
+* Some email clients won't or can’t handle HTML at all
+* Some people explicitly choose plaintext just by preference or accessibility
+* Spam filters tend to like emails with a plain text alternative (but if you use this gem to help you spam people, I
+  will yell at you)
 
 ## Installation
 
@@ -14,28 +21,268 @@ gem 'ghostwriter'
 
 And then execute:
 
-    $ bundle
+    bundle
 
-Or install it yourself as:
+Or install it manually with:
 
-    $ gem install ghostwriter
+    gem install ghostwriter
 
 ## Usage
 
-TODO: Write usage instructions here
+Create a `Ghostwriter::Writer` and call `#textify` with the html you want modified:
+
+```ruby
+html = '<html><body><p>This is some markup <a href="tenjin.ca">and a link</a></p><p>Other tags translate, too</p></body></html>'
+
+Ghostwriter::Writer.new.textify(html)
+```
+
+Produces:
+
+```
+This is some markup and a link (tenjin.ca)
+
+Other tags translate, too
+```
+
+### Links
+
+Links are converted to the link text followed by the link target in brackets:
+
+```html
+
+<html>
+<body>
+Visit our <a href="https://example.com">Website</a>
+<body>
+</html>
+```
+
+Becomes:
+
+```
+Visit our Website (https://example.com)
+```
+
+#### Relative Links
+
+Since emails are wholly distinct from your web address, relative links might break.
+
+To avoid this problem, either use the `<base>` header tag:
+
+```html
+
+<html>
+<head>
+   <base href="https://www.example.com">
+</head>
+<body>
+Use the base tag to <a href="/contact">expand</a> links.
+</body>
+</html>
+```
+
+Becomes:
+
+```
+Use the base tag to expand (https://www.example.com/contact) links
+```
+
+Or you can use the `link_base` configuration:
+
+```ruby
+Ghostwriter::Writer.new(link_base: 'tenjin.ca').textify(html)
+```
+
+### Images
+
+Images with alt text are converted:
+
+```html
+<img src="logo.jpg" alt="ACME Anvils" />
+```
+
+Becomes:
+
+```
+ACME Anvils (logo.jpg)
+```
+
+But images lacking alt text or with a presentation ARIA role are ignored:
+
+```html
+<!-- these will just become an empty string -->
+<img src="decoration.jpg">
+<img src="logo.jpg" role="presentation">
+```
+
+And images with data URIs won't include the data portion.
+
+```html
+<img src="data:image/gif;base64,R0lGODdhIwAjAMZ/AAkMBxETEBUUDBoaExkaGCIcFx4fGCEfFCcfECkjHiUlHiglGikmFjAqFi8pJCsrJT8sCjMzLDUzJzs0GjkzLTszKTM1Mzg4MD48Mzs+O0tAIElCJ1NCGVdBHUtEMkNFQjlHTFJDOkdGPT1ISUxLRENOT1tMI01PTGdLKk1RU0hTVEtTT0NVVFRTTExYWE9YVGhVP1VZXGFYTWhaMFRcWHFYL1FdXV1dRHdZMVRgYFhgXFdiY11hY1tkX31hJltmZ2pnWnloLGFrbG9oYXlqN3NqTnBqWHxqRItvRIh0Nod0ToF2U5J4LX55Xm97e4B5aZqAQpGAdqOCOZKEYZ2FOJyEVoyKbqiOXpySbLCVcLCXaKWbdKCdfZyhi66dksGdc76fbbije7mkdLOmgq6ogrCpibyvirexisWvhs2vgsGyiLq1lce1lMC5ks28nsfBmcHDq9bAl9PDmMnFo9TGh8zIoM7Jm9vLs9nRo93QqtfSquLQpdXUs+fdterlw////ywAAAAAIwAjAAAH/oArOTo6PYaGOz08P0KMOTZCOzw7PzY/Pz2JPYSDhTSFPTSXPY0tIiIfJz05o5Q/O7A5moc6O4Q0oS8uQisXGCItwTItP5OxOrKjhzSfLzYvgz85ERQXJKcSIkZeJDqOl43StrSEKzo2LhkOGBISDw40JyIVFVEyorBCkZmwtCsrtnLQSJCAwoMFCiwoiECPAr0TjPrtECJwXLMVNARlUCBhQAEFC2SsgWPGDBs3d2RcorSD1SVGr3qskOkihoIH70DO0cOHDx48evD0KQONmQ0aORZJE3VLRYoPBRwoUCCCSx07eoL+xLNnj5UfNFry4BHuR6EcK0qkKJFhAYUE/g+cdHlz1efPrnvM2MjhQlYOWTxktXThIoUKhQoKDHBi5Y0dO0CD5smzJ46NvWJfjYW1w4WKEiWkKkgw9UYdPXTo8Mn6042bvX9pTHoFa5GKzykekP5owEidN1u6PKnzMw+QJ3ttUPr7qKUs0C5KHOyoAMMaNWrmjKlSRYscMFm+nBBUybkLSYsIl3DxwAgcKwWMzGnz5kqTK1e09AEDI0uGE8rJEgNfsuxVggoujGABF1xMoYAVc9RRhxxq5JGVHn3EEYcIGfT1igvGKLfDZyWMkMINa5QhQRNz9CQhT1n5URmHJ8Sygw2BSWLDbaCpgEFPNzxBV4QwApVhHBhg/vABZ0pJIhuCoI0wQhFlkLEGGWfQ9wZ2W6KRBhoUJKncKyK2tMOBPI6wwAxltInlG1uKcQUUV3xpwQUXACSJjbCAxgJoJShggBVtnmGGlm/M4UYcX14QQQQ1PpJjUjmsd5sKCg5gBRdkYMlGG2KwoUYWWYARxgXVnODXqmP9CWgJIESwxhJTbEHGGGbMsSWpaRRBQQQXpPKIiJOgg+BnI4AwwhxcHFHrGGN0KYYYaEhAzQX/7flIDMqx4CoIJY7QxhpY0GorXXXwkUcRj1Lg7gfMDavcCSx4BqsIHpyxRhtT1FCDEmNgF4YY1j6KZ4eXXTast9GVcAIHG2TZRhlT/qCAAg5IZIzCA+1QQ0EGKbgAG7c0pPOAAgQcwEQSZ2R5RhlYVIFEFVccAQEAAASgWEIrXEZYDDHQYAEBAQSAcxBUbCExGWVsMfMVCHSA89QCbHBDX4QRRsPURuMcQBBQYLHGHGuwoYUYVdQQxAIOBCCACVLUgDMBS7rwwgtENHDAAEYLMIAAHhABRRVYKFEDDjjU0AA9HiQhxQQOCDC1BXe/UAQVVATRwAIDDGCAAAd0EAQTTEgBBQ4IIFSBFHFPdYEIFJBAQOUE1K5AAyZgnsQME/jNwAG/e7QBFT4sYEABBiQv6ANDDLDCCwPULr0ADYyeOQcMLMAAAxNAIQUHJwckYEDn5CfvgAEKvECA3+R7nrwB2k+ggQkmaLB3++Sz3zkMIawQCAA7" alt="Data picture"/>
+```
+
+Becomes:
+
+```
+Data picture (embedded)
+```
+
+
+### Lists
+
+Lists are converted, too. They are padded with newlines and are given simple markers:
+
+```html
+
+<ul>
+   <li>Planes</li>
+   <li>Trains</li>
+   <li>Automobiles</li>
+</ul>
+<ol>
+   <li>I get knocked down</li>
+   <li>I get up again</li>
+   <li>Never gonna keep me down</li>
+</ol>
+```
+
+Becomes:
+
+```
+
+- Planes
+- Trains
+- Automobiles
+
+1. I get knocked down
+2. I get up again
+3. Never gonna keep me down
+
+```
+
+### Tables
+
+Tables are still often used in email structuring because support for more modern HTML and CSS is inconsistent. If your
+table is purely presentational, mark it with `role="presentation"`. See below for details.
+
+For real data tables, Ghostwriter tries to maintain table structure for simple tables:
+
+```html
+
+<table>
+   <thead>
+   <tr>
+      <th>Ship</th>
+      <th>Captain</th>
+   </tr>
+   </thead>
+   <tbody>
+   <tr>
+      <td>Enterprise</td>
+      <td>Jean-Luc Picard</td>
+   </tr>
+   <tr>
+      <td>TARDIS</td>
+      <td>The Doctor</td>
+   </tr>
+   <tr>
+      <td>Planet Express Ship</td>
+      <td>Turanga Leela</td>
+   </tr>
+   </tbody>
+</table>
+```
+
+Becomes:
+
+```
+| Ship                | Captain         |
+|---------------------|-----------------|
+| Enterprise          | Jean-Luc Picard |
+| TARDIS              | The Doctor      |
+| Planet Express Ship | Turanga Leela   |
+```
 
-## Development
+### Presentation ARIA Role
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+Lists and tables with `role="presentation"` will be treated as a simple container and the normal behaviour will be
+suppressed.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+```html
+
+<table role="presentation">
+   <tr>
+      <td>The table is a lie</td>
+   </tr>
+</table>
+<ul role="presentation">
+   <li>No such list</li>
+</ul>
+```
+
+Becomes:
+
+```
+The table is a lie
+No such list
+```
+
+### Mail Gem Example
+
+To use `#textify` with the [mail](https://github.com/mikel/mail) gem, just provide the text-part by pasisng the html
+through Ghostwriter:
+
+```ruby
+require 'mail'
+
+html        = 'My email and a <a href="https://tenjin.ca">link</a>'
+ghostwriter = Ghostwriter::Writer.new
+
+Mail.deliver do
+   to 'bob@example.com'
+   from 'dot@example.com'
+   subject 'Using Ghostwriter with Mail'
+
+   html_part do
+      content_type 'text/html; charset=UTF-8'
+      body html
+   end
+
+   text_part do
+      body ghostwriter.textify(html)
+   end
+end
+
+```
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/ghostwriter. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+Bug reports and pull requests are welcome on GitHub at https://github.com/TenjinInc/ghostwriter
+
+This project is intended to be a friendly space for collaboration, and contributors are expected to adhere to the
+[Contributor Covenant](contributor-covenant.org) code of conduct.
+
+### Core Developers
+
+After checking out the repo, run `bundle install` to install dependencies. Then, run `rake spec` to run the tests. You
+can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
+#### Local Install
+
+To install this gem onto your local machine only, run
+
+`bundle exec rake install`
+
+#### Gem Release
+
+To release a gem to the world at large
+
+1. Update the version number in `version.rb`,
+2. Run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push
+   the `.gem` file to [rubygems.org](https://rubygems.org).
+3. Do a wee dance
 
 ## License
 
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
-

data/RELEASE_NOTES.md

@@ -0,0 +1,77 @@
+# Release Notes
+
+## 1.0.0 (2021-03-21)
+
+### Major
+
+* Moved `link_base` parameter to constructor
+* Moved input HTML parameter to `#textify`
+
+### Minor
+
+* Treats tables and lists with role="presentation" as simple containers
+* Now handles ordered and unordered lists
+* Images are now replaced with their alt text
+
+### Bugfixes
+
+* none
+
+## 0.4.2 (2021-03-17)
+
+### Major
+
+* none
+
+### Minor
+
+* none
+
+### Bugfixes
+
+* Works with links using `tel:` and `mailto:` schemas.
+
+## 0.4.1 (2021-03-17)
+
+### Major
+
+* none
+
+### Minor
+
+* No longer provides link target in brackets after link text when they are the same
+
+### Bugfixes
+
+* Added explicit testing for HTML entity interpretation
+
+## 0.4.0 (2021-03-16)
+
+### Major
+
+* Updated gem dependencies
+
+### Minor
+
+* Updated docs
+* Added support for tables
+
+### Bugfixes
+
+* none
+
+## 0.3.0 (2016-03-06)
+
+### Major
+
+* Renamed to Ghostwriter
+
+### Minor
+
+* Docs: Added instruction for using textify with mail gem
+
+### Bugfixes
+
+* none
+
+

data/Rakefile

@@ -1,6 +1,8 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec

data/bin/console

@@ -1,7 +1,9 @@
 #!/usr/bin/env ruby
+#
+# frozen_string_literal: true
 
-require "bundler/setup"
-require "ghostwriter"
+require 'bundler/setup'
+require 'ghostwriter'
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
@@ -10,5 +12,5 @@ require "ghostwriter"
 # require "pry"
 # Pry.start
 
-require "irb"
+require 'irb'
 IRB.start

data/bin/setup

@@ -1,7 +1,6 @@
-#!/usr/bin/env bash
+#!/bin/bash
 set -euo pipefail
 IFS=$'\n\t'
-set -vx
 
 bundle install
 

data/dirt-textify.gemspec

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'ghostwriter/version'
+
+Gem::Specification.new do |spec|
+   spec.name    = 'ghostwriter'
+   spec.version = Ghostwriter::VERSION
+   spec.authors = ['Robin Miller']
+   spec.email   = ['robin@tenjin.ca']
+
+   spec.summary     = 'Intelligently extracts plaintext from an HTML document.'
+   spec.description = <<~DESC
+      Transforms HTML into plaintext while preserving legibility and functionality.
+   DESC
+   spec.homepage = 'https://github.com/TenjinInc/ghostwriter'
+   spec.license  = 'MIT'
+
+   spec.files = `git ls-files -z`.split("\x0").reject do |f|
+      f.match(%r{^(test|spec|features)/})
+   end
+
+   spec.bindir        = 'exe'
+   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+   spec.require_paths = ['lib']
+
+   spec.required_ruby_version = '~> 2.4'
+
+   spec.add_dependency 'nokogiri', '= 1.8.4'
+
+   spec.add_development_dependency 'bundler', '~> 2.2'
+   spec.add_development_dependency 'rake', '~> 13.0'
+   spec.add_development_dependency 'rspec', '~> 3.3'
+   spec.add_development_dependency 'rubocop', '~> 1.11'
+   spec.add_development_dependency 'rubocop-performance', '~> 1.10'
+end

data/lib/ghostwriter.rb

@@ -1,5 +1,5 @@
-require "ghostwriter/version"
+# frozen_string_literal: true
 
-module Ghostwriter
-  # Your code goes here...
-end
+require 'ghostwriter/version'
+require 'ghostwriter/writer'
+require 'nokogiri'

data/lib/ghostwriter/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Ghostwriter
-  VERSION = "0.1.0.placeholder"
+   VERSION = '1.0.0'
 end

data/lib/ghostwriter/writer.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+module Ghostwriter
+   # Main Ghostwriter converter object.
+   class Writer
+      # Creates a new ghostwriter
+      #
+      # @param [String] link_base the url to prefix relative links with
+      def initialize(link_base: '')
+         @link_base   = link_base
+         @list_marker = '-'
+      end
+
+      # Strips HTML down to plain text.
+      #
+      # @param html [String] the HTML to be convert to text
+      #
+      # @return converted text
+      def textify(html)
+         doc = Nokogiri::HTML(normalize_whitespace(html).gsub('</p>', "</p>\n\n"))
+
+         doc.search('style, script').remove
+
+         replace_anchors(doc)
+         replace_images(doc)
+
+         simple_replace(doc, '*[role="presentation"]', "\n")
+
+         replace_headers(doc)
+         replace_lists(doc)
+         replace_tables(doc)
+
+         simple_replace(doc, 'hr', "\n----------\n")
+         simple_replace(doc, 'br', "\n")
+
+         # doc.search('p').each do |link_node|
+         #   link_node.inner_html = link_node.inner_html + "\n\n"
+         # end
+
+         # trim, but only single-space character
+         doc.text.gsub(/^ +| +$/, '')
+      end
+
+      private
+
+      def normalize_whitespace(html)
+         html.gsub(/\s/, ' ').squeeze(' ')
+      end
+
+      def replace_anchors(doc)
+         doc.search('a').each do |link_node|
+            href = get_link_target(link_node, get_link_base(doc))
+
+            link_node.inner_html = if link_matches(href, link_node.inner_html)
+                                      href.to_s
+                                   else
+                                      "#{ link_node.inner_html } (#{ href })"
+                                   end
+         end
+      end
+
+      def link_matches(first, second)
+         first.to_s.gsub(%r{^https?://}, '').chomp('/') == second.gsub(%r{^https?://}, '').chomp('/')
+      end
+
+      def get_link_base(doc)
+         # <base> node is unique by W3C spec
+         base_node = doc.search('base').first
+
+         base_node ? base_node['href'] : @link_base
+      end
+
+      def get_link_target(link_node, base)
+         href = URI(link_node['href'])
+         if href.absolute?
+            href
+         else
+            base + href.to_s
+         end
+      rescue URI::InvalidURIError
+         link_node['href'].gsub(/^(tel|mailto):/, '').strip
+      end
+
+      def replace_headers(doc)
+         doc.search('header, h1, h2, h3, h4, h5, h6').each do |node|
+            node.inner_html = "-- #{ node.inner_html } --\n".squeeze(' ')
+         end
+      end
+
+      def replace_images(doc)
+         doc.search('img[role=presentation]').remove
+
+         doc.search('img').each do |img_node|
+            src = img_node['src']
+            alt = img_node['alt']
+
+            src = 'embedded' if src.start_with? 'data:'
+
+            img_node.replace("#{ alt } (#{ src })") unless alt.nil? || alt.empty?
+         end
+      end
+
+      def replace_lists(doc)
+         doc.search('ul, ol').each do |list_node|
+            list_node.search('./li').each_with_index do |list_item, i|
+               marker = if list_node.node_name == 'ol'
+                           "#{ i + 1 }."
+                        else
+                           @list_marker
+                        end
+
+               list_item.inner_html = "#{ marker } #{ list_item.inner_html }\n".squeeze(' ')
+            end
+
+            list_node.replace("\n#{ list_node.inner_html }\n")
+         end
+      end
+
+      def replace_tables(doc)
+         doc.css('table').each do |table|
+            column_sizes = calculate_column_sizes(table)
+
+            table.search('./thead/tr', './tbody/tr', './tr').each do |row|
+               replace_table_nodes(row, column_sizes)
+
+               row.inner_html = "#{ row.inner_html }|\n"
+            end
+
+            add_table_header_underline(table, column_sizes)
+
+            table.inner_html = "#{ table.inner_html }\n"
+         end
+      end
+
+      def calculate_column_sizes(table)
+         column_sizes = table.search('tr').collect do |row|
+            row.search('th', 'td').collect do |node|
+               node.inner_html.length
+            end
+         end
+
+         column_sizes.transpose.collect(&:max)
+      end
+
+      def add_table_header_underline(table, column_sizes)
+         table.search('./thead').each do |row|
+            header_bottom = "|#{ column_sizes.collect { |len| ('-' * (len + 2)) }.join('|') }|"
+
+            row.inner_html = "#{ row.inner_html }#{ header_bottom }\n"
+         end
+      end
+
+      def replace_table_nodes(row, column_sizes)
+         row.search('th', 'td').each_with_index do |node, i|
+            new_content = "| #{ node.inner_html }".squeeze(' ')
+
+            # +2 for the extra spacing between text and pipe
+            node.inner_html = new_content.ljust(column_sizes[i] + 2)
+         end
+      end
+
+      def simple_replace(doc, tag, replacement)
+         doc.search(tag).each do |node|
+            node.replace(node.inner_html + replacement)
+         end
+      end
+   end
+end

metadata

@@ -1,59 +1,102 @@
 --- !ruby/object:Gem::Specification
 name: ghostwriter
 version: !ruby/object:Gem::Version
-  version: 0.1.0.placeholder
+  version: 1.0.0
 platform: ruby
 authors:
 - Robin Miller
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-02-26 00:00:00.000000000 Z
+date: 2021-03-22 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.8.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.8.4
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '2.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
-description: Transforms HTML into plaintext while preserving legibility and functionality.
-  Previously known as dirt-textify.
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rubocop-performance
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+description: 'Transforms HTML into plaintext while preserving legibility and functionality.
+
+'
 email:
 - robin@tenjin.ca
 executables: []
@@ -62,17 +105,21 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".rubocop.yml"
+- ".ruby-version"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE.txt
 - README.md
+- RELEASE_NOTES.md
 - Rakefile
 - bin/console
 - bin/setup
-- ghostwriter.gemspec
+- dirt-textify.gemspec
 - lib/ghostwriter.rb
 - lib/ghostwriter/version.rb
+- lib/ghostwriter/writer.rb
 homepage: https://github.com/TenjinInc/ghostwriter
 licenses:
 - MIT
@@ -83,17 +130,16 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - "~>"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.6
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Intelligently extracts plaintext from an HTML document.

data/ghostwriter.gemspec

@@ -1,25 +0,0 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'ghostwriter/version'
-
-Gem::Specification.new do |spec|
-  spec.name          = "ghostwriter"
-  spec.version       = Ghostwriter::VERSION
-  spec.authors       = ["Robin Miller"]
-  spec.email         = ["robin@tenjin.ca"]
-
-  spec.summary       = %q{Intelligently extracts plaintext from an HTML document.}
-  spec.description   = %q{Transforms HTML into plaintext while preserving legibility and functionality. Previously known as dirt-textify.}
-  spec.homepage      = 'https://github.com/TenjinInc/ghostwriter'
-  spec.license       = "MIT"
-
-  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
-
-  spec.add_development_dependency "bundler", "~> 1.11"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
-end