klipbook 3.0.0 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 307a672f1458f7fc37543aaa4157b70bc247086d
-  data.tar.gz: 8c1586619868e3517043d4ec610165dd22802ef9
+SHA256:
+  metadata.gz: 404f5bf771069c94bf0c7b2081168d52a83b50d90519f0bcc03743e56cd88639
+  data.tar.gz: 8e5f90442fd8c81add6818016f3b1c7aad537920b1bee88407e8190b20284a93
 SHA512:
-  metadata.gz: f67d8ceefa6a239fb61c3df9319c5ab5aceb85474a4edb0520975ded47f3c7f5d33a38287c854dd57345e459c239c5bc5aa2554939268a9031ca86924313a84b
-  data.tar.gz: 195d8294a0d2fdbf8b6f176f6314b0f478eacac11c768c9965bdec4134e2afa088416722c904c4cf35ff2a2a5eab15010d63853c4cff4558bdf9e53b00b036f3
+  metadata.gz: b7fc57d83fcb14116102c2def8d357afd090dd321af905e78022257dd802fb39f8708dd9ce2f11c81dba291d98b268914272109959ca0e47fdc8e66ef7156359
+  data.tar.gz: 48199b739319ef3b85318257dfa784d0f66ef728f53323b7be00915258e42a654cb6c1da68c9384fed196df57df13c7e51c89fe46811b4a931edf4c64c719fc7

data/.travis.yml

@@ -1,3 +1,3 @@
 language: ruby
 rvm:
-  - 2.3.1
+  - 2.5.2

data/CHANGELOG.txt

@@ -1,45 +1,54 @@
+== 4.0.0 / 2018-11-07
+
+* Breaking changes
+
+  * Remove site scraping feature.
+
+* Minor changes
+
+  * Tested on Ruby 2.5.2
+
 == 3.0.0 / 2016-09-18
 
 * Feature Changes
 
-	* Added markdown as an export format.
+  * Added markdown as an export format.
 
 * Breaking changes
 
-	* Changed command flags to make things more consistent. 
-	* Changed JSON export to export into individual files rather than a single file. Again for consistency with other export formats.
+  * Changed command flags to make things more consistent.
+  * Changed JSON export to export into individual files rather than a single file. Again for consistency with other export formats.
 
 == 2.1.3 / 2014-07-25
 
 * Minor changes
 
-	* Removed Rainbow, RR, pry-debugger gems
-	* Updated dependent gems
-	* Updated tests to use RSpec 3
-	* Tested on Ruby 2.1.2
+  * Removed Rainbow, RR, pry-debugger gems
+  * Updated dependent gems
+  * Updated tests to use RSpec 3
+  * Tested on Ruby 2.1.2
 
 == 2.1.0 / 2013-12-11
 
 * Feature changes
 
-	* Reworked parameters for better clarity. tosjon and tohtml are now supported.
+  * Reworked parameters for better clarity. tosjon and tohtml are now supported.
 
 == 2.0.0 / 2013-02-12
 
 * Feature changes
 
-	* Renamed the collate command to pretty print which describes what it does better.
+  * Renamed the collate command to pretty print which describes what it does better.
 
-	* Introduces a _new_ collate command that collates all clippings into a single JSON
+  * Introduces a _new_ collate command that collates all clippings into a single JSON
 file.
 
 * Other unexposed changes
 
-	* Internally refactored much of the codebase into a more sensible structure.
+  * Internally refactored much of the codebase into a more sensible structure.
 
 == 1.0.2 / 2012-12-12
 
 * Bug fixes
 
-	* Updated site scraping code to ensure it still works
-
+  * Updated site scraping code to ensure it still works

data/Gemfile.lock

@@ -1,97 +1,72 @@
 PATH
   remote: .
   specs:
-    klipbook (3.0.0)
+    klipbook (4.0.0)
       commander
-      mechanize
 
 GEM
   remote: https://rubygems.org/
   specs:
-    aruba (0.14.2)
-      childprocess (~> 0.5.6)
+    aruba (0.14.6)
+      childprocess (>= 0.6.3, < 0.10.0)
       contracts (~> 0.9)
       cucumber (>= 1.3.19)
       ffi (~> 1.9.10)
       rspec-expectations (>= 2.99)
       thor (~> 0.19)
-    builder (3.2.2)
-    byebug (9.0.5)
-    childprocess (0.5.9)
+    backports (3.11.4)
+    builder (3.2.3)
+    byebug (10.0.2)
+    childprocess (0.9.0)
       ffi (~> 1.0, >= 1.0.11)
-    coderay (1.1.1)
-    commander (4.4.0)
-      highline (~> 1.7.2)
-    contracts (0.14.0)
-    cucumber (2.4.0)
+    coderay (1.1.2)
+    commander (4.4.7)
+      highline (~> 2.0.0)
+    contracts (0.16.0)
+    cucumber (3.1.2)
       builder (>= 2.1.2)
-      cucumber-core (~> 1.5.0)
+      cucumber-core (~> 3.2.0)
+      cucumber-expressions (~> 6.0.1)
       cucumber-wire (~> 0.0.1)
-      diff-lcs (>= 1.1.3)
-      gherkin (~> 4.0)
+      diff-lcs (~> 1.3)
+      gherkin (~> 5.1.0)
       multi_json (>= 1.7.5, < 2.0)
       multi_test (>= 0.1.2)
-    cucumber-core (1.5.0)
-      gherkin (~> 4.0)
+    cucumber-core (3.2.1)
+      backports (>= 3.8.0)
+      cucumber-tag_expressions (~> 1.1.0)
+      gherkin (~> 5.0)
+    cucumber-expressions (6.0.1)
+    cucumber-tag_expressions (1.1.1)
     cucumber-wire (0.0.1)
-    diff-lcs (1.2.5)
-    domain_name (0.5.20160826)
-      unf (>= 0.0.5, < 1.0.0)
-    ffi (1.9.14)
-    gherkin (4.0.0)
-    highline (1.7.8)
-    http-cookie (1.0.2)
-      domain_name (~> 0.5)
-    mechanize (2.7.5)
-      domain_name (~> 0.5, >= 0.5.1)
-      http-cookie (~> 1.0)
-      mime-types (>= 1.17.2)
-      net-http-digest_auth (~> 1.1, >= 1.1.1)
-      net-http-persistent (~> 2.5, >= 2.5.2)
-      nokogiri (~> 1.6)
-      ntlm-http (~> 0.1, >= 0.1.1)
-      webrobots (>= 0.0.9, < 0.2)
-    method_source (0.8.2)
-    mime-types (3.1)
-      mime-types-data (~> 3.2015)
-    mime-types-data (3.2016.0521)
-    mini_portile2 (2.1.0)
-    multi_json (1.12.1)
+    diff-lcs (1.3)
+    ffi (1.9.25)
+    gherkin (5.1.0)
+    highline (2.0.0)
+    method_source (0.9.1)
+    multi_json (1.13.1)
     multi_test (0.1.2)
-    net-http-digest_auth (1.4)
-    net-http-persistent (2.9.4)
-    nokogiri (1.6.8)
-      mini_portile2 (~> 2.1.0)
-      pkg-config (~> 1.1.7)
-    ntlm-http (0.1.1)
-    pkg-config (1.1.7)
-    pry (0.10.4)
+    pry (0.12.0)
       coderay (~> 1.1.0)
-      method_source (~> 0.8.1)
-      slop (~> 3.4)
-    pry-byebug (3.4.0)
-      byebug (~> 9.0)
+      method_source (~> 0.9.0)
+    pry-byebug (3.6.0)
+      byebug (~> 10.0)
       pry (~> 0.10)
-    rake (10.3.2)
-    rspec (3.5.0)
-      rspec-core (~> 3.5.0)
-      rspec-expectations (~> 3.5.0)
-      rspec-mocks (~> 3.5.0)
-    rspec-core (3.5.3)
-      rspec-support (~> 3.5.0)
-    rspec-expectations (3.5.0)
+    rake (12.3.1)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.5.0)
-    rspec-mocks (3.5.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.5.0)
-    rspec-support (3.5.0)
-    slop (3.6.0)
-    thor (0.19.1)
-    unf (0.1.4)
-      unf_ext
-    unf_ext (0.0.7.2)
-    webrobots (0.1.2)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    thor (0.20.0)
 
 PLATFORMS
   ruby
@@ -106,4 +81,4 @@ DEPENDENCIES
   rspec
 
 BUNDLED WITH
-   1.12.4
+   1.16.4

data/README.md

@@ -12,13 +12,7 @@ of a clippings file" />
 
 ## What sources does it support?
 
-Klipbook can take your highlights from a clippings file off a physical Kindle device or from the [Kindle highlights site](https://kindle.amazon.com/your_highlights) itself.
-
-### When should I use the file and when should I use the site?
-
-If your books are purchased from Amazon directly then use the site. The site is the canonical source and will contain all of your notes and highlights from across all of your Kindle devices and applications. Currently the Amazon highlights site only shows clippings for books you've purchased on Amazon.
-
-If your books are personal documents and you've read them on your Kindle device itself then you should use the file from that device as a source i.e. your Kindle device only keeps clippings that you make directly on it.
+Klipbook can take your highlights from a clippings file off a physical Kindle device.
 
 ## How does it work?
 
@@ -37,19 +31,9 @@ Copy your clippings file (called "My Clippings.txt" on a 3rd generation Kindle)
 Then specify the path to your clippings via:
 
 ```sh
-$ klipbook --from-file "My Clippings.txt"
-```
-
-### Reading from the highlights site
-
-Specify your Amazon credentials to klipbook and it will scrape the site and output a clippings file.
-
-```sh
-$ klipbook --from-site my-username@blah.com:my-password
+$ klipbook list --from-file "My Clippings.txt"
 ```
 
-Note that the scraping requires a network connection (obviously) and can take a while so please be patient.
-
 ## List
 
 The `list` command lists the books available in the specified source.
@@ -79,7 +63,7 @@ Klipbook will not overwrite an exiting file by default. You can change this with
 You can export clippings into a pretty html file for each book.
 
 ```sh
-$ klipbook export ---from-file "My Clippings.txt" --output-format html
+$ klipbook export --from-file "My Clippings.txt" --format html
 ```
 
 ### Export to JSON
@@ -87,7 +71,7 @@ $ klipbook export ---from-file "My Clippings.txt" --output-format html
 You can export clippings into a JSON file for each book.
 
 ```sh
-$ klipbook export --from-file "My Clippings.txt" --output-format json
+$ klipbook export --from-file "My Clippings.txt" --format json
 ```
 
 ### Export to Markdown
@@ -95,7 +79,7 @@ $ klipbook export --from-file "My Clippings.txt" --output-format json
 You can export clippings into a markdown file for each book.
 
 ```sh
-$ klipbook export --from-file "My Clippings.txt" --output-format markdown
+$ klipbook export --from-file "My Clippings.txt" --format markdown
 ```
 
 ## Set defaults
@@ -107,7 +91,6 @@ This is a YAML file and you can specify default values for the source and the ou
 ```sh
 $ cat ~/.klipbookrc
 
-:from_site: my-username@blah.com:my-password
 :output_dir: /path/to/my/default/output/directory
 ```
 
@@ -127,7 +110,7 @@ Klipbook has been tested on clippings files from 3rd generation Kindles and the
 
 ## Tested platforms
 
-Klipbook has been tested on Mac OS X El Capitan using Ruby 2.3.1.
+Klipbook has been tested on Mac OS High Sierra using Ruby 2.5.1.
 
 ## Contributing to Klipbook
 
@@ -135,4 +118,4 @@ Fork the project on [Github](https://github.com/grassdog/klipbook), add tests fo
 
 ## Copyright
 
-Copyright (c) 2016 Ray Grasso. See LICENSE.txt for further details.
+Copyright (c) 2018 Ray Grasso. See LICENSE.txt for further details.

data/Rakefile

@@ -9,10 +9,6 @@ desc 'Run specs'
 RSpec::Core::RakeTask.new(:spec)
 
 Cucumber::Rake::Task.new(:features) do |t|
-  t.cucumber_opts = '--format pretty --tags ~@slow'
-end
-
-Cucumber::Rake::Task.new(:allfeatures) do |t|
   t.cucumber_opts = '--format pretty'
 end
 

data/klipbook.gemspec

@@ -28,7 +28,6 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "commander"
-  spec.add_dependency "mechanize"
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"

data/lib/klipbook.rb

@@ -22,5 +22,3 @@ require 'klipbook/sources/kindle_device/entry_parser'
 require 'klipbook/sources/kindle_device/entry'
 require 'klipbook/sources/kindle_device/file'
 
-require 'klipbook/sources/amazon_site/site_scraper'
-require 'klipbook/sources/amazon_site/book_scraper'

data/lib/klipbook/cli.rb

@@ -13,19 +13,17 @@ module Klipbook
       program :version, Klipbook::VERSION
       program :description, "Klipbook exports the clippings you've saved on your Kindle into JSON, Markdown, or pretty HTML"
 
-      program :help, 'Source', "You must specify either `--from-file` or `--from-site` as an input."
+      program :help, 'Source', "You must specify `--from-file` as an input."
       program :help, 'Config', "Note that command options can be stored in a file called ~/.klipbookrc. This file is YAML formatted and options should be snake case e.g.\n\n" +
-        ":from_site: my-kindle-user@blah.com:my-kindle-password\n" +
         ":output_dir: ~/my/default/output/directory\n"
 
       default_command :help
 
       command :list do |c|
         c.syntax = "klipbook list"
-        c.description = "List the books on the site or in the clippings file"
+        c.description = "List the books in the clippings file"
 
         c.option '--from-file FILE', String, "Input clippings file"
-        c.option '--from-site username:password', String, "Credentials for Kindle highlights site"
         c.option '-c', '--count COUNT', Integer, "Maximum number of books to list (default is #{Config::DEFAULT_MAXBOOKS})"
 
         c.action do |_args, options|
@@ -40,7 +38,6 @@ module Klipbook
         c.description = 'Export book clippings'
 
         c.option '--from-file FILE', String, "Input clippings file"
-        c.option '--from-site username:password', String, "Credentials for Kindle highlights site"
         c.option '-c', '--count COUNT', Integer, "Maximum number of books to list (default is #{Config::DEFAULT_MAXBOOKS})"
         c.option '--format FORMAT', "Format to export in [html, markdown, or json]"
         c.option '--output-dir DIRECTORY', "Directory to export files to (default pwd)"
@@ -59,9 +56,7 @@ module Klipbook
     private
 
     def merge_config(options, config)
-      if options.from_site || options.from_file
-        [:from_site, :from_file].each { |key| config.delete(key) }
-      end
+      config.delete(:from_file) if options.from_file
 
       options.default config
     end

data/lib/klipbook/commands/command.rb

@@ -22,8 +22,8 @@ module Klipbook
       end
 
       def exit_unless_valid_source(options)
-        unless options.from_file || options.from_site
-          logger.error "Error: You must specify either `--from-file` or `--from-site` as an input."
+        unless options.from_file
+          logger.error "Error: You must specify `--from-file` as an input."
           exit 127
         end
       end

data/lib/klipbook/sources/source.rb

@@ -4,24 +4,11 @@ module Klipbook
       def self.build(options)
         if options.from_file
           file_source(options.from_file, options.count)
-        elsif options.from_site
-          site_source(options.from_site, options.count)
         else
           raise "Unknown source type"
         end
       end
 
-      def self.site_source(credentials, max_books)
-        unless credentials =~ /(.+):(.+)/
-          logger.error "Error: your credentials need to be in username:password format."
-          exit 127
-        end
-
-        username = $1
-        password = $2
-        Sources::AmazonSite::SiteScraper.new(username, password, max_books)
-      end
-
       def self.file_source(file, max_books)
         Sources::KindleDevice::File.new(File.read(file), max_books)
       end

data/lib/klipbook/version.rb

@@ -1,3 +1,3 @@
 module Klipbook
-  VERSION = '3.0.0'
+  VERSION = '4.0.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: klipbook
 version: !ruby/object:Gem::Version
-  version: 3.0.0
+  version: 4.0.0
 platform: ruby
 authors:
 - Ray Grasso
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-20 00:00:00.000000000 Z
+date: 2018-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: commander
@@ -24,20 +24,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: mechanize
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -133,7 +119,6 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
-- ".ruby-version"
 - ".travis.yml"
 - CHANGELOG.txt
 - Gemfile
@@ -157,8 +142,6 @@ files:
 - lib/klipbook/commands/list.rb
 - lib/klipbook/config.rb
 - lib/klipbook/logger.rb
-- lib/klipbook/sources/amazon_site/book_scraper.rb
-- lib/klipbook/sources/amazon_site/site_scraper.rb
 - lib/klipbook/sources/book.rb
 - lib/klipbook/sources/clipping.rb
 - lib/klipbook/sources/kindle_device/entry.rb
@@ -190,7 +173,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Klipbook creates a nice html summary of the clippings you've created on your

data/.ruby-version

@@ -1 +0,0 @@
-2.3.1

data/lib/klipbook/sources/amazon_site/book_scraper.rb

@@ -1,65 +0,0 @@
-module Klipbook::Sources
-  module AmazonSite
-    class BookScraper
-
-      def scrape_book(page)
-        page.search(".//div[@class='bookMain yourHighlightsHeader']").map { |element| build_book(page, element) }
-      end
-
-      private
-
-      def build_book(page, element)
-        Klipbook::Book.new.tap do |b|
-          b.asin = element.attribute("id").value.gsub(/_[0-9]+$/, "")
-          b.author = element.xpath("span[@class='author']").text.gsub("\n", "").gsub(" by ", "").strip
-          b.title = element.xpath("span/a").text
-          b.last_update = extract_last_update(element.xpath("div[@class='lastHighlighted']").text)
-          b.clippings = scrape_clippings(page)
-        end
-      end
-
-      def extract_last_update(text)
-        text = text.gsub('Last annotated on ', '')
-        DateTime.parse(text)
-      end
-
-      def scrape_clippings(page)
-        page.search(".//div[@class='highlightRow yourHighlight']").map { |element| build_clipping(element) }.flatten
-      end
-
-      def build_clipping(element)
-        location = extract_location(element)
-        annotation_id = element.xpath("form/input[@id='annotation_id']").attribute("value").value
-        note_text = element.xpath("p/span[@class='noteContent']").text
-
-        highlight = Klipbook::Clipping.new.tap do |c|
-          c.annotation_id = annotation_id
-          c.text = element.xpath("span[@class='highlight']").text
-          c.type = :highlight
-          c.location = location
-        end
-
-        if note_text.blank?
-          highlight
-        else
-          note = Klipbook::Clipping.new.tap do |c|
-            c.annotation_id = annotation_id
-            c.text = note_text
-            c.type = :note
-            c.location = location
-          end
-
-          [highlight, note]
-        end
-      end
-
-      def extract_location(element)
-        if element.xpath("a[@class='k4pcReadMore readMore linkOut']").attribute("href").value =~ /location=([0-9]+)$/
-          $1.to_i
-        else
-          0
-        end
-      end
-    end
-  end
-end

data/lib/klipbook/sources/amazon_site/site_scraper.rb

@@ -1,80 +0,0 @@
-require 'mechanize'
-
-module Klipbook::Sources
-  module AmazonSite
-    class SiteScraper
-      def initialize(username, password, max_books,
-                     book_scraper=Klipbook::Sources::AmazonSite::BookScraper.new,
-                     message_stream=$stdout)
-        @username = username
-        @password = password
-        @max_books = max_books
-        @message_stream = message_stream
-        @agent = Mechanize.new do |a|
-          a.user_agent_alias = 'Mac Safari'
-        end
-        @book_scraper = book_scraper
-      end
-
-      def books
-        @books ||= fetch_up_to_max_books
-      end
-
-      private
-
-      def fetch_up_to_max_books
-        scrape_books(fetch_first_books_page)
-      end
-
-      def fetch_first_books_page
-        welcome_page = get_welcome_page
-
-        raise 'Invalid Username or password' unless welcome_page.title == 'Amazon Kindle: Home'
-
-        @agent.click(welcome_page.link_with(:text => /Your Highlights/))
-      end
-
-      def get_welcome_page
-        @message_stream.puts 'Logging into site'
-
-        begin
-          page = @agent.get("https://www.amazon.com/ap/signin?openid.return_to=https%3A%2F%2Fkindle.amazon.com%3A443%2Fauthenticate%2Flogin_callback%3Fwctx%3D%252F&pageId=amzn_kindle&openid.mode=checkid_setup&openid.ns=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0&openid.identity=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&openid.pape.max_auth_age=0&openid.assoc_handle=amzn_kindle&openid.claimed_id=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select")
-        rescue
-          raise 'Could not connect to Amazon Kindle Site'
-        end
-
-        login_form = page.form('signIn').tap do |f|
-          f.email = @username
-          f.password = @password
-        end
-
-        @agent.submit(login_form)
-      end
-
-      def scrape_books(page)
-        books = []
-        @message_stream.print 'Fetching books '
-
-        @max_books.times do |count|
-          @message_stream.print '.'
-          books << @book_scraper.scrape_book(page)
-          page = get_next_page(page)
-          break unless page
-        end
-
-        puts ' Done!'
-
-        books.flatten
-      end
-
-      def get_next_page(page)
-        next_book_link = page.link_with(:dom_id => "nextBookLink")
-        if next_book_link
-          @agent.click(next_book_link)
-        else
-          nil
-        end
-      end
-    end
-  end
-end