klipbook 1.0.1 → 1.0.2

data/CHANGELOG.txt

@@ -0,0 +1,6 @@
+== 1.0.2 / 2012-12-12
+
+* Bug fixes
+
+	* Updated site scraping code to ensure it still works
+

data/Gemfile.lock

@@ -16,7 +16,7 @@ GEM
       gherkin (~> 2.11.0)
       json (>= 1.4.6)
     diff-lcs (1.1.3)
-    domain_name (0.5.3)
+    domain_name (0.5.6)
       unf (~> 0.0.3)
     ffi (1.1.5)
     gherkin (2.11.2)
@@ -51,7 +51,7 @@ GEM
     method_source (0.8)
     mime-types (1.19)
     net-http-digest_auth (1.2.1)
-    net-http-persistent (2.7)
+    net-http-persistent (2.8)
     nokogiri (1.5.5)
     ntlm-http (0.1.1)
     pry (0.9.10)

data/README.md

@@ -18,8 +18,8 @@ canonical source and will contain all of your notes and highlights from across a
 your Kindle devices and applications. Currently the Amazon highlights site only shows
 clippings for books you've purchased on Amazon.
 
-If your books are personal documents and you've read them on your kindle device
-itself then you should use the file from that device as a source i.e. you Kindle
+If your books are personal documents and you've read them on your Kindle device
+itself then you should use the file from that device as a source i.e. your Kindle
 device only keeps clippings that you made directly on it.
 
 ## How does it work?
@@ -40,23 +40,28 @@ Then write out a list of your clippings via:
     $ klipbook collate "file:My Clippings.txt"
 
 This command will write the collected clippings for the latest annotated book to a file in the current
-directory. You can override the output directory with the `--output-dir` flag.
+directory. You can override the output directory with the `--output-dir` switch.
+
+You can also specify a maximum count of books you'd like collated with the `--num-books` switch.
 
 Klipbook will not overwrite an exiting file by default. You can change this with
 the `--force` flag.
 
-You can also specify a maximum count of books you'd like collated with the `--num-books` flag.
-
 #### From the site
 
-Specify your amazon username and password to klipbook and it will scrape the site and
+Specify your Amazon username and password to klipbook and it will scrape the site and
 output a clippings file.
 
     $ klipbook collate site:my-username@blah.com:my-password
     
 The same flags above apply.
 
-If you don't feel like having your credentials in your shell history you can set your
+Note that the scraping requires a network connection (obviously) and can take a while
+so please be patient.
+
+### Set up defaults
+
+If you don't feel like having your Amazon credentials in your shell history you can set your
 default source in the klipbook rc file: `~/.klipbookrc`.
 
 This is simply a YAML file and you can specify default values for the source and the
@@ -78,6 +83,9 @@ The list command lists the books avaiable in the specified source.
     [1] The Big Sleep by Raymond Chandler
     [2] How to jump out of a plane without a parachute and survive by Rip Rockjaw
 
+By default it will only list the latest book. This can be overrided with the
+`--num-books` switch.
+
 ## Installation
 
 Klipbook is a Ruby gem. To install simply run:

data/klipbook.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "klipbook"
-  s.version = "1.0.1"
+  s.version = "1.0.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Ray Grasso"]
-  s.date = "2012-09-08"
+  s.date = "2012-12-12"
   s.description = "Process your Kindle clippings file to generate a nicely formatted compilation of the clippings of the books you've read"
   s.email = "ray.grasso@gmail.com"
   s.executables = ["klipbook"]
@@ -21,6 +21,7 @@ Gem::Specification.new do |s|
     ".document",
     ".rspec",
     ".yardopts",
+    "CHANGELOG.txt",
     "Gemfile",
     "Gemfile.lock",
     "Guardfile",
@@ -49,7 +50,7 @@ Gem::Specification.new do |s|
     "lib/klipbook/output/html_summary_writer.rb",
     "lib/klipbook/printer.rb",
     "lib/klipbook/sources/amazon_site/book_scraper.rb",
-    "lib/klipbook/sources/amazon_site/scraper.rb",
+    "lib/klipbook/sources/amazon_site/site_scraper.rb",
     "lib/klipbook/sources/kindle_device/entry.rb",
     "lib/klipbook/sources/kindle_device/entry_parser.rb",
     "lib/klipbook/sources/kindle_device/file.rb",

data/lib/klipbook.rb

@@ -7,7 +7,7 @@ require 'klipbook/sources/kindle_device/entry_parser'
 require 'klipbook/sources/kindle_device/entry'
 require 'klipbook/sources/kindle_device/file'
 
-require 'klipbook/sources/amazon_site/scraper'
+require 'klipbook/sources/amazon_site/site_scraper'
 require 'klipbook/sources/amazon_site/book_scraper'
 
 require 'klipbook/invalid_source_error'

data/lib/klipbook/fetcher.rb

@@ -9,7 +9,7 @@ module Klipbook
       elsif (source_spec =~ /site:(.+):(.+)/)
         username = $1
         password = $2
-        @source = Klipbook::Sources::AmazonSite::Scraper.new(username, password, max_books)
+        @source = Klipbook::Sources::AmazonSite::SiteScraper.new(username, password, max_books)
       else
         raise InvalidSourceError("Unrecognised source type. Only 'file' and 'site' are supported")
       end

data/lib/klipbook/sources/amazon_site/{scraper.rb → site_scraper.rb}

@@ -2,7 +2,7 @@ require 'mechanize'
 
 module Klipbook::Sources
   module AmazonSite
-    class Scraper
+    class SiteScraper
       def initialize(username, password, max_books,
                      book_scraper=Klipbook::Sources::AmazonSite::BookScraper.new,
                      message_stream=$stdout)
@@ -10,29 +10,31 @@ module Klipbook::Sources
         @password = password
         @max_books = max_books
         @message_stream = message_stream
-        @agent = Mechanize.new
+        @agent = Mechanize.new do |a|
+          a.user_agent_alias = 'Mac Safari'
+        end
         @book_scraper = book_scraper
       end
 
       def books
-        @books ||= fetch_books
+        @books ||= fetch_up_to_max_books
       end
 
       private
 
-      def fetch_books
-        login_form = login
-
-        signin_submission = @agent.submit(login_form)
+      def fetch_up_to_max_books
+        scrape_books(fetch_first_books_page)
+      end
 
-        raise 'Invalid Username or password' unless signin_submission.title == "Amazon Kindle: Home"
+      def fetch_first_books_page
+        welcome_page = get_welcome_page
 
-        page = @agent.click(signin_submission.link_with(:text => /Your Highlights/))
+        raise 'Invalid Username or password' unless welcome_page.title == 'Amazon Kindle: Home'
 
-        scrape_books(page)
+        @agent.click(welcome_page.link_with(:text => /Your Highlights/))
       end
 
-      def login
+      def get_welcome_page
         @message_stream.puts 'Logging into site'
 
         begin
@@ -41,12 +43,12 @@ module Klipbook::Sources
           raise 'Could not connect to Amazon Kindle Site'
         end
 
-        login_form = page.form('signIn')
-
-        login_form.email = @username
-        login_form.password = @password
+        login_form = page.form('signIn').tap do |f|
+          f.email = @username
+          f.password = @password
+        end
 
-        login_form
+        @agent.submit(login_form)
       end
 
       def scrape_books(page)
@@ -66,9 +68,9 @@ module Klipbook::Sources
       end
 
       def get_next_page(page)
-        ret = page.search(".//a[@id='nextBookLink']").first
-        if ret and ret.attribute("href")
-          @agent.get("https://kindle.amazon.com" + ret.attribute("href").value)
+        next_book_link = page.link_with(:dom_id => "nextBookLink")
+        if next_book_link
+          @agent.click(next_book_link)
         else
           nil
         end

data/lib/klipbook/version.rb

@@ -1,3 +1,3 @@
 module Klipbook
-  VERSION = '1.0.1'
+  VERSION = '1.0.2'
 end

data/spec/lib/klipbook/fetcher_spec.rb

@@ -20,12 +20,12 @@ describe Klipbook::Fetcher do
     let(:fetcher) { Klipbook::Fetcher.new('site:username@example.com:password', 2) }
 
     before(:each) do
-      stub(Klipbook::Sources::AmazonSite::Scraper).new { fake_source }
+      stub(Klipbook::Sources::AmazonSite::SiteScraper).new { fake_source }
     end
 
     it 'creates a site scraper with the provided credentials' do
       subject
-      Klipbook::Sources::AmazonSite::Scraper.should have_received.new('username@example.com', 'password', 2)
+      Klipbook::Sources::AmazonSite::SiteScraper.should have_received.new('username@example.com', 'password', 2)
     end
 
     describe '#fetch_books' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: klipbook
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-08 00:00:00.000000000 Z
+date: 2012-12-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: gli
@@ -264,6 +264,7 @@ files:
 - .document
 - .rspec
 - .yardopts
+- CHANGELOG.txt
 - Gemfile
 - Gemfile.lock
 - Guardfile
@@ -292,7 +293,7 @@ files:
 - lib/klipbook/output/html_summary_writer.rb
 - lib/klipbook/printer.rb
 - lib/klipbook/sources/amazon_site/book_scraper.rb
-- lib/klipbook/sources/amazon_site/scraper.rb
+- lib/klipbook/sources/amazon_site/site_scraper.rb
 - lib/klipbook/sources/kindle_device/entry.rb
 - lib/klipbook/sources/kindle_device/entry_parser.rb
 - lib/klipbook/sources/kindle_device/file.rb
@@ -324,7 +325,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -4508748763061441065
+      hash: -4595890160696464397
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: