klipbook 0.3.0 → 1.0.0

data/features/support/env.rb

@@ -1,8 +1,12 @@
 require 'rubygems'
 require 'bundler/setup'
 
+require 'aruba/cucumber'
+
 $LOAD_PATH << File.expand_path('../../../lib', __FILE__)
 require 'klipbook'
 require 'fileutils'
 
-TEST_DIR = File.expand_path('../../../tmp/test', __FILE__)
+Before('@slow') do
+  @aruba_io_wait_seconds = 40
+end

data/klipbook.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "klipbook"
-  s.version = "0.3.0"
+  s.version = "1.0.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Ray Grasso"]
-  s.date = "2011-12-29"
+  s.date = "2012-09-08"
   s.description = "Process your Kindle clippings file to generate a nicely formatted compilation of the clippings of the books you've read"
   s.email = "ray.grasso@gmail.com"
   s.executables = ["klipbook"]
@@ -29,26 +29,40 @@ Gem::Specification.new do |s|
     "Rakefile",
     "bin/klipbook",
     "example.png",
-    "features/list_books.feature",
-    "features/print_book_summary.feature",
-    "features/step_definitions/klipbook_steps.rb",
+    "features/collate.feature",
+    "features/fixtures/clippings-for-three-books.txt",
+    "features/list.feature",
+    "features/step_definitions/collate_steps.rb",
+    "features/step_definitions/list_steps.rb",
     "features/support/env.rb",
     "klipbook.gemspec",
     "lib/klipbook.rb",
     "lib/klipbook/blank.rb",
-    "lib/klipbook/book_summary.erb",
-    "lib/klipbook/book_summary.rb",
-    "lib/klipbook/cli.rb",
+    "lib/klipbook/book.rb",
     "lib/klipbook/clipping.rb",
-    "lib/klipbook/clippings_file.rb",
-    "lib/klipbook/clippings_parser.rb",
-    "lib/klipbook/runner.rb",
+    "lib/klipbook/collator.rb",
+    "lib/klipbook/config.rb",
+    "lib/klipbook/fetcher.rb",
+    "lib/klipbook/invalid_source_error.rb",
+    "lib/klipbook/output/book_helpers.rb",
+    "lib/klipbook/output/html_book_summary.erb",
+    "lib/klipbook/output/html_summary_writer.rb",
+    "lib/klipbook/printer.rb",
+    "lib/klipbook/sources/amazon_site/book_scraper.rb",
+    "lib/klipbook/sources/amazon_site/scraper.rb",
+    "lib/klipbook/sources/kindle_device/entry.rb",
+    "lib/klipbook/sources/kindle_device/entry_parser.rb",
+    "lib/klipbook/sources/kindle_device/file.rb",
+    "lib/klipbook/sources/kindle_device/file_parser.rb",
     "lib/klipbook/version.rb",
-    "spec/lib/klipbook/book_summary_spec.rb",
-    "spec/lib/klipbook/clipping_spec.rb",
-    "spec/lib/klipbook/clippings_file_spec.rb",
-    "spec/lib/klipbook/clippings_parser_spec.rb",
-    "spec/lib/klipbook/runner_spec.rb",
+    "spec/lib/klipbook/book_spec.rb",
+    "spec/lib/klipbook/collator_spec.rb",
+    "spec/lib/klipbook/fetcher_spec.rb",
+    "spec/lib/klipbook/output/html_summary_writer_spec.rb",
+    "spec/lib/klipbook/printer_spec.rb",
+    "spec/lib/klipbook/sources/kindle_device/entry_parser_spec.rb",
+    "spec/lib/klipbook/sources/kindle_device/file_parser_spec.rb",
+    "spec/lib/klipbook/sources/kindle_device/file_spec.rb",
     "spec/spec_helper.rb",
     "spec/support/rspec2.rb",
     "spec/support/with_rr.rb"
@@ -56,58 +70,61 @@ Gem::Specification.new do |s|
   s.homepage = "https://github.com/grassdog/klipbook"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.10"
+  s.rubygems_version = "1.8.23"
   s.summary = "Klipbook creates a nice html summary of the clippings you've created on your Kindle."
 
   if s.respond_to? :specification_version then
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<thor>, [">= 0"])
+      s.add_runtime_dependency(%q<gli>, [">= 0"])
+      s.add_runtime_dependency(%q<mechanize>, [">= 0"])
+      s.add_runtime_dependency(%q<rainbow>, [">= 0"])
       s.add_development_dependency(%q<rspec>, [">= 0"])
       s.add_development_dependency(%q<rr>, [">= 0"])
       s.add_development_dependency(%q<bundler>, [">= 0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
       s.add_development_dependency(%q<rcov>, [">= 0"])
+      s.add_development_dependency(%q<pry>, [">= 0"])
       s.add_development_dependency(%q<cucumber>, [">= 0"])
+      s.add_development_dependency(%q<aruba>, [">= 0"])
       s.add_development_dependency(%q<guard>, [">= 0"])
       s.add_development_dependency(%q<guard-rspec>, [">= 0"])
       s.add_development_dependency(%q<guard-cucumber>, [">= 0"])
-      s.add_development_dependency(%q<rb-inotify>, [">= 0"])
-      s.add_development_dependency(%q<rb-fsevent>, [">= 0"])
-      s.add_development_dependency(%q<rb-fchange>, [">= 0"])
-      s.add_development_dependency(%q<growl_notify>, [">= 0"])
+      s.add_development_dependency(%q<terminal-notifier-guard>, [">= 0"])
     else
-      s.add_dependency(%q<thor>, [">= 0"])
+      s.add_dependency(%q<gli>, [">= 0"])
+      s.add_dependency(%q<mechanize>, [">= 0"])
+      s.add_dependency(%q<rainbow>, [">= 0"])
       s.add_dependency(%q<rspec>, [">= 0"])
       s.add_dependency(%q<rr>, [">= 0"])
       s.add_dependency(%q<bundler>, [">= 0"])
       s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
       s.add_dependency(%q<rcov>, [">= 0"])
+      s.add_dependency(%q<pry>, [">= 0"])
       s.add_dependency(%q<cucumber>, [">= 0"])
+      s.add_dependency(%q<aruba>, [">= 0"])
       s.add_dependency(%q<guard>, [">= 0"])
       s.add_dependency(%q<guard-rspec>, [">= 0"])
       s.add_dependency(%q<guard-cucumber>, [">= 0"])
-      s.add_dependency(%q<rb-inotify>, [">= 0"])
-      s.add_dependency(%q<rb-fsevent>, [">= 0"])
-      s.add_dependency(%q<rb-fchange>, [">= 0"])
-      s.add_dependency(%q<growl_notify>, [">= 0"])
+      s.add_dependency(%q<terminal-notifier-guard>, [">= 0"])
     end
   else
-    s.add_dependency(%q<thor>, [">= 0"])
+    s.add_dependency(%q<gli>, [">= 0"])
+    s.add_dependency(%q<mechanize>, [">= 0"])
+    s.add_dependency(%q<rainbow>, [">= 0"])
     s.add_dependency(%q<rspec>, [">= 0"])
     s.add_dependency(%q<rr>, [">= 0"])
     s.add_dependency(%q<bundler>, [">= 0"])
     s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
     s.add_dependency(%q<rcov>, [">= 0"])
+    s.add_dependency(%q<pry>, [">= 0"])
     s.add_dependency(%q<cucumber>, [">= 0"])
+    s.add_dependency(%q<aruba>, [">= 0"])
     s.add_dependency(%q<guard>, [">= 0"])
     s.add_dependency(%q<guard-rspec>, [">= 0"])
     s.add_dependency(%q<guard-cucumber>, [">= 0"])
-    s.add_dependency(%q<rb-inotify>, [">= 0"])
-    s.add_dependency(%q<rb-fsevent>, [">= 0"])
-    s.add_dependency(%q<rb-fchange>, [">= 0"])
-    s.add_dependency(%q<growl_notify>, [">= 0"])
+    s.add_dependency(%q<terminal-notifier-guard>, [">= 0"])
   end
 end
 

data/lib/klipbook/book.rb

@@ -0,0 +1,18 @@
+module Klipbook
+  class Book
+    attr_accessor :asin, :author, :title, :last_update, :clippings
+
+    def initialize
+      yield self if block_given?
+    end
+
+    def title_and_author
+      author_txt = author ? " by #{author}" : ''
+      "#{title}#{author_txt}"
+    end
+
+    def get_binding
+      binding
+    end
+  end
+end

data/lib/klipbook/clipping.rb

@@ -1,15 +1,9 @@
-require 'ostruct'
-require 'date'
-
 module Klipbook
-  class Clipping < OpenStruct
-    def initialize(attributes)
-      super(attributes)
-      self.added_on = DateTime.strptime(self.added_on, '%A, %B %d, %Y, %I:%M %p') if self.added_on
-    end
+  class Clipping
+    attr_accessor :annotation_id, :text, :location, :type, :page
 
-    def <=>(other)
-      (self.location || 0) <=> (other.location || 0)
+    def initialize
+      yield self if block_given?
     end
   end
 end

data/lib/klipbook/collator.rb

@@ -0,0 +1,17 @@
+module Klipbook
+  class Collator
+    def initialize(books, summary_writer=Klipbook::Output::HtmlSummaryWriter.new)
+      @books = books
+      @summary_writer = summary_writer
+    end
+
+    def collate_books(output_dir, force, message_stream=$stdout)
+      message_stream.puts "Using output directory: #{output_dir}"
+
+      @books.each do |book|
+        @summary_writer.write(book, output_dir, force)
+      end
+    end
+  end
+end
+

data/lib/klipbook/config.rb

@@ -0,0 +1,22 @@
+module Klipbook
+  class Config
+    def initialize(config_file_name='.klipbookrc')
+      @config_file_name = config_file_name
+    end
+
+    def read
+      merge_config_from_rc_file({})
+    end
+
+    def merge_config_from_rc_file(config)
+      config_file = File.join(File.expand_path(ENV['HOME']), @config_file_name)
+
+      if config_file && File.exist?(config_file)
+        require 'yaml'
+        config.merge!(File.open(config_file) { |file| YAML::load(file) })
+      end
+
+      config
+    end
+  end
+end

data/lib/klipbook/fetcher.rb

@@ -0,0 +1,29 @@
+module Klipbook
+  class Fetcher
+    def initialize(source_spec, max_books)
+      raise InvalidSourceError unless valid_source(source_spec)
+
+      if (source_spec =~ /file:(.+)/)
+        raw_file = File.open($1, 'r')
+        @source = Klipbook::Sources::KindleDevice::File.new(raw_file.read.strip, max_books)
+      elsif (source_spec =~ /site:(.+):(.+)/)
+        username = $1
+        password = $2
+        @source = Klipbook::Sources::AmazonSite::Scraper.new(username, password, max_books)
+      else
+        raise InvalidSourceError("Unrecognised source type. Only 'file' and 'site' are supported")
+      end
+    end
+
+    def fetch_books
+      @source.books
+    end
+
+    private
+
+    def valid_source(source_spec)
+      source_spec =~ /(file:|site:.+:.+)/
+    end
+  end
+end
+

data/lib/klipbook/invalid_source_error.rb

@@ -0,0 +1,12 @@
+class InvalidSourceError < RuntimeError
+
+  DEFAULT_MESSAGE = "Please provide a valid source.\n" +
+                    "e.g.\n" +
+                    "     file:path/to/my-clippings-file.txt\n" +
+                    "     site:my-kindle-user@blah.com:my-kindle-password"
+
+  def initialize(msg = DEFAULT_MESSAGE)
+    super
+  end
+end
+

data/lib/klipbook/output/book_helpers.rb

@@ -0,0 +1,12 @@
+module Klipbook::Output
+  module BookHelpers
+    def location_html(location)
+      if self.asin
+        "<a href=\"kindle://book?action=open&asin=#{asin}&location=#{location}\">loc #{location}</a>"
+      else
+        "loc #{location}"
+      end
+    end
+  end
+end
+

data/lib/klipbook/{book_summary.erb → output/html_book_summary.erb}

@@ -3,7 +3,7 @@
   <head>
     <meta charset="utf-8"/>
     <meta name="generator" content="Klipbook"/>
-    <title><%= @clippings.first.title %> - Collated Kindle Clippings</title>
+    <title><%= title %> - Collated Kindle Clippings</title>
     <style type="text/css">
       body {
         color: #333;
@@ -26,18 +26,44 @@
 
       h1 {
         line-height: 1.1em;
+        font-weight: bold;
+      }
+
+      .byline {
+        width: 730px;
       }
 
       h2 {
+        float: left;
         line-height: 1.1em;
         font-size: 1.3em;
         color: #555;
+        font-weight: bold;
         font-style: italic;
       }
 
+      h3 {
+        float: right;
+        font-style: italic;
+        font-weight: normal;
+        color: #555;
+        font-size: 1.1em;
+      }
+
+      .clearfix:before,
+      .clearfix:after {
+        content: " ";
+        display: table;
+      }
+
+      .clearfix:after {
+        clear: both;
+      }
+
       ul {
         margin-top: 2em;
-        width: 43em;
+        width: 690px;
+        padding-left: 40px;
       }
 
       ul li {
@@ -51,8 +77,8 @@
 
       ul li footer {
         text-align: right;
-        font-size: .85em;
         color: #8C8C8C;
+        margin-left: 20em;
       }
 
       li.note p {
@@ -60,24 +86,53 @@
       }
 
       footer {
-        font-size: .85em;
-        margin-left: 20em;
+        font-size: .8em;
       }
 
       footer span {
         font-style: italic;
       }
+
+      footer a {
+        text-decoration: underline;
+        color: #8C8C8C;
+      }
+
+      footer a, footer a:link, footer a:visited {
+        color: #8C8C8C;
+      }
+
+      footer a:hover {
+        color: #084ab7;
+      }
+
+      body > footer {
+        font-size: .8em;
+        margin-left: 0;
+        width: 690px;
+        text-align: center;
+        padding-left: 40px;
+      }
     </style>
   </head>
   <body>
 
-    <h1><%= @clippings.first.title %></h1>
-    <% unless @author.blank? %>
+    <h1><%= title %></h1>
+
+    <div class="byline">
+    <% unless author.blank? %>
       <h2>by <%= author %></h2>
     <% end %>
 
+    <% if last_update %>
+      <h3>Last updated: <%= last_update.strftime('%e %B %Y') %></h3>
+    <% end %>
+    </div>
+
+    <div class="clearfix"></div>
+
     <ul>
-      <% @clippings.each do |clipping| %>
+      <% clippings.each do |clipping| %>
         <% unless clipping.text.blank? %>
         <li class="<%= clipping.type %>">
           <p>
@@ -85,8 +140,7 @@
           </p>
           <footer>
             <%= clipping.type %>
-            <% if include_pages && clipping.page %> on page <%= clipping.page %><% end %>
-            <% if clipping.location %> @ loc <%= clipping.location %><% end %>
+            <% if clipping.location %> @ <%= location_html(clipping.location) %><% end %>
           </footer>
         </li>
         <% end %>
@@ -94,7 +148,7 @@
     </ul>
 
     <footer>
-      Generated by <a href="https://github.com/grassdog/klipbook">Klipbook <%= Klipbook::VERSION %></a> on <span><%= DateTime.now.strftime('%e %b %Y at %l:%M %P') %></span>
+      <%= clippings.count %> clippings &bull; Generated by <a href="https://github.com/grassdog/klipbook">Klipbook <%= Klipbook::VERSION %></a> on <span><%= DateTime.now.strftime('%e %b %Y at %l:%M %P') %></span>
     </footer>
   </body>
 </html>

data/lib/klipbook/output/html_summary_writer.rb

@@ -0,0 +1,42 @@
+require 'erb'
+
+module Klipbook::Output
+  class HtmlSummaryWriter
+    def initialize(message_stream=$stdout)
+      @message_stream = message_stream
+    end
+
+    def write(book, output_dir, force)
+      require 'rainbow'
+
+      book.extend Klipbook::Output::BookHelpers
+
+      filename = filename_for_book(book)
+      filepath = File.join(output_dir, filename)
+
+      if !force && File.exists?(filepath)
+        @message_stream.puts("Skipping ".foreground(:yellow) + filename)
+        return
+      end
+
+      @message_stream.puts("Writing ".foreground(:green) + filename)
+      File.open(filepath, 'w') do |f|
+        f.write generate_html(book)
+      end
+    end
+
+    private
+
+    def filename_for_book(book)
+      "#{book.title_and_author}.html"
+    end
+
+    def generate_html(book)
+      ERB.new(template, 0, '%<>').result(book.get_binding)
+    end
+
+    def template
+      @template ||= File.read(File.join(File.dirname(__FILE__), 'html_book_summary.erb'))
+    end
+  end
+end

data/lib/klipbook/printer.rb

@@ -0,0 +1,18 @@
+module Klipbook
+  class Printer
+    def initialize(books)
+      @books = books
+    end
+
+    def print(output=$stdout)
+      if @books.empty?
+        output.puts 'No books available'
+      else
+        output.puts 'Book list:'
+        @books.each_with_index do |book, index|
+          output.puts "[#{index + 1}] #{book.title_and_author}"
+        end
+      end
+    end
+  end
+end

data/lib/klipbook/sources/amazon_site/book_scraper.rb

@@ -0,0 +1,67 @@
+require 'mechanize'
+
+module Klipbook::Sources
+  module AmazonSite
+    class BookScraper
+
+      def scrape_book(page)
+        page.search(".//div[@class='bookMain yourHighlightsHeader']").map { |element| build_book(page, element) }
+      end
+
+      private
+
+      def build_book(page, element)
+        Klipbook::Book.new do |b|
+          b.asin = element.attribute("id").value.gsub(/_[0-9]+$/, "")
+          b.author = element.xpath("span[@class='author']").text.gsub("\n", "").gsub(" by ", "").strip
+          b.title = element.xpath("span/a").text
+          b.last_update = extract_last_update(element.xpath("div[@class='lastHighlighted']").text)
+          b.clippings = scrape_clippings(page)
+        end
+      end
+
+      def extract_last_update(text)
+        text = text.gsub('Last annotated on ', '')
+        DateTime.parse(text)
+      end
+
+      def scrape_clippings(page)
+        page.search(".//div[@class='highlightRow yourHighlight']").map { |element| build_clipping(element) }.flatten
+      end
+
+      def build_clipping(element)
+        location = extract_location(element)
+        annotation_id = element.xpath("form/input[@id='annotation_id']").attribute("value").value
+        note_text = element.xpath("p/span[@class='noteContent']").text
+
+        highlight = Klipbook::Clipping.new do |c|
+          c.annotation_id = annotation_id
+          c.text = element.xpath("span[@class='highlight']").text
+          c.type = :highlight
+          c.location = location
+        end
+
+        if note_text.blank?
+          highlight
+        else
+          note = Klipbook::Clipping.new do |c|
+            c.annotation_id = annotation_id
+            c.text = note_text
+            c.type = :note
+            c.location = location
+          end
+
+          [highlight, note]
+        end
+      end
+
+      def extract_location(element)
+        if element.xpath("a[@class='k4pcReadMore readMore linkOut']").attribute("href").value =~ /location=([0-9]+)$/
+          $1.to_i
+        else
+          0
+        end
+      end
+    end
+  end
+end

data/lib/klipbook/sources/amazon_site/scraper.rb

@@ -0,0 +1,78 @@
+require 'mechanize'
+
+module Klipbook::Sources
+  module AmazonSite
+    class Scraper
+      def initialize(username, password, max_books,
+                     book_scraper=Klipbook::Sources::AmazonSite::BookScraper.new,
+                     message_stream=$stdout)
+        @username = username
+        @password = password
+        @max_books = max_books
+        @message_stream = message_stream
+        @agent = Mechanize.new
+        @book_scraper = book_scraper
+      end
+
+      def books
+        @books ||= fetch_books
+      end
+
+      private
+
+      def fetch_books
+        login_form = login
+
+        signin_submission = @agent.submit(login_form)
+
+        raise 'Invalid Username or password' unless signin_submission.title == "Amazon Kindle: Home"
+
+        page = @agent.click(signin_submission.link_with(:text => /Your Highlights/))
+
+        scrape_books(page)
+      end
+
+      def login
+        @message_stream.puts 'Logging into site'
+
+        begin
+          page = @agent.get("https://www.amazon.com/ap/signin?openid.return_to=https%3A%2F%2Fkindle.amazon.com%3A443%2Fauthenticate%2Flogin_callback%3Fwctx%3D%252F&pageId=amzn_kindle&openid.mode=checkid_setup&openid.ns=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0&openid.identity=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&openid.pape.max_auth_age=0&openid.assoc_handle=amzn_kindle&openid.claimed_id=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select")
+        rescue
+          raise 'Could not connect to Amazon Kindle Site'
+        end
+
+        login_form = page.form('signIn')
+
+        login_form.email = @username
+        login_form.password = @password
+
+        login_form
+      end
+
+      def scrape_books(page)
+        books = []
+        @message_stream.print 'Fetching books '
+
+        @max_books.times do |count|
+          @message_stream.print '.'
+          books << @book_scraper.scrape_book(page)
+          page = get_next_page(page)
+          break unless page
+        end
+
+        puts ' Done!'
+
+        books.flatten
+      end
+
+      def get_next_page(page)
+        ret = page.search(".//a[@id='nextBookLink']").first
+        if ret and ret.attribute("href")
+          @agent.get("https://kindle.amazon.com" + ret.attribute("href").value)
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/klipbook/sources/kindle_device/entry.rb

@@ -0,0 +1,11 @@
+module Klipbook::Sources
+  module KindleDevice
+    class Entry
+      attr_accessor :title, :author, :type, :location, :page, :added_on, :text
+
+      def initialize
+        yield self if block_given?
+      end
+    end
+  end
+end