kindle 0.1.3 → 0.7.0.beta2

data/features/support/aruba.rb

@@ -0,0 +1 @@
+require 'aruba/cucumber'

data/features/support/highlight_steps.rb

@@ -0,0 +1,13 @@
+require "kindle"
+require 'cucumber/timecop'
+
+Given(/^a database exists$/) do
+  Kindle::Models::Book.delete_all
+  Kindle::Models::Highlight.delete_all
+  book = Kindle::Models::Book.create! title: "Zen", author: "Monk", highlight_count: 1
+  highlight = book.highlights.create! highlight: "Reach for enlightenment"
+end
+
+Given(/^time is frozen$/) do
+  Timecop.freeze(Time.local(1990))
+end

data/kindle.gemspec

@@ -1,6 +1,6 @@
 # -*- encoding: utf-8 -*-
 $:.push File.expand_path("../lib", __FILE__)
-require "kindle/version"
+require_relative "lib/kindle"
 
 Gem::Specification.new do |s|
   s.name        = "kindle"
@@ -8,9 +8,9 @@ Gem::Specification.new do |s|
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Matt Petty"]
   s.email       = ["matt@kizmeta.com"]
-  s.homepage    = ""
-  s.summary     = %q{Manage your kindle highlights with ruby}
-  s.description = %q{Manage your kindle highlights with ruby}
+  s.homepage    = "https://github.com/lodestone/kindle"
+  s.summary     = %q{Manage your kindle highlights with ruby, output in JSON, Markdown, and CSV formats}
+  s.description = %q{Manage your Amazon Kindle highlights: Sync and cache to an ActiveRecord database and output in various formats}
 
   s.rubyforge_project = "kindle"
 
@@ -18,7 +18,11 @@ Gem::Specification.new do |s|
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
-  s.add_dependency "nokogiri"
-  s.add_dependency "highline"
-  s.add_dependency "mechanize"
+  s.add_runtime_dependency "gli"
+  s.add_runtime_dependency "sqlite3"
+  s.add_runtime_dependency "activerecord", "~>5.0.0"
+  s.add_runtime_dependency "nokogiri"
+  s.add_runtime_dependency "mechanize"
+  s.add_runtime_dependency "rainbow"
+  s.add_runtime_dependency "pry"
 end

data/lib/kindle.rb

@@ -1,20 +1,21 @@
-require 'nokogiri'
-require 'mechanize'
-require_relative 'kindle/highlight'
-require_relative 'kindle/highlights_parser'
+require "active_record"
+require_relative "kindle/settings"
+require_relative "kindle/migrations/initializer"
+require_relative "kindle/migrations/base_migration"
+require_relative "kindle/models/highlight"
+require_relative "kindle/models/book"
+require_relative "kindle/parser/agent"
+require_relative "kindle/parser/annotations"
+require_relative "kindle/remote/book"
+require_relative "kindle/remote/highlight"
+require_relative "kindle/exports/markdown"
+require_relative "kindle/exports/json"
+require_relative "kindle/exports/csv"
 
-module Kindle
-
-  class Highlights
-
-    def initialize(options = {})
-      options.each { |k,v| instance_variable_set("@#{k}", v) }
-    end
-
-    def fetch_highlights
-      HighlightsParser.new(login: @login, password: @password).get_highlights
-    end
-
-  end
+# TODO: Handle multiple environments
+ActiveRecord::Base.establish_connection(adapter: "sqlite3", database: Kindle::Settings::KINDLE_DATABASE_FILENAME)
 
+module Kindle
+  VERSION = "0.7.0.beta2"
+  include Models
 end

data/lib/kindle/cli.rb

@@ -0,0 +1,115 @@
+require_relative "../kindle"
+require "pry"
+require "gli"
+require "rainbow"
+
+# TODO: Create a test and pull request for this monkey patch
+#  that handles an empty file without blowing up.
+module GLI
+  module AppSupport
+    def parse_config # :nodoc:
+      config = { 'commands' => {} }
+      if @config_file && File.exist?(@config_file)
+        require 'yaml'
+        yaml = YAML.load(File.open(@config_file).read)
+        config.merge!(yaml||{})
+      end
+      config
+    end
+  end
+end
+
+module Kindle
+  class CLI
+    extend GLI::App
+
+    sort_help :manually
+    version Kindle::VERSION
+    hide_commands_without_desc true
+    config_file ".kindle/kindlerc.yml"
+    program_desc Rainbow("Fetch and query your Amazon Kindle Highlights").cyan
+
+    flag [:username, :u]
+    flag [:password, :p]
+    flag [:domain,   :d], default_value: "amazon.com"
+
+    # NOTE: Commenting out these descriptions to *hide* the option from help
+    # desc "Initialize the highlights database"
+    # long_desc "Creates a SQLITE3 database to store/cache your highlights"
+    command :initdb do |c|
+      c.action do |global_options, options, args|
+        begin
+          puts Rainbow("\nInitializing the database...").green
+          Kindle::Migrations::Initializer.new
+        rescue ActiveRecord::StatementInvalid
+          puts Rainbow("Looks like the database is already created, skipping...").red
+        end
+      end
+    end
+
+    desc Rainbow("Initialize the database and config files").yellow
+    long_desc <<~EOD
+      Creates the ~/.kindle/kindle.db SQLITE file
+      Creates the ~/.kindle directory and a default config file
+    EOD
+    command :init do |c|
+      c.action do |global_options, options, args|
+        commands[:initdb].execute(global_options, options, args)
+        commands[:initconfig].execute(global_options, options, args)
+      end
+    end
+
+    desc Rainbow("Export Highlights as JSON, CSV, or Markdown").yellow
+    long_desc "Output your highlights database in JSON, CSV, or Markdown format"
+    command :highlights do |highlights|
+
+      highlights.desc Rainbow("Refresh the database of highlights").yellow
+      highlights.long_desc "Scrape your current Kindle highlights and update the local database"
+      highlights.command :update do |update|
+        update.action do
+          Kindle::Parser::Annotations.new
+        end
+      end
+
+      highlights.desc Rainbow("Export highlights and books as JSON").yellow
+      highlights.command :json do |json|
+        json.action do
+          puts Kindle::Exports::Json.new(Kindle::Models::Highlight.includes(:book).all)
+        end
+      end
+
+      highlights.desc Rainbow("Export highlights and books as CSV").yellow
+      highlights.command :csv do |csv|
+        csv.action do
+          puts Kindle::Exports::Csv.new(Kindle::Models::Highlight.includes(:book).all)
+        end
+      end
+
+      highlights.desc Rainbow("Export highlights and books as Markdown").yellow
+      highlights.command :markdown do |markdown|
+        markdown.action do
+          puts Kindle::Exports::Markdown.new(Kindle::Models::Book.order("title ASC").all)
+        end
+      end
+    end
+
+    desc Rainbow("Open an interactive console").yellow
+    long_desc %{
+      Open an interactive console with both Book and Highlight objects available.
+    }
+    command :console do |c|
+      c.action do |global_options,options,args|
+        Pry.start Kindle, prompt: [proc { "Kindle :) " }]
+      end
+    end
+
+    # NOTE: Fuckery to hide the auto generated actions:
+    helpcmd = commands[:help]
+    helpcmd.instance_variable_set("@description", Rainbow(helpcmd.instance_variable_get("@description")).yellow)
+    commands[:initconfig].instance_variable_set("@description", nil)
+    commands[:initconfig].instance_variable_set("@hide_commands_without_desc", true)
+
+    exit run(ARGV)
+
+  end
+end

data/lib/kindle/exports/csv.rb

@@ -0,0 +1,15 @@
+module Kindle
+  module Exports
+    class Csv
+      def initialize(highlights)
+        @document = "highlight,book title,book author,book asin\n"
+        highlights.each do |h|
+          @document << "\"#{h.highlight}\",\"#{h.book.title}\",\"#{h.book.author}\",#{h.book.asin}\n"
+        end
+      end
+      def to_s
+        @document
+      end
+    end
+  end
+end

data/lib/kindle/exports/json.rb

@@ -0,0 +1,12 @@
+module Kindle
+  module Exports
+    class Json
+      def initialize(highlights)
+        @document = highlights.to_json(include: :book)
+      end
+      def to_s
+        @document
+      end
+    end
+  end
+end

data/lib/kindle/exports/markdown.rb

@@ -0,0 +1,18 @@
+module Kindle
+  module Exports
+    class Markdown
+      def initialize(books)
+        @document = "## Kindle Highlights\n\n"
+        books.each do |book|
+          @document << "\n### #{book.title} by #{book.author}\n\n"
+          book.highlights.each do |highlight|
+            @document << "> #{highlight.highlight}\n\n"
+          end
+        end
+      end
+      def to_s
+        @document
+      end
+    end
+  end
+end

data/lib/kindle/migrations/base_migration.rb

@@ -0,0 +1,22 @@
+class CreateBaseStructure < ActiveRecord::Migration[5.0]
+
+  def self.up
+    create_table :books do |t|
+      t.string :asin, :title, :author
+      t.integer :highlight_count
+      t.timestamps
+    end
+    create_table :highlights do |t|
+      t.text :highlight
+      t.string :amazon_id
+      t.integer :book_id
+      t.timestamps
+    end
+  end
+
+  def self.down
+    drop_table :books
+    drop_table :highlights
+  end
+
+end

data/lib/kindle/migrations/initializer.rb

@@ -0,0 +1,13 @@
+require_relative "base_migration"
+
+module Kindle
+  module Migrations
+    class Initializer
+      def initialize
+        # ActiveRecord::Base.configurations = YAML::load(IO.read(ENV['HOME']+'/.kindle/database.yml'))
+        # ActiveRecord::Base.establish_connection("development")
+        CreateBaseStructure.migrate(:up)
+      end
+    end
+  end
+end

data/lib/kindle/models/book.rb

@@ -0,0 +1,26 @@
+module Kindle
+  module Models
+    class Book < ActiveRecord::Base
+
+      attribute :asin, :string
+      attribute :highlight_count, :integer
+      attribute :title, :string
+      attribute :author, :string
+
+      has_many :highlights
+
+      # def initialize(asin, options = {})
+      #   @asin            = asin
+      #   @title           = options[:title]
+      #   @author          = options[:author]
+      #   @highlight_count = options[:highlight_count]
+      #   @highlights      = []
+      # end
+
+      # def to_csv
+      #   "#{id},#{asin},#{author},#{title},#{highlight},highlight_count"
+      # end
+
+    end
+  end
+end

data/lib/kindle/models/highlight.rb

@@ -0,0 +1,24 @@
+module Kindle
+  module Models
+    class Highlight < ActiveRecord::Base
+
+      attribute :amazon_id, :integer
+      attribute :highlight, :text
+
+      belongs_to :book
+
+      # def to_csv
+      #   "#{id},#{asin},#{author},#{title},#{highlight},highlight_count"
+      # end
+      #
+      # def to_json
+      #   {id: id, highlight: highlight, asin: asin, title: title, author: author, highlight_count: highlight_count}.to_json
+      # end
+
+      # def as_json
+      #   {id: id, highlight: highlight, asin: book.asin, title: book.title, author: book.author}
+      # end
+
+    end
+  end
+end

data/lib/kindle/parser/agent.rb

@@ -0,0 +1,13 @@
+module Kindle
+  module Parser
+    class Agent
+      def agent
+        return @agent if @agent
+        @agent = Mechanize.new
+        @agent.redirect_ok = true
+        @agent.user_agent_alias = 'Mac Mozilla'
+        @agent
+      end
+    end
+  end
+end

data/lib/kindle/parser/annotations.rb

@@ -0,0 +1,130 @@
+require "mechanize"
+require "nokogiri"
+require "json"
+require_relative "agent"
+require_relative "../remote/highlight"
+require_relative "../remote/book"
+require_relative "../models/highlight"
+require_relative "../models/book"
+
+module Kindle
+  module Parser
+    class Annotations
+
+      SETTINGS = Kindle::Settings.new
+
+      attr_accessor :page, :highlights
+      attr_reader :books
+
+      def initialize(options={})
+        @username   = SETTINGS.username || options[:username]
+        @password   = SETTINGS.password || options[:password]
+        @books      = []
+        @highlights = []
+        load_highlights
+      end
+
+      def load_highlights
+        login
+        fetch_highlights
+        collect_authoritative_highlights
+      end
+
+      private
+
+      def agent
+        @agent ||= Kindle::Parser::Agent.new.agent
+      end
+
+      def login
+        @page = agent.get(SETTINGS.url + "/login")
+        login_form = page.forms.first
+        login_form.email    = @username
+        login_form.password = @password
+        @page = login_form.submit
+      end
+
+      # TODO: Handle CAPTCHA
+      # def handle_captcha
+      #   if page.css("#ap_captcha_img")
+      #     puts "CAPTCHA LOGIN ERROR"
+      #     save_and_open_page
+      #     p page.link_with(text: /See a new challenge/).resolved_uri.to_s
+      #     agent.cookie_jar.save("cookies.txt", format: :cookiestxt)
+      #   end
+      #   save_and_open_page
+      #   rescue => exception
+      #   end
+      # end
+
+      def collect_authoritative_highlights
+        # NOTE: This fetch may fail if the highlight count is realy large.
+        books.each do |book|
+          kb = Kindle::Book.find_or_create_by(asin: book.asin, title: book.title, author: book.author)
+          if kb.highlight_count != book.highlight_count
+            kb.highlight_count = book.highlight_count
+            kb.save!
+          end
+          url = "#{SETTINGS.url}/kcw/highlights?asin=#{book.asin}&cursor=0&count=#{book.highlight_count}"
+          bpage = agent.get url
+          items = JSON.parse(bpage.body).fetch("items", [])
+          book.highlights = items.map do |item|
+            kb.highlights.find_or_create_by(highlight: item["highlight"])
+            # TODO: FIXME: amazon_id: item["embeddedId"]
+          end
+        end
+      end
+
+      def fetch_highlights(state={current_upcoming: [], asins: []})
+        page = agent.get("#{SETTINGS.url}/your_highlights")
+        initialize_state_with_page state, page
+        highlights = extract_highlights_from_page(page, state)
+        begin
+          page = get_the_next_page(state, highlights.flatten)
+          new_highlights = extract_highlights_from_page(page, state)
+          highlights << new_highlights
+        end until new_highlights.length == 0
+        highlights.flatten
+      end
+
+      def extract_highlights_from_page(page, state)
+        return [] if page.css(".yourHighlight").length == 0
+        page.css(".yourHighlight").map do |hl|
+          parse_highlight(hl, state)
+        end
+      end
+
+      def initialize_state_with_page(state, page)
+        return if (page/".yourHighlight").length == 0
+        state[:current_asin] = (page/".yourHighlightsHeader").first.attributes["id"].value.split("_").first
+        state[:current_upcoming] = (page/".upcoming").first.text.split(',') rescue []
+        state[:title] = (page/".yourHighlightsHeader .title").text.to_s.strip
+        state[:author] = (page/".yourHighlightsHeader .author").text.to_s.strip
+        highlights_on_page = (page/".yourHighlightsHeader")
+        state[:current_offset] = (highlights_on_page.collect{|h| h.attributes['id'].value }).first.split('_').last
+        highlight_count = (highlights_on_page/".highlightCount#{state[:current_asin]}").text.to_i
+        state[:current_number_of_highlights] = highlight_count
+        @books << Kindle::Remote::Book.new(state[:current_asin], {author: state[:author], title: state[:title], highlight_count: highlight_count})
+      end
+
+      def get_the_next_page(state, previously_extracted_highlights = [])
+        asins           = previously_extracted_highlights.map(&:asin).uniq
+        asins_string    = asins.collect { |asin| "used_asins[]=#{asin}" } * '&'
+        upcoming_string = state[:current_upcoming].map { |l| "upcoming_asins[]=#{l}" } * '&'
+        url = "#{SETTINGS.url}/your_highlights/next_book?#{asins_string}&current_offset=#{state[:current_offset]}&#{upcoming_string}"
+        ajax_headers = { 'X-Requested-With' => 'XMLHttpRequest', 'Host' => "kindle.#{SETTINGS.domain}" }
+        page = agent.get(url,[],"#{SETTINGS.url}/your_highlight", ajax_headers)
+        initialize_state_with_page state, page
+        page
+      end
+
+      def parse_highlight(hl, state)
+        highlight_id   = hl.xpath('//*[@id="annotation_id"]').first["value"]
+        highlight_text = (hl/".highlight").text
+        asin           = (hl/".asin").text
+        highlight = Kindle::Remote::Highlight.new(amazon_id: highlight_id, highlight: highlight_text, asin: asin)
+        highlight
+      end
+    end
+  end
+end