kindle_manager 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 73c95f02f9f63d413e423a014c592451b3856fbe
-  data.tar.gz: 12d55a94e7a21ea0b748619a1a936d3a02caeaab
+  metadata.gz: 5df40793942c890de33041d60b6e27efe076a695
+  data.tar.gz: 31a5cbce41c6b166de7954cb876f041fc4ddf3ec
 SHA512:
-  metadata.gz: c3f7f1da0d291473403a887fca2537f29843e73cd1400e40d52090d9bad9bead1e21b4b7f86f73c88e5c9a23f397ee6c7b8f66d1b93f39972c361690e66a3f06
-  data.tar.gz: 49a31303b797f8c844133b36e37a0523aea887f72ef69ec61626df06321693ae2d89bb669415b19ea3fbc3a9f07ca89636bc0bd8425086476277d87ad1b1a408
+  metadata.gz: 4d9ac40336b50f1f6f10c201f11eef301eb11ae373e8bd9fd88c5167edd00e4cf6edc95fda3b82f2436d440d1dff8f9ca8e459708967d66fd0756086927a6b30
+  data.tar.gz: 315a3ae71152310075fe4d855355bd93c280af92c6d400f2bbb5ae2333941380f598b8092373f3aa3d63b0a371162e1a050407e772b437abe181a96b28b17c11

data/.gitignore

@@ -10,8 +10,8 @@
 /.env
 /.byebug_history
 /.ruby-version
-/downloads/
 /spec/fixtures/files/
+/spec/fixtures/tmp/
 
 # rspec failure tracking
 .rspec_status

data/README.md

@@ -54,7 +54,7 @@ In console
 
 ```ruby
 require 'kindle_manager'
-client = KindleManager::Client.new(verbose: true, limit: 1000)
+client = KindleManager::Client.new(keep_cookie: true, verbose: true, limit: 1000)
 client.fetch_kindle_list
 
 books = client.load_kindle_books
@@ -89,7 +89,7 @@ In console
 
 ```ruby
 require 'kindle_manager'
-client = KindleManager::Client.new(verbose: true, limit: 10)
+client = KindleManager::Client.new(keep_cookie: true, verbose: true, limit: 10)
 client.fetch_kindle_highlights
 
 books = client.load_kindle_highlights

data/kindle_manager.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "amazon_auth", "~> 0.3.2"
+  spec.add_runtime_dependency "amazon_auth", "~> 0.4.0"
   spec.add_development_dependency "bundler", "~> 1.14"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"

data/lib/kindle_manager/adapters/base_adapter.rb

@@ -10,7 +10,7 @@ module KindleManager
       extend(AmazonAuth::SessionExtension)
 
       @store = KindleManager::FileStore.new(options.merge(session: @session))
-      log "Directory for downloaded pages is #{store.base_dir}"
+      log "Directory for downloaded pages is #{store.target_dir}"
     end
 
     def limit

data/lib/kindle_manager/adapters/books_adapter.rb

@@ -22,6 +22,8 @@ module KindleManager
         if session.first('.navHeader_myx')
           log "Page found '#{session.first('.navHeader_myx').text}'"
           break
+        else
+          submit_signin_form
         end
       end
     end
@@ -29,10 +31,10 @@ module KindleManager
     def load_next_kindle_list
       wait_for_selector('.contentCount_myx')
       current_loop = 0
+      last_page_scroll_offset = page_scroll_offset
       while current_loop <= max_scroll_attempts
-        if limit && limit < number_of_fetched_books
-          break
-        elsif has_more_button?
+        break if limit && limit < number_of_fetched_books
+        if has_more_button?
           snapshot_page
           current_loop = 0
 
@@ -46,9 +48,16 @@ module KindleManager
           session.execute_script "window.scrollBy(0,10000)"
         end
         sleep fetching_interval
+        if last_page_scroll_offset == page_scroll_offset
+          log "Stopping loading because 'page_scroll_offset' didn't change after a loop"
+          break
+        else
+          debug "last_page_scroll_offset:#{last_page_scroll_offset} new page_scroll_offset:#{page_scroll_offset}"
+        end
+        last_page_scroll_offset = page_scroll_offset
         current_loop += 1
       end
-      log "Stopped loading"
+      log "Stopped loading. You may want to resume with 'client.adapter.load_next_kindle_list'"
       snapshot_page
     end
 
@@ -58,7 +67,11 @@ module KindleManager
         parser = KindleManager::BooksParser.new(file)
         books += parser.parse
       end
-      books.uniq(&:asin)
+      books.sort_by{|b| [-b.date.to_time.to_i, -b.fetched_at.to_i] }.uniq(&:asin)
+    end
+
+    def page_scroll_offset
+      session.evaluate_script('window.pageYOffset')
     end
 
     def has_more_button?

data/lib/kindle_manager/adapters/highlights_adapter.rb

@@ -100,7 +100,7 @@ module KindleManager
         parser = KindleManager::HighlightsParser.new(file)
         books += parser.parse
       end
-      books.reject(&:invalid?).uniq(&:asin)
+      books.reject(&:invalid?).sort_by{|b| [-b.last_annotated_on.to_time.to_i, -b.fetched_at.to_i] }.uniq(&:asin)
     end
 
     def snapshot_page(message = nil)

data/lib/kindle_manager/client.rb

@@ -2,7 +2,7 @@ module KindleManager
   class Client
     include AmazonAuth::CommonExtension
 
-    attr_accessor :adapter
+    attr_accessor :adapter, :options
 
     def initialize(options = {})
       @options = options
@@ -14,10 +14,6 @@ module KindleManager
       @_session ||= @client.session
     end
 
-    def sign_in
-      @client.sign_in
-    end
-
     def fetch_kindle_list
       sign_in
       set_adapter(:books, @options.merge(session: session))
@@ -25,8 +21,7 @@ module KindleManager
     end
 
     def fetch_kindle_highlights
-      session.visit KindleManager::HighlightsAdapter::KINDLE_HIGHLIGHT_URL
-      @client.submit_signin_form
+      sign_in KindleManager::HighlightsAdapter::KINDLE_HIGHLIGHT_URL
       set_adapter(:highlights, @options.merge(session: session))
       adapter.fetch
     end

data/lib/kindle_manager/file_store.rb

@@ -1,6 +1,8 @@
 module KindleManager
   class FileStore
-    attr_accessor :dir_name, :session
+    TIME_FORMAT_FOR_FILENAME = '%Y%m%d%H%M%S'
+
+    attr_accessor :sub_dir, :dir_name, :session
 
     def initialize(options = {})
       @sub_dir = options.fetch(:sub_dir, 'books').to_s
@@ -11,28 +13,20 @@ module KindleManager
       @session = options.fetch(:session, nil)
     end
 
-    def downloads_dir
-      'downloads'
-    end
-
-    def root_dir
-      File.join(downloads_dir, @sub_dir)
-    end
-
-    def base_dir
-      File.join(root_dir, @dir_name)
+    def target_dir
+      File.join(sub_dir, dir_name)
     end
 
     def list_work_dirs
-      Dir["#{root_dir}/*"].select{|f| File.directory? f }
+      Dir[File.join(Capybara.save_path, sub_dir,'*')].select{|f| File.directory? f }
     end
 
     def find_latest_dir_name
       list_work_dirs.sort.last.to_s.split('/').last
     end
 
-    def list_html_files(dir = nil)
-      Dir[File.join(base_dir,'*.html')].select{|f| File.file? f }
+    def list_html_files
+      Dir[File.join(Capybara.save_path, target_dir,'*.html')].select{|f| File.file? f }
     end
 
     def html_path(time)
@@ -52,7 +46,7 @@ module KindleManager
     private
 
       def build_filepath(time, ext)
-        File.join(base_dir, "#{time.strftime('%Y%m%d%H%M%S')}#{(time.usec / 1000.0).round.to_s.rjust(3,'0')}.#{ext}")
+        File.join(target_dir, "#{time.strftime(TIME_FORMAT_FOR_FILENAME)}#{(time.usec / 1000.0).round.to_s.rjust(3,'0')}.#{ext}")
       end
   end
 end

data/lib/kindle_manager/parsers/base_parser.rb

@@ -1,8 +1,15 @@
 module KindleManager
   class BaseParser
+    attr_accessor :fetched_at
 
     def initialize(filepath, options = {})
       @filepath = filepath
+
+      @fetched_at = if File.basename(@filepath) =~ /\A\d{14}/
+        Time.strptime(File.basename(@filepath)[0..14], KindleManager::FileStore::TIME_FORMAT_FOR_FILENAME)
+      else
+        File.ctime(@filepath)
+      end
     end
 
     def doc

data/lib/kindle_manager/parsers/books_parser.rb

@@ -1,10 +1,12 @@
 module KindleManager
   class BooksParser < BaseParser
     class BookRow
+
       include KindleManager::Parsers::Common
 
-      def initialize(node)
+      def initialize(node, options = {})
         @node = node
+        @fetched_at = options[:fetched_at]
       end
 
       def inspect
@@ -45,7 +47,9 @@ module KindleManager
     end
 
     def parse
-      @_parsed ||= doc.css("div[id^='contentTabList_']").map{|e| BookRow.new(e) }
+      @_parsed ||= begin
+        doc.css("div[id^='contentTabList_']").map{|e| BookRow.new(e, fetched_at: fetched_at) }
+      end
     end
   end
 end

data/lib/kindle_manager/parsers/common.rb

@@ -1,6 +1,11 @@
 module KindleManager
   module Parsers
     module Common
+      extend ActiveSupport::Concern
+
+      included do
+        attr_accessor :fetched_at
+      end
 
       def parse_date(date_text)
         begin

data/lib/kindle_manager/parsers/highlights_parser.rb

@@ -3,8 +3,9 @@ module KindleManager
     class BookWithNote
       include KindleManager::Parsers::Common
 
-      def initialize(node)
+      def initialize(node, options = {})
         @node = node
+        @fetched_at = options[:fetched_at]
       end
 
       def inspect
@@ -81,7 +82,7 @@ module KindleManager
 
     def parse
       @_parsed ||= begin
-        result = doc.css('.kp-notebook-annotation-container').map{|e| BookWithNote.new(e) }
+        result = doc.css('.kp-notebook-annotation-container').map{|e| BookWithNote.new(e, fetched_at: fetched_at) }
         puts "[DEBUG] This page(#{@filepath}) has many books. asin -> #{result.map(&:asin).join(',')}" if result.size >= 2
         puts "[DEBUG] Incomplete page(#{@filepath}). asin:#{result.first.asin} #{result.first.title} (#{result.first.count_summary['text'].inspect})" if result.any?(&:invalid?)
         result

data/lib/kindle_manager/version.rb

@@ -1,3 +1,3 @@
 module KindleManager
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kindle_manager
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Kazuho Yamaguchi
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-07-05 00:00:00.000000000 Z
+date: 2017-07-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: amazon_auth
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.2
+        version: 0.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.2
+        version: 0.4.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement