kindle_manager 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 38b4108ec337ceae8173babf86fb0adac4550f06f68aab4913228b411ff9ae85
-  data.tar.gz: d517f89c420a47a9403d2caafa476f83b6c9f7056dd0963a7ee096f5e3893507
+  metadata.gz: a490117c413d9e85320298a280d32da6a1c0af1f97059af1cc89affbd40fc284
+  data.tar.gz: 6b1b31d1d72eedbe4605ea6b61ae243b624e36971aba14c31389f092f9a121f8
 SHA512:
-  metadata.gz: 55a646ff5b8d6c7487a83ae6468763b783e5787a91a4b12be622c90b603ebbe66a0ac39e4c18c9ad3038a5d4f7a54855ef53123e5db760f49678af37c29925ed
-  data.tar.gz: 69ae1b4a50c7708625f74b2aa027aee315e81103f06f14f24e4f0b216fdd81ca53d49fe9c1beea177e14b8d740b3df493dd0c5af440dee8b04d5bbddca79d38e
+  metadata.gz: a8d03445f3b6a517c1dffba41a214e37340a40e641936521c36905109c5977cfe84f7239f365a18dd503cc07ecc766f31e647f0f3e5666997f43e7e0123b2a74
+  data.tar.gz: 5a0fad36eeda000586c97cf343a83792f5ee98c12110fd248a24a5f9327590c01ba16b6c945e44725c7bcb281e5ccd2fb69ad2f71e9fa1b0481b1b571e2a3373

data/README.md

@@ -142,11 +142,6 @@ Login and password: `login: 'xxx', password: 'yyy'`
 
 Output debug log: `debug: true`
 
-## Issues
-
-There may be problems with capybara 3.  
-Use older version with `gem 'capybara', '~> 2.18.0'` in that case.
-
 ## TODO
 
 - Limit the number of fetching books by date

data/kindle_manager.gemspec

@@ -21,7 +21,8 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "amazon_auth", "~> 0.6.0"
+  spec.add_runtime_dependency "amazon_auth", "~> 0.8.0"
+  spec.add_runtime_dependency "webdrivers"
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"

data/lib/kindle_manager/adapters/books_adapter.rb

@@ -1,6 +1,8 @@
 module KindleManager
   class BooksAdapter < BaseAdapter
-    URL_FOR_KINDLE_CONTENTS = 'https://www.amazon.co.jp/gp/digital/fiona/manage?ie=UTF8&ref_=nav_youraccount_myk'
+    def url_for_kindle_contents
+      "https://www.#{ENV['AMAZON_DOMAIN']}/hz/mycd/digital-console/contentlist/booksAll/dateDsc/"
+    end
 
     def fetch
       go_to_kindle_management_page
@@ -10,17 +12,17 @@ module KindleManager
         puts "[ERROR] #{e}"
         puts e.backtrace
         puts
-        puts "Retry manually -> client.adapter.load_next_kindle_list or client.session etc."
+        puts "Investigate the error using 'client.session', 'client.doc' etc."
       end
     end
 
     def go_to_kindle_management_page
       log "Visiting kindle management page"
       3.times do
-        session.visit URL_FOR_KINDLE_CONTENTS
-        wait_for_selector('.navHeader_title_myx')
-        if session.has_css?('.navHeader_title_myx')
-          log "Page found '#{session.first('.navHeader_title_myx').text}'"
+        session.visit url_for_kindle_contents
+        wait_for_selector('#content-page-title')
+        if session.has_css?('#content-page-title')
+          log "Page found '#{session.first('#content-page-title').text}'"
           break
         else
           submit_signin_form
@@ -29,34 +31,15 @@ module KindleManager
     end
 
     def load_next_kindle_list
-      wait_for_selector('.contentCount_myx')
-      current_loop = 0
-      last_page_scroll_offset = page_scroll_offset
-      while current_loop <= max_scroll_attempts
-        break if limit && limit < number_of_fetched_books
-        if has_more_button?
-          snapshot_page
-          current_loop = 0
-
-          log "Clicking 'Show More'"
-          session.execute_script "window.scrollBy(0,-800)"
-          show_more_button.click
-          sleep fetching_interval
-        else
-          log "Loading books with scrolling #{current_loop+1}"
-          session.execute_script "window.scrollBy(0,10000)"
-        end
+      wait_for_selector('#CONTENT_COUNT')
+      loop do
+        snapshot_page
+        break if current_page == max_page
+        break if limit && limit <= number_of_fetched_books
+        session.first("#page-#{current_page + 1}").click
         sleep fetching_interval
-        if last_page_scroll_offset == page_scroll_offset
-          log "Stopping loading because 'page_scroll_offset' didn't change after a loop"
-          break
-        else
-          debug "last_page_scroll_offset:#{last_page_scroll_offset} new page_scroll_offset:#{page_scroll_offset}"
-        end
-        last_page_scroll_offset = page_scroll_offset
-        current_loop += 1
       end
-      log "Stopped loading. You may want to resume with 'client.adapter.load_next_kindle_list'"
+      log "Stopped loading. You may want to continue with 'client.session', 'client.doc' etc."
       snapshot_page
     end
 
@@ -69,33 +52,26 @@ module KindleManager
       books.sort_by{|b| [-b.date.to_time.to_i, -b.fetched_at.to_i] }.uniq(&:asin)
     end
 
-    def page_scroll_offset
-      session.evaluate_script('window.pageYOffset')
-    end
-
-    def has_more_button?
-      !!show_more_button
+    def current_page
+      doc.css('#pagination a.active').first.text.to_i
     end
 
-    def show_more_button
-      session.all('.contentTableShowMore_myx').find{|e| e['outerHTML'].match(/cnt_shw_more/) }
+    def max_page
+      @_max_page ||= doc.css('#pagination a').last.text.to_i
     end
 
     def number_of_fetched_books
-      re = (AmazonInfo.domain =~ /\.jp\z/ ? /(\d+)〜(\d+)/ : /(\d+) - (\d+)/)
-      wait_for_selector('.contentCount_myx')
-      text = doc.css('.contentCount_myx').text
+      re = (AmazonInfo.domain =~ /\.jp\z/ ? /(\d+)から(\d+)/ : / (\d+) to (\d+) /)
+      wait_for_selector('#CONTENT_COUNT')
+      text = doc.css('#CONTENT_COUNT').text
+      log "Number of books: [#{text}]"
       m = text.match(re)
       return m[2].to_i if m.present?
       raise("Couldn't get the number of fetched books [#{text}]")
     end
 
-    def loading?
-      session.first('.myx-popover-loading-wrapper').present?
-    end
-
     def snapshot_page
-      if (text = doc.css('.contentCount_myx').try!(:text)).present?
+      if (text = doc.css('#CONTENT_COUNT').try!(:text)).present?
         log "Current page [#{text.to_s.gsub(/[[:space:]]+/, ' ').strip}]"
       end
       store.record_page

data/lib/kindle_manager/parsers/books_parser.rb

@@ -13,28 +13,33 @@ module KindleManager
         "#<#{self.class.name}:#{self.object_id} #{self.to_hash}>"
       end
 
+      def title_node
+        # Possible to use "div[id^='content-title-']"
+        @_title_node ||= @node.css('.digital_entity_title').first
+      end
+
       def asin
-        @_asin ||= @node['name'].gsub(/\AcontentTabList_/, '')
+        @_asin ||= title_node.attributes['id'].value.remove('content-title-')
       end
 
       def title
-        @_title ||= @node.css("div[id^='title']").text
+        @_title ||= title_node.text
       end
 
       def tag
-        @_tag ||= @node.css("div[id^='listViewTitleTag']").css('.myx-text-bold').first.text.strip
+        @_tag ||= @node.css('.information_row.tags').first&.text&.strip
       end
 
       def author
-        @_author ||= @node.css("div[id^='author']").text
+        @_author ||= @node.css("div[id^='content-author-']").text
       end
 
       def date
-        @_date ||= parse_date(@node.css("div[id^='date']").text)
+        @_date ||= parse_date(@node.css("div[id^='content-acquired-date-']").text)
       end
 
       def collection_count
-        @_collection_count ||= @node.css(".collectionsCount .myx-collection-count").first.text.strip.to_i
+        @_collection_count ||= @node.css('.dropdown_count').first&.text&.strip.to_i
       end
 
       def to_hash
@@ -48,7 +53,7 @@ module KindleManager
 
     def parse
       @_parsed ||= begin
-        doc.css("div[id^='contentTabList_']").map{|e| BookRow.new(e, fetched_at: fetched_at) }
+        doc.css('#CONTENT_LIST table tbody tr').map{|e| BookRow.new(e, fetched_at: fetched_at) }
       end
     end
   end

data/lib/kindle_manager/parsers/common.rb

@@ -11,7 +11,7 @@ module KindleManager
         begin
           Date.parse(date_text)
         rescue ArgumentError => e
-          m = date_text.match(/\A(?<year>\d{4})年(?<month>\d{1,2})月(?<day>\d{1,2})日/)
+          m = date_text.match(/(?<year>\d{4})年(?<month>\d{1,2})月(?<day>\d{1,2})日/)
           m = date_text.match(/(?<month>\d{1,2})月\D+(?<day>\d{1,2}),\D+(?<year>\d{4})/) if m.nil?
           raise("Failed to parse date [#{date_text}]") if m.nil?
           Date.new(m[:year].to_i, m[:month].to_i, m[:day].to_i)

data/lib/kindle_manager/version.rb

@@ -1,3 +1,3 @@
 module KindleManager
-  VERSION = "0.7.0"
+  VERSION = "0.8.0"
 end

data/lib/kindle_manager.rb

@@ -9,6 +9,7 @@ require "kindle_manager/parsers/common"
 require "kindle_manager/parsers/base_parser"
 require "kindle_manager/parsers/books_parser"
 require "kindle_manager/parsers/highlights_parser"
+require 'webdrivers'
 
 module KindleManager
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kindle_manager
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Kazuho Yamaguchi
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-12-05 00:00:00.000000000 Z
+date: 2022-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: amazon_auth
@@ -16,14 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: 0.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: webdrivers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -127,7 +141,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: Scrape information of kindle books from amazon site