kindle_manager 0.6.4 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 94b27bae732e8c8a50d463380a7fbcc499ad5ea46f810bed185ed469f3ae0c8b
-  data.tar.gz: 8f902074660f6774d95a32e37afafa96e06cbfedfc7cc9ee20b215f3f83364e9
+  metadata.gz: a490117c413d9e85320298a280d32da6a1c0af1f97059af1cc89affbd40fc284
+  data.tar.gz: 6b1b31d1d72eedbe4605ea6b61ae243b624e36971aba14c31389f092f9a121f8
 SHA512:
-  metadata.gz: d759c1c3b06ec52cb47dfc5ebbdfbe29693899758b55b66853c0a6d088edf5c2e53e0a383d7939d6021f64b009c05d03bc3b4d9501753579194bd190dddc0fe0
-  data.tar.gz: c4bf70715590d1e9c7840e9736a1bfcca77369ed0370af5afa1f31dee2a32b423769f39fd52db71c8a14e915eea78bd2c7d89e122c4b7875587c51540448bb3e
+  metadata.gz: a8d03445f3b6a517c1dffba41a214e37340a40e641936521c36905109c5977cfe84f7239f365a18dd503cc07ecc766f31e647f0f3e5666997f43e7e0123b2a74
+  data.tar.gz: 5a0fad36eeda000586c97cf343a83792f5ee98c12110fd248a24a5f9327590c01ba16b6c945e44725c7bcb281e5ccd2fb69ad2f71e9fa1b0481b1b571e2a3373

data/.circleci/config.yml

@@ -0,0 +1,21 @@
+version: 2.1
+
+jobs:
+  test:
+    parameters:
+      ruby-version:
+        type: string
+    docker:
+      - image: cimg/ruby:<< parameters.ruby-version >>-browsers
+    steps:
+      - checkout
+      - run: bundle install
+      - run: bundle exec rspec
+
+workflows:
+  all-tests:
+    jobs:
+      - test:
+          matrix:
+            parameters:
+              ruby-version: ["2.7.5", "3.0.3"]

data/README.md

@@ -1,7 +1,7 @@
 # KindleManager
 
 [![Gem Version](https://badge.fury.io/rb/kindle_manager.svg)](https://badge.fury.io/rb/kindle_manager)
-[![Build Status](https://travis-ci.org/kyamaguchi/kindle_manager.svg?branch=master)](https://travis-ci.org/kyamaguchi/kindle_manager)
+[![CircleCI](https://circleci.com/gh/kyamaguchi/kindle_manager.svg?style=svg)](https://circleci.com/gh/kyamaguchi/kindle_manager)
 
 Scrape information of kindle books & highlights from amazon site
 
@@ -34,7 +34,7 @@ Or install it yourself as:
 
 ### Setup
 
-[chromedriver](https://sites.google.com/a/chromium.org/chromedriver/downloads) is required. Please [download chromedriver](http://chromedriver.storage.googleapis.com/index.html) and update chromedriver regularly.  
+[chromedriver](https://sites.google.com/chromium.org/driver/) is required. Please [download chromedriver](https://chromedriver.storage.googleapis.com/index.html) and update chromedriver regularly.  
 
 Create _.env_ following the instructions of https://github.com/kyamaguchi/amazon_auth
 
@@ -142,11 +142,6 @@ Login and password: `login: 'xxx', password: 'yyy'`
 
 Output debug log: `debug: true`
 
-## Issues
-
-There may be problems with capybara 3.  
-Use older version with `gem 'capybara', '~> 2.18.0'` in that case.
-
 ## TODO
 
 - Limit the number of fetching books by date

data/kindle_manager.gemspec

@@ -21,9 +21,10 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "amazon_auth", "~> 0.4.0"
+  spec.add_runtime_dependency "amazon_auth", "~> 0.8.0"
+  spec.add_runtime_dependency "webdrivers"
   spec.add_development_dependency "bundler"
-  spec.add_development_dependency "rake", "~> 13.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
   spec.add_development_dependency "byebug"
 end

data/lib/kindle_manager/adapters/books_adapter.rb

@@ -1,6 +1,8 @@
 module KindleManager
   class BooksAdapter < BaseAdapter
-    URL_FOR_KINDLE_CONTENTS = 'https://www.amazon.co.jp/gp/digital/fiona/manage?ie=UTF8&ref_=nav_youraccount_myk'
+    def url_for_kindle_contents
+      "https://www.#{ENV['AMAZON_DOMAIN']}/hz/mycd/digital-console/contentlist/booksAll/dateDsc/"
+    end
 
     def fetch
       go_to_kindle_management_page
@@ -10,17 +12,17 @@ module KindleManager
         puts "[ERROR] #{e}"
         puts e.backtrace
         puts
-        puts "Retry manually -> client.adapter.load_next_kindle_list or client.session etc."
+        puts "Investigate the error using 'client.session', 'client.doc' etc."
       end
     end
 
     def go_to_kindle_management_page
       log "Visiting kindle management page"
       3.times do
-        session.visit URL_FOR_KINDLE_CONTENTS
-        wait_for_selector('.navHeader_myx')
-        if session.first('.navHeader_myx')
-          log "Page found '#{session.first('.navHeader_myx').text}'"
+        session.visit url_for_kindle_contents
+        wait_for_selector('#content-page-title')
+        if session.has_css?('#content-page-title')
+          log "Page found '#{session.first('#content-page-title').text}'"
           break
         else
           submit_signin_form
@@ -29,34 +31,15 @@ module KindleManager
     end
 
     def load_next_kindle_list
-      wait_for_selector('.contentCount_myx')
-      current_loop = 0
-      last_page_scroll_offset = page_scroll_offset
-      while current_loop <= max_scroll_attempts
-        break if limit && limit < number_of_fetched_books
-        if has_more_button?
-          snapshot_page
-          current_loop = 0
-
-          log "Clicking 'Show More'"
-          session.execute_script "window.scrollBy(0,-800)"
-          show_more_button.click
-          sleep fetching_interval
-        else
-          log "Loading books with scrolling #{current_loop+1}"
-          session.execute_script "window.scrollBy(0,10000)"
-        end
+      wait_for_selector('#CONTENT_COUNT')
+      loop do
+        snapshot_page
+        break if current_page == max_page
+        break if limit && limit <= number_of_fetched_books
+        session.first("#page-#{current_page + 1}").click
         sleep fetching_interval
-        if last_page_scroll_offset == page_scroll_offset
-          log "Stopping loading because 'page_scroll_offset' didn't change after a loop"
-          break
-        else
-          debug "last_page_scroll_offset:#{last_page_scroll_offset} new page_scroll_offset:#{page_scroll_offset}"
-        end
-        last_page_scroll_offset = page_scroll_offset
-        current_loop += 1
       end
-      log "Stopped loading. You may want to resume with 'client.adapter.load_next_kindle_list'"
+      log "Stopped loading. You may want to continue with 'client.session', 'client.doc' etc."
       snapshot_page
     end
 
@@ -69,33 +52,26 @@ module KindleManager
       books.sort_by{|b| [-b.date.to_time.to_i, -b.fetched_at.to_i] }.uniq(&:asin)
     end
 
-    def page_scroll_offset
-      session.evaluate_script('window.pageYOffset')
-    end
-
-    def has_more_button?
-      !!show_more_button
+    def current_page
+      doc.css('#pagination a.active').first.text.to_i
     end
 
-    def show_more_button
-      session.all('.contentTableShowMore_myx').find{|e| e['outerHTML'].match(/cnt_shw_more/) }
+    def max_page
+      @_max_page ||= doc.css('#pagination a').last.text.to_i
     end
 
     def number_of_fetched_books
-      re = (AmazonInfo.domain =~ /\.jp\z/ ? /(\d+)〜(\d+)/ : /(\d+) - (\d+)/)
-      wait_for_selector('.contentCount_myx')
-      text = doc.css('.contentCount_myx').text
+      re = (AmazonInfo.domain =~ /\.jp\z/ ? /(\d+)から(\d+)/ : / (\d+) to (\d+) /)
+      wait_for_selector('#CONTENT_COUNT')
+      text = doc.css('#CONTENT_COUNT').text
+      log "Number of books: [#{text}]"
       m = text.match(re)
       return m[2].to_i if m.present?
       raise("Couldn't get the number of fetched books [#{text}]")
     end
 
-    def loading?
-      session.first('.myx-popover-loading-wrapper').present?
-    end
-
     def snapshot_page
-      if (text = doc.css('.contentCount_myx').try!(:text)).present?
+      if (text = doc.css('#CONTENT_COUNT').try!(:text)).present?
         log "Current page [#{text.to_s.gsub(/[[:space:]]+/, ' ').strip}]"
       end
       store.record_page

data/lib/kindle_manager/parsers/books_parser.rb

@@ -13,28 +13,33 @@ module KindleManager
         "#<#{self.class.name}:#{self.object_id} #{self.to_hash}>"
       end
 
+      def title_node
+        # Possible to use "div[id^='content-title-']"
+        @_title_node ||= @node.css('.digital_entity_title').first
+      end
+
       def asin
-        @_asin ||= @node['name'].gsub(/\AcontentTabList_/, '')
+        @_asin ||= title_node.attributes['id'].value.remove('content-title-')
       end
 
       def title
-        @_title ||= @node.css("div[id^='title']").text
+        @_title ||= title_node.text
       end
 
       def tag
-        @_tag ||= @node.css("div[id^='listViewTitleTag']").css('.myx-text-bold').first.text.strip
+        @_tag ||= @node.css('.information_row.tags').first&.text&.strip
       end
 
       def author
-        @_author ||= @node.css("div[id^='author']").text
+        @_author ||= @node.css("div[id^='content-author-']").text
       end
 
       def date
-        @_date ||= parse_date(@node.css("div[id^='date']").text)
+        @_date ||= parse_date(@node.css("div[id^='content-acquired-date-']").text)
       end
 
       def collection_count
-        @_collection_count ||= @node.css(".collectionsCount .myx-collection-count").first.text.strip.to_i
+        @_collection_count ||= @node.css('.dropdown_count').first&.text&.strip.to_i
       end
 
       def to_hash
@@ -48,7 +53,7 @@ module KindleManager
 
     def parse
       @_parsed ||= begin
-        doc.css("div[id^='contentTabList_']").map{|e| BookRow.new(e, fetched_at: fetched_at) }
+        doc.css('#CONTENT_LIST table tbody tr').map{|e| BookRow.new(e, fetched_at: fetched_at) }
       end
     end
   end

data/lib/kindle_manager/parsers/common.rb

@@ -11,7 +11,7 @@ module KindleManager
         begin
           Date.parse(date_text)
         rescue ArgumentError => e
-          m = date_text.match(/\A(?<year>\d{4})年(?<month>\d{1,2})月(?<day>\d{1,2})日/)
+          m = date_text.match(/(?<year>\d{4})年(?<month>\d{1,2})月(?<day>\d{1,2})日/)
           m = date_text.match(/(?<month>\d{1,2})月\D+(?<day>\d{1,2}),\D+(?<year>\d{4})/) if m.nil?
           raise("Failed to parse date [#{date_text}]") if m.nil?
           Date.new(m[:year].to_i, m[:month].to_i, m[:day].to_i)

data/lib/kindle_manager/version.rb

@@ -1,3 +1,3 @@
 module KindleManager
-  VERSION = "0.6.4"
+  VERSION = "0.8.0"
 end

data/lib/kindle_manager.rb

@@ -9,6 +9,7 @@ require "kindle_manager/parsers/common"
 require "kindle_manager/parsers/base_parser"
 require "kindle_manager/parsers/books_parser"
 require "kindle_manager/parsers/highlights_parser"
+require 'webdrivers'
 
 module KindleManager
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kindle_manager
 version: !ruby/object:Gem::Version
-  version: 0.6.4
+  version: 0.8.0
 platform: ruby
 authors:
 - Kazuho Yamaguchi
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-05-27 00:00:00.000000000 Z
+date: 2022-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: amazon_auth
@@ -16,14 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.0
+        version: 0.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.0
+        version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: webdrivers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -42,30 +56,30 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '13.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '13.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -87,9 +101,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".gitignore"
 - ".rspec"
-- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -112,7 +126,7 @@ homepage: https://github.com/kyamaguchi/kindle_manager
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -127,8 +141,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
-signing_key: 
+rubygems_version: 3.2.33
+signing_key:
 specification_version: 4
 summary: Scrape information of kindle books from amazon site
 test_files: []

data/.travis.yml

@@ -1,7 +0,0 @@
-sudo: false
-language: ruby
-rvm:
-  - 2.4.5
-  - 2.5.3
-  - 2.6.0
-before_install: gem install bundler