kindle_manager 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cb02e5a3a67d8232fbff330bdc508c7b2a841e09
+  data.tar.gz: 2641e09f54bb46cf8ca1dfe3b70950ce213ab43c
+SHA512:
+  metadata.gz: c695ef3285f6d530a8ad13ba16bf75105f782736bb0eccd05cdf97171789474caa51c385da23a16a6bce9d21f5764975d8f7bb5ef682ca7d9f2f12cdf9964b10
+  data.tar.gz: f80af23783b9af78e220cfb16aa42a6340d86ed9c667db43831aa1ac41de1c5869d73c46f23df80b38debc02c1b7d232952f558fd04932c2bc60facd348017a0

data/.gitignore

@@ -0,0 +1,16 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/.env
+/.byebug_history
+/downloads/
+/spec/fixtures/files/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.2.4
+before_install: gem install bundler -v 1.14.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in kindle_manager.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Kazuho Yamaguchi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,65 @@
+# KindleManager
+
+Scrape information of kindle books from amazon site
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'kindle_manager'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install kindle_manager
+
+## Usage
+
+### Setup
+
+Create _.env_ following the instructions of https://github.com/kyamaguchi/amazon_auth
+
+And `Dotenv.load` or `gem 'dotenv-rails'` may be required when you use this in your app.
+
+### Run
+
+```
+bin/console
+```
+
+In console
+
+```
+client = KindleManager::Client.new
+client.fetch_kindle_list
+books = client.load_kindle_books
+
+client.quit
+```
+
+#### Options
+
+Debug print: `client = KindleManager::Client.new(debug: true)`
+
+Limit fetching with number of fetched books: `client = KindleManager::Client.new(limit: 100)`
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/kyamaguchi/kindle_manager.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "kindle_manager"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/kindle_manager.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'kindle_manager/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "kindle_manager"
+  spec.version       = KindleManager::VERSION
+  spec.authors       = ["Kazuho Yamaguchi"]
+  spec.email         = ["kzh.yap@gmail.com"]
+
+  spec.summary       = %q{Scrape information of kindle books from amazon site}
+  spec.description   = %q{Scrape information of kindle books from amazon site}
+  spec.homepage      = "https://github.com/kyamaguchi/kindle_manager"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "amazon_auth", "~> 0.1.2"
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "byebug"
+end

data/lib/kindle_manager.rb

@@ -0,0 +1,9 @@
+require 'byebug'
+require 'amazon_auth'
+require "kindle_manager/version"
+require "kindle_manager/client"
+require "kindle_manager/file_store"
+require "kindle_manager/list_parser"
+
+module KindleManager
+end

data/lib/kindle_manager/client.rb

@@ -0,0 +1,116 @@
+module KindleManager
+  class Client
+    attr_accessor :page
+
+    def initialize(options = {})
+      @debug = options.fetch(:debug, false)
+      @limit = options.fetch(:limit, nil)
+      begin
+        @client = AmazonAuth::Client.new
+      rescue => e
+        puts "Please setup credentials of amazon_auth gem with folloing its instruction."
+        puts
+        raise e
+      end
+    end
+
+    def store
+      # Create file store without session(page) by default
+      @store ||= KindleManager::FileStore.new(nil, latest: true)
+    end
+
+    def setup_file_store_with_session
+      @store = KindleManager::FileStore.new(page)
+    end
+
+    def fetch_kindle_list
+      sign_in
+      setup_file_store_with_session
+      go_to_kindle_management_page
+      begin
+        load_next_kindle_list
+      rescue => e
+        byebug
+        # retry ?
+        puts e
+      end
+    end
+
+    def load_kindle_books
+      books = []
+      store.list_html_files.each do |file|
+        parser = KindleManager::ListParser.new(file)
+        books += parser.book_list
+      end
+      books.uniq(&:asin)
+    end
+
+    def sign_in
+      @page = @client.sign_in
+    end
+
+    def go_to_kindle_management_page
+      wait_for_selector('#shopAllLinks')
+      page.within('#shopAllLinks') do
+        page.find('a', text: 'コンテンツと端末の管理').click
+      end
+      page
+    end
+
+    def load_next_kindle_list
+      wait_for_selector('.contentCount_myx')
+      @current_loop = 0
+      while @current_loop <= 12 # max attempts
+        if @limit && @limit < number_of_fetched_books
+          break
+        elsif has_more_button?
+          debug_print_page
+          @current_loop = 0
+
+          puts "Clicking もっと表示"
+          page.execute_script "window.scrollBy(0,-800)"
+          page.click_on('もっと表示')
+          sleep 1
+          raise('Clicking of more button may have failed') if has_more_button?
+        else
+          puts "Scrolling #{@current_loop}"
+          page.execute_script "window.scrollBy(0,10000)"
+        end
+        sleep 5
+        @current_loop += 1
+      end
+      debug_print_page
+    end
+
+    def quit
+      page.driver.quit
+    end
+
+    def wait_for_selector(selector, seconds = 3)
+      seconds.times { sleep(1) unless page.first(selector) }
+    end
+
+    def has_more_button?
+      page.all('#contentTable_showMore_myx').map(&:text).include?('もっと表示')
+    end
+
+    def number_of_fetched_books
+      m = page.first('.contentCount_myx').text.match(/(\d+)を表示中/)
+      m.nil? ? nil : m[1].to_i
+    end
+
+    def loading?
+      page.first('.myx-popover-loading-wrapper').present?
+    end
+
+    def debug_print_page
+      if @debug
+        store.record_page
+        puts Time.current.strftime("%Y-%m-%d %H:%M:%S")
+        puts "Loop: #{@current_loop}"
+        puts page.first('.contentCount_myx').text if page.first('.contentCount_myx')
+        puts
+      end
+    end
+  end
+end

data/lib/kindle_manager/file_store.rb

@@ -0,0 +1,60 @@
+module KindleManager
+  class FileStore
+    attr_accessor :dir_name
+
+    def initialize(session, options = {})
+      @dir_name = options.fetch(:dir_name) do
+        options[:latest] ? find_latest_dir_name : Time.current.strftime("%Y%m%d%H%M%S")
+      end
+      @session = session
+    end
+
+    def base_dir
+      File.join(self.class.downloads_dir, @dir_name)
+    end
+
+    def self.downloads_dir
+      'downloads'
+    end
+
+    def html_path(time)
+      build_filepath(time, 'html')
+    end
+
+    def image_path(time)
+      build_filepath(time, 'png')
+    end
+
+    def record_page
+      time = Time.current
+      @session.save_page(html_path(time))
+      @session.save_screenshot(image_path(time))
+    end
+
+    def self.list_download_dirs
+      Dir["#{downloads_dir}/*"].select{|f| File.directory? f }
+    end
+
+    def self.list_html_files(dir = nil)
+      if dir
+        Dir[File.join(downloads_dir, dir,'*.html')].select{|f| File.file? f }
+      else
+        Dir["#{downloads_dir}/*/*.html"].select{|f| File.file? f }
+      end
+    end
+
+    def list_html_files
+      self.class.list_html_files(@dir_name)
+    end
+
+    def find_latest_dir_name
+      self.class.list_download_dirs.sort.last.split('/').last
+    end
+
+    private
+
+      def build_filepath(time, ext)
+        File.join(base_dir, "#{time.strftime('%Y%m%d%H%M%S')}#{(time.usec / 1000.0).round.to_s.rjust(3,'0')}.#{ext}")
+      end
+  end
+end

data/lib/kindle_manager/list_parser.rb

@@ -0,0 +1,64 @@
+module KindleManager
+  class ListParser
+    class BookRow
+      def initialize(node)
+        @node = node
+      end
+
+      def inspect
+        "#<#{self.class.name}:#{self.object_id} #{self.to_hash}>"
+      end
+
+      def asin
+        @_asin ||= @node['name'].gsub(/\AcontentTabList_/, '')
+      end
+
+      def title
+        @_title ||= @node.css("div[id^='title']").text
+      end
+
+      def tag
+        @_tag ||= @node.css("div[id^='listViewTitleTag']").css('.myx-text-bold').first.text.strip
+      end
+
+      def author
+        @_author ||= @node.css("div[id^='author']").text
+      end
+
+      def date
+        @_date ||= begin
+          m = @node.css("div[id^='date']").text.match(/\A(?<year>\d{4})年(?<month>\d{1,2})月(?<day>\d{1,2})日\z/)
+          Date.new(m[:year].to_i, m[:month].to_i, m[:day].to_i)
+        end
+      end
+
+      def collection_count
+        @_collection_count ||= @node.css(".collectionsCount .myx-collection-count").first.text.strip.to_i
+      end
+
+      def to_hash
+        hash = {}
+        %w[asin title tag author date collection_count].each do |f|
+          hash[f] = send(f)
+        end
+        hash
+      end
+    end
+
+    def initialize(filepath, options = {})
+      @filepath = filepath
+    end
+
+    def book_list
+      @book_list ||= doc.css("div[id^='contentTabList_']").map{|e| BookRow.new(e) }
+    end
+
+    def doc
+      @doc ||= Nokogiri::HTML(body)
+    end
+
+    def body
+      @body ||= File.read(@filepath)
+    end
+  end
+end

data/lib/kindle_manager/version.rb

@@ -0,0 +1,3 @@
+module KindleManager
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,129 @@
+--- !ruby/object:Gem::Specification
+name: kindle_manager
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Kazuho Yamaguchi
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-03-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: amazon_auth
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Scrape information of kindle books from amazon site
+email:
+- kzh.yap@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- kindle_manager.gemspec
+- lib/kindle_manager.rb
+- lib/kindle_manager/client.rb
+- lib/kindle_manager/file_store.rb
+- lib/kindle_manager/list_parser.rb
+- lib/kindle_manager/version.rb
+homepage: https://github.com/kyamaguchi/kindle_manager
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Scrape information of kindle books from amazon site
+test_files: []