bookmeter_exporter 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 6bdd15cf3580a2b9112053f97c67b50388f8d0c6341a0f73f77f81665fca568d
+  data.tar.gz: 906480de373aacdf3ab810151cdf7887b2cabcf3969fa4082f7213c36e282891
+SHA512:
+  metadata.gz: cdf26fd81c41dc05cf5875515f0422332b43e249f5bf89c6b099a22e1b1af346e7574b865ac3f8d6ef42a9476434ae25d5055ba90b191a9c8fea4070c6e924ca
+  data.tar.gz: 11f56ecd657fff8879b6299012125bf9f0e64eb9d61d09f425f38217a182f018ee1658bf6d2cc49f894332d44c7bd035758366f302e7209153c68f6c09242a8c

data/.github/workflows/main.yml

@@ -0,0 +1,19 @@
+name: Ruby
+
+on: [push,pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby: [2.6, 2.7, 3.0]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby}}
+        bundler-cache: true
+    - name: Run the default task
+      run: bundle exec rake

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+books.csv

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,23 @@
+AllCops:
+  TargetRubyVersion: 2.6
+  NewCops: enable
+
+Metrics/AbcSize:
+  Max: 30
+
+Metrics/MethodLength:
+  Max: 20
+
+Style/Documentation:
+  Enabled: false
+
+Style/StringLiterals:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Style/StringLiteralsInInterpolation:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Layout/LineLength:
+  Max: 120

data/.ruby-version

@@ -0,0 +1 @@
+3.0.2

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## [Unreleased]
+
+## [0.1.0] - 2021-07-24
+
+- Initial release

data/Gemfile

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gemspec
+
+gem "rake", "~> 13.0"
+
+gem "rspec", "~> 3.0"
+
+gem "rubocop", "~> 1.7"
+gem "rubocop-rake"
+gem "rubocop-rspec"

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2021 Ikuo Degawa
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,82 @@
+# bookmeter_exporter
+
+A command line tool to export read books data from https://bookmeter.com as CSV
+
+[読書メーター](https://bookmeter.com)から読み終わった本をエクスポートするCLIツール
+
+## Requirements
+
+* Ruby >= 2.6 and bundler
+* `chromedriver` in $PATH and same version of Google Chrome
+  * Download: https://chromedriver.chromium.org/downloads
+* 読書メーターのアカウントにメールアドレスとパスワードが設定されていること
+
+## Installation
+
+```bash
+$ gem install bookmeter_exporter
+```
+
+## Usage
+
+```bash
+$ bookmeter_exporter
+Commands:
+  bookmeter_exporter export EMAIL    # This command exports all read books of an account as CSV.
+  bookmeter_exporter help [COMMAND]  # Describe available commands or one specific command
+  bookmeter_exporter version         # Display version info
+```
+
+Example:
+
+```bash
+$ bookmeter_exporter export ikuwow@example.com
+Password for EMAIL: # Type the password of your account
+Starting Chrome... # Chrome window opens
+Login success.
+Book count: XXX
+XX books fetched...
+YY books fetched...
+[...]
+Books are successfully exported as './books.csv'.
+```
+
+## CSV Format
+
+|ASIN|Read Date|Review|
+|---|---|---|
+|ex. B071K5WM6P|ex. 2021/07/31|ex. This book is awesome!|
+
+Ex.
+
+```csv
+4408167967,2021/06/02,ここまで読んだ
+B09366V8V2,2021/05/13,""
+B093KCX58C,2021/05/12,""
+```
+
+## Miscellaneous
+
+### [ブクログ](https://booklog.jp)へのインポート
+
+ブクログはCSVファイルから本をインポートすることが出来ます:
+https://booklog.zendesk.com/hc/ja/articles/360048930533-他の読書管理サイトからブクログへデータを移行したいです
+
+bookmeter_exporterで出力したCSVをExcel等でフォーマットを整えるとよいです。
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/ikuwow/bookmeter_exporter.
+
+## Wishlist
+
+* More tests
+* More fields in CSV
+
+## License
+
+MIT

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]

data/_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-cayman

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "bookmeter_exporter"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/bookmeter_exporter.gemspec

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative "lib/bookmeter_exporter/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "bookmeter_exporter"
+  spec.version       = BookmeterExporter::VERSION
+  spec.authors       = ["Ikuo Degawa"]
+  spec.email         = ["degawa@ikuwow.com"]
+
+  spec.summary       = "Export user history of bookmeter.com"
+  spec.homepage      = "https://ikuwow.github.io/bookmeter_exporter/"
+  spec.license       = "MIT"
+  spec.required_ruby_version = ">= 2.6"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/ikuwow/bookmeter_exporter"
+  spec.metadata["changelog_uri"] = "https://github.com/ikuwow/bookmeter_exporter/blob/master/CHANGELOG.md"
+
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "selenium-webdriver", "~> 3.142.2"
+  spec.add_dependency "thor", "~> 1.1.0"
+end

data/exe/bookmeter_exporter

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+require "bookmeter_exporter"
+
+BookmeterExporter::CLI.start(ARGV)

data/lib/bookmeter_exporter.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require_relative "bookmeter_exporter/version"
+require_relative "bookmeter_exporter/cli"
+
+module BookmeterExporter
+  class Error < StandardError; end
+  # Your code goes here...
+end

data/lib/bookmeter_exporter/books.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module BookmeterExporter
+  Book = Struct.new(:asin, :read_date, :review)
+
+  class Books
+    extend Forwardable
+
+    def_delegator :@books, :[]
+    def_delegator :@books, :<<
+
+    def initialize
+      @books = []
+    end
+
+    def to_csv
+      CSV.generate do |csv|
+        @books.each do |book|
+          csv << [book.asin, book.read_date, book.review]
+        end
+      end
+    end
+  end
+end

data/lib/bookmeter_exporter/cli.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "thor"
+require "csv"
+require_relative "crawler"
+require_relative "version"
+
+module BookmeterExporter
+  class CLI < Thor
+    class << self
+      def exit_on_failure?
+        true
+      end
+    end
+
+    desc "export EMAIL", "This command exports all read books of an account as CSV."
+    option :destination, aliases: :d, desc: "path where CSV file is saved to"
+    def export(email)
+      password = ask("Password for #{email}:", echo: false)
+      puts ""
+
+      crawler = BookmeterExporter::Crawler.new(email, password)
+      books = crawler.crawl
+
+      destination = options[:destination] || "./books.csv"
+      IO.write(destination, books.to_csv)
+      puts "Books are successfully exported as '#{destination}'."
+    end
+
+    desc "version", "Display version info"
+    def version
+      puts VERSION
+    end
+  end
+end

data/lib/bookmeter_exporter/crawler.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require "selenium-webdriver"
+require "uri"
+require "cgi"
+
+require_relative "books"
+
+module BookmeterExporter
+  class Crawler
+    def initialize(email, password)
+      @email = email
+      @password = password
+      @root_url = URI("https://bookmeter.com/")
+      @user_id = nil
+    end
+
+    def crawl
+      start_webdriver
+      login
+      book_urls = collect_book_urls
+      fetch_books(book_urls)
+    end
+
+    private
+
+    def start_webdriver
+      puts "Starting Chrome..."
+      @driver = Selenium::WebDriver.for :chrome
+      @wait = Selenium::WebDriver::Wait.new(timeout: 10)
+    end
+
+    def login
+      @driver.get @root_url.merge("/login")
+      @driver.find_element(:id, "session_email_address").send_keys @email
+      @driver.find_element(:id, "session_password").send_keys @password
+      @driver.find_element(:css, "form[action='/login'] button[type=submit]").click
+      @wait.until { @driver.current_url == @root_url.merge("/home").to_s }
+
+      puts "Login success."
+
+      @user_id = @driver.find_element(:css, ".personal-account__data__link").attribute("href")[%r{/([0-9]+)$}, 1]
+    end
+
+    def fetch_books(book_urls)
+      books = Books.new
+      fetched_books_count = 0
+      book_urls.each do |url|
+        books << fetch_book(url)
+        fetched_books_count += 1
+        puts "#{fetched_books_count} books fetched..." if (fetched_books_count % 10).zero?
+      end
+      puts "All books fetched."
+      books
+    end
+
+    def collect_book_urls
+      go_read_books
+
+      last_page_url = @driver.find_element(:css, "ul.bm-pagination").find_element(:link_text, "最後").attribute("href")
+      last_page = CGI.parse(URI.parse(last_page_url).query)["page"].shift.to_i
+
+      urls = []
+      (1..last_page).each do |page|
+        page_url = @root_url.merge("/users/#{@user_id}/books/read?page=#{page}")
+        @driver.get page_url
+
+        book_list_css_selector = ".book-list--grid ul.book-list__group"
+        @wait.until { @driver.current_url == page_url.to_s }
+        @driver.find_elements(:css, book_list_css_selector).each do |ul|
+          ul.find_elements(:css, "li .detail__title").each do |li|
+            urls << li.find_element(:tag_name, "a").attribute("href")
+          end
+        end
+      end
+
+      puts "Book count: #{urls.count}"
+      urls
+    end
+
+    def go_read_books
+      read_books_url = @root_url.merge("/users/#{@user_id}/books/read")
+      @driver.get read_books_url
+      @wait.until { sleep 3 }
+    end
+
+    def fetch_book(url)
+      @driver.get url
+      @wait.until do
+        %r{/books/[0-9]+$}.match(@driver.current_url)
+      end
+
+      book_asin = @driver.find_element(:css, ".sidebar__group .group__image a").attribute("href")
+                         .gsub(%r{https://www.amazon.co.jp/dp/product/(.+)/.*}, '\1')
+      read_date = @driver.find_element(:css, ".read-book__date").text
+      review_text = @driver.find_element(:css, ".read-book__content").text
+
+      Book.new(book_asin, read_date, review_text)
+    end
+  end
+end

data/lib/bookmeter_exporter/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module BookmeterExporter
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,95 @@
+--- !ruby/object:Gem::Specification
+name: bookmeter_exporter
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ikuo Degawa
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2021-07-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: selenium-webdriver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.142.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.142.2
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+description:
+email:
+- degawa@ikuwow.com
+executables:
+- bookmeter_exporter
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/main.yml"
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".ruby-version"
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- _config.yml
+- bin/console
+- bin/setup
+- bookmeter_exporter.gemspec
+- exe/bookmeter_exporter
+- lib/bookmeter_exporter.rb
+- lib/bookmeter_exporter/books.rb
+- lib/bookmeter_exporter/cli.rb
+- lib/bookmeter_exporter/crawler.rb
+- lib/bookmeter_exporter/version.rb
+homepage: https://ikuwow.github.io/bookmeter_exporter/
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://ikuwow.github.io/bookmeter_exporter/
+  source_code_uri: https://github.com/ikuwow/bookmeter_exporter
+  changelog_uri: https://github.com/ikuwow/bookmeter_exporter/blob/master/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.6'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.25
+signing_key:
+specification_version: 4
+summary: Export user history of bookmeter.com
+test_files: []