bookmeter_scraper 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6c328c66bbd91ea36ee0471c01f4e16a69ffd347
-  data.tar.gz: d1d5fde5a9d223c0aada00c670d1feed0b5e9c3a
+  metadata.gz: a8d0b08d6fe209f678ebda3c621b41412a575aef
+  data.tar.gz: 1d9db389c1c0ac92216c00e76f7a025327ed77c7
 SHA512:
-  metadata.gz: 9a2c3a6149faa92850aca03c455bd5ccd95f8eddaee005e4befd4daa328c340e171df5018803f9feb0ebb0f7fdf630f91d9b9259c52fb358f5b3ccf5570595e7
-  data.tar.gz: dcbf2db1efa63928b1c00a00d35af13ffba46345578c84b40b26a9b320ba69acd899c66c9319af5e15894a3d4e5ccc5fadbf64136fbbcbea804a0097f0729251
+  metadata.gz: 9df797a4925a93e8cb8a7f07e38bbd4b1288acadd27a2d0e32fd3e0161120a843dd3b2c5c184c7116e90bb6d8b52a2cacfc246a04f7ede74b6081359882f2d51
+  data.tar.gz: ba4d5f59e35034facda467d469a0f2ef3cfef10829f2e75a82af6a839233fe2ff0e588a9c1535db25bc48eb24a2d08fd81d614b67f1c92d66b2656b0603fc303

data/README.ja.md

@@ -28,7 +28,16 @@ require 'bookmeter_scraper'
 
 ### ログイン
 
-書籍情報、お気に入り / お気に入られユーザ情報を取得するには、`Bookmeter.log_in` でログインしておく必要があります。
+書籍情報、お気に入り / お気に入られユーザ情報を取得するには、`Bookmeter.log_in` または `Bookmeter#log_in` でログインしておく必要があります。
+
+ログイン情報の入力には以下の 2 通りの方法があります。
+
+1. 引数として渡す
+2. `config.yml` へ記述しておく
+
+#### 1. 引数として渡す
+
+以下のように `Bookmeter.log_in` へメールアドレスとパスワードを引数として渡すことで、ログインできます。
 
 ```ruby
 bookmeter = BookmeterScraper::Bookmeter.log_in('example@example.com', 'password')
@@ -42,6 +51,22 @@ bookmeter = BookmeterScraper::Bookmeter.new
 bookmeter.log_in('example@example.com', 'password')
 ```
 
+#### 2. `config.yml` へ記述しておく
+
+まず、以下のように YAML ファイル `config.yml` を記述し、Ruby スクリプトを実行する場所と同じディレクトリに置きます。
+
+```yml
+mail: example@example.com
+password: your_password
+```
+
+次に、引数なしで `Bookmeter.log_in` または `Bookmeter#log_in` を呼ぶと、`config.yml` からログイン情報を読みとり、ログインできます。
+
+```ruby
+bookmeter = BookmeterScraper::Bookmeter.log_in
+bookmeter.logged_in?    # true
+```
+
 ### 書籍情報の取得
 
 以下の書籍情報
@@ -62,10 +87,17 @@ books = bookmeter.read_books        # ログインユーザの「読んだ本」
 bookmeter.read_books('01010101')    # 他のユーザの ID を指定して、そのユーザの「読んだ本」を取得
 ```
 
-書籍情報は書名 `name` と読了日（初読了日と再読日の両方）の配列 `read_dates` を属性として持つ `Struct` の配列として取得できます。
+書籍情報は
+
+- 書名 `name`
+- 著者 `author`
+- 読了日（初読了日と再読日の両方）の配列 `read_dates`
+
+を属性として持つ `Struct` の配列として取得できます。
 
 ```ruby
 books[0].name
+books[0].author
 books[0].read_dates
 ```
 
@@ -95,6 +127,7 @@ books = bookmeter.read_books_in(2016, 1, '01010101')    # ID で指定した他
 ```ruby
 books = bookmeter.reading_books    # ログインユーザの「読んでる本」を取得
 books[0].name
+books[0].author
 books[0].read_dates    # 読了日の Array は空
 
 bookmeter.tsundoku     # ログインユーザの「積読本」を取得

data/README.md

@@ -32,10 +32,19 @@ require 'bookmeter_scraper'
 
 ### Log in
 
-You need to log in Bookmeter to get books and followings / followers information by `Bookmeter.log_in`:
+You need to log in Bookmeter to get books and followings / followers information by `Bookmeter.log_in` or `Bookmeter#log_in`.
+
+There are 2 ways to input authentication information:
+
+1. Passing as arguments
+2. Writing out to `config.yml`
+
+#### 1. Passing as arguments
+
+You can log in Bookmeter by passing mail address and password to `Bookmeter.log_in`:
 
 ```ruby
-bookmeter = BookmeterScraper::Bookmeter.log_in('example@example.com', 'password')
+bookmeter = BookmeterScraper::Bookmeter.log_in('example@example.com', 'your_password')
 bookmeter.logged_in?    # true
 ```
 
@@ -46,6 +55,23 @@ bookmeter = BookmeterScraper::Bookmeter.new
 bookmeter.log_in('example@example.com', 'password')
 ```
 
+#### 2. Writing out to `config.yml`
+
+Create `config.yml` as followings and save it to the same directory as your Ruby script:
+
+```yml
+mail: example@example.com
+password: your_password
+```
+
+Now you can log in Bookmeter by calling `Bookmeter.log_in` or `Bookmeter#log_in` with no arguments:
+
+```ruby
+bookmeter = BookmeterScraper::Bookmeter.log_in
+bookmeter.logged_in?    # true
+```
+
+
 ### Get books information
 
 You can get books information:

data/bookmeter_scraper.gemspec

@@ -18,11 +18,13 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.10"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.4"
-  spec.add_development_dependency "webmock", "~> 1.22"
+  spec.required_ruby_version = '>= 2.0'
 
-  spec.add_dependency "yasuri", "~> 0.0"
-  spec.add_dependency "mechanize", "~> 2.7"
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "webmock"
+
+  spec.add_dependency "mechanize"
+  spec.add_dependency "yasuri"
 end

data/lib/bookmeter_scraper.rb

@@ -1,2 +1,3 @@
 require 'bookmeter_scraper/bookmeter'
+require 'bookmeter_scraper/configuration'
 require 'bookmeter_scraper/version'

data/lib/bookmeter_scraper/bookmeter.rb

@@ -1,16 +1,39 @@
+require 'forwardable'
 require 'mechanize'
 require 'yasuri'
 
 module BookmeterScraper
   class Bookmeter
+    DEFAULT_CONFIG_PATH = './config.yml'.freeze
+
     ROOT_URI  = 'http://bookmeter.com'.freeze
     LOGIN_URI = "#{ROOT_URI}/login".freeze
 
     PROFILE_ATTRIBUTES = %i(name gender age blood_type job address url description first_day elapsed_days read_books_count read_pages_count reviews_count bookshelfs_count)
     Profile = Struct.new(*PROFILE_ATTRIBUTES)
 
-    BOOK_ATTRIBUTES = %i(name read_dates)
+    BOOK_ATTRIBUTES = %i(name author read_dates)
     Book = Struct.new(*BOOK_ATTRIBUTES)
+    class Books
+      extend Forwardable
+
+      def_delegator :@books, :[]
+      def_delegator :@books, :[]=
+      def_delegator :@books, :<<
+      def_delegator :@books, :each
+      def_delegator :@books, :flatten!
+
+      def initialize; @books = []; end
+
+      def concat(books)
+        books.each do |book|
+          next if @books.any? { |b| b.name == book.name && b.author == book.author }
+          @books << book
+        end
+      end
+
+      def to_a; @books; end
+    end
 
     USER_ATTRIBUTES = %i(name id)
     User = Struct.new(*USER_ATTRIBUTES)
@@ -71,7 +94,7 @@ module BookmeterScraper
       "#{ROOT_URI}/u/#{user_id}/favorited_user"
     end
 
-    def self.log_in(mail, password)
+    def self.log_in(mail = nil, password = nil)
       Bookmeter.new.tap do |bookmeter|
         bookmeter.log_in(mail, password)
       end
@@ -81,22 +104,26 @@ module BookmeterScraper
     def initialize(agent = nil)
       @agent = agent.nil? ? Bookmeter.new_agent : agent
       @logged_in = false
+      @log_in_user_id = nil
+      @book_pages = {}
     end
 
-    def log_in(mail, password)
+    def log_in(mail = nil, password = nil)
       raise BookmeterError if @agent.nil?
 
-      next_page = nil
+      config = Configuration.new(DEFAULT_CONFIG_PATH) if mail.nil? && password.nil?
+
+      page_after_submitting_form = nil
       page = @agent.get(LOGIN_URI) do |page|
-        next_page = page.form_with(action: '/login') do |form|
-          form.field_with(name: 'mail').value = mail
-          form.field_with(name: 'password').value = password
+        page_after_submitting_form = page.form_with(action: '/login') do |form|
+          form.field_with(name: 'mail').value     = config ? config.mail     : mail
+          form.field_with(name: 'password').value = config ? config.password : password.to_s
         end.submit
       end
-      @logged_in = next_page.uri.to_s == ROOT_URI + '/'
+      @logged_in = page_after_submitting_form.uri.to_s == ROOT_URI + '/'
       return unless logged_in?
 
-      mypage = next_page.link_with(text: 'マイページ').click
+      mypage = page_after_submitting_form.link_with(text: 'マイページ').click
       @log_in_user_id = extract_user_id(mypage)
     end
 
@@ -124,32 +151,32 @@ module BookmeterScraper
     def read_books(user_id = @log_in_user_id)
       books = get_books(user_id, :read_books_uri)
       books.each { |b| yield b } if block_given?
-      books
+      books.to_a
     end
 
     def read_books_in(year, month, user_id = @log_in_user_id)
       date = Time.local(year, month)
       books = get_read_books(user_id, date)
       books.each { |b| yield b } if block_given?
-      books
+      books.to_a
     end
 
     def reading_books(user_id = @log_in_user_id)
       books = get_books(user_id, :reading_books_uri)
       books.each { |b| yield b } if block_given?
-      books
+      books.to_a
     end
 
     def tsundoku(user_id = @log_in_user_id)
       books = get_books(user_id, :tsundoku_uri)
       books.each { |b| yield b } if block_given?
-      books
+      books.to_a
     end
 
     def wish_list(user_id = @log_in_user_id)
       books = get_books(user_id, :wish_list_uri)
       books.each { |b| yield b } if block_given?
-      books
+      books.to_a
     end
 
     def followings(user_id = @log_in_user_id)
@@ -164,7 +191,9 @@ module BookmeterScraper
 
     def self.new_agent
       agent = Mechanize.new do |a|
-        a.user_agent_alias = 'Mac Safari'
+        a.user_agent_alias = Mechanize::AGENT_ALIASES.keys.reject do |ua_alias|
+          %w(Android iPad iPhone Mechanize).include?(ua_alias)
+        end.sample
       end
     end
 
@@ -173,7 +202,7 @@ module BookmeterScraper
     end
 
     def get_books(user_id, uri_method)
-      books = []
+      books = Books.new
       scraped_pages = scrape_book_pages(user_id, uri_method)
       scraped_pages.each do |page|
         books << get_book_structs(page)
@@ -183,7 +212,7 @@ module BookmeterScraper
     end
 
     def get_read_books(user_id, target_ym)
-      result = []
+      result = Books.new
       scrape_book_pages(user_id, :read_books_uri).each do |page|
         first_book_date = get_read_date(page['book_1_link'])
         last_book_date  = get_last_book_date(page)
@@ -217,7 +246,7 @@ module BookmeterScraper
     end
 
     def get_target_books(target_ym, page)
-      target_books = []
+      target_books = Books.new
 
       1.upto(NUM_BOOKS_PER_PAGE) do |i|
         next if page["book_#{i}_link"].empty?
@@ -245,7 +274,8 @@ module BookmeterScraper
           end
         end
         book_name = get_book_name(page["book_#{i}_link"])
-        book = Book.new(book_name, read_dates)
+        book_author = get_book_author(page["book_#{i}_link"])
+        book = Book.new(book_name, book_author, read_dates)
         target_books << book
       end
 
@@ -282,28 +312,35 @@ module BookmeterScraper
       books_root.inject(@agent, books_page)
     end
 
-    def get_book_name(book_link)
-      @agent.get(ROOT_URI + book_link).search('#title').text
+    def get_book_page(book_uri)
+      @book_pages[book_uri] = @agent.get(ROOT_URI + book_uri) unless @book_pages[book_uri]
+      @book_pages[book_uri]
+    end
+
+    def get_book_name(book_uri)
+      get_book_page(book_uri).search('#title').text
+    end
+
+    def get_book_author(book_uri)
+      get_book_page(book_uri).search('#author_name').text
     end
 
-    def get_read_date(book_link)
-      book_page = @agent.get(ROOT_URI + book_link)
+    def get_read_date(book_uri)
       book_date = Yasuri.struct_date '//*[@id="book_edit_area"]/form[1]/div[2]' do
         text_year  '//*[@id="read_date_y"]/option[1]', truncate: /\d+/, proc: :to_i
         text_month '//*[@id="read_date_m"]/option[1]', truncate: /\d+/, proc: :to_i
         text_day   '//*[@id="read_date_d"]/option[1]', truncate: /\d+/, proc: :to_i
       end
-      book_date.inject(@agent, book_page)
+      book_date.inject(@agent, get_book_page(book_uri))
     end
 
-    def get_reread_date(book_link)
-      book_page = @agent.get(ROOT_URI + book_link)
+    def get_reread_date(book_uri)
       book_reread_date = Yasuri.struct_reread_date '//*[@id="book_edit_area"]/div/form[1]/div[2]' do
         text_reread_year  '//div[@class="reread_box"]/form[1]/div[2]/select[1]/option[1]', truncate: /\d+/, proc: :to_i
         text_reread_month '//div[@class="reread_box"]/form[1]/div[2]/select[2]/option[1]', truncate: /\d+/, proc: :to_i
         text_reread_day   '//div[@class="reread_box"]/form[1]/div[2]/select[3]/option[1]', truncate: /\d+/, proc: :to_i
       end
-      book_reread_date.inject(@agent, book_page)
+      book_reread_date.inject(@agent, get_book_page(book_uri))
     end
 
     def get_book_structs(page)
@@ -329,7 +366,8 @@ module BookmeterScraper
         end
 
         book_name = get_book_name(page["book_#{i}_link"])
-        book = Book.new(book_name, read_dates)
+        book_author = get_book_author(page["book_#{i}_link"])
+        book = Book.new(book_name, book_author, read_dates)
         books << book
       end
 

data/lib/bookmeter_scraper/configuration.rb

@@ -0,0 +1,19 @@
+require 'yaml'
+
+module BookmeterScraper
+  class Configuration
+    attr_reader :mail, :password
+
+    def initialize(config_file)
+      config = YAML.load_file(config_file)
+      unless config.has_key?('mail') && config.has_key?('password')
+        raise ConfigurationError, "#{config_file}: Invalid configuration file"
+      end
+
+      @mail     = config['mail']
+      @password = config['password']
+    end
+  end
+
+  class ConfigurationError < StandardError; end
+end

data/lib/bookmeter_scraper/version.rb

@@ -1,3 +1,3 @@
 module BookmeterScraper
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

metadata

@@ -1,99 +1,99 @@
 --- !ruby/object:Gem::Specification
 name: bookmeter_scraper
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Kohei Yamamoto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-02-26 00:00:00.000000000 Z
+date: 2016-03-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.4'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.4'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.22'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.22'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: yasuri
+  name: mechanize
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.0'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: mechanize
+  name: yasuri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '0'
 description: Bookmeter scraping library
 email:
 - kymmt90@gmail.com
@@ -116,6 +116,7 @@ files:
 - exe/bookmeter_scraper
 - lib/bookmeter_scraper.rb
 - lib/bookmeter_scraper/bookmeter.rb
+- lib/bookmeter_scraper/configuration.rb
 - lib/bookmeter_scraper/version.rb
 homepage: https://github.com/kymmt90/bookmeter_scraper
 licenses:
@@ -129,7 +130,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -137,7 +138,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Bookmeter scraping library