rakuten_securities_scraper 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 4597241d08e13fa2afc21854b6bef9a2948633244b7fd19355f55e63a891fc7d
+  data.tar.gz: 3ff338627402785d5d992f3de6eb8356306b956e2958cdf3afe5498e43cb4e0b
+SHA512:
+  metadata.gz: bfbc83f71accbf4fd07eb0aefb4665fe937d41780e7a8491f227424431f026d96e6c4c1da78363a5d1418cbe09fa3ab489fc6c6b6e0a402cfeeaaad0dbc7a043
+  data.tar.gz: 061317f5fe6878f06b43b1fe6988ec63cd95457cefa447568479fdf68d794f8e4ebf87ce20d71913806d609766a4b5f4e44b547548720daea3488d2ede437942

data/.gitignore

@@ -0,0 +1,29 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/.bundle
+/.secret_key
+/.secret-key
+/.envrc
+/.vscode
+/log/*
+/tmp/*
+!/log/.keep
+!/tmp/.keep
+/shared/certs/*
+.DS_Store
+/db/.DS_Store
+/config/.DS_Store
+/storage/*
+!/storage/.keep
+/lib/json/*
+.byebug_history
+/.idea/
+/cloud_sql_proxy
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,219 @@
+AllCops:
+  EnabledByDefault: true
+
+##################### Style ##################################
+
+# redirect_to xxx and return のイディオムを維持したい
+Style/AndOr:
+  Enabled: false
+
+# 日本語のコメントを許可する
+Style/AsciiComments:
+  Enabled: false
+
+Style/SymbolArray:
+  Enabled: false
+
+Style/WordArray:
+  Enabled: false
+
+Style/MixinGrouping:
+  Enabled: false
+
+Style/RescueStandardError:
+  Enabled: false
+
+Style/MethodCallWithArgsParentheses:
+  Enabled: false
+
+Style/DocumentationMethod:
+  Enabled: false
+
+Style/CollectionMethods:
+  PreferredMethods:
+    detect: "detect"
+    find: "detect"
+    inject: "inject"
+    reduce: "inject"
+
+# ドキュメントの無い public class を許可する
+Style/Documentation:
+  Enabled: false
+
+# !! のイディオムは積極的に使う
+Style/DoubleNegation:
+  Enabled: false
+
+# MethodDefParentheses:
+#   EnforcedStyle: require_no_parentheses
+  
+# メソッドチェーンの改行は末尾に . を入れる
+# REPL に貼り付けた際の暴発を防ぐため
+Layout/DotPosition:
+  EnforcedStyle: trailing
+
+
+Layout/EmptyLineAfterGuardClause:
+   Enabled: false
+
+# 明示的に else で nil を返すのは分かりやすいので許可する
+Style/EmptyElse:
+  EnforcedStyle: empty
+
+# 桁揃えが綺麗にならないことが多いので migration は除外
+Layout/ExtraSpacing:
+  Exclude:
+    - "db/migrate/*.rb"
+
+# いずれかに揃えるのならば `sprintf` や `format` より String#% が好きです
+Style/FormatString:
+  EnforcedStyle: percent
+
+# if 文の中に 3 行程度のブロックを書くぐらいは許容した方が現実的
+Style/GuardClause:
+  MinBodyLength: 5
+
+# rake タスクの順序の hash は rocket を許可する
+Style/HashSyntax:
+  EnforcedStyle: ruby19
+  Exclude:
+    - "**/*.rake"
+    - "Rakefile"
+
+# 条件式の方を意識させたい場合には後置の if/unless を使わない方が分かりやすい
+Style/IfUnlessModifier:
+  Enabled: false
+
+# private/protected は一段深くインデントする
+Style/PreferredHashMethods:
+  EnforcedStyle: short
+
+# scope 等は複数行でも lambda ではなく ->{} で揃えた方が見た目が綺麗
+Style/Lambda:
+  Enabled: false
+
+# 1_000_000 と区切り文字が 2 個以上必要になる場合のみ _ 区切りを必須にする
+Style/NumericLiterals:
+  MinDigits: 7
+
+# 正規表現にマッチさせた時の特殊変数の置き換えは Regex.last_match ではなく
+# 名前付きキャプチャを使って参照したいので auto-correct しない
+Style/PerlBackrefs:
+  AutoCorrect: false
+
+# 特に model 内において、ローカル変数とメソッド呼び出しの区別をつけた方が分かりやすい場合が多い
+Style/RedundantSelf:
+  Enabled: false
+
+Style/FrozenStringLiteralComment:
+  Enabled: false
+
+# 受け取り側で multiple assignment しろというのを明示
+Style/RedundantReturn:
+  AllowMultipleReturnValues: true
+
+# spec 内は見た目が綺麗になるので許可
+Style/Semicolon:
+  Exclude:
+    - "spec/**/*"
+
+# fail と使い分ける必要ナシ
+Style/SignalException:
+  EnforcedStyle: only_raise
+
+Style/MethodDefParentheses:
+  Enabled: false
+
+# `||` も align に使うことがあるので追加する
+# Style/SpaceAroundOperators:
+#   MultiSpaceAllowedForOperators:
+#     - "="
+#     - "=>"
+#     - "||"
+
+# * 式展開したい場合に書き換えるのが面倒
+# * 文章ではダブルクォートよりもシングルクォートの方が頻出する
+# ことから EnforcedStyle: double_quotes 推奨
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+
+# auto-correct 時に Style/StringLiterals とカニバって無限ループになる (v0.28.0)
+Style/StringLiteralsInInterpolation:
+  Enabled: false
+
+# いくらなんでも inject { |a, e| } は短すぎるので分かりやすい名前をつけたい
+Style/SingleLineBlockParams:
+  Enabled: false
+
+# * migrate
+# * jbuilder
+# * model の association
+# * controller の callback
+# 辺りの桁揃えで引っかかるので全体的にチェックしない
+# Style/SingleSpaceBeforeFirstArg:
+#   Enabled: false
+
+
+##################### Lint ##################################
+
+# * 同名のメソッドがある場合にローカル変数に `_` を付ける
+# * 一時変数として `_` を付ける
+# というテクニックは頻出する
+Lint/UnderscorePrefixedVariableName:
+  Enabled: false
+
+# 子クラスで実装させるつもりのメソッドで引っかかるので
+Lint/UnusedMethodArgument:
+  Enabled: false
+
+##################### Metrics ##################################
+
+Metrics/AbcSize:
+  Enabled: false
+
+# 6 は強すぎるので緩める
+Metrics/CyclomaticComplexity:
+  Max: 10
+
+Metrics/PerceivedComplexity:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Layout/LineLength:
+  Enabled: false
+  
+
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/ClassLength:
+  Enabled: false
+
+Naming/HeredocDelimiterNaming:
+   Enabled: false
+
+Naming/AccessorMethodName:
+  Enabled: false
+
+Metrics/ModuleLength:
+  Enabled: false
+
+Lint/UnusedBlockArgument:
+  Enabled: false
+
+Lint/RescueException:
+  Enabled: false
+
+Naming/RescuedExceptionsVariableName:
+  Enabled: false
+
+Bundler/GemComment:
+  Enabled: false
+
+Style/SafeNavigation:
+  Enabled: false
+
+Lint/NumberConversion:
+  Enabled: false

data/.travis.yml

@@ -0,0 +1,6 @@
+---
+language: ruby
+cache: bundler
+rvm:
+  - 2.7.0
+before_install: gem install bundler -v 2.1.4

data/Gemfile

@@ -0,0 +1,10 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in rakuten_securities.gemspec
+gemspec
+
+gem "rake", "~> 12.0"
+gem "rspec", "~> 3.0"
+gem "rubocop", "0.82.0"
+gem "rubocop-performance", "1.5.2"
+gem "selenium-webdriver", "3.142.7"

data/Gemfile.lock

@@ -0,0 +1,61 @@
+PATH
+  remote: .
+  specs:
+    rakuten_securities_scraper (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.0)
+    childprocess (3.0.0)
+    diff-lcs (1.3)
+    jaro_winkler (1.5.4)
+    parallel (1.19.1)
+    parser (2.7.1.2)
+      ast (~> 2.4.0)
+    rainbow (3.0.0)
+    rake (12.3.3)
+    rexml (3.2.4)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+    rubocop (0.82.0)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.7.0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      rexml
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-performance (1.5.2)
+      rubocop (>= 0.71.0)
+    ruby-progressbar (1.10.1)
+    rubyzip (2.3.0)
+    selenium-webdriver (3.142.7)
+      childprocess (>= 0.5, < 4.0)
+      rubyzip (>= 1.2.2)
+    unicode-display_width (1.7.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rake (~> 12.0)
+  rakuten_securities_scraper!
+  rspec (~> 3.0)
+  rubocop (= 0.82.0)
+  rubocop-performance (= 1.5.2)
+  selenium-webdriver (= 3.142.7)
+
+BUNDLED WITH
+   2.1.4

data/README.md

@@ -0,0 +1,36 @@
+# RakutenSecuritiesScraper
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/rakuten_securities_scraper`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'rakuten_securities_scraper'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install rakuten_securities_scraper
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/rakuten_securities_scraper.
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "rakuten_securities_scraper"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/rakuten_securities_scraper.rb

@@ -0,0 +1,136 @@
+require "rakuten_securities_scraper/version"
+
+module RakutenSecuritiesScraper
+  class Error < StandardError; end
+  class RakutenScraper
+    def initialize login_id, login_pw
+      @login = {
+        login_id: login_id,
+        login_pw: login_pw
+      }
+    end
+
+    def all_history
+      driver = get_selenium_driver(:chrome)
+      wait = Selenium::WebDriver::Wait.new(timeout: 100)
+      url = "https://www.rakuten-sec.co.jp/ITS/V_ACT_Login.html"
+      driver.get url
+      driver.find_element(id: "form-login-id").send_keys @login[:login_id]
+      driver.find_element(id: "form-login-pass").send_keys @login[:login_pw]
+      driver.find_element(id: "login-btn").click
+      wait.until { driver.find_element(id: "header-menu-button-asset") }
+      driver.action.move_to(driver.find_element(id: "header-menu-button-asset")).perform
+      wait.until { driver.find_element(xpath: "/html/body/div[2]/div/div/div[2]/ul[1]/li[1]/ul/li[1]/a") }
+      driver.find_element(xpath: "/html/body/div[2]/div/div/div[2]/ul[1]/li[1]/ul/li[1]/a").click
+
+      sleep 3
+      span_select = Selenium::WebDriver::Support::Select.new(driver.find_element(:id, "termCd"))
+      span_select.select_by(:value, "ALL")
+
+      driver.find_element(css: "input[type='image']").click
+      sleep 2
+      rows = driver.find_elements(xpath: "//tr")
+      page_num = rows[8].text.scan(/(.+)件中/)[0][0].to_i
+      trade_table_data driver, page_num
+    end
+
+    def todays_history
+      driver = get_selenium_driver(:chrome)
+      wait = Selenium::WebDriver::Wait.new(timeout: 100)
+      url = "https://www.rakuten-sec.co.jp/ITS/V_ACT_Login.html"
+      driver.get url
+      driver.find_element(id: "form-login-id").send_keys @login[:login_id]
+      driver.find_element(id: "form-login-pass").send_keys @login[:login_pw]
+      driver.find_element(id: "login-btn").click
+      wait.until { driver.find_element(id: "header-menu-button-asset") }
+      driver.action.move_to(driver.find_element(id: "header-menu-button-asset")).perform
+      wait.until { driver.find_element(xpath: "/html/body/div[2]/div/div/div[2]/ul[1]/li[1]/ul/li[1]/a") }
+      driver.find_element(xpath: "/html/body/div[2]/div/div/div[2]/ul[1]/li[1]/ul/li[1]/a").click
+
+      driver.find_element(css: "input[type='image']").click
+      sleep 2
+      rows = driver.find_elements(xpath: "//tr")
+      begin
+        page_num = rows[8].text.scan(/(.+)件中/)[0][0].to_i
+      rescue
+        return "no data"
+      end
+      trade_table_data driver, page_num
+    end
+
+    def trade_table_data driver, page_num
+      run_times = page_num.divmod(20)
+      trade_records = []
+      (1..run_times[0]-1).each do |f|
+        rows = driver.find_elements(xpath: "//tr")
+        rows[10..29].map do |row|
+          cells = row.find_elements(:css, "td").map { |a| a.text.strip.gsub(",", "") }
+          trade_records << {
+            bit_date: cells[0].split("\n")[0],
+            gain_date: cells[0].split("\n")[1],
+            stock_name: cells[1].split("\n")[0],
+            code: cells[1].split("\n")[1].split(" ")[0],
+            market: cells[1].split("\n")[1].split(" ")[1],
+            account: cells[2],
+            trade_type: cells[3].split("\n")[0],
+            buy_sell: cells[3].split("\n")[1],
+            credit: cells[4].split("\n")[0],
+            due: cells[4].split("\n")[1],
+            stocks: cells[5].split("\n")[0].gsub(" 株", ""),
+            price: cells[5].split("\n")[1],
+            fee: cells[6].split("\n")[0],
+            tax: cells[6].split("\n")[1],
+            charge: cells[7].split("\n")[0],
+            tax_type: cells[7].split("\n")[1],
+            total_price: cells[8]
+          }
+          return trade_records if page_num < 20
+        end
+        driver.find_element(link: "次の20件").click
+      end
+      sleep 2
+      driver.find_element(link: "次の#{run_times[1]}件").click
+      rows = driver.find_elements(xpath: "//tr")
+      rows[10.."1#{run_times[1]}".to_i-1].map do |row|
+        cells = row.find_elements(:css, "td").map { |a| a.text.strip.gsub(",", "") }
+        trade_records << {
+          bit_date: cells[0].split("\n")[0],
+          gain_date: cells[0].split("\n")[1],
+          stock_name: cells[1].split("\n")[0],
+          code: cells[1].split("\n")[1].split(" ")[0],
+          market: cells[1].split("\n")[1].split(" ")[1],
+          account: cells[2],
+          trade_type: cells[3].split("\n")[0],
+          buy_sell: cells[3].split("\n")[1],
+          credit: cells[4].split("\n")[0],
+          due: cells[4].split("\n")[1],
+          stocks: cells[5].split("\n")[0].gsub(" 株", ""),
+          price: cells[5].split("\n")[1],
+          fee: cells[6].split("\n")[0],
+          tax: cells[6].split("\n")[1],
+          charge: cells[7].split("\n")[0],
+          tax_type: cells[7].split("\n")[1],
+          total_price: cells[8]
+        }
+      end
+      trade_records
+    end
+
+    def get_selenium_driver(mode = :chrome)
+      case mode
+      when :firefox_remote_capabilities
+        firefox_capabilities = Selenium::WebDriver::Remote::Capabilities.firefox
+        Selenium::WebDriver.for(:remote, url: "http://hub:4444/wd/hub", desired_capabilities: firefox_capabilities)
+      when :firefox
+        Selenium::WebDriver.for :firefox
+      else
+        options = Selenium::WebDriver::Chrome::Options.new
+        # options.add_argument("--ignore-certificate-errors")
+        options.add_argument("--disable-popup-blocking")
+        options.add_argument("--disable-translate")
+        options.add_argument("-headless")
+        Selenium::WebDriver.for :chrome, options: options
+      end
+    end
+  end
+end

data/lib/rakuten_securities_scraper/version.rb

@@ -0,0 +1,3 @@
+module RakutenSecuritiesScraper
+  VERSION = "0.1.0"
+end

data/rakuten_securities_scraper.gemspec

@@ -0,0 +1,26 @@
+require_relative "lib/rakuten_securities_scraper/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "rakuten_securities_scraper"
+  spec.version       = RakutenSecuritiesScraper::VERSION
+  spec.authors       = ["FUMI-POPPIN"]
+  spec.email         = ["fumitake.kawasaki@el-soul.com"]
+
+  spec.summary       = "楽天証券スクレイパー / Rakuten Securities Scraper"
+  spec.description   = "楽天証券スクレイパー / Rakuten Securities Scraper"
+  spec.homepage      = "https://github.com/elsoul/rakuten_securities"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.7.0")
+  spec.license = "MIT"
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/elsoul/rakuten_securities_scraper"
+  spec.metadata["changelog_uri"] = "https://github.com/elsoul/rakuten_securities_scraper"
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files =
+    Dir.chdir(File.expand_path(__dir__)) do
+      `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+    end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+end

metadata

@@ -0,0 +1,59 @@
+--- !ruby/object:Gem::Specification
+name: rakuten_securities_scraper
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- FUMI-POPPIN
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2020-05-10 00:00:00.000000000 Z
+dependencies: []
+description: 楽天証券スクレイパー / Rakuten Securities Scraper
+email:
+- fumitake.kawasaki@el-soul.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/rakuten_securities_scraper.rb
+- lib/rakuten_securities_scraper/version.rb
+- rakuten_securities_scraper.gemspec
+homepage: https://github.com/elsoul/rakuten_securities
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/elsoul/rakuten_securities
+  source_code_uri: https://github.com/elsoul/rakuten_securities_scraper
+  changelog_uri: https://github.com/elsoul/rakuten_securities_scraper
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.3
+signing_key: 
+specification_version: 4
+summary: 楽天証券スクレイパー / Rakuten Securities Scraper
+test_files: []