sakamichi_scraper 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 47b8be892d241fb3f745274fd2dad85bf15ca93216ab19bab45ba77a6f962bef
-  data.tar.gz: d18194978ae187c6beee80ce3cd7b18452f096a2d7bad7db767316fdbbc27baf
+  metadata.gz: 11a74e79888199fbe85816294b9ef042e1373cffbee3a43ad8d05bcf873e42e6
+  data.tar.gz: c33c5dcc257be160498d32a58c47aca1051f85d7c1ace6ea98f010658bada8dd
 SHA512:
-  metadata.gz: 671f27b24edb3e8a1ce6135201c3d25d3bde9e38b83cfed26223b2626f0491f0f1e4b9a6b59e62cdfd7886eb4aa79c16ee01cd1f927dde4c20f27823e3b4784f
-  data.tar.gz: 248730c77f4c4f104829cee1878a7b9a5cd6acd13058c1af25afa408ee2e476363730bd11329a20c2d92e4f05bd5569b3a93e004c5656fe167fc452aa3c511d2
+  metadata.gz: abd5b9724b17161fc77d9276896872cf6141e2655bf57485a352acf0c4b7fa9a0c999010c478a76b2f5ba01a2b49db0369472b40271183c1fb7b5def7a89a78a
+  data.tar.gz: 200345d5e406e879ee66df8689bd852cc134d97a107a4e08dc56bbb7a5f1916ecf6493f91bd28a15192d161311750a3807dcdc77a02dc5bbbca61a5d641bbf34

data/.github/workflows/ruby.yml

@@ -17,17 +17,20 @@ jobs:
   test:
 
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: ['2.6', '2.7']
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Ruby
-    # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
-    # change this to (see https://github.com/ruby/setup-ruby#versioning):
-    # uses: ruby/setup-ruby@v1
-      uses: ruby/setup-ruby@ec106b438a1ff6ff109590de34ddc62c540232e0
-      with:
-        ruby-version: 2.7
-    - name: Install dependencies
-      run: bundle install
-    - name: Run tests
-      run: bundle exec rspec spec/*
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
+        # change this to (see https://github.com/ruby/setup-ruby#versioning):
+        # uses: ruby/setup-ruby@v1
+        uses: ruby/setup-ruby@473e4d8fe5dd94ee328fdfca9f8c9c7afc9dae5e
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+          bundler: 2.1.4
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+      - name: Run tests
+        run: bundle exec rspec spec/*

data/.gitignore

@@ -7,6 +7,7 @@
 /spec/reports/
 /tmp/
 /vendor/
+/img/
 
 # ignore files needed only for development
 .ruby-version
@@ -17,3 +18,5 @@ Gemfile.lock
 
 # ignore genereted file from IDE
 .idea/
+sakamichi_scraper.iml
+.rubocop_todo.yml

data/.rubocop.yml

@@ -0,0 +1,46 @@
+inherit_from: .rubocop_todo.yml
+
+# Hashの key/value の記述方法についてkey: value 形式を有効とする
+Style/HashSyntax:
+  EnforcedStyle: ruby19
+
+# 変数展開やエスケープの必要がない場合はダブルクォートを利用する
+StringLiterals:
+  EnforcedStyle: double_quotes
+
+# パーセント記法の記号について とりあえずすべて()
+PercentLiteralDelimiters:
+  PreferredDelimiters:
+    '%':  ()
+    '%i': ()
+    '%q': ()
+    '%Q': ()
+    '%r': ()
+    '%s': ()
+    '%w': ()
+    '%W': ()
+    '%x': ()
+
+# 式展開中でもダブルクォートを使う
+Style/StringLiteralsInInterpolation:
+  EnforcedStyle: double_quotes
+
+# 日本語コメントを許容する
+Style/AsciiComments:
+  Enabled: false
+
+# 鬱陶しいのでfrozen_string_literalは書かなくてもいいことにする
+Style/FrozenStringLiteralComment:
+  Enabled: false
+
+# required_ruby_versionはまだ指定しない
+Gemspec/RequiredRubyVersion:
+  Enabled: false
+
+# initialize classにsuperを強制しない
+Lint/MissingSuper:
+  Enabled: false
+
+#メソッド行上限を20行にする
+Metrics/MethodLength:
+  Max: 20

data/Gemfile

@@ -1,6 +1,6 @@
 source "https://rubygems.org"
 
-git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 
 # Specify your gem's dependencies in sakamichi_scraper.gemspec
 gemspec

data/README.md

@@ -1,8 +1,6 @@
 # SakamichiScraper
-
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/sakamichi_scraper`. To experiment with that code, run `bin/console` for an interactive prompt.
-
-TODO: Delete this and the text above, and describe your gem
+## about me
+This gem is used to scrape information from Sakamichi group's(Hinatazaka46, Sakurazaka46) blog.
 
 ## Installation
 
@@ -25,25 +23,43 @@ Or install it yourself as:
 ### get title of blog
 
 #### Hinatazaka46
+
 ```ruby
 hinata = SakamichiScraper::Hinatazaka.new
 
 # get newest title of article
-hinata.get_newest_blog_title
+hinata.newest_blog_title
 
 # get recent title, member name, timestamp of article
-hinata.get_recent_blog_info
+hinata.recent_blog_info
 ```
 
 #### Sakurazaka46
+
 ```ruby
 sakura = SakamichiScraper::Sakurazaka.new
 
 # get newest title of article
-sakura.get_newest_blog_title
+sakura.newest_blog_title
 
 # get recent title, member name, timestamp of article
-sakura.get_recent_blog_info
+sakura.recent_blog_info
+```
+
+### get images
+NOTE: Any images obtained by executing the following method will be stored in folder `img/<today(yyyymmdd)>`.
+#### Hinatazaka46
+
+```ruby
+# get images from newest article
+hinata.picture_in_newest_article
+```
+
+#### Sakurazaka46
+
+```ruby
+# get images from newest article
+sakura.picture_in_newest_article
 ```
 
 ## Development

data/config/url.yml

@@ -4,4 +4,7 @@ hinatazaka:
 sakurazaka:
   blog_top_page: "https://sakurazaka46.com/s/s46/diary/blog"
   blog_list_page: "https://sakurazaka46.com/s/s46/diary/blog/list"
+  exclude_img_path:
+    - /files/14/s46/img/com-logo_pc.svg
+    - /files/14/s46/img/jasrac.jpg
 keyakizaka:

data/lib/sakamichi_scraper/base.rb

@@ -1,14 +1,27 @@
+require "fileutils"
+
 module SakamichiScraper
   class Base
-    def init_url(group_name, yml_key)
+    def initialize(group_name)
+      @group_name = group_name
+      @home_page = "https://#{@group_name}46.com"
+    end
+
+    def init_url_from_yml(group_name, yml_key)
       url = YAML.load_file("config/url.yml")["#{group_name}"]["#{yml_key}"]
       get_content(url)
     end
 
     def get_content(url)
-      URI.open(url, "User-Agent" => "Chrome/86.0.4240.80") do |f|
-        f.read
-      end
+      URI.open(url, "User-Agent" => "Ruby/2.7.1", &:read)
+    end
+
+    def blog_top_page
+      init_url_from_yml(@group_name, "blog_top_page")
+    end
+
+    def blog_list_page
+      init_url_from_yml(@group_name, "blog_list_page")
     end
 
     def format_content(content)
@@ -16,7 +29,48 @@ module SakamichiScraper
     end
 
     def format_timestamp(datetime)
-      DateTime.parse(datetime).strftime('%Y-%-m-%-d %-H:%-M')
+      DateTime.parse(datetime).strftime("%Y-%-m-%-d %-H:%-M")
+    end
+
+    def exec_date
+      @exec_date ||= Time.now.strftime("%Y%m%d")
+    end
+
+    def mkdir_today_file_path
+      FileUtils.mkdir_p(image_file_path)
+    end
+
+    def exclude_img_path(group_name)
+      YAML.load_file("config/url.yml")["#{group_name}"]["exclude_img_path"]
+    end
+
+    def image_file_path
+      "img/#{@group_name}/#{exec_date}"
+    end
+
+    def image_urls_from_article_url(article_html, class_name)
+      [].tap do |url|
+        Nokogiri.parse(article_html, nil, nil).css("#{class_name} img").each do |c|
+          image_url = c.attribute("src").value
+          url << case @group_name
+                 when "sakurazaka"
+                   "#{@home_page}#{image_url}"
+                 else
+                   image_url
+                 end
+        end
+      end
+    end
+
+    def download_images_from_url_list(image_urls)
+      image_urls.each do |image_url|
+        dest_image_path = "#{image_file_path}/#{image_url[%r([^/]+$)]}"
+        File.open(dest_image_path, "w") do |pass|
+          URI.parse(image_url).open do |img|
+            pass.write(img.read)
+          end
+        end
+      end
     end
   end
 end

data/lib/sakamichi_scraper/hinatazaka.rb

@@ -1,24 +1,25 @@
-require "sakamichi_scraper/base"
+require_relative "base"
 
 module SakamichiScraper
   class Hinatazaka < Base
-    def get_blog_top_page_title
-      html = get_blog_top_page
-      Nokogiri::HTML.parse(html, nil, nil).title
+    def initialize
+      super("hinatazaka")
     end
 
-    def get_newest_blog_title
-      html = get_blog_top_page
-      scraped_title = Nokogiri.parse(html, nil, nil)
+    def blog_top_page_title
+      Nokogiri::HTML.parse(blog_top_page, nil, nil).title
+    end
+
+    def newest_blog_title
+      scraped_title = Nokogiri.parse(blog_top_page, nil, nil)
                         .at_css(".p-blog-main__head > .c-blog-main__title")
                         .content
       format_content(scraped_title)
     end
 
-    def get_recent_blog_info
+    def recent_blog_info
       res = []
-      html = get_blog_top_page
-      Nokogiri.parse(html, nil, nil).css(".p-blog-top__list > li").each do |c|
+      Nokogiri.parse(blog_top_page, nil, nil).css(".p-blog-top__list > li").each do |c|
         info_arr = c.content.strip.split("\n").reject { |i| i.blank? }
         info = {
           member: info_arr[0],
@@ -29,10 +30,27 @@ module SakamichiScraper
       end
     end
 
+    def picture_in_newest_article
+      newest_article_url = article_urls_from_list_page(blog_top_page).first
+      article_html = get_content(newest_article_url)
+      image_urls = image_urls_from_article_url(article_html, "div.c-blog-article__text")
+
+      mkdir_today_file_path unless Dir.exist?(image_file_path)
+      download_images_from_url_list(image_urls)
+    end
+
     private
 
-    def get_blog_top_page
-      init_url("hinatazaka", "blog_top_page")
+    def article_urls_from_list_page(html)
+      [].tap do |array|
+        Nokogiri.parse(html, nil, nil).css(".p-blog-top__list > li").each do |c|
+          array << "#{@home_page}#{c.css("a")[0][:href]}".match(/(.*)\?.*$/)[1]
+        end
+      end
+    end
+
+    def blog_top_page
+      init_url_from_yml(@group_name, "blog_top_page")
     end
   end
 end

data/lib/sakamichi_scraper/sakurazaka.rb

@@ -1,21 +1,22 @@
-require "sakamichi_scraper/base"
+require_relative "base"
 
 module SakamichiScraper
   class Sakurazaka < Base
-    def get_blog_top_page_title
-      html = get_blog_top_page
-      Nokogiri::HTML.parse(html, nil, nil).title
+    def initialize
+      super("sakurazaka")
     end
 
-    def get_newest_blog_title
-      html = get_blog_list_page
-      Nokogiri.parse(html, nil, nil).css(".inner.title-wrap > .title").first.children.to_s
+    def blog_top_page_title
+      Nokogiri::HTML.parse(blog_top_page, nil, nil).title
     end
 
-    def get_recent_blog_info
+    def newest_blog_title
+      Nokogiri.parse(blog_list_page, nil, nil).at_css(".date-title > .title").children.to_s
+    end
+
+    def recent_blog_info
       res = []
-      html = get_blog_top_page
-      Nokogiri.parse(html, nil, nil).css(".com-blog-part.box4.fxpc > li").each do |c|
+      Nokogiri.parse(blog_list_page, nil, nil).css(".com-blog-part.box4.fxpc > li").each do |c|
         info = {
           member: c.css(".prof-in.fx > .name").children.to_s,
           title:  c.css(".date-title > .title").children.to_s,
@@ -25,14 +26,23 @@ module SakamichiScraper
       end
     end
 
-    private
+    def picture_in_newest_article
+      newest_article_url = article_urls_from_list_page(blog_list_page).first
+      article_html = get_content(newest_article_url)
+      image_urls = image_urls_from_article_url(article_html, "div.box-article")
 
-    def get_blog_top_page
-      init_url("sakurazaka", "blog_top_page")
+      mkdir_today_file_path unless Dir.exist?(image_file_path)
+      download_images_from_url_list(image_urls)
     end
 
-    def get_blog_list_page
-      init_url("sakurazaka", "blog_list_page")
+    private
+
+    def article_urls_from_list_page(html)
+      [].tap do |array|
+        Nokogiri.parse(html, nil, nil).css(".com-blog-part.box4.fxpc > li").each do |c|
+          array << "#{@home_page}#{c.css("a")[0][:href]}"
+        end
+      end
     end
   end
 end

data/lib/sakamichi_scraper/version.rb

@@ -1,3 +1,3 @@
 module SakamichiScraper
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/sakamichi_scraper.gemspec

@@ -1,5 +1,4 @@
-
-lib = File.expand_path("../lib", __FILE__)
+lib = File.expand_path("lib", __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "sakamichi_scraper/version"
 
@@ -9,25 +8,25 @@ Gem::Specification.new do |spec|
   spec.authors       = ["oyrarumakan"]
   spec.email         = ["ryo19911030@hotmail.co.jp"]
 
-  spec.summary       = %q{ Write a short summary, because RubyGems requires one.}
-  spec.description   = %q{ Write a longer description or delete this line.}
+  spec.summary       = "scrape gem from Sakamichi group's(Hinatazaka46, Sakurazaka46) blog"
+  spec.description   = "This gem is used to scrape info from Sakamichi group's(Hinatazaka46, Sakurazaka46) blog."
   spec.homepage      = "https://github.com/oyrarumakan/sakamichi_scraper"
   spec.license       = "MIT"
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+  spec.files         = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
   spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.executables   = spec.files.grep(%r(^exe/)) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
   spec.add_dependency "activesupport", "6.0.3.4"
   spec.add_dependency "nokogiri", "1.10.10"
   spec.add_development_dependency "bundler", "2.1.4"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "pry", "0.13.1"
-
+  spec.add_development_dependency "rake", "~> 12.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rubocop", "1.11.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sakamichi_scraper
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - oyrarumakan
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-11-10 00:00:00.000000000 Z
+date: 2021-04-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -52,20 +52,34 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 2.1.4
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.13.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.13.1
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -81,20 +95,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.0'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.13.1
+        version: 1.11.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.13.1
-description: " Write a longer description or delete this line."
+        version: 1.11.0
+description: This gem is used to scrape info from Sakamichi group's(Hinatazaka46,
+  Sakurazaka46) blog.
 email:
 - ryo19911030@hotmail.co.jp
 executables: []
@@ -104,6 +119,7 @@ files:
 - ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rspec"
+- ".rubocop.yml"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -141,5 +157,5 @@ requirements: []
 rubygems_version: 3.1.2
 signing_key:
 specification_version: 4
-summary: Write a short summary, because RubyGems requires one.
+summary: scrape gem from Sakamichi group's(Hinatazaka46, Sakurazaka46) blog
 test_files: []