dmm-crawler 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: '06925a3fd6d470aaaf623d6b2da213c5b8ff631f'
-  data.tar.gz: 40dfc7eb3824eade889b6218d8c2042c4c62ae02
+  metadata.gz: d81d554887372647840ee289931bb8e48fdf7773
+  data.tar.gz: cb08a2dbecf8d0a05d622e263a028d4a30ee8078
 SHA512:
-  metadata.gz: 32e43bd62c3af54c67b60eda38d7455f2209a9d3b2351b7893b94ec9329964c0a38c1ac9975fb3d6b720d2cd5d8be3c0c00a3d927c6021b1c2a9849f64bc3520
-  data.tar.gz: 42b519cd6268171d10634953c9392b5fdd1b220067dd7ca36c2e8ca1b2496c8d3b147026eb965417c52d7bde2a656caf0d39679d5c3a0fe18b79653f82008767
+  metadata.gz: 9cc563c97f1391290b6a9d43b57053bc8b661336f49a42b43e6c140f0538b083349e0e8136925348fd13720383fe958a8a40f54c8eefb87a478216bb4caaa6dc
+  data.tar.gz: '01795b372af4159cda9661d50dcb76f30fa1ab1cacb34337572d89425abe5d2243e20c9d54e252c4fac8e41f6c12f2a78855f6f4e96fb7d91736511f34ee1313'

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Change logs
 
+## 0.1.3
+- Add crawling more information.
+
 ## 0.1.2
 - Extract Attributes klass from Ranking klass.
 - Support art page.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    dmm-crawler (0.1.2)
+    dmm-crawler (0.1.3)
       mechanize
 
 GEM

data/lib/dmm-crawler/attributes.rb

@@ -15,6 +15,7 @@ module DMMCrawler
         description,
         description_raw,
         submedia,
+        informations,
         tags
       ]
     end
@@ -51,7 +52,7 @@ module DMMCrawler
         @element.search('.summary .summary__txt').text
       else
         @element.search('.rank-desc').text
-      end
+      end.strip
     end
 
     def description_raw
@@ -74,6 +75,19 @@ module DMMCrawler
         .delete('-')
     end
 
+    def informations
+      keys = extract_text(@element.search('.m-productInformation .productInformation__item .informationList__ttl'))
+      values = extract_text(@element.search('.m-productInformation .productInformation__item .informationList__txt'))
+
+      keys.zip(values).map { |key, value| { key: key, value: value } }
+    end
+
+    def extract_text(elements)
+      elements
+        .select { |element| element.text.strip != 'ジャンル' }
+        .map { |element| element.children.text.strip }
+    end
+
     def tags
       if art_page?
         @element.search('.genreTagList .genreTagList__item a').map { |e| e.text.strip }

data/lib/dmm-crawler/ranking.rb

@@ -9,10 +9,13 @@ module DMMCrawler
 
     def arts
       arts = page.search('.rank-rankListItem.fn-setPurchaseChange').map do |element|
-        Attributes.new(element, @submedia).to_a
+        sleep 1
+        url = File.join(BASE_URL, element.search('.rank-name a').first.attributes['href'].value)
+        page = @agent.get(url)
+        Attributes.new(page, @submedia).to_a
       end
 
-      arts.map.with_index(1) do |(title, title_link, image_url, description, description_raw, submedia, tags), rank|
+      arts.map.with_index(1) do |(title, title_link, image_url, description, description_raw, submedia, informations, tags), rank|
         {
           title: "#{rank}位: #{title}",
           title_link: title_link,
@@ -20,6 +23,7 @@ module DMMCrawler
           description: description,
           description_raw: description_raw,
           submedia: submedia,
+          informations: informations,
           tags: tags
         }
       end

data/lib/dmm-crawler/version.rb

@@ -1,3 +1,3 @@
 module DMMCrawler
-  VERSION = '0.1.2'.freeze
+  VERSION = '0.1.3'.freeze
 end

data/spec/dmm-crawler/ranking_spec.rb

@@ -20,8 +20,7 @@ describe DMMCrawler::Ranking do
 
       let(:term) { '24' }
 
-      it { is_expected.not_to be_empty }
-      it { is_expected.to all(include(:title, :title_link, :image_url, :description, :description_raw, :submedia, :tags)) }
+      it { is_expected.to all(include(:title, :title_link, :image_url, :description, :description_raw, :submedia, :informations, :tags)) }
     end
 
     context 'with not registered argument' do

data/spec/spec_helper.rb

@@ -1,4 +1,5 @@
 require 'dmm-crawler'
+require 'pry'
 
 RSpec.configure do |config|
   config.order = 'random'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dmm-crawler
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Satoshi Ohmori
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-01 00:00:00.000000000 Z
+date: 2017-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -140,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.12
+rubygems_version: 2.6.10
 signing_key: 
 specification_version: 4
 summary: Show DMM and DMM.R18's crawled data