dmm-crawler 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bde3c371a800c5ea7438e38d62c21792771899c3
-  data.tar.gz: f446060552d5900de241ce4785201e9c612b72d1
+  metadata.gz: 0fbdb558032e848d11224c9030642f04dfee44bf
+  data.tar.gz: ccb2cc27b9aafe47ca5331ae092889345fd177f9
 SHA512:
-  metadata.gz: ea920b8c0258998de73cb1cb2973fe09c9f4272424751dee6c1a5ba27f0f327f379a8d13449f761ea5fdfe1d7e7d6e1e9e312a0a5d170d8a241907d27ea5e205
-  data.tar.gz: c425232a8d22c6f1a7b6e476211fb7c4b483952f04f9ddeeb30fe6eae1a2b73688c25bf7659e06f8601e11a4ded8ca700a35442a6adc2ec736badd27c9ee374b
+  metadata.gz: 72f5cd5902904c4fe4100fbacf5374bc28070406112d5ca82a5abd909a5eb3c5d7882b77ca905ce60254c9a8b0daf56d74a7df7fc7c7ae77d54c631cad0e50d0
+  data.tar.gz: 70f8c48093ab24a88c1923312db6c6e2131c2704c04f1d6d20ac603d90b7efa9df1b237313cdfb6be1357064aec89a1c15c4fe831112ae78ea8b4ef8a39be200

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Change logs
 
+## 0.2.1
+- Add an attribute for author.
+
+## 0.2.0
+- Make optional configuration settable for Mechanize.
+- Fix a bug where data-src is not found
+- Do not fix types of submedia.
+- Update `README.md`.
+
+### Breaking Changes
+- Do not use passed value of submedia type.
+
 ## 0.1.5
 - Do not crawl columns related to description.
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    dmm-crawler (0.2.0)
+    dmm-crawler (0.2.1)
       mechanize
 
 GEM
@@ -27,11 +27,11 @@ GEM
     mime-types (3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2016.0521)
-    mini_portile2 (2.2.0)
+    mini_portile2 (2.3.0)
     net-http-digest_auth (1.4.1)
     net-http-persistent (2.9.4)
-    nokogiri (1.8.0)
-      mini_portile2 (~> 2.2.0)
+    nokogiri (1.8.1)
+      mini_portile2 (~> 2.3.0)
     ntlm-http (0.1.1)
     parser (2.4.0.0)
       ast (~> 2.2)

data/lib/dmm-crawler/attributes.rb

@@ -1,7 +1,7 @@
 module DMMCrawler
   class Attributes
     def initialize(url)
-      @element = Agent.instance.agent.get(url)
+      @page = Agent.instance.agent.get(url)
     end
 
     def to_a
@@ -10,6 +10,7 @@ module DMMCrawler
         title_link,
         image_url,
         submedia,
+        author,
         informations,
         tags
       ]
@@ -19,16 +20,24 @@ module DMMCrawler
 
     def title
       if art_page?
-        @element.search('.productTitle__txt span').remove
-        @element.search('.productTitle__txt').text.strip
+        @page.search('.productTitle__txt span').remove
+        @page.search('.productTitle__txt').text.strip
       else
-        @element.search('.rank-name').first.text.strip
+        @page.search('.rank-name').first.text.strip
+      end
+    end
+
+    def title_link
+      if art_page?
+        @page.uri.to_s
+      else
+        File.join(BASE_URL, @page.search('.rank-name').first.search('a').first.attributes.first[1].value)
       end
     end
 
     def image_url
       if art_page?
-        attrs = @element.search('.productPreview__item img').last.attributes
+        attrs = @page.search('.productPreview__item img').last.attributes
 
         if attrs['data-src']
           attrs['data-src'].value
@@ -36,20 +45,12 @@ module DMMCrawler
           attrs['src'].value
         end
       else
-        @element.search('img').last.attributes['src'].value
-      end
-    end
-
-    def title_link
-      if art_page?
-        @element.uri.to_s
-      else
-        File.join(BASE_URL, @element.search('.rank-name').first.search('a').first.attributes.first[1].value)
+        @page.search('img').last.attributes['src'].value
       end
     end
 
     def submedia
-      @element
+      @page
         .search('.productAttribute-listItem .c_icon_productGenre')
         .first
         .attributes['class']
@@ -58,9 +59,13 @@ module DMMCrawler
         .delete('-')
     end
 
+    def author
+      @page.search('p.circleProductTitle__main').text.gsub('作品一覧', '')
+    end
+
     def informations
-      keys = extract_text(@element.search('.m-productInformation .productInformation__item .informationList__ttl'))
-      values = extract_text(@element.search('.m-productInformation .productInformation__item .informationList__txt'))
+      keys = extract_text(@page.search('.m-productInformation .productInformation__item .informationList__ttl'))
+      values = extract_text(@page.search('.m-productInformation .productInformation__item .informationList__txt'))
 
       information = keys.zip(values)
       series = information.find { |array| array.first == 'シリーズ' }
@@ -73,22 +78,22 @@ module DMMCrawler
       information.map { |key, value| { key: key, value: value } }
     end
 
-    def extract_text(elements)
-      elements
-        .select { |element| element.text.strip != 'ジャンル' }
-        .map { |element| element.children.text.strip }
-    end
-
     def tags
       if art_page?
-        @element.search('.genreTagList .genreTagList__item a').map { |e| e.text.strip }
+        @page.search('.genreTagList .genreTagList__item a').map { |e| e.text.strip }
       else
-        @element.search('.rank-labelListItem').map { |e| e.search('a').text.strip }
+        @page.search('.rank-labelListItem').map { |e| e.search('a').text.strip }
       end
     end
 
+    def extract_text(elements)
+      elements
+        .select { |element| element.text.strip != 'ジャンル' }
+        .map { |element| element.children.text.strip }
+    end
+
     def art_page?
-      @element.search('.rank-name').empty?
+      @page.search('.rank-name').empty?
     end
   end
 end

data/lib/dmm-crawler/ranking.rb

@@ -14,12 +14,13 @@ module DMMCrawler
         Attributes.new(url).to_a
       end
 
-      arts.map.with_index(1) do |(title, title_link, image_url, submedia, informations, tags), rank|
+      arts.map.with_index(1) do |(title, title_link, image_url, submedia, author, informations, tags), rank|
         {
           title: "#{rank}位: #{title}",
           title_link: title_link,
           image_url: image_url,
           submedia: submedia,
+          author: author,
           informations: informations,
           tags: tags
         }

data/lib/dmm-crawler/version.rb

@@ -1,3 +1,3 @@
 module DMMCrawler
-  VERSION = '0.2.0'.freeze
+  VERSION = '0.2.1'.freeze
 end

data/spec/dmm-crawler/ranking_spec.rb

@@ -21,7 +21,7 @@ describe DMMCrawler::Ranking do
 
       let(:term) { '24' }
 
-      it { is_expected.to all(include(:title, :title_link, :image_url, :submedia, :informations, :tags)) }
+      it { is_expected.to all(include(:title, :title_link, :image_url, :submedia, :author, :informations, :tags)) }
     end
 
     context 'with not registered argument' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dmm-crawler
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Satoshi Ohmori