dmm-crawler 0.1.5 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b6d65185ef93c93354c8eb9967b6b60ca64650b2
-  data.tar.gz: e9043312de49e3a70f54bf18d6c5fc64b1772a07
+  metadata.gz: bde3c371a800c5ea7438e38d62c21792771899c3
+  data.tar.gz: f446060552d5900de241ce4785201e9c612b72d1
 SHA512:
-  metadata.gz: 38e501d44bbef30dbfe1efdadf5e696bcfd9d4cc7580c03bf7b9feff1eabef385b5a8751d45219ddba6da1c659f08120f203dc3cc0359d242696ee452fd5582f
-  data.tar.gz: 48720a7e5b861dfc505da33779f8f1c43cb1b0b31b2ff385a8ed92a3fe696879bf98191ed60a3aae89eb8329782d963bb0f8f34c47768f2f237233530d9e2e10
+  metadata.gz: ea920b8c0258998de73cb1cb2973fe09c9f4272424751dee6c1a5ba27f0f327f379a8d13449f761ea5fdfe1d7e7d6e1e9e312a0a5d170d8a241907d27ea5e205
+  data.tar.gz: c425232a8d22c6f1a7b6e476211fb7c4b483952f04f9ddeeb30fe6eae1a2b73688c25bf7659e06f8601e11a4ded8ca700a35442a6adc2ec736badd27c9ee374b

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    dmm-crawler (0.1.5)
+    dmm-crawler (0.2.0)
       mechanize
 
 GEM
@@ -10,7 +10,7 @@ GEM
     ast (2.3.0)
     coderay (1.1.1)
     diff-lcs (1.3)
-    domain_name (0.5.20170223)
+    domain_name (0.5.20170404)
       unf (>= 0.0.5, < 1.0.0)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
@@ -27,11 +27,11 @@ GEM
     mime-types (3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2016.0521)
-    mini_portile2 (2.1.0)
+    mini_portile2 (2.2.0)
     net-http-digest_auth (1.4.1)
     net-http-persistent (2.9.4)
-    nokogiri (1.7.0.1)
-      mini_portile2 (~> 2.1.0)
+    nokogiri (1.8.0)
+      mini_portile2 (~> 2.2.0)
     ntlm-http (0.1.1)
     parser (2.4.0.0)
       ast (~> 2.2)
@@ -65,7 +65,7 @@ GEM
     slop (3.6.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.2)
+    unf_ext (0.0.7.4)
     unicode-display_width (1.1.3)
     webrobots (0.1.2)
 
@@ -81,4 +81,4 @@ DEPENDENCIES
   rubocop (~> 0.47)
 
 BUNDLED WITH
-   1.15.1
+   1.16.0.pre.2

data/README.md

@@ -15,6 +15,7 @@ gem 'dmm-crawler'
 ```
 
 ## Usage
+
 From the bot in invited Slack's room.
 
 ```ruby
@@ -22,14 +23,19 @@ require 'dmm-crawler'
 
 include DMMCrawler
 
-Ranking.new(term: '24', submedia: 'cg').arts
+client = Client.new do |agent|
+  agent.ignore_bad_chunking = false
+end
 
+client.rankings(term: '24', submedia: 'cg')
 # =>
 # {
 #   title: "title",
-#   url: 'URL for title',
-#   image_url: 'Link to title's main image',
-#   tags: ['tag1', 'tag2']
+#   title_link: "title url",
+#   image_url: "Link to title"s main image",
+#   submedia: "cg",
+#   informations: [{key: 'key', value: 'value'}],
+#   tags: ["tag1", "tag2"]
 # }
 ```
 

data/doc/ja/README.md

@@ -3,6 +3,7 @@
 # DMM Crawler
 
 ## DMM Crawlerとは
+
 DMM.R18のクロールしたデータを取得するgemです。現在、**同人**のランキングにのみ対応しております。
 
 ## インストール
@@ -14,6 +15,7 @@ gem 'dmm-crawler'
 ```
 
 ## 使い方
+
 データを使いたい`.rb`ファイルで以下を実行したらクロールしたデータが取得出来ます。
 
 ```ruby
@@ -21,14 +23,19 @@ require 'dmm-crawler'
 
 include DMMCrawler
 
-Ranking.new(term: '24', submedia: 'cg').arts
+client = Client.new do |agent|
+  agent.ignore_bad_chunking = false
+end
 
+client.rankings(term: '24', submedia: 'cg')
 # =>
 # {
 #   title: "タイトル",
-#   url: '作品のURL',
-#   image_url: '作品のメイン画像へのURL',
-#   tags: ['タグ1', 'タグ2']
+#   title_link: "タイトルURL",
+#   image_url: "画像URL",
+#   submedia: "cg",
+#   informations: [{key: 'key', value: 'value'}],
+#   tags: ["タグ1", "タグ2"]
 # }
 ```
 

data/lib/dmm-crawler.rb

@@ -7,4 +7,5 @@ end
 require 'dmm-crawler/agent'
 require 'dmm-crawler/attributes'
 require 'dmm-crawler/ranking'
+require 'dmm-crawler/client'
 require 'dmm-crawler/version'

data/lib/dmm-crawler/attributes.rb

@@ -1,10 +1,7 @@
 module DMMCrawler
   class Attributes
-    def initialize(element, submedia = nil)
-      @agent = Agent.instance.agent
-      @element = element
-
-      @submedia = submedia
+    def initialize(url)
+      @element = Agent.instance.agent.get(url)
     end
 
     def to_a
@@ -31,7 +28,13 @@ module DMMCrawler
 
     def image_url
       if art_page?
-        @element.search('.productPreview__item img').last.attributes['data-src'].value
+        attrs = @element.search('.productPreview__item img').last.attributes
+
+        if attrs['data-src']
+          attrs['data-src'].value
+        else
+          attrs['src'].value
+        end
       else
         @element.search('img').last.attributes['src'].value
       end
@@ -46,8 +49,6 @@ module DMMCrawler
     end
 
     def submedia
-      return @submedia if @submedia
-
       @element
         .search('.productAttribute-listItem .c_icon_productGenre')
         .first

data/lib/dmm-crawler/client.rb

@@ -0,0 +1,13 @@
+module DMMCrawler
+  class Client
+    def initialize
+      @agent = Agent.instance.agent
+
+      yield @agent if block_given?
+    end
+
+    def rankings(arguments)
+      Ranking.new(arguments.merge!(agent: @agent)).arts
+    end
+  end
+end

data/lib/dmm-crawler/ranking.rb

@@ -1,18 +1,17 @@
 module DMMCrawler
   class Ranking
     def initialize(arguments)
+      @agent = discriminate_agent(arguments[:agent])
       @term = discriminate_term(arguments[:term])
-      @submedia = discriminate_submedia(arguments[:submedia])
+      @submedia = arguments[:submedia]
       @url = File.join(BASE_URL, "/dc/doujin/-/ranking-all/=/sort=popular/submedia=#{@submedia}/term=#{@term}")
-      @agent = Agent.instance.agent
     end
 
     def arts
       arts = page.search('.rank-rankListItem.fn-setPurchaseChange').map do |element|
         sleep 1
         url = File.join(BASE_URL, element.search('.rank-name a').first.attributes['href'].value)
-        page = @agent.get(url)
-        Attributes.new(page, @submedia).to_a
+        Attributes.new(url).to_a
       end
 
       arts.map.with_index(1) do |(title, title_link, image_url, submedia, informations, tags), rank|
@@ -38,8 +37,8 @@ module DMMCrawler
       raise TypeError
     end
 
-    def discriminate_submedia(submedia)
-      return submedia if %w(all comic cg game voice).include?(submedia)
+    def discriminate_agent(agent)
+      return agent if agent.is_a?(Mechanize)
       raise TypeError
     end
   end

data/lib/dmm-crawler/version.rb

@@ -1,3 +1,3 @@
 module DMMCrawler
-  VERSION = '0.1.5'.freeze
+  VERSION = '0.2.0'.freeze
 end

data/spec/dmm-crawler/ranking_spec.rb

@@ -1,8 +1,9 @@
 describe DMMCrawler::Ranking do
   let(:attachments) { described_class.new(arguments).arts }
 
+  let(:agent) { DMMCrawler::Agent.instance.agent }
   let(:submedia) { 'cg' }
-  let(:arguments) { { submedia: submedia, term: term } }
+  let(:arguments) { { submedia: submedia, term: term, agent: agent } }
 
   describe '#arts' do
     after { sleep 2 }
@@ -26,7 +27,7 @@ describe DMMCrawler::Ranking do
     context 'with not registered argument' do
       subject { -> { attachments } }
 
-      let(:term) { 'hoge' }
+      let(:term) { nil }
 
       it { is_expected.to raise_error(TypeError) }
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dmm-crawler
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.0
 platform: ruby
 authors:
 - Satoshi Ohmori
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-24 00:00:00.000000000 Z
+date: 2017-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -116,6 +116,7 @@ files:
 - lib/dmm-crawler.rb
 - lib/dmm-crawler/agent.rb
 - lib/dmm-crawler/attributes.rb
+- lib/dmm-crawler/client.rb
 - lib/dmm-crawler/ranking.rb
 - lib/dmm-crawler/version.rb
 - spec/dmm-crawler/ranking_spec.rb
@@ -140,7 +141,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.10
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Show DMM and DMM.R18's crawled data