dmm-crawler 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 99738f48c076aa4e2569f97fab7c9ad20500aa0f
-  data.tar.gz: aaa5cf428459e5310449a03cfce45fcf87310de9
+  metadata.gz: 9348e5bbc1d9fdd3687fa75aad5f99c5ddd26eef
+  data.tar.gz: acda09a89a2d16b311422d54e61f5df157b0b2f4
 SHA512:
-  metadata.gz: 58ca9321e140ba4327b2e9d49a062fe37a23a0c0feeb2d8c914054f1f9888fb151ffa6e87f3977268f63c3e024ad380275c011107b99f9189f70474a83600162
-  data.tar.gz: f8b08887860332a742ab5301edaf48361a72e2cd85da2e6a7e5a915978e00712c65600f64e00d1e0791cae3ce1102f0d7f495b2512e140518bd27e30d92ce9f3
+  metadata.gz: be3d69e605d26e625eff51f373e5c6cbf4f9ac0182f93ad981d56fb366e441a4c062b0e0b187725679589c75bf90c11cdcc019385b6faa582bd5612011aa8332
+  data.tar.gz: 5695dd753f88ba79580a6258e40b00d6e98b37f031717831966c398d0a2751dac04b7f001db94dc928d5d2e2b30c9500c2a7143c09eb06042cce145859e36830

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Change logs
 
+## 0.0.6
+- Fix the bug where description is empty.
+- Refactor codes related to attributes for the art.
+
 ## 0.0.5
 - Add a function to fetch description.
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    dmm-crawler (0.0.5)
+    dmm-crawler (0.0.6)
       mechanize
 
 GEM

data/lib/dmm-crawler/agent.rb

@@ -1,11 +1,14 @@
+require 'singleton'
+
 module DMMCrawler
   class Agent
-    attr_accessor :agent
+    include Singleton
 
-    def initialize
-      @agent = ::Mechanize.new
-      @agent.request_headers = { 'Accept-Encoding' => '' }
-      @agent.ignore_bad_chunking = true
+    def agent
+      agent = ::Mechanize.new
+      agent.request_headers = { 'Accept-Encoding' => '' }
+      agent.ignore_bad_chunking = true
+      agent
     end
   end
 end

data/lib/dmm-crawler/ranking.rb

@@ -5,19 +5,13 @@ module DMMCrawler
     def initialize(arguments)
       @term = discriminate_term(arguments[:term])
       @submedia = discriminate_submedia(arguments[:submedia])
-      @url = "#{BASE_URL}/dc/doujin/-/ranking-all/=/sort=popular/submedia=#{@submedia}/term=#{@term}"
-      @agent = Agent.new.agent
+      @url = File.join(BASE_URL, "/dc/doujin/-/ranking-all/=/sort=popular/submedia=#{@submedia}/term=#{@term}")
+      @agent = Agent.instance.agent
     end
 
     def arts
       arts = page.search('.rank-rankListItem.fn-setPurchaseChange').map do |element|
-        [
-          element.search('.rank-name').first.text.strip,
-          element.search('img').last.attributes['src'].value,
-          "#{BASE_URL}#{element.search('.rank-name').first.search('a').first.attributes.first[1].value}",
-          element.search('.rank-desc').text,
-          element.search('.rank-labelListItem').map { |e| e.search('a').text.strip }
-        ]
+        Attribute.new(element).to_a
       end
 
       arts.map.with_index(1) do |(title, image_url, title_link, description, tags), rank|
@@ -31,6 +25,53 @@ module DMMCrawler
       end
     end
 
+    class Attribute
+      def initialize(element)
+        @element = element
+        @agent = Agent.instance.agent
+      end
+
+      def to_a
+        [
+          title,
+          title_link,
+          image_url,
+          description,
+          tags
+        ]
+      end
+
+      private
+
+      def title
+        @element.search('.rank-name').first.text.strip
+      end
+
+      def image_url
+        @element.search('img').last.attributes['src'].value
+      end
+
+      def title_link
+        File.join(BASE_URL, @element.search('.rank-name').first.search('a').first.attributes.first[1].value)
+      end
+
+      def description
+        @element.search('.rank-desc').text.nil? ? fetch_description : @element.search('.rank-desc').text
+      end
+
+      def fetch_description
+        url = File.join(BASE_URL, @element.search('.rank-name a').first.attributes['href'].value)
+        page = @agent.get(url)
+        page.search('.summary .summary__txt').to_s.gsub(/\sclass=".*"/, '')
+      end
+
+      def tags
+        @element.search('.rank-labelListItem').map { |e| e.search('a').text.strip }
+      end
+    end
+
+    private_constant :Attribute
+
     private
 
     def page

data/lib/dmm-crawler/version.rb

@@ -1,3 +1,3 @@
 module DMMCrawler
-  VERSION = '0.0.5'.freeze
+  VERSION = '0.0.6'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dmm-crawler
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Satoshi Ohmori
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-20 00:00:00.000000000 Z
+date: 2017-05-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -139,7 +139,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.12
 signing_key: 
 specification_version: 4
 summary: Show DMM and DMM.R18's crawled data