dmm-crawler 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5df41735d4f8eedf776285760e2cc02e4bcc7c39
+  data.tar.gz: 85f963c4432c166a3d5b9896099babaffc9cd407
+SHA512:
+  metadata.gz: 79819ff75becdf9790590491d15afc27f641c2c61b68c9a8f5049fc971121eebc6b4894d114ecec719d62497f8eb5cd6e45d7260d1b31736d19b63f7cdb3f1d8
+  data.tar.gz: bfe581c5bff9f8a69910fac9821ff86e9822df122a409cdae86199229db302ccc4885d337479ac559dd9db2d53dc97f0d9c84b57cd08ea0e8500033c2845da70

data/.gitignore

@@ -0,0 +1,2 @@
+/vendor
+/pkg

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,21 @@
+LineLength:
+  Enabled: false
+
+Metrics/AbcSize:
+  Enabled: false
+
+CyclomaticComplexity:
+  Enabled: false
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Max: 12
+
+Style/Documentation:
+  Enabled: false
+
+Style/FileName:
+  Exclude:
+    - lib/dmm-crawler.rb

data/.travis.yml

@@ -0,0 +1,15 @@
+language: ruby
+
+rvm:
+  - ruby-2.2.2
+  - ruby-2.3.0
+  - ruby-2.4.0
+  - ruby-2.4.1
+
+before_install:
+  - gem install bundler
+
+script: 'bundle exec rake spec'
+
+notifications:
+  slack: sachin21dev:J8sIAeYdBknOGbFHtrphiobX

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,84 @@
+PATH
+  remote: .
+  specs:
+    dmm-crawler (0.0.1)
+      mechanize
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.3.0)
+    coderay (1.1.1)
+    diff-lcs (1.3)
+    domain_name (0.5.20170223)
+      unf (>= 0.0.5, < 1.0.0)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    mechanize (2.7.5)
+      domain_name (~> 0.5, >= 0.5.1)
+      http-cookie (~> 1.0)
+      mime-types (>= 1.17.2)
+      net-http-digest_auth (~> 1.1, >= 1.1.1)
+      net-http-persistent (~> 2.5, >= 2.5.2)
+      nokogiri (~> 1.6)
+      ntlm-http (~> 0.1, >= 0.1.1)
+      webrobots (>= 0.0.9, < 0.2)
+    method_source (0.8.2)
+    mime-types (3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2016.0521)
+    mini_portile2 (2.1.0)
+    net-http-digest_auth (1.4.1)
+    net-http-persistent (2.9.4)
+    nokogiri (1.7.0.1)
+      mini_portile2 (~> 2.1.0)
+    ntlm-http (0.1.1)
+    parser (2.4.0.0)
+      ast (~> 2.2)
+    powerpack (0.1.1)
+    pry (0.10.4)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rainbow (2.2.1)
+    rake (10.5.0)
+    rspec (3.5.0)
+      rspec-core (~> 3.5.0)
+      rspec-expectations (~> 3.5.0)
+      rspec-mocks (~> 3.5.0)
+    rspec-core (3.5.4)
+      rspec-support (~> 3.5.0)
+    rspec-expectations (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-mocks (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-support (3.5.0)
+    rubocop (0.47.1)
+      parser (>= 2.3.3.1, < 3.0)
+      powerpack (~> 0.1)
+      rainbow (>= 1.99.1, < 3.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.0, >= 1.0.1)
+    ruby-progressbar (1.8.1)
+    slop (3.6.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.2)
+    unicode-display_width (1.1.3)
+    webrobots (0.1.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.14)
+  dmm-crawler!
+  pry (= 0.10.4)
+  rake (~> 10.0)
+  rspec (= 3.5.0)
+  rubocop (~> 0.47)
+
+BUNDLED WITH
+   1.14.6

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Satoshi Ohmori
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,11 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+gem_helper = Bundler::GemHelper.new
+
+desc "Create tag #{gem_helper.send(:version_tag)}"
+task :tag do
+  gem_helper.send(:tag_version) unless gem_helper.send(:already_tagged?)
+end

data/dmm-crawler.gemspec

@@ -0,0 +1,27 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'dmm-crawler/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'dmm-crawler'
+  spec.version       = DMMCrawler::VERSION
+  spec.authors       = ['Satoshi Ohmori']
+  spec.email         = ['sachin21dev@gmail.com']
+
+  spec.summary       = "Show DMM and DMM.R18's crawled data"
+  spec.description   = "Show DMM and DMM.R18's crawled data. e.g. ranking"
+  spec.homepage      = 'https://github.com/sachin21/dmm-crawler'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+
+  spec.require_paths = ['lib']
+
+  spec.add_runtime_dependency 'mechanize'
+
+  spec.add_development_dependency 'bundler', '~> 1.14'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rubocop', '~> 0.47'
+  spec.add_development_dependency 'pry', '0.10.4'
+  spec.add_development_dependency 'rspec', '3.5.0'
+end

data/lib/dmm-crawler.rb

@@ -0,0 +1,8 @@
+require 'mechanize'
+
+module DMMCrawler
+end
+
+require 'dmm-crawler/agent'
+require 'dmm-crawler/ranking'
+require 'dmm-crawler/version'

data/lib/dmm-crawler/agent.rb

@@ -0,0 +1,11 @@
+module DMMCrawler
+  class Agent
+    attr_accessor :agent
+
+    def initialize
+      @agent = ::Mechanize.new
+      @agent.request_headers = { 'Accept-Encoding' => '' }
+      @agent.ignore_bad_chunking = true
+    end
+  end
+end

data/lib/dmm-crawler/ranking.rb

@@ -0,0 +1,47 @@
+module DMMCrawler
+  class Ranking
+    BASE_URL = 'http://www.dmm.co.jp'.freeze
+    COLORS = %w(7acc28 9dcc28 cc2828 cc6e28 28a9cc 3428cc cc28b4 c0cc28 2886cc 28cca9).map { |hex| "##{hex}" }.freeze
+
+    def initialize(arguments)
+      @term = discriminate_term(arguments[:term])
+      @submedia = discriminate_submedia(arguments[:submedia])
+      @url = "#{BASE_URL}/dc/doujin/-/ranking-all/=/sort=popular/submedia=#{@submedia}/term=#{@term}"
+      @agent = Agent.new.agent
+    end
+
+    def books
+      books = page.search('.rank-rankListItem.fn-setPurchaseChange').map do |element|
+        [
+          element.search('.rank-name').first.text.strip,
+          element.search('img').last.attributes['src'].value,
+          "#{BASE_URL}#{element.search('.rank-name').first.search('a').first.attributes.first[1].value}"
+        ]
+      end
+      books.take(10).zip(COLORS).map.with_index(1) { |((title, image, url), color), rank| { title: "#{rank}位: #{title}", title_link: url, image_url: image, color: color } }
+    end
+
+    private
+
+    def credentials
+      {
+        api_id:       ENV['API_ID'],
+        affiliate_id: ENV['AFFILIATE_ID']
+      }
+    end
+
+    def page
+      @agent.get(@url)
+    end
+
+    def discriminate_term(term)
+      return term if %w(24 weekly monthly total).include?(term)
+      raise TypeError
+    end
+
+    def discriminate_submedia(submedia)
+      return submedia if %w(all weekly cg game voice).include?(submedia)
+      raise TypeError
+    end
+  end
+end

data/lib/dmm-crawler/version.rb

@@ -0,0 +1,3 @@
+module DMMCrawler
+  VERSION = '0.0.1'.freeze
+end

data/spec/dmm-crawler/ranking_spec.rb

@@ -0,0 +1,25 @@
+describe DMMCrawler::Ranking do
+  let(:attachments) { described_class.new(arguments).books }
+
+  let(:submedia) { 'cg' }
+  let(:arguments) { { submedia: submedia, term: term } }
+
+  describe '#books' do
+    context 'with 24 argument' do
+      subject { attachments }
+
+      let(:term) { '24' }
+
+      it { is_expected.not_to be_empty }
+      it { is_expected.to all(include(:title, :title_link, :image_url, :color)) }
+    end
+
+    context 'with not registered argument' do
+      subject { -> { attachments } }
+
+      let(:term) { 'hoge' }
+
+      it { is_expected.to raise_error(TypeError) }
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+require 'dmm-crawler'
+
+RSpec.configure do |config|
+  config.order = 'random'
+  config.expect_with :rspec do |rspec|
+    rspec.syntax = :expect
+  end
+end

metadata

@@ -0,0 +1,143 @@
+--- !ruby/object:Gem::Specification
+name: dmm-crawler
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Satoshi Ohmori
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-04-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.47'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.47'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.10.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.10.4
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.5.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.5.0
+description: Show DMM and DMM.R18's crawled data. e.g. ranking
+email:
+- sachin21dev@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- Rakefile
+- dmm-crawler.gemspec
+- lib/dmm-crawler.rb
+- lib/dmm-crawler/agent.rb
+- lib/dmm-crawler/ranking.rb
+- lib/dmm-crawler/version.rb
+- spec/dmm-crawler/ranking_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/sachin21/dmm-crawler
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: Show DMM and DMM.R18's crawled data
+test_files: []