miteru 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '08e49cafafb4f0ca46b1aad40e1de0ce09e1098dd49395aa38ac98fd0d08e09f'
-  data.tar.gz: ace78f1898e2ef84573abb7ef8b624819d6819eb7924a21e6fb6189c918cfe98
+  metadata.gz: 4b6ffb792e263519ab645ef14235c5fb6cfa08919375bbaf2064528515dacd43
+  data.tar.gz: a67289780e2d42dfbbcd0a4b8142487e3ba22c5aa5addeef5707fe107161ecb5
 SHA512:
-  metadata.gz: a0cd62ad104dda985f0de5d023295079f43638553c08ba11a4c3c67e4271423d50860f6eb0349c10c2a25cc91ed57c231e452434ae657c848ba890499b0dcb76
-  data.tar.gz: 69f50f7a12ab6433f6ec1cb517779aaa92ab2f714f261f9ac8786e859a7d9b1c955ef503d7e355166f5481010ce7666ffe14f87c906f8011fbf2a80ca81cf487
+  metadata.gz: 5e43966e7560b60cc4e09930ef9f390862747380419267b0135dc7b86edf3aec83f1fb18732e3553a8adb805fc08c70654327c89bf973c171d3aa1bc59c70b4b
+  data.tar.gz: 906207242b7cafbba681f19d73df74b4e3c5160d8982e0144cc5d2f30c53917593e3abde329b3666400f2f27db9055c55bef4f880df5f20cc973add7e077ea6c

data/README.md

@@ -1,5 +1,6 @@
 # Miteru
 
+[![Gem Version](https://badge.fury.io/rb/miteru.svg)](https://badge.fury.io/rb/miteru)
 [![Build Status](https://travis-ci.org/ninoseki/miteru.svg?branch=master)](https://travis-ci.org/ninoseki/miteru)
 [![Maintainability](https://api.codeclimate.com/v1/badges/d90e1b5bbdd9663a17d1/maintainability)](https://codeclimate.com/github/ninoseki/miteru/maintainability)
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/miteru/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/miteru?branch=master)

data/lib/miteru.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require "miteru/error"
-require "miteru/downloader"
+require "miteru/http_client"
 require "miteru/website"
 require "miteru/crawler"
 require "miteru/cli"

data/lib/miteru/cli.rb

@@ -27,7 +27,7 @@ module Miteru
         zip_files.each do |path|
           target_url = "#{url}/#{path}"
           begin
-            destination = Downloader.download(target_url, base_dir)
+            destination = HTTPClient.download(target_url, base_dir)
             puts "Download #{target_url} as #{destination}"
           rescue Down::Error => e
             puts "Failed to download: #{target_url} (#{e})"

data/lib/miteru/http_client.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "http"
+require "down/http"
+require "securerandom"
+
+module Miteru
+  class HTTPClient
+    DEFAULT_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
+    attr_reader :ssl_context
+    def initialize
+      ctx = OpenSSL::SSL::SSLContext.new
+      ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      @ssl_context = ctx
+    end
+
+    def download(url, base_dir)
+      destination = download_to(base_dir, save_filename)
+      down = Down::Http.new(default_options) { |client| client.headers(default_headers) }
+      down.download(url, destination: destination)
+      destination
+    end
+
+    def self.download(url, base_dir = "/tmp")
+      new.download(url, base_dir)
+    end
+
+    def get(url)
+      HTTP.headers(default_headers).get(url, default_options)
+    end
+
+    def self.get(url)
+      new.get url
+    end
+
+    private
+
+    def default_headers
+      { user_agent: DEFAULT_UA }
+    end
+
+    def default_options
+      { ssl_context: ssl_context }
+    end
+
+    def save_filename
+      "#{SecureRandom.alphanumeric}.zip"
+    end
+
+    def download_to(base_dir, save_filename)
+      "#{base_dir}/#{save_filename}"
+    end
+  end
+end

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.3.1"
+  VERSION = "0.3.2"
 end

data/lib/miteru/website.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "http"
 require "oga"
 
 module Miteru
@@ -49,9 +48,7 @@ module Miteru
     end
 
     def get
-      ctx = OpenSSL::SSL::SSLContext.new
-      ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
-      HTTP.get(url, ssl_context: ctx)
+      HTTPClient.get url
     end
 
     def doc

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-09-19 00:00:00.000000000 Z
+date: 2018-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -213,8 +213,8 @@ files:
 - lib/miteru.rb
 - lib/miteru/cli.rb
 - lib/miteru/crawler.rb
-- lib/miteru/downloader.rb
 - lib/miteru/error.rb
+- lib/miteru/http_client.rb
 - lib/miteru/version.rb
 - lib/miteru/website.rb
 - miteru.gemspec

data/lib/miteru/downloader.rb

@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-require "down"
-require "securerandom"
-
-module Miteru
-  class Downloader
-    attr_reader :url, :base_dir
-    def initialize(url, base_dir = "/tmp")
-      @url = url
-      @base_dir = base_dir
-    end
-
-    def save_filename
-      "#{SecureRandom.alphanumeric}.zip"
-    end
-
-    def destination
-      @destination ||= "#{base_dir}/#{save_filename}"
-    end
-
-    def download
-      Down.download(url, destination: destination)
-      destination
-    end
-
-    def self.download(url, base_dir = "/tmp")
-      new(url, base_dir).download
-    end
-  end
-end