web_loader 2.2.0 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 803b7f81240d732a305705cc0fa46610781688ce19203f0b7d8710822b088558
-  data.tar.gz: ec8ed1467ab473fa5fbae5a370304e3bb2281e525ecfc4f54c752828f49c1d00
+  metadata.gz: 03a86b4f610a1575be326e19740897204b75e2dbe41e1eac56ee1c98d9c44558
+  data.tar.gz: b83a8f147007e94e4fac89abd4297066727dfe6ebf0a68052569f97cece0f223
 SHA512:
-  metadata.gz: 51c54de50e0faa9a4886cc3c84efbf88887003c12e8cafcc9cfa1ea9c5ae5beeb795a2795d08d48031dd373f6cb56de1e28116c7d0450a18b288c04e81e54e0f
-  data.tar.gz: 62cd1ecd932a88fb51f919a5565833e5d14bbef94414acdcc55c90ab075fb387d0d98050753d05438bcaf3d8a371b731a2b146ac57cb98d4420a2b9ff97ffccc
+  metadata.gz: 27395f10236456d0780ee08c28d5b1c9fd447acab50261732eb24c4bb88011eb823381d01005ac6faf4d5e580d85dd1e16c69038c4ca1f4d3ed257613aeb1daf
+  data.tar.gz: 24a7b60226b63fcc7c1fda0a44ca11fbb14d10176936f93b0684fc0866289cbae73058fc7c4f485579fa83850520d6f9cc13112cddaff60d733e009b06ded5cc

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    web_loader (2.2.0)
+    web_loader (2.3.0)
 
 GEM
   remote: https://rubygems.org/

data/lib/web_loader/command.rb

@@ -36,8 +36,6 @@ module WebLoader
 
       # ドライバーのセットアップ
       @driver = driver
-      @driver.user_agent = @user_agent
-      @driver.binary = @binary
     end
 
     attr_reader :load_cache_page
@@ -67,15 +65,9 @@ module WebLoader
 
       ##### サーバーからロード
       log("Load server: #{url}")
-      # uri = URI.parse(url)
-      # http = Net::HTTP.new(uri.host, uri.port)
-      # if uri.scheme == 'https'
-      #   http.use_ssl = true
-      #   http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-      # end
-      # @response = nil
       begin
-        #        @response = http.get(uri.request_uri, 'User-Agent' => @user_agent) # request_uri=path + '?' + query
+        @driver.user_agent = @user_agent
+        @driver.binary = @binary
         @response = @driver.fetch(url)
       rescue Net::ReadTimeout
         # タイムアウトした場合リトライ可能ならばsleepした後に再度ロード実行
@@ -110,51 +102,7 @@ module WebLoader
         # それ以外は対応した例外を発生
         log("error #{url}", true)
       end
-
       result
-
-          # ##### レスポンスの処理
-      # result = nil
-      # case @response
-      # when Net::HTTPSuccess
-      #   # @responseがNet::HTTPSuccessのサブクラスの場合成功とみなし読み込んだ内容を返す
-      #   body = @response.body
-      #   unless @binary
-      #     # デフォルトでは ASCII-8BITが帰ってくる。
-      #     # Content-Typeのcharsetとみなす。
-      #     # https://bugs.ruby-lang.org/issues/2567
-      #     encoding = @response.type_params['charset']
-      #     body = toutf8(body, encoding)
-      #   end
-      #
-      #   if @use_cache || @always_write_cache
-      #     log("Write cache: #{url}")
-      #     Cache.write(@cache_dir, url, @response.code, body)
-      #   end
-      #   result = body
-      # when Net::HTTPRedirection
-      #   result = load(to_redirect_url(uri, @response['location']), redirect_count - 1)
-      #   # when Net::HTTPNotFound
-      #   #   result = nil
-      # when Net::HTTPTooManyRequests, Net::ReadTimeout
-      #   # 上記以外のレスポンスの場合、リトライ可能ならばsleepした後に再度ロード実行
-      #   if retry_count > 0
-      #     sleep_for = 10
-      #     if @response.is_a?(Net::HTTPTooManyRequests)
-      #       # HTTPTooManyRequestsならばretry-afterで指定された値を取得。
-      #       sleep_for = @response.header['retry-after'].to_i + 10
-      #       log("Rate limit: #{uri} #{@response.header.to_hash} (429 Too Many Requests). Sleeping #{sleep_for} seconds and retry (##{retry_count}).")
-      #     else
-      #       log("Unknown response: #{uri} #{@response.inspect}. Sleeping #{sleep_for} seconds and retry (##{retry_count}).")
-      #     end
-      #     sleep sleep_for
-      #     result = load(url, redirect_count , retry_count - 1)
-      #   end
-      # else
-      #   # それ以外は対応した例外を発生
-      #   log("error #{url}", true)
-      # end
-      # result
     end
 
     private

data/lib/web_loader/downloader.rb

@@ -18,6 +18,8 @@ module WebLoader
       drivers = ['pureruby', 'selenium']
       opt.on('-d DRIVER', '--driver=DRIVER', drivers, drivers.join("|") + "(default pureruby)") {|v| opts[:d] = v }
       opt.on("--disable-cache", "Disable cache") {|v| opts[:disable_cache] = v }
+      opt.on('--user-agent=USERAGENT', 'Set User-Agent header') {|v| opts[:user_agent] = v }
+      opt.on('-b', '--binary', 'Download binary files') {|v| opts[:binary] = v }
       opt.parse!(argv)
       if argv.empty?
         puts "Error: URL is required."
@@ -39,6 +41,12 @@ module WebLoader
       if @opts[:disable_cache]
         loader.use_cache = false
       end
+      if @opts[:user_agent]
+        loader.user_agent = @opts[:user_agent]
+      end
+      if @opts[:binary]
+        loader.binary = true
+      end
       loader.load(url)
     end
 

data/lib/web_loader/drivers/selenium_driver.rb

@@ -38,7 +38,7 @@ module WebLoader
 
           content_type = driver.execute_script("return document.contentType;")
 
-          body = @binary ? page_source.b : driver.page_source
+          body = @binary ? driver.page_source.b : driver.page_source
           response = WebLoader::Response.new(status: 200,
                                              headers: {
                                                'Content-Type' => content_type

data/lib/web_loader/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module WebLoader
-  VERSION = "2.2.0"
+  VERSION = "2.3.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: web_loader
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 2.3.0
 platform: ruby
 authors:
 - src