web_loader 2.1.0 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7bd0d0702828cbd7f4150c4dbf5f89144a1b5794a2e393fa9c6c9b0768e0577b
-  data.tar.gz: ddd8af0a7a5827b9da3d12cb60e7fbb55841c5b7ed29d1668c0ba5507987c4ac
+  metadata.gz: 03a86b4f610a1575be326e19740897204b75e2dbe41e1eac56ee1c98d9c44558
+  data.tar.gz: b83a8f147007e94e4fac89abd4297066727dfe6ebf0a68052569f97cece0f223
 SHA512:
-  metadata.gz: 1b65f8df8b55fe2340cc6a7dd50885c7223aa3246748599b55eb28903ca09c97af854b8861a53973ffb37a2e81b3faaf9b837dc23f77c9ddd543bbe70aa8693c
-  data.tar.gz: 4464a9fba104605f4658f912c6910a16341410a511dc441e78357e6ab17a892c42286e465dc51e876ed4bb1334fe3a28418fac381afaf1e5d6aaf961a864a4e1
+  metadata.gz: 27395f10236456d0780ee08c28d5b1c9fd447acab50261732eb24c4bb88011eb823381d01005ac6faf4d5e580d85dd1e16c69038c4ca1f4d3ed257613aeb1daf
+  data.tar.gz: 24a7b60226b63fcc7c1fda0a44ca11fbb14d10176936f93b0684fc0866289cbae73058fc7c4f485579fa83850520d6f9cc13112cddaff60d733e009b06ded5cc

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    web_loader (2.1.0)
+    web_loader (2.3.0)
 
 GEM
   remote: https://rubygems.org/

data/exe/wl

@@ -4,58 +4,21 @@ require "web_loader"
 require 'web_loader/drivers/selenium_driver'
 require 'optparse'
 
-class Wl
-  def self.run(argv)
-    STDOUT.sync = true
-    opts = {}
-    opt = OptionParser.new(argv)
-    opt.banner = "Usage: #{opt.program_name} [-h|--help] [Options] <URL> "
-    opt.version = WebLoader::VERSION
-    opt.separator('')
-    opt.separator("Options:")
-    opt.on_head('-h', '--help', 'Show this message') do |v|
-      puts opt.help
-      exit
-    end
-    opt.on('-v', '--verbose', 'Verbose message') {|v| opts[:v] = v}
-    drivers = ['pureruby', 'selenium']
-    opt.on('-d DRIVER', '--driver=DRIVER', drivers, drivers.join("|") + "(default pureruby)") {|v| opts[:d] = v }
-    opt.on("--disable-cache", "Disable cache") {|v| opts[:disable_cache] = v }
-    opt.parse!(argv)
-    if argv.empty?
-      puts "Error: URL is required."
-      puts opt.help
-      exit
-    end
-    command = Wl.new(opts)
-    url = argv[0]
-    command.execute(url)
-  end
 
-  def initialize(opts)
-    @opts = opts
-  end
+result = WebLoader::Downloader.run(ARGV)
+puts result
 
-  def execute(url)
-    driver = create_driver
-    loader = WebLoader::Command.new(driver)
-    if @opts[:disable_cache]
-      loader.use_cache = false
-    end
-    result = loader.load(url)
-    puts result
-  end
-
-  private
-  def create_driver
-    case @opts[:d]
-    when 'selenium'
-      driver = WebLoader::Drivers::SeleniumDriver.new
-    else
-      driver = WebLoader::Drivers::HttpDriver.new
-    end
-    driver
-  end
-end
-
-Wl.run(ARGV)
+# custom downloader example(selenium with custom wait proc)
+#     wl --driver=selenium https://www.example.com
+#
+# class MyDownloader < WebLoader::Downloader
+#   def create_wait_proc
+#     #    puts "Using custom wait proc..."
+#     proc do |driver|
+#       # Example wait condition: wait until the document is fully loaded
+#       driver.execute_script("return document.readyState") == "complete"
+#     end
+#   end
+# end
+# result = MyDownloader.run(ARGV)
+# puts result

data/lib/web_loader/command.rb

@@ -36,8 +36,6 @@ module WebLoader
 
       # ドライバーのセットアップ
       @driver = driver
-      @driver.user_agent = @user_agent
-      @driver.binary = @binary
     end
 
     attr_reader :load_cache_page
@@ -67,15 +65,9 @@ module WebLoader
 
       ##### サーバーからロード
       log("Load server: #{url}")
-      # uri = URI.parse(url)
-      # http = Net::HTTP.new(uri.host, uri.port)
-      # if uri.scheme == 'https'
-      #   http.use_ssl = true
-      #   http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-      # end
-      # @response = nil
       begin
-        #        @response = http.get(uri.request_uri, 'User-Agent' => @user_agent) # request_uri=path + '?' + query
+        @driver.user_agent = @user_agent
+        @driver.binary = @binary
         @response = @driver.fetch(url)
       rescue Net::ReadTimeout
         # タイムアウトした場合リトライ可能ならばsleepした後に再度ロード実行
@@ -110,51 +102,7 @@ module WebLoader
         # それ以外は対応した例外を発生
         log("error #{url}", true)
       end
-
       result
-
-          # ##### レスポンスの処理
-      # result = nil
-      # case @response
-      # when Net::HTTPSuccess
-      #   # @responseがNet::HTTPSuccessのサブクラスの場合成功とみなし読み込んだ内容を返す
-      #   body = @response.body
-      #   unless @binary
-      #     # デフォルトでは ASCII-8BITが帰ってくる。
-      #     # Content-Typeのcharsetとみなす。
-      #     # https://bugs.ruby-lang.org/issues/2567
-      #     encoding = @response.type_params['charset']
-      #     body = toutf8(body, encoding)
-      #   end
-      #
-      #   if @use_cache || @always_write_cache
-      #     log("Write cache: #{url}")
-      #     Cache.write(@cache_dir, url, @response.code, body)
-      #   end
-      #   result = body
-      # when Net::HTTPRedirection
-      #   result = load(to_redirect_url(uri, @response['location']), redirect_count - 1)
-      #   # when Net::HTTPNotFound
-      #   #   result = nil
-      # when Net::HTTPTooManyRequests, Net::ReadTimeout
-      #   # 上記以外のレスポンスの場合、リトライ可能ならばsleepした後に再度ロード実行
-      #   if retry_count > 0
-      #     sleep_for = 10
-      #     if @response.is_a?(Net::HTTPTooManyRequests)
-      #       # HTTPTooManyRequestsならばretry-afterで指定された値を取得。
-      #       sleep_for = @response.header['retry-after'].to_i + 10
-      #       log("Rate limit: #{uri} #{@response.header.to_hash} (429 Too Many Requests). Sleeping #{sleep_for} seconds and retry (##{retry_count}).")
-      #     else
-      #       log("Unknown response: #{uri} #{@response.inspect}. Sleeping #{sleep_for} seconds and retry (##{retry_count}).")
-      #     end
-      #     sleep sleep_for
-      #     result = load(url, redirect_count , retry_count - 1)
-      #   end
-      # else
-      #   # それ以外は対応した例外を発生
-      #   log("error #{url}", true)
-      # end
-      # result
     end
 
     private

data/lib/web_loader/downloader.rb

@@ -0,0 +1,74 @@
+
+module WebLoader
+  class Downloader
+
+    def self.run(argv)
+      STDOUT.sync = true
+      opts = {}
+      opt = OptionParser.new(argv)
+      opt.banner = "Usage: #{opt.program_name} [-h|--help] [Options] <URL> "
+      opt.version = WebLoader::VERSION
+      opt.separator('')
+      opt.separator("Options:")
+      opt.on_head('-h', '--help', 'Show this message') do |v|
+        puts opt.help
+        exit
+      end
+      opt.on('-v', '--verbose', 'Verbose message') {|v| opts[:v] = v}
+      drivers = ['pureruby', 'selenium']
+      opt.on('-d DRIVER', '--driver=DRIVER', drivers, drivers.join("|") + "(default pureruby)") {|v| opts[:d] = v }
+      opt.on("--disable-cache", "Disable cache") {|v| opts[:disable_cache] = v }
+      opt.on('--user-agent=USERAGENT', 'Set User-Agent header') {|v| opts[:user_agent] = v }
+      opt.on('-b', '--binary', 'Download binary files') {|v| opts[:binary] = v }
+      opt.parse!(argv)
+      if argv.empty?
+        puts "Error: URL is required."
+        puts opt.help
+        exit
+      end
+      command = self.new(opts)
+      url = argv[0]
+      command.execute(url)
+    end
+
+    def initialize(opts)
+      @opts = opts
+    end
+
+    def execute(url)
+      driver = create_driver
+      loader = WebLoader::Command.new(driver)
+      if @opts[:disable_cache]
+        loader.use_cache = false
+      end
+      if @opts[:user_agent]
+        loader.user_agent = @opts[:user_agent]
+      end
+      if @opts[:binary]
+        loader.binary = true
+      end
+      loader.load(url)
+    end
+
+    private
+    def create_driver
+      case @opts[:d]
+      when 'selenium'
+        driver = WebLoader::Drivers::SeleniumDriver.new
+        driver.wait_proc = create_wait_proc
+      else
+        driver = WebLoader::Drivers::HttpDriver.new
+      end
+      driver
+    end
+
+    def create_wait_proc
+      # proc do |driver|
+      #   # Example wait condition: wait until the document is fully loaded
+      #   ready_state = driver.execute_script('return document.readyState')
+      #   ready_state == 'complete'
+      # end
+    end
+  end
+
+end

data/lib/web_loader/drivers/selenium_driver.rb

@@ -38,7 +38,7 @@ module WebLoader
 
           content_type = driver.execute_script("return document.contentType;")
 
-          body = @binary ? page_source.b : driver.page_source
+          body = @binary ? driver.page_source.b : driver.page_source
           response = WebLoader::Response.new(status: 200,
                                              headers: {
                                                'Content-Type' => content_type

data/lib/web_loader/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module WebLoader
-  VERSION = "2.1.0"
+  VERSION = "2.3.0"
 end

data/lib/web_loader.rb

@@ -4,6 +4,7 @@ require_relative "web_loader/version"
 require_relative "web_loader/utils"
 require_relative "web_loader/cache"
 require_relative "web_loader/command"
+require_relative "web_loader/downloader"
 require_relative "web_loader/response"
 require_relative "web_loader/drivers/base_driver"
 require_relative "web_loader/drivers/http_driver"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: web_loader
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.3.0
 platform: ruby
 authors:
 - src
@@ -39,6 +39,7 @@ files:
 - lib/web_loader.rb
 - lib/web_loader/cache.rb
 - lib/web_loader/command.rb
+- lib/web_loader/downloader.rb
 - lib/web_loader/drivers/base_driver.rb
 - lib/web_loader/drivers/http_driver.rb
 - lib/web_loader/drivers/selenium_driver.rb