web_loader 1.8.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1434106ba64ce88a81a2c382b330d92d82525162638d02ee0cd8a11958f50acd
-  data.tar.gz: c55bafdbea2ae8fda51272087683e4a0f157c275a001cdcc357a086add410afc
+  metadata.gz: 7bd0d0702828cbd7f4150c4dbf5f89144a1b5794a2e393fa9c6c9b0768e0577b
+  data.tar.gz: ddd8af0a7a5827b9da3d12cb60e7fbb55841c5b7ed29d1668c0ba5507987c4ac
 SHA512:
-  metadata.gz: 7139ac3325dd3daaf23bc6aed6d0b645c65df7fc943ace0ccf730c70ec3e30b7f6aa6e677198697b2414e334279a43a06669fc21d8b40b8bcebf215d0689e5f6
-  data.tar.gz: bad9b7b7e4fce6a6f5c2d9947c0ac44072556a5cdcbf8205a689e2240e34bb5562f69a7dcbd09412ca89cbd5249530eb6806a978aecb82a455fa77a5eddbe5f6
+  metadata.gz: 1b65f8df8b55fe2340cc6a7dd50885c7223aa3246748599b55eb28903ca09c97af854b8861a53973ffb37a2e81b3faaf9b837dc23f77c9ddd543bbe70aa8693c
+  data.tar.gz: 4464a9fba104605f4658f912c6910a16341410a511dc441e78357e6ab17a892c42286e465dc51e876ed4bb1334fe3a28418fac381afaf1e5d6aaf961a864a4e1

data/.idea/copilot.data.migration.ask.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="AskMigrationStateService">
+    <option name="migrationStatus" value="COMPLETED" />
+  </component>
+</project>

data/.idea/copilot.data.migration.ask2agent.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Ask2AgentMigrationStateService">
+    <option name="migrationStatus" value="COMPLETED" />
+  </component>
+</project>

data/.idea/copilot.data.migration.edit.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="EditMigrationStateService">
+    <option name="migrationStatus" value="COMPLETED" />
+  </component>
+</project>

data/Gemfile

@@ -2,9 +2,11 @@
 
 source "https://rubygems.org"
 
-# Specify your gem's dependencies in web_loader.gemspec
+# Specify your gem's dependencies in wl.gemspec
 gemspec
 
 gem "rake", "~> 13.0"
 
 gem "minitest", "~> 5.0"
+
+gem 'selenium-webdriver'

data/Gemfile.lock

@@ -1,13 +1,24 @@
 PATH
   remote: .
   specs:
-    web_loader (1.8.0)
+    web_loader (2.1.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    minitest (5.26.1)
+    base64 (0.3.0)
+    logger (1.7.0)
+    minitest (5.26.2)
     rake (13.3.1)
+    rexml (3.4.4)
+    rubyzip (3.2.2)
+    selenium-webdriver (4.38.0)
+      base64 (~> 0.2)
+      logger (~> 1.4)
+      rexml (~> 3.2, >= 3.2.5)
+      rubyzip (>= 1.2.2, < 4.0)
+      websocket (~> 1.0)
+    websocket (1.2.11)
 
 PLATFORMS
   arm64-darwin-24
@@ -16,6 +27,7 @@ PLATFORMS
 DEPENDENCIES
   minitest (~> 5.0)
   rake (~> 13.0)
+  selenium-webdriver
   web_loader!
 
 BUNDLED WITH

data/exe/wl

@@ -0,0 +1,61 @@
+#!/usr/bin/env ruby
+
+require "web_loader"
+require 'web_loader/drivers/selenium_driver'
+require 'optparse'
+
+class Wl
+  def self.run(argv)
+    STDOUT.sync = true
+    opts = {}
+    opt = OptionParser.new(argv)
+    opt.banner = "Usage: #{opt.program_name} [-h|--help] [Options] <URL> "
+    opt.version = WebLoader::VERSION
+    opt.separator('')
+    opt.separator("Options:")
+    opt.on_head('-h', '--help', 'Show this message') do |v|
+      puts opt.help
+      exit
+    end
+    opt.on('-v', '--verbose', 'Verbose message') {|v| opts[:v] = v}
+    drivers = ['pureruby', 'selenium']
+    opt.on('-d DRIVER', '--driver=DRIVER', drivers, drivers.join("|") + "(default pureruby)") {|v| opts[:d] = v }
+    opt.on("--disable-cache", "Disable cache") {|v| opts[:disable_cache] = v }
+    opt.parse!(argv)
+    if argv.empty?
+      puts "Error: URL is required."
+      puts opt.help
+      exit
+    end
+    command = Wl.new(opts)
+    url = argv[0]
+    command.execute(url)
+  end
+
+  def initialize(opts)
+    @opts = opts
+  end
+
+  def execute(url)
+    driver = create_driver
+    loader = WebLoader::Command.new(driver)
+    if @opts[:disable_cache]
+      loader.use_cache = false
+    end
+    result = loader.load(url)
+    puts result
+  end
+
+  private
+  def create_driver
+    case @opts[:d]
+    when 'selenium'
+      driver = WebLoader::Drivers::SeleniumDriver.new
+    else
+      driver = WebLoader::Drivers::HttpDriver.new
+    end
+    driver
+  end
+end
+
+Wl.run(ARGV)

data/lib/web_loader/command.rb

@@ -1,7 +1,6 @@
 require 'open-uri'
 require 'net/http'
 require 'uri'
-require 'kconv'
 
 module WebLoader
   class Command
@@ -23,7 +22,7 @@ module WebLoader
       File.binwrite(file, content)
     end
 
-    def initialize
+    def initialize(driver = ::WebLoader::Drivers::HttpDriver.new)
       @use_cache = true
       @load_cache_page = false #キャッシュを読み込んだかどうか
       @cache_dir = File.expand_path(CACHE_DIR)
@@ -34,12 +33,18 @@ module WebLoader
       @always_write_cache = false
       @response = nil
       @logger = nil
+
+      # ドライバーのセットアップ
+      @driver = driver
+      @driver.user_agent = @user_agent
+      @driver.binary = @binary
     end
 
     attr_reader :load_cache_page
     attr_accessor :use_cache, :cache_dir, :binary, :user_agent, :verbose
     attr_accessor :cache_limit
     attr_accessor :always_write_cache
+    attr_accessor :driver
     attr_reader :response
     attr_accessor :logger
 
@@ -62,15 +67,16 @@ module WebLoader
 
       ##### サーバーからロード
       log("Load server: #{url}")
-      uri = URI.parse(url)
-      http = Net::HTTP.new(uri.host, uri.port)
-      if uri.scheme == 'https'
-        http.use_ssl = true
-        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-      end
-      @response = nil
+      # uri = URI.parse(url)
+      # http = Net::HTTP.new(uri.host, uri.port)
+      # if uri.scheme == 'https'
+      #   http.use_ssl = true
+      #   http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      # end
+      # @response = nil
       begin
-        @response = http.get(uri.request_uri, 'User-Agent' => @user_agent) # request_uri=path + '?' + query
+        #        @response = http.get(uri.request_uri, 'User-Agent' => @user_agent) # request_uri=path + '?' + query
+        @response = @driver.fetch(url)
       rescue Net::ReadTimeout
         # タイムアウトした場合リトライ可能ならばsleepした後に再度ロード実行
         log("Read timeout: #{url}")
@@ -82,38 +88,21 @@ module WebLoader
 
       ##### レスポンスの処理
       result = nil
-      case @response
-      when Net::HTTPSuccess
-        # @responseがNet::HTTPSuccessのサブクラスの場合成功とみなし読み込んだ内容を返す
+      if response.ok?
         body = @response.body
-        unless @binary
-          # デフォルトでは ASCII-8BITが帰ってくる。
-          # Content-Typeのcharsetとみなす。
-          # https://bugs.ruby-lang.org/issues/2567
-          encoding = @response.type_params['charset']
-          body = toutf8(body, encoding)
-        end
-
         if @use_cache || @always_write_cache
           log("Write cache: #{url}")
-          Cache.write(@cache_dir, url, @response.code, body)
+          Cache.write(@cache_dir, url, @response.status, body)
         end
         result = body
-      when Net::HTTPRedirection
-        result = load(to_redirect_url(uri, @response['location']), redirect_count - 1)
-        # when Net::HTTPNotFound
-        #   result = nil
-      when Net::HTTPTooManyRequests, Net::ReadTimeout
+      elsif response.redirect?
+        result = load(to_redirect_url(URI.parse(url), @response.headers['location']), redirect_count - 1)
+      elsif response.rate_limited?
         # 上記以外のレスポンスの場合、リトライ可能ならばsleepした後に再度ロード実行
         if retry_count > 0
-          sleep_for = 10
-          if @response.is_a?(Net::HTTPTooManyRequests)
-            # HTTPTooManyRequestsならばretry-afterで指定された値を取得。
-            sleep_for = @response.header['retry-after'].to_i + 10
-            log("Rate limit: #{uri} #{@response.header.to_hash} (429 Too Many Requests). Sleeping #{sleep_for} seconds and retry (##{retry_count}).")
-          else
-            log("Unknown response: #{uri} #{@response.inspect}. Sleeping #{sleep_for} seconds and retry (##{retry_count}).")
-          end
+          # HTTPTooManyRequestsならばretry-afterで指定された値を取得。
+          sleep_for = @response.header['retry-after'].to_i + 10
+          log("Rate limit: #{uri} #{@response.header.to_hash} (429 Too Many Requests). Sleeping #{sleep_for} seconds and retry (##{retry_count}).")
           sleep sleep_for
           result = load(url, redirect_count , retry_count - 1)
         end
@@ -121,7 +110,51 @@ module WebLoader
         # それ以外は対応した例外を発生
         log("error #{url}", true)
       end
+
       result
+
+          # ##### レスポンスの処理
+      # result = nil
+      # case @response
+      # when Net::HTTPSuccess
+      #   # @responseがNet::HTTPSuccessのサブクラスの場合成功とみなし読み込んだ内容を返す
+      #   body = @response.body
+      #   unless @binary
+      #     # デフォルトでは ASCII-8BITが帰ってくる。
+      #     # Content-Typeのcharsetとみなす。
+      #     # https://bugs.ruby-lang.org/issues/2567
+      #     encoding = @response.type_params['charset']
+      #     body = toutf8(body, encoding)
+      #   end
+      #
+      #   if @use_cache || @always_write_cache
+      #     log("Write cache: #{url}")
+      #     Cache.write(@cache_dir, url, @response.code, body)
+      #   end
+      #   result = body
+      # when Net::HTTPRedirection
+      #   result = load(to_redirect_url(uri, @response['location']), redirect_count - 1)
+      #   # when Net::HTTPNotFound
+      #   #   result = nil
+      # when Net::HTTPTooManyRequests, Net::ReadTimeout
+      #   # 上記以外のレスポンスの場合、リトライ可能ならばsleepした後に再度ロード実行
+      #   if retry_count > 0
+      #     sleep_for = 10
+      #     if @response.is_a?(Net::HTTPTooManyRequests)
+      #       # HTTPTooManyRequestsならばretry-afterで指定された値を取得。
+      #       sleep_for = @response.header['retry-after'].to_i + 10
+      #       log("Rate limit: #{uri} #{@response.header.to_hash} (429 Too Many Requests). Sleeping #{sleep_for} seconds and retry (##{retry_count}).")
+      #     else
+      #       log("Unknown response: #{uri} #{@response.inspect}. Sleeping #{sleep_for} seconds and retry (##{retry_count}).")
+      #     end
+      #     sleep sleep_for
+      #     result = load(url, redirect_count , retry_count - 1)
+      #   end
+      # else
+      #   # それ以外は対応した例外を発生
+      #   log("error #{url}", true)
+      # end
+      # result
     end
 
     private

data/lib/web_loader/drivers/base_driver.rb

@@ -0,0 +1,17 @@
+module WebLoader
+  module Drivers
+    class BaseDriver
+      def initialize
+        @user_agent = nil
+        @binary = false
+      end
+
+      attr_accessor :user_agent, :binary
+
+      def fetch(url)
+        raise NotImplementedError, 'Subclasses must implement the fetch method'
+      end
+
+    end
+  end
+end

data/lib/web_loader/drivers/http_driver.rb

@@ -0,0 +1,40 @@
+require 'open-uri'
+require 'net/http'
+require 'uri'
+require 'kconv'
+
+
+module WebLoader
+  module Drivers
+    class HttpDriver < WebLoader::Drivers::BaseDriver
+
+      def fetch(url)
+        uri = URI.parse(url)
+        http = Net::HTTP.new(uri.host, uri.port)
+        if uri.scheme == 'https'
+          http.use_ssl = true
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        end
+        response = http.get(uri.request_uri, 'User-Agent' => @user_agent) # request_uri=path + '?' + query
+        create_response(response)
+      end
+
+      private
+      def create_response(response)
+        body = response.body
+        unless @binary
+          # デフォルトでは ASCII-8BITが帰ってくる。
+          # Content-Typeのcharsetとみなす。
+          # https://bugs.ruby-lang.org/issues/2567
+          encoding = response.type_params['charset']
+          body = ::WebLoader::Utils.toutf8(body, encoding)
+        end
+        ::WebLoader::Response.new(
+          status: response.code.to_i,
+          headers: response.each_header.to_h,
+          body: body
+        )
+      end
+    end
+  end
+end

data/lib/web_loader/drivers/selenium_driver.rb

@@ -0,0 +1,62 @@
+module WebLoader
+  module Drivers
+    class SeleniumDriver < ::WebLoader::Drivers::BaseDriver
+
+      def initialize
+        super
+        @wait_proc = nil
+        @wait_seconds = 3
+      end
+
+      attr_accessor :wait_proc
+      attr_accessor :wait_seconds
+
+      def fetch(url)
+        require 'selenium-webdriver'
+
+        #        puts "SeleniumDriver fetching URL: #{url}"
+
+        options = Selenium::WebDriver::Chrome::Options.new
+        options.add_argument("--headless")
+        options.add_argument("--disable-gpu")
+        options.add_argument("--no-sandbox")
+        options.add_argument("--user-agent=#{@user_agent}") if @user_agent
+
+        begin
+
+          driver = Selenium::WebDriver.for(:chrome, options: options)
+          driver.navigate.to url
+
+          wait = Selenium::WebDriver::Wait.new(timeout: 10)
+
+          # ページ全体のロード完了を待つ
+          if @wait_proc
+            wait.until { @wait_proc.call(driver) }
+          else
+            sleep @wait_seconds
+          end
+
+          content_type = driver.execute_script("return document.contentType;")
+
+          body = @binary ? page_source.b : driver.page_source
+          response = WebLoader::Response.new(status: 200,
+                                             headers: {
+                                               'Content-Type' => content_type
+                                             }, body: body)
+          response
+        ensure
+          driver.quit if driver
+        end
+      end
+
+      private
+      def create_response(body)
+        # デフォルトは成功200
+        status = 200
+        # redirected = driver.current_url != original_url
+        # status = 300 if redirected # 簡易的にリダイレクト扱い
+        new(status: status, headers: {}, body: body)
+      end
+    end
+  end
+end

data/lib/web_loader/response.rb

@@ -0,0 +1,22 @@
+module WebLoader
+  class Response
+    include WebLoader::Utils
+
+
+    def self.from_selenium(driver, original_url)
+
+    end
+
+    def initialize(status:, headers: {}, body: nil)
+      @status  = status.to_i
+      @headers = headers || {}
+      @body    = body
+    end
+
+    attr_reader :status, :headers, :body
+
+    def ok?; (200..299).include?(@status); end
+    def redirect?; (300..399).include?(@status); end
+    def rate_limited?; @status == 429; end
+  end
+end

data/lib/web_loader/utils.rb

@@ -67,6 +67,7 @@ module WebLoader
       end
       result
     end
+    module_function :toutf8
 
     def to_redirect_url(orig_uri, location)
       redirect_url = location
@@ -75,5 +76,6 @@ module WebLoader
       end
       redirect_url
     end
+    module_function :to_redirect_url
   end
 end

data/lib/web_loader/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module WebLoader
-  VERSION = "1.8.0"
+  VERSION = "2.1.0"
 end

data/lib/web_loader.rb

@@ -4,6 +4,10 @@ require_relative "web_loader/version"
 require_relative "web_loader/utils"
 require_relative "web_loader/cache"
 require_relative "web_loader/command"
+require_relative "web_loader/response"
+require_relative "web_loader/drivers/base_driver"
+require_relative "web_loader/drivers/http_driver"
+require_relative "web_loader/drivers/selenium_driver"
 
 module WebLoader
   class Error < StandardError; end

data/test_wl.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+bundle exec ruby exe/wl "$@"

data/web_loader.iml

@@ -10,6 +10,285 @@
     </content>
     <orderEntry type="jdk" jdkName="rbenv: 3.2.6" jdkType="RUBY_SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="module-library">
+      <library name="base64 (v0.3.0) [path][gem]" type="rubylib">
+        <properties>
+          <option name="additionalInfo">
+            <AdditionalInfo>
+              <option name="authors" value="該当なし" />
+              <option name="email" value="該当なし" />
+              <option name="homepage" value="該当なし" />
+              <option name="summary" value="該当なし" />
+            </AdditionalInfo>
+          </option>
+          <option name="fromPath" value="true" />
+          <option name="name" value="base64" />
+          <option name="requirePaths">
+            <list>
+              <option value="lib" />
+            </list>
+          </option>
+          <option name="url" value="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/base64-0.3.0" />
+          <option name="version" value="0.3.0" />
+        </properties>
+        <CLASSES>
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/base64-0.3.0/lib" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/base64-0.3.0/sig" />
+        </CLASSES>
+        <JAVADOC />
+        <SOURCES>
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/base64-0.3.0/lib" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/base64-0.3.0/sig" />
+        </SOURCES>
+      </library>
+    </orderEntry>
+    <orderEntry type="module-library">
+      <library name="logger (v1.7.0) [path][gem]" type="rubylib">
+        <properties>
+          <option name="additionalInfo">
+            <AdditionalInfo>
+              <option name="authors" value="該当なし" />
+              <option name="email" value="該当なし" />
+              <option name="homepage" value="該当なし" />
+              <option name="summary" value="該当なし" />
+            </AdditionalInfo>
+          </option>
+          <option name="fromPath" value="true" />
+          <option name="name" value="logger" />
+          <option name="requirePaths">
+            <list>
+              <option value="lib" />
+            </list>
+          </option>
+          <option name="url" value="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/logger-1.7.0" />
+          <option name="version" value="1.7.0" />
+        </properties>
+        <CLASSES>
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/logger-1.7.0/lib" />
+        </CLASSES>
+        <JAVADOC />
+        <SOURCES>
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/logger-1.7.0/lib" />
+        </SOURCES>
+      </library>
+    </orderEntry>
+    <orderEntry type="module-library">
+      <library name="minitest (v5.26.2) [path][gem]" type="rubylib">
+        <properties>
+          <option name="additionalInfo">
+            <AdditionalInfo>
+              <option name="authors" value="該当なし" />
+              <option name="email" value="該当なし" />
+              <option name="homepage" value="該当なし" />
+              <option name="summary" value="該当なし" />
+            </AdditionalInfo>
+          </option>
+          <option name="fromPath" value="true" />
+          <option name="name" value="minitest" />
+          <option name="requirePaths">
+            <list>
+              <option value="lib" />
+            </list>
+          </option>
+          <option name="url" value="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/minitest-5.26.2" />
+          <option name="version" value="5.26.2" />
+        </properties>
+        <CLASSES>
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/minitest-5.26.2/lib" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/minitest-5.26.2/test" />
+        </CLASSES>
+        <JAVADOC />
+        <SOURCES>
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/minitest-5.26.2/lib" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/minitest-5.26.2/test" />
+        </SOURCES>
+        <excluded>
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/minitest-5.26.2/test" />
+        </excluded>
+      </library>
+    </orderEntry>
+    <orderEntry type="module-library">
+      <library name="rake (v13.3.1) [path][gem]" type="rubylib">
+        <properties>
+          <option name="additionalInfo">
+            <AdditionalInfo>
+              <option name="authors" value="該当なし" />
+              <option name="email" value="該当なし" />
+              <option name="homepage" value="該当なし" />
+              <option name="summary" value="該当なし" />
+            </AdditionalInfo>
+          </option>
+          <option name="fromPath" value="true" />
+          <option name="name" value="rake" />
+          <option name="requirePaths">
+            <list>
+              <option value="lib" />
+            </list>
+          </option>
+          <option name="url" value="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rake-13.3.1" />
+          <option name="version" value="13.3.1" />
+        </properties>
+        <CLASSES>
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rake-13.3.1/doc" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rake-13.3.1/exe" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rake-13.3.1/lib" />
+        </CLASSES>
+        <JAVADOC />
+        <SOURCES>
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rake-13.3.1/doc" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rake-13.3.1/exe" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rake-13.3.1/lib" />
+        </SOURCES>
+        <excluded>
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rake-13.3.1/doc" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rake-13.3.1/exe" />
+        </excluded>
+      </library>
+    </orderEntry>
+    <orderEntry type="module-library">
+      <library name="rexml (v3.4.4) [path][gem]" type="rubylib">
+        <properties>
+          <option name="additionalInfo">
+            <AdditionalInfo>
+              <option name="authors" value="該当なし" />
+              <option name="email" value="該当なし" />
+              <option name="homepage" value="該当なし" />
+              <option name="summary" value="該当なし" />
+            </AdditionalInfo>
+          </option>
+          <option name="fromPath" value="true" />
+          <option name="name" value="rexml" />
+          <option name="requirePaths">
+            <list>
+              <option value="lib" />
+            </list>
+          </option>
+          <option name="url" value="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rexml-3.4.4" />
+          <option name="version" value="3.4.4" />
+        </properties>
+        <CLASSES>
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rexml-3.4.4/doc" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rexml-3.4.4/lib" />
+        </CLASSES>
+        <JAVADOC />
+        <SOURCES>
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rexml-3.4.4/doc" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rexml-3.4.4/lib" />
+        </SOURCES>
+        <excluded>
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rexml-3.4.4/doc" />
+        </excluded>
+      </library>
+    </orderEntry>
+    <orderEntry type="module-library">
+      <library name="rubyzip (v3.2.2) [path][gem]" type="rubylib">
+        <properties>
+          <option name="additionalInfo">
+            <AdditionalInfo>
+              <option name="authors" value="該当なし" />
+              <option name="email" value="該当なし" />
+              <option name="homepage" value="該当なし" />
+              <option name="summary" value="該当なし" />
+            </AdditionalInfo>
+          </option>
+          <option name="fromPath" value="true" />
+          <option name="name" value="rubyzip" />
+          <option name="requirePaths">
+            <list>
+              <option value="lib" />
+            </list>
+          </option>
+          <option name="url" value="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rubyzip-3.2.2" />
+          <option name="version" value="3.2.2" />
+        </properties>
+        <CLASSES>
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rubyzip-3.2.2/lib" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rubyzip-3.2.2/samples" />
+        </CLASSES>
+        <JAVADOC />
+        <SOURCES>
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rubyzip-3.2.2/lib" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rubyzip-3.2.2/samples" />
+        </SOURCES>
+        <excluded>
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rubyzip-3.2.2/samples" />
+        </excluded>
+      </library>
+    </orderEntry>
+    <orderEntry type="module-library">
+      <library name="selenium-webdriver (v4.38.0) [path][gem]" type="rubylib">
+        <properties>
+          <option name="additionalInfo">
+            <AdditionalInfo>
+              <option name="authors" value="該当なし" />
+              <option name="email" value="該当なし" />
+              <option name="homepage" value="該当なし" />
+              <option name="summary" value="該当なし" />
+            </AdditionalInfo>
+          </option>
+          <option name="fromPath" value="true" />
+          <option name="name" value="selenium-webdriver" />
+          <option name="requirePaths">
+            <list>
+              <option value="lib" />
+            </list>
+          </option>
+          <option name="url" value="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/selenium-webdriver-4.38.0" />
+          <option name="version" value="4.38.0" />
+        </properties>
+        <CLASSES>
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/selenium-webdriver-4.38.0/bin" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/selenium-webdriver-4.38.0/lib" />
+        </CLASSES>
+        <JAVADOC />
+        <SOURCES>
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/selenium-webdriver-4.38.0/bin" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/selenium-webdriver-4.38.0/lib" />
+        </SOURCES>
+        <excluded>
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/selenium-webdriver-4.38.0/bin" />
+        </excluded>
+      </library>
+    </orderEntry>
+    <orderEntry type="module-library">
+      <library name="websocket (v1.2.11) [path][gem]" type="rubylib">
+        <properties>
+          <option name="additionalInfo">
+            <AdditionalInfo>
+              <option name="authors" value="該当なし" />
+              <option name="email" value="該当なし" />
+              <option name="homepage" value="該当なし" />
+              <option name="summary" value="該当なし" />
+            </AdditionalInfo>
+          </option>
+          <option name="fromPath" value="true" />
+          <option name="name" value="websocket" />
+          <option name="requirePaths">
+            <list>
+              <option value="lib" />
+            </list>
+          </option>
+          <option name="url" value="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/websocket-1.2.11" />
+          <option name="version" value="1.2.11" />
+        </properties>
+        <CLASSES>
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/websocket-1.2.11/lib" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/websocket-1.2.11/spec" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/websocket-1.2.11/.github" />
+        </CLASSES>
+        <JAVADOC />
+        <SOURCES>
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/websocket-1.2.11/lib" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/websocket-1.2.11/spec" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/websocket-1.2.11/.github" />
+        </SOURCES>
+        <excluded>
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/websocket-1.2.11/spec" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/websocket-1.2.11/.github" />
+        </excluded>
+      </library>
+    </orderEntry>
+    <orderEntry type="library" scope="PROVIDED" name="bundler (v2.6.3, rbenv: 3.2.6) [gem]" level="application" />
   </component>
   <component name="RakeTasksCache">
     <option name="myRootTask">

metadata

@@ -1,26 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: web_loader
 version: !ruby/object:Gem::Version
-  version: 1.8.0
+  version: 2.1.0
 platform: ruby
 authors:
 - src
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-11-12 00:00:00.000000000 Z
+date: 2025-11-28 00:00:00.000000000 Z
 dependencies: []
 description: Web loader.
 email:
 - src@srcw.net
 executables:
-- web_loader
+- wl
 extensions: []
 extra_rdoc_files: []
 files:
 - ".DS_Store"
 - ".idea/.gitignore"
 - ".idea/copilot.data.migration.agent.xml"
+- ".idea/copilot.data.migration.ask.xml"
+- ".idea/copilot.data.migration.ask2agent.xml"
+- ".idea/copilot.data.migration.edit.xml"
 - ".idea/inspectionProfiles/Project_Default.xml"
 - ".idea/misc.xml"
 - ".idea/modules.xml"
@@ -32,14 +35,19 @@ files:
 - README.md
 - Rakefile
 - build.sh
-- exe/web_loader
+- exe/wl
 - lib/web_loader.rb
 - lib/web_loader/cache.rb
 - lib/web_loader/command.rb
+- lib/web_loader/drivers/base_driver.rb
+- lib/web_loader/drivers/http_driver.rb
+- lib/web_loader/drivers/selenium_driver.rb
+- lib/web_loader/response.rb
 - lib/web_loader/utils.rb
 - lib/web_loader/version.rb
 - sig/web_loader.rbs
 - test.sh
+- test_wl.sh
 - web_loader.iml
 homepage: https://srcw.net
 licenses:

data/exe/web_loader

@@ -1,3 +0,0 @@
-#!/usr/bin/env ruby
-
-require "web_loader"