web_loader 2.1.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7bd0d0702828cbd7f4150c4dbf5f89144a1b5794a2e393fa9c6c9b0768e0577b
-  data.tar.gz: ddd8af0a7a5827b9da3d12cb60e7fbb55841c5b7ed29d1668c0ba5507987c4ac
+  metadata.gz: 803b7f81240d732a305705cc0fa46610781688ce19203f0b7d8710822b088558
+  data.tar.gz: ec8ed1467ab473fa5fbae5a370304e3bb2281e525ecfc4f54c752828f49c1d00
 SHA512:
-  metadata.gz: 1b65f8df8b55fe2340cc6a7dd50885c7223aa3246748599b55eb28903ca09c97af854b8861a53973ffb37a2e81b3faaf9b837dc23f77c9ddd543bbe70aa8693c
-  data.tar.gz: 4464a9fba104605f4658f912c6910a16341410a511dc441e78357e6ab17a892c42286e465dc51e876ed4bb1334fe3a28418fac381afaf1e5d6aaf961a864a4e1
+  metadata.gz: 51c54de50e0faa9a4886cc3c84efbf88887003c12e8cafcc9cfa1ea9c5ae5beeb795a2795d08d48031dd373f6cb56de1e28116c7d0450a18b288c04e81e54e0f
+  data.tar.gz: 62cd1ecd932a88fb51f919a5565833e5d14bbef94414acdcc55c90ab075fb387d0d98050753d05438bcaf3d8a371b731a2b146ac57cb98d4420a2b9ff97ffccc

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    web_loader (2.1.0)
+    web_loader (2.2.0)
 
 GEM
   remote: https://rubygems.org/

data/exe/wl

@@ -4,58 +4,21 @@ require "web_loader"
 require 'web_loader/drivers/selenium_driver'
 require 'optparse'
 
-class Wl
-  def self.run(argv)
-    STDOUT.sync = true
-    opts = {}
-    opt = OptionParser.new(argv)
-    opt.banner = "Usage: #{opt.program_name} [-h|--help] [Options] <URL> "
-    opt.version = WebLoader::VERSION
-    opt.separator('')
-    opt.separator("Options:")
-    opt.on_head('-h', '--help', 'Show this message') do |v|
-      puts opt.help
-      exit
-    end
-    opt.on('-v', '--verbose', 'Verbose message') {|v| opts[:v] = v}
-    drivers = ['pureruby', 'selenium']
-    opt.on('-d DRIVER', '--driver=DRIVER', drivers, drivers.join("|") + "(default pureruby)") {|v| opts[:d] = v }
-    opt.on("--disable-cache", "Disable cache") {|v| opts[:disable_cache] = v }
-    opt.parse!(argv)
-    if argv.empty?
-      puts "Error: URL is required."
-      puts opt.help
-      exit
-    end
-    command = Wl.new(opts)
-    url = argv[0]
-    command.execute(url)
-  end
 
-  def initialize(opts)
-    @opts = opts
-  end
+result = WebLoader::Downloader.run(ARGV)
+puts result
 
-  def execute(url)
-    driver = create_driver
-    loader = WebLoader::Command.new(driver)
-    if @opts[:disable_cache]
-      loader.use_cache = false
-    end
-    result = loader.load(url)
-    puts result
-  end
-
-  private
-  def create_driver
-    case @opts[:d]
-    when 'selenium'
-      driver = WebLoader::Drivers::SeleniumDriver.new
-    else
-      driver = WebLoader::Drivers::HttpDriver.new
-    end
-    driver
-  end
-end
-
-Wl.run(ARGV)
+# custom downloader example(selenium with custom wait proc)
+#     wl --driver=selenium https://www.example.com
+#
+# class MyDownloader < WebLoader::Downloader
+#   def create_wait_proc
+#     #    puts "Using custom wait proc..."
+#     proc do |driver|
+#       # Example wait condition: wait until the document is fully loaded
+#       driver.execute_script("return document.readyState") == "complete"
+#     end
+#   end
+# end
+# result = MyDownloader.run(ARGV)
+# puts result

data/lib/web_loader/downloader.rb

@@ -0,0 +1,66 @@
+
+module WebLoader
+  class Downloader
+
+    def self.run(argv)
+      STDOUT.sync = true
+      opts = {}
+      opt = OptionParser.new(argv)
+      opt.banner = "Usage: #{opt.program_name} [-h|--help] [Options] <URL> "
+      opt.version = WebLoader::VERSION
+      opt.separator('')
+      opt.separator("Options:")
+      opt.on_head('-h', '--help', 'Show this message') do |v|
+        puts opt.help
+        exit
+      end
+      opt.on('-v', '--verbose', 'Verbose message') {|v| opts[:v] = v}
+      drivers = ['pureruby', 'selenium']
+      opt.on('-d DRIVER', '--driver=DRIVER', drivers, drivers.join("|") + "(default pureruby)") {|v| opts[:d] = v }
+      opt.on("--disable-cache", "Disable cache") {|v| opts[:disable_cache] = v }
+      opt.parse!(argv)
+      if argv.empty?
+        puts "Error: URL is required."
+        puts opt.help
+        exit
+      end
+      command = self.new(opts)
+      url = argv[0]
+      command.execute(url)
+    end
+
+    def initialize(opts)
+      @opts = opts
+    end
+
+    def execute(url)
+      driver = create_driver
+      loader = WebLoader::Command.new(driver)
+      if @opts[:disable_cache]
+        loader.use_cache = false
+      end
+      loader.load(url)
+    end
+
+    private
+    def create_driver
+      case @opts[:d]
+      when 'selenium'
+        driver = WebLoader::Drivers::SeleniumDriver.new
+        driver.wait_proc = create_wait_proc
+      else
+        driver = WebLoader::Drivers::HttpDriver.new
+      end
+      driver
+    end
+
+    def create_wait_proc
+      # proc do |driver|
+      #   # Example wait condition: wait until the document is fully loaded
+      #   ready_state = driver.execute_script('return document.readyState')
+      #   ready_state == 'complete'
+      # end
+    end
+  end
+
+end

data/lib/web_loader/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module WebLoader
-  VERSION = "2.1.0"
+  VERSION = "2.2.0"
 end

data/lib/web_loader.rb

@@ -4,6 +4,7 @@ require_relative "web_loader/version"
 require_relative "web_loader/utils"
 require_relative "web_loader/cache"
 require_relative "web_loader/command"
+require_relative "web_loader/downloader"
 require_relative "web_loader/response"
 require_relative "web_loader/drivers/base_driver"
 require_relative "web_loader/drivers/http_driver"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: web_loader
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.2.0
 platform: ruby
 authors:
 - src
@@ -39,6 +39,7 @@ files:
 - lib/web_loader.rb
 - lib/web_loader/cache.rb
 - lib/web_loader/command.rb
+- lib/web_loader/downloader.rb
 - lib/web_loader/drivers/base_driver.rb
 - lib/web_loader/drivers/http_driver.rb
 - lib/web_loader/drivers/selenium_driver.rb