web_loader 2.0.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +3 -1
- data/Gemfile.lock +14 -2
- data/exe/wl +24 -0
- data/lib/web_loader/downloader.rb +66 -0
- data/lib/web_loader/drivers/base_driver.rb +17 -0
- data/lib/web_loader/drivers/http_driver.rb +19 -9
- data/lib/web_loader/drivers/selenium_driver.rb +62 -0
- data/lib/web_loader/response.rb +1 -20
- data/lib/web_loader/version.rb +1 -1
- data/lib/web_loader.rb +3 -0
- data/test_wl.sh +3 -0
- data/web_loader.iml +213 -8
- metadata +8 -4
- data/exe/web_loader +0 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 803b7f81240d732a305705cc0fa46610781688ce19203f0b7d8710822b088558
|
|
4
|
+
data.tar.gz: ec8ed1467ab473fa5fbae5a370304e3bb2281e525ecfc4f54c752828f49c1d00
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 51c54de50e0faa9a4886cc3c84efbf88887003c12e8cafcc9cfa1ea9c5ae5beeb795a2795d08d48031dd373f6cb56de1e28116c7d0450a18b288c04e81e54e0f
|
|
7
|
+
data.tar.gz: 62cd1ecd932a88fb51f919a5565833e5d14bbef94414acdcc55c90ab075fb387d0d98050753d05438bcaf3d8a371b731a2b146ac57cb98d4420a2b9ff97ffccc
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
|
@@ -1,13 +1,24 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
web_loader (2.
|
|
4
|
+
web_loader (2.2.0)
|
|
5
5
|
|
|
6
6
|
GEM
|
|
7
7
|
remote: https://rubygems.org/
|
|
8
8
|
specs:
|
|
9
|
-
|
|
9
|
+
base64 (0.3.0)
|
|
10
|
+
logger (1.7.0)
|
|
11
|
+
minitest (5.26.2)
|
|
10
12
|
rake (13.3.1)
|
|
13
|
+
rexml (3.4.4)
|
|
14
|
+
rubyzip (3.2.2)
|
|
15
|
+
selenium-webdriver (4.38.0)
|
|
16
|
+
base64 (~> 0.2)
|
|
17
|
+
logger (~> 1.4)
|
|
18
|
+
rexml (~> 3.2, >= 3.2.5)
|
|
19
|
+
rubyzip (>= 1.2.2, < 4.0)
|
|
20
|
+
websocket (~> 1.0)
|
|
21
|
+
websocket (1.2.11)
|
|
11
22
|
|
|
12
23
|
PLATFORMS
|
|
13
24
|
arm64-darwin-24
|
|
@@ -16,6 +27,7 @@ PLATFORMS
|
|
|
16
27
|
DEPENDENCIES
|
|
17
28
|
minitest (~> 5.0)
|
|
18
29
|
rake (~> 13.0)
|
|
30
|
+
selenium-webdriver
|
|
19
31
|
web_loader!
|
|
20
32
|
|
|
21
33
|
BUNDLED WITH
|
data/exe/wl
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
require "web_loader"
|
|
4
|
+
require 'web_loader/drivers/selenium_driver'
|
|
5
|
+
require 'optparse'
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
result = WebLoader::Downloader.run(ARGV)
|
|
9
|
+
puts result
|
|
10
|
+
|
|
11
|
+
# custom downloader example(selenium with custom wait proc)
|
|
12
|
+
# wl --driver=selenium https://www.example.com
|
|
13
|
+
#
|
|
14
|
+
# class MyDownloader < WebLoader::Downloader
|
|
15
|
+
# def create_wait_proc
|
|
16
|
+
# # puts "Using custom wait proc..."
|
|
17
|
+
# proc do |driver|
|
|
18
|
+
# # Example wait condition: wait until the document is fully loaded
|
|
19
|
+
# driver.execute_script("return document.readyState") == "complete"
|
|
20
|
+
# end
|
|
21
|
+
# end
|
|
22
|
+
# end
|
|
23
|
+
# result = MyDownloader.run(ARGV)
|
|
24
|
+
# puts result
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
|
|
2
|
+
module WebLoader
|
|
3
|
+
class Downloader
|
|
4
|
+
|
|
5
|
+
def self.run(argv)
|
|
6
|
+
STDOUT.sync = true
|
|
7
|
+
opts = {}
|
|
8
|
+
opt = OptionParser.new(argv)
|
|
9
|
+
opt.banner = "Usage: #{opt.program_name} [-h|--help] [Options] <URL> "
|
|
10
|
+
opt.version = WebLoader::VERSION
|
|
11
|
+
opt.separator('')
|
|
12
|
+
opt.separator("Options:")
|
|
13
|
+
opt.on_head('-h', '--help', 'Show this message') do |v|
|
|
14
|
+
puts opt.help
|
|
15
|
+
exit
|
|
16
|
+
end
|
|
17
|
+
opt.on('-v', '--verbose', 'Verbose message') {|v| opts[:v] = v}
|
|
18
|
+
drivers = ['pureruby', 'selenium']
|
|
19
|
+
opt.on('-d DRIVER', '--driver=DRIVER', drivers, drivers.join("|") + "(default pureruby)") {|v| opts[:d] = v }
|
|
20
|
+
opt.on("--disable-cache", "Disable cache") {|v| opts[:disable_cache] = v }
|
|
21
|
+
opt.parse!(argv)
|
|
22
|
+
if argv.empty?
|
|
23
|
+
puts "Error: URL is required."
|
|
24
|
+
puts opt.help
|
|
25
|
+
exit
|
|
26
|
+
end
|
|
27
|
+
command = self.new(opts)
|
|
28
|
+
url = argv[0]
|
|
29
|
+
command.execute(url)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def initialize(opts)
|
|
33
|
+
@opts = opts
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def execute(url)
|
|
37
|
+
driver = create_driver
|
|
38
|
+
loader = WebLoader::Command.new(driver)
|
|
39
|
+
if @opts[:disable_cache]
|
|
40
|
+
loader.use_cache = false
|
|
41
|
+
end
|
|
42
|
+
loader.load(url)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
def create_driver
|
|
47
|
+
case @opts[:d]
|
|
48
|
+
when 'selenium'
|
|
49
|
+
driver = WebLoader::Drivers::SeleniumDriver.new
|
|
50
|
+
driver.wait_proc = create_wait_proc
|
|
51
|
+
else
|
|
52
|
+
driver = WebLoader::Drivers::HttpDriver.new
|
|
53
|
+
end
|
|
54
|
+
driver
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def create_wait_proc
|
|
58
|
+
# proc do |driver|
|
|
59
|
+
# # Example wait condition: wait until the document is fully loaded
|
|
60
|
+
# ready_state = driver.execute_script('return document.readyState')
|
|
61
|
+
# ready_state == 'complete'
|
|
62
|
+
# end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
module WebLoader
|
|
2
|
+
module Drivers
|
|
3
|
+
class BaseDriver
|
|
4
|
+
def initialize
|
|
5
|
+
@user_agent = nil
|
|
6
|
+
@binary = false
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
attr_accessor :user_agent, :binary
|
|
10
|
+
|
|
11
|
+
def fetch(url)
|
|
12
|
+
raise NotImplementedError, 'Subclasses must implement the fetch method'
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -6,14 +6,7 @@ require 'kconv'
|
|
|
6
6
|
|
|
7
7
|
module WebLoader
|
|
8
8
|
module Drivers
|
|
9
|
-
class HttpDriver
|
|
10
|
-
|
|
11
|
-
def initialize
|
|
12
|
-
@user_agent = nil
|
|
13
|
-
@binary = false
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
attr_accessor :user_agent, :binary
|
|
9
|
+
class HttpDriver < WebLoader::Drivers::BaseDriver
|
|
17
10
|
|
|
18
11
|
def fetch(url)
|
|
19
12
|
uri = URI.parse(url)
|
|
@@ -23,7 +16,24 @@ module WebLoader
|
|
|
23
16
|
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
|
24
17
|
end
|
|
25
18
|
response = http.get(uri.request_uri, 'User-Agent' => @user_agent) # request_uri=path + '?' + query
|
|
26
|
-
|
|
19
|
+
create_response(response)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
private
|
|
23
|
+
def create_response(response)
|
|
24
|
+
body = response.body
|
|
25
|
+
unless @binary
|
|
26
|
+
# デフォルトでは ASCII-8BITが帰ってくる。
|
|
27
|
+
# Content-Typeのcharsetとみなす。
|
|
28
|
+
# https://bugs.ruby-lang.org/issues/2567
|
|
29
|
+
encoding = response.type_params['charset']
|
|
30
|
+
body = ::WebLoader::Utils.toutf8(body, encoding)
|
|
31
|
+
end
|
|
32
|
+
::WebLoader::Response.new(
|
|
33
|
+
status: response.code.to_i,
|
|
34
|
+
headers: response.each_header.to_h,
|
|
35
|
+
body: body
|
|
36
|
+
)
|
|
27
37
|
end
|
|
28
38
|
end
|
|
29
39
|
end
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
module WebLoader
|
|
2
|
+
module Drivers
|
|
3
|
+
class SeleniumDriver < ::WebLoader::Drivers::BaseDriver
|
|
4
|
+
|
|
5
|
+
def initialize
|
|
6
|
+
super
|
|
7
|
+
@wait_proc = nil
|
|
8
|
+
@wait_seconds = 3
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
attr_accessor :wait_proc
|
|
12
|
+
attr_accessor :wait_seconds
|
|
13
|
+
|
|
14
|
+
def fetch(url)
|
|
15
|
+
require 'selenium-webdriver'
|
|
16
|
+
|
|
17
|
+
# puts "SeleniumDriver fetching URL: #{url}"
|
|
18
|
+
|
|
19
|
+
options = Selenium::WebDriver::Chrome::Options.new
|
|
20
|
+
options.add_argument("--headless")
|
|
21
|
+
options.add_argument("--disable-gpu")
|
|
22
|
+
options.add_argument("--no-sandbox")
|
|
23
|
+
options.add_argument("--user-agent=#{@user_agent}") if @user_agent
|
|
24
|
+
|
|
25
|
+
begin
|
|
26
|
+
|
|
27
|
+
driver = Selenium::WebDriver.for(:chrome, options: options)
|
|
28
|
+
driver.navigate.to url
|
|
29
|
+
|
|
30
|
+
wait = Selenium::WebDriver::Wait.new(timeout: 10)
|
|
31
|
+
|
|
32
|
+
# ページ全体のロード完了を待つ
|
|
33
|
+
if @wait_proc
|
|
34
|
+
wait.until { @wait_proc.call(driver) }
|
|
35
|
+
else
|
|
36
|
+
sleep @wait_seconds
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
content_type = driver.execute_script("return document.contentType;")
|
|
40
|
+
|
|
41
|
+
body = @binary ? page_source.b : driver.page_source
|
|
42
|
+
response = WebLoader::Response.new(status: 200,
|
|
43
|
+
headers: {
|
|
44
|
+
'Content-Type' => content_type
|
|
45
|
+
}, body: body)
|
|
46
|
+
response
|
|
47
|
+
ensure
|
|
48
|
+
driver.quit if driver
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
private
|
|
53
|
+
def create_response(body)
|
|
54
|
+
# デフォルトは成功200
|
|
55
|
+
status = 200
|
|
56
|
+
# redirected = driver.current_url != original_url
|
|
57
|
+
# status = 300 if redirected # 簡易的にリダイレクト扱い
|
|
58
|
+
new(status: status, headers: {}, body: body)
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
data/lib/web_loader/response.rb
CHANGED
|
@@ -2,28 +2,9 @@ module WebLoader
|
|
|
2
2
|
class Response
|
|
3
3
|
include WebLoader::Utils
|
|
4
4
|
|
|
5
|
-
def self.from_net_http(response, binary)
|
|
6
|
-
body = response.body
|
|
7
|
-
unless binary
|
|
8
|
-
# デフォルトでは ASCII-8BITが帰ってくる。
|
|
9
|
-
# Content-Typeのcharsetとみなす。
|
|
10
|
-
# https://bugs.ruby-lang.org/issues/2567
|
|
11
|
-
encoding = response.type_params['charset']
|
|
12
|
-
body = ::WebLoader::Utils.toutf8(body, encoding)
|
|
13
|
-
end
|
|
14
|
-
new(
|
|
15
|
-
status: response.code.to_i,
|
|
16
|
-
headers: response.each_header.to_h,
|
|
17
|
-
body: body
|
|
18
|
-
)
|
|
19
|
-
end
|
|
20
5
|
|
|
21
6
|
def self.from_selenium(driver, original_url)
|
|
22
|
-
|
|
23
|
-
status = 200
|
|
24
|
-
# redirected = driver.current_url != original_url
|
|
25
|
-
# status = 300 if redirected # 簡易的にリダイレクト扱い
|
|
26
|
-
new(status: status, headers: {}, body: driver.page_source)
|
|
7
|
+
|
|
27
8
|
end
|
|
28
9
|
|
|
29
10
|
def initialize(status:, headers: {}, body: nil)
|
data/lib/web_loader/version.rb
CHANGED
data/lib/web_loader.rb
CHANGED
|
@@ -4,8 +4,11 @@ require_relative "web_loader/version"
|
|
|
4
4
|
require_relative "web_loader/utils"
|
|
5
5
|
require_relative "web_loader/cache"
|
|
6
6
|
require_relative "web_loader/command"
|
|
7
|
+
require_relative "web_loader/downloader"
|
|
7
8
|
require_relative "web_loader/response"
|
|
9
|
+
require_relative "web_loader/drivers/base_driver"
|
|
8
10
|
require_relative "web_loader/drivers/http_driver"
|
|
11
|
+
require_relative "web_loader/drivers/selenium_driver"
|
|
9
12
|
|
|
10
13
|
module WebLoader
|
|
11
14
|
class Error < StandardError; end
|
data/test_wl.sh
ADDED
data/web_loader.iml
CHANGED
|
@@ -11,7 +11,69 @@
|
|
|
11
11
|
<orderEntry type="jdk" jdkName="rbenv: 3.2.6" jdkType="RUBY_SDK" />
|
|
12
12
|
<orderEntry type="sourceFolder" forTests="false" />
|
|
13
13
|
<orderEntry type="module-library">
|
|
14
|
-
<library name="
|
|
14
|
+
<library name="base64 (v0.3.0) [path][gem]" type="rubylib">
|
|
15
|
+
<properties>
|
|
16
|
+
<option name="additionalInfo">
|
|
17
|
+
<AdditionalInfo>
|
|
18
|
+
<option name="authors" value="該当なし" />
|
|
19
|
+
<option name="email" value="該当なし" />
|
|
20
|
+
<option name="homepage" value="該当なし" />
|
|
21
|
+
<option name="summary" value="該当なし" />
|
|
22
|
+
</AdditionalInfo>
|
|
23
|
+
</option>
|
|
24
|
+
<option name="fromPath" value="true" />
|
|
25
|
+
<option name="name" value="base64" />
|
|
26
|
+
<option name="requirePaths">
|
|
27
|
+
<list>
|
|
28
|
+
<option value="lib" />
|
|
29
|
+
</list>
|
|
30
|
+
</option>
|
|
31
|
+
<option name="url" value="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/base64-0.3.0" />
|
|
32
|
+
<option name="version" value="0.3.0" />
|
|
33
|
+
</properties>
|
|
34
|
+
<CLASSES>
|
|
35
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/base64-0.3.0/lib" />
|
|
36
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/base64-0.3.0/sig" />
|
|
37
|
+
</CLASSES>
|
|
38
|
+
<JAVADOC />
|
|
39
|
+
<SOURCES>
|
|
40
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/base64-0.3.0/lib" />
|
|
41
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/base64-0.3.0/sig" />
|
|
42
|
+
</SOURCES>
|
|
43
|
+
</library>
|
|
44
|
+
</orderEntry>
|
|
45
|
+
<orderEntry type="module-library">
|
|
46
|
+
<library name="logger (v1.7.0) [path][gem]" type="rubylib">
|
|
47
|
+
<properties>
|
|
48
|
+
<option name="additionalInfo">
|
|
49
|
+
<AdditionalInfo>
|
|
50
|
+
<option name="authors" value="該当なし" />
|
|
51
|
+
<option name="email" value="該当なし" />
|
|
52
|
+
<option name="homepage" value="該当なし" />
|
|
53
|
+
<option name="summary" value="該当なし" />
|
|
54
|
+
</AdditionalInfo>
|
|
55
|
+
</option>
|
|
56
|
+
<option name="fromPath" value="true" />
|
|
57
|
+
<option name="name" value="logger" />
|
|
58
|
+
<option name="requirePaths">
|
|
59
|
+
<list>
|
|
60
|
+
<option value="lib" />
|
|
61
|
+
</list>
|
|
62
|
+
</option>
|
|
63
|
+
<option name="url" value="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/logger-1.7.0" />
|
|
64
|
+
<option name="version" value="1.7.0" />
|
|
65
|
+
</properties>
|
|
66
|
+
<CLASSES>
|
|
67
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/logger-1.7.0/lib" />
|
|
68
|
+
</CLASSES>
|
|
69
|
+
<JAVADOC />
|
|
70
|
+
<SOURCES>
|
|
71
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/logger-1.7.0/lib" />
|
|
72
|
+
</SOURCES>
|
|
73
|
+
</library>
|
|
74
|
+
</orderEntry>
|
|
75
|
+
<orderEntry type="module-library">
|
|
76
|
+
<library name="minitest (v5.26.2) [path][gem]" type="rubylib">
|
|
15
77
|
<properties>
|
|
16
78
|
<option name="additionalInfo">
|
|
17
79
|
<AdditionalInfo>
|
|
@@ -28,20 +90,20 @@
|
|
|
28
90
|
<option value="lib" />
|
|
29
91
|
</list>
|
|
30
92
|
</option>
|
|
31
|
-
<option name="url" value="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/minitest-5.26.
|
|
32
|
-
<option name="version" value="5.26.
|
|
93
|
+
<option name="url" value="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/minitest-5.26.2" />
|
|
94
|
+
<option name="version" value="5.26.2" />
|
|
33
95
|
</properties>
|
|
34
96
|
<CLASSES>
|
|
35
|
-
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/minitest-5.26.
|
|
36
|
-
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/minitest-5.26.
|
|
97
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/minitest-5.26.2/lib" />
|
|
98
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/minitest-5.26.2/test" />
|
|
37
99
|
</CLASSES>
|
|
38
100
|
<JAVADOC />
|
|
39
101
|
<SOURCES>
|
|
40
|
-
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/minitest-5.26.
|
|
41
|
-
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/minitest-5.26.
|
|
102
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/minitest-5.26.2/lib" />
|
|
103
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/minitest-5.26.2/test" />
|
|
42
104
|
</SOURCES>
|
|
43
105
|
<excluded>
|
|
44
|
-
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/minitest-5.26.
|
|
106
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/minitest-5.26.2/test" />
|
|
45
107
|
</excluded>
|
|
46
108
|
</library>
|
|
47
109
|
</orderEntry>
|
|
@@ -83,6 +145,149 @@
|
|
|
83
145
|
</excluded>
|
|
84
146
|
</library>
|
|
85
147
|
</orderEntry>
|
|
148
|
+
<orderEntry type="module-library">
|
|
149
|
+
<library name="rexml (v3.4.4) [path][gem]" type="rubylib">
|
|
150
|
+
<properties>
|
|
151
|
+
<option name="additionalInfo">
|
|
152
|
+
<AdditionalInfo>
|
|
153
|
+
<option name="authors" value="該当なし" />
|
|
154
|
+
<option name="email" value="該当なし" />
|
|
155
|
+
<option name="homepage" value="該当なし" />
|
|
156
|
+
<option name="summary" value="該当なし" />
|
|
157
|
+
</AdditionalInfo>
|
|
158
|
+
</option>
|
|
159
|
+
<option name="fromPath" value="true" />
|
|
160
|
+
<option name="name" value="rexml" />
|
|
161
|
+
<option name="requirePaths">
|
|
162
|
+
<list>
|
|
163
|
+
<option value="lib" />
|
|
164
|
+
</list>
|
|
165
|
+
</option>
|
|
166
|
+
<option name="url" value="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rexml-3.4.4" />
|
|
167
|
+
<option name="version" value="3.4.4" />
|
|
168
|
+
</properties>
|
|
169
|
+
<CLASSES>
|
|
170
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rexml-3.4.4/doc" />
|
|
171
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rexml-3.4.4/lib" />
|
|
172
|
+
</CLASSES>
|
|
173
|
+
<JAVADOC />
|
|
174
|
+
<SOURCES>
|
|
175
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rexml-3.4.4/doc" />
|
|
176
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rexml-3.4.4/lib" />
|
|
177
|
+
</SOURCES>
|
|
178
|
+
<excluded>
|
|
179
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rexml-3.4.4/doc" />
|
|
180
|
+
</excluded>
|
|
181
|
+
</library>
|
|
182
|
+
</orderEntry>
|
|
183
|
+
<orderEntry type="module-library">
|
|
184
|
+
<library name="rubyzip (v3.2.2) [path][gem]" type="rubylib">
|
|
185
|
+
<properties>
|
|
186
|
+
<option name="additionalInfo">
|
|
187
|
+
<AdditionalInfo>
|
|
188
|
+
<option name="authors" value="該当なし" />
|
|
189
|
+
<option name="email" value="該当なし" />
|
|
190
|
+
<option name="homepage" value="該当なし" />
|
|
191
|
+
<option name="summary" value="該当なし" />
|
|
192
|
+
</AdditionalInfo>
|
|
193
|
+
</option>
|
|
194
|
+
<option name="fromPath" value="true" />
|
|
195
|
+
<option name="name" value="rubyzip" />
|
|
196
|
+
<option name="requirePaths">
|
|
197
|
+
<list>
|
|
198
|
+
<option value="lib" />
|
|
199
|
+
</list>
|
|
200
|
+
</option>
|
|
201
|
+
<option name="url" value="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rubyzip-3.2.2" />
|
|
202
|
+
<option name="version" value="3.2.2" />
|
|
203
|
+
</properties>
|
|
204
|
+
<CLASSES>
|
|
205
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rubyzip-3.2.2/lib" />
|
|
206
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rubyzip-3.2.2/samples" />
|
|
207
|
+
</CLASSES>
|
|
208
|
+
<JAVADOC />
|
|
209
|
+
<SOURCES>
|
|
210
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rubyzip-3.2.2/lib" />
|
|
211
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rubyzip-3.2.2/samples" />
|
|
212
|
+
</SOURCES>
|
|
213
|
+
<excluded>
|
|
214
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/rubyzip-3.2.2/samples" />
|
|
215
|
+
</excluded>
|
|
216
|
+
</library>
|
|
217
|
+
</orderEntry>
|
|
218
|
+
<orderEntry type="module-library">
|
|
219
|
+
<library name="selenium-webdriver (v4.38.0) [path][gem]" type="rubylib">
|
|
220
|
+
<properties>
|
|
221
|
+
<option name="additionalInfo">
|
|
222
|
+
<AdditionalInfo>
|
|
223
|
+
<option name="authors" value="該当なし" />
|
|
224
|
+
<option name="email" value="該当なし" />
|
|
225
|
+
<option name="homepage" value="該当なし" />
|
|
226
|
+
<option name="summary" value="該当なし" />
|
|
227
|
+
</AdditionalInfo>
|
|
228
|
+
</option>
|
|
229
|
+
<option name="fromPath" value="true" />
|
|
230
|
+
<option name="name" value="selenium-webdriver" />
|
|
231
|
+
<option name="requirePaths">
|
|
232
|
+
<list>
|
|
233
|
+
<option value="lib" />
|
|
234
|
+
</list>
|
|
235
|
+
</option>
|
|
236
|
+
<option name="url" value="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/selenium-webdriver-4.38.0" />
|
|
237
|
+
<option name="version" value="4.38.0" />
|
|
238
|
+
</properties>
|
|
239
|
+
<CLASSES>
|
|
240
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/selenium-webdriver-4.38.0/bin" />
|
|
241
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/selenium-webdriver-4.38.0/lib" />
|
|
242
|
+
</CLASSES>
|
|
243
|
+
<JAVADOC />
|
|
244
|
+
<SOURCES>
|
|
245
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/selenium-webdriver-4.38.0/bin" />
|
|
246
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/selenium-webdriver-4.38.0/lib" />
|
|
247
|
+
</SOURCES>
|
|
248
|
+
<excluded>
|
|
249
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/selenium-webdriver-4.38.0/bin" />
|
|
250
|
+
</excluded>
|
|
251
|
+
</library>
|
|
252
|
+
</orderEntry>
|
|
253
|
+
<orderEntry type="module-library">
|
|
254
|
+
<library name="websocket (v1.2.11) [path][gem]" type="rubylib">
|
|
255
|
+
<properties>
|
|
256
|
+
<option name="additionalInfo">
|
|
257
|
+
<AdditionalInfo>
|
|
258
|
+
<option name="authors" value="該当なし" />
|
|
259
|
+
<option name="email" value="該当なし" />
|
|
260
|
+
<option name="homepage" value="該当なし" />
|
|
261
|
+
<option name="summary" value="該当なし" />
|
|
262
|
+
</AdditionalInfo>
|
|
263
|
+
</option>
|
|
264
|
+
<option name="fromPath" value="true" />
|
|
265
|
+
<option name="name" value="websocket" />
|
|
266
|
+
<option name="requirePaths">
|
|
267
|
+
<list>
|
|
268
|
+
<option value="lib" />
|
|
269
|
+
</list>
|
|
270
|
+
</option>
|
|
271
|
+
<option name="url" value="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/websocket-1.2.11" />
|
|
272
|
+
<option name="version" value="1.2.11" />
|
|
273
|
+
</properties>
|
|
274
|
+
<CLASSES>
|
|
275
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/websocket-1.2.11/lib" />
|
|
276
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/websocket-1.2.11/spec" />
|
|
277
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/websocket-1.2.11/.github" />
|
|
278
|
+
</CLASSES>
|
|
279
|
+
<JAVADOC />
|
|
280
|
+
<SOURCES>
|
|
281
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/websocket-1.2.11/lib" />
|
|
282
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/websocket-1.2.11/spec" />
|
|
283
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/websocket-1.2.11/.github" />
|
|
284
|
+
</SOURCES>
|
|
285
|
+
<excluded>
|
|
286
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/websocket-1.2.11/spec" />
|
|
287
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.2.0/gems/websocket-1.2.11/.github" />
|
|
288
|
+
</excluded>
|
|
289
|
+
</library>
|
|
290
|
+
</orderEntry>
|
|
86
291
|
<orderEntry type="library" scope="PROVIDED" name="bundler (v2.6.3, rbenv: 3.2.6) [gem]" level="application" />
|
|
87
292
|
</component>
|
|
88
293
|
<component name="RakeTasksCache">
|
metadata
CHANGED
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: web_loader
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.
|
|
4
|
+
version: 2.2.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- src
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2025-11-
|
|
11
|
+
date: 2025-11-28 00:00:00.000000000 Z
|
|
12
12
|
dependencies: []
|
|
13
13
|
description: Web loader.
|
|
14
14
|
email:
|
|
15
15
|
- src@srcw.net
|
|
16
16
|
executables:
|
|
17
|
-
-
|
|
17
|
+
- wl
|
|
18
18
|
extensions: []
|
|
19
19
|
extra_rdoc_files: []
|
|
20
20
|
files:
|
|
@@ -35,16 +35,20 @@ files:
|
|
|
35
35
|
- README.md
|
|
36
36
|
- Rakefile
|
|
37
37
|
- build.sh
|
|
38
|
-
- exe/
|
|
38
|
+
- exe/wl
|
|
39
39
|
- lib/web_loader.rb
|
|
40
40
|
- lib/web_loader/cache.rb
|
|
41
41
|
- lib/web_loader/command.rb
|
|
42
|
+
- lib/web_loader/downloader.rb
|
|
43
|
+
- lib/web_loader/drivers/base_driver.rb
|
|
42
44
|
- lib/web_loader/drivers/http_driver.rb
|
|
45
|
+
- lib/web_loader/drivers/selenium_driver.rb
|
|
43
46
|
- lib/web_loader/response.rb
|
|
44
47
|
- lib/web_loader/utils.rb
|
|
45
48
|
- lib/web_loader/version.rb
|
|
46
49
|
- sig/web_loader.rbs
|
|
47
50
|
- test.sh
|
|
51
|
+
- test_wl.sh
|
|
48
52
|
- web_loader.iml
|
|
49
53
|
homepage: https://srcw.net
|
|
50
54
|
licenses:
|
data/exe/web_loader
DELETED