webrobots 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/dependabot.yml +14 -0
- data/.github/workflows/actions.lock +38 -0
- data/.github/workflows/ci.yml +89 -0
- data/Gemfile +3 -1
- data/LICENSE.txt +1 -1
- data/README.rdoc +12 -1
- data/Rakefile +1 -0
- data/lib/webrobots/version.rb +1 -1
- data/lib/webrobots.rb +10 -5
- data/test/helper.rb +8 -1
- data/test/simplecov_start.rb +3 -0
- data/test/test_webrobots.rb +87 -26
- data/test/vcr_cassettes/major_sites.yml +2006 -0
- data/webrobots.gemspec +7 -3
- metadata +96 -33
- data/.travis.yml +0 -15
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 53157954049eb7dc8acdff6d7c6c8a7c60f45add22581eacf063c2dc18d11fd8
|
|
4
|
+
data.tar.gz: 81496d4a807cccd9fd43d87fe3e374b1e5985064612355f8ddafe9ef85ac8bdf
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 96d3d17a236c342bc1e77fd458c4383ec1bb2584b0b63e968bebcb94484201e41a679f41601ddae5178f799c9c16756dff17e4fadbf6b43c66dc541f92ef22f6
|
|
7
|
+
data.tar.gz: 9736bc2a4c90b843c24e23c6ac71a9e98490ba39d275572d1465613565848e2b4d6a8bcb48cac1a3968ec34f57b55fa90617dd1a11969e07c38937ecc20fa878
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# This file is machine-generated by `gh actions-lock`.
|
|
2
|
+
# Do not edit by hand; run `gh actions-lock` to update.
|
|
3
|
+
# Docs: https://gh.io/actions-lockfile
|
|
4
|
+
version: 'v0.0.3'
|
|
5
|
+
workflows:
|
|
6
|
+
'.github/workflows/ci.yml':
|
|
7
|
+
- 'actions/checkout@v7.0.1'
|
|
8
|
+
- 'ruby/setup-ruby@v1.338.0'
|
|
9
|
+
- 'rubygems/release-gem@v1.4.1'
|
|
10
|
+
- 'step-security/harden-runner@v2.22.4'
|
|
11
|
+
dependencies:
|
|
12
|
+
'actions/checkout@v7.0.1':
|
|
13
|
+
ref: 'v7.0.1'
|
|
14
|
+
commit: 'sha1-3d3c42e5aac5ba805825da76410c181273ba90b1'
|
|
15
|
+
owner_id: 44036562
|
|
16
|
+
repo_id: 197814629
|
|
17
|
+
'ruby/setup-ruby@v1.338.0':
|
|
18
|
+
ref: 'v1.338.0'
|
|
19
|
+
commit: 'sha1-b3aaceaef7e0ee231b499e7b5f49c03fc1d32199'
|
|
20
|
+
owner_id: 210414
|
|
21
|
+
repo_id: 231208785
|
|
22
|
+
'rubygems/configure-rubygems-credentials@dc5a8d8553e6ee01fc26761a49e99e733d17954a':
|
|
23
|
+
ref: 'v2.1.0'
|
|
24
|
+
commit: 'sha1-dc5a8d8553e6ee01fc26761a49e99e733d17954a'
|
|
25
|
+
owner_id: 208761
|
|
26
|
+
repo_id: 631633510
|
|
27
|
+
'rubygems/release-gem@v1.4.1':
|
|
28
|
+
ref: 'v1.4.1'
|
|
29
|
+
commit: 'sha1-7f9650160c1a4e7989fdc9855807bdbd421d8b6b'
|
|
30
|
+
owner_id: 208761
|
|
31
|
+
repo_id: 731435765
|
|
32
|
+
uses:
|
|
33
|
+
- 'rubygems/configure-rubygems-credentials@dc5a8d8553e6ee01fc26761a49e99e733d17954a'
|
|
34
|
+
'step-security/harden-runner@v2.22.4':
|
|
35
|
+
ref: 'v2.22.4'
|
|
36
|
+
commit: 'sha1-1d88384fd173af6ecb012dfd0264a306b711241a'
|
|
37
|
+
owner_id: 88700172
|
|
38
|
+
repo_id: 422287306
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [master]
|
|
6
|
+
tags: ["v*"]
|
|
7
|
+
pull_request:
|
|
8
|
+
|
|
9
|
+
permissions:
|
|
10
|
+
contents: read
|
|
11
|
+
|
|
12
|
+
concurrency:
|
|
13
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
|
14
|
+
cancel-in-progress: true
|
|
15
|
+
|
|
16
|
+
jobs:
|
|
17
|
+
test:
|
|
18
|
+
name: Ruby ${{ matrix.ruby }}
|
|
19
|
+
runs-on: ubuntu-latest
|
|
20
|
+
timeout-minutes: 15
|
|
21
|
+
continue-on-error: ${{ matrix.ruby == 'ruby-head' }}
|
|
22
|
+
strategy:
|
|
23
|
+
fail-fast: false
|
|
24
|
+
matrix:
|
|
25
|
+
ruby: ["3.3", "3.4", "4.0", ruby-head]
|
|
26
|
+
steps:
|
|
27
|
+
- name: Harden runner
|
|
28
|
+
uses: step-security/harden-runner@v2.22.4
|
|
29
|
+
with:
|
|
30
|
+
egress-policy: audit
|
|
31
|
+
- uses: actions/checkout@v7.0.1
|
|
32
|
+
with:
|
|
33
|
+
persist-credentials: false
|
|
34
|
+
- name: Verify action dependencies
|
|
35
|
+
if: matrix.ruby == '4.0'
|
|
36
|
+
env:
|
|
37
|
+
GH_TOKEN: ${{ github.token }}
|
|
38
|
+
run: |
|
|
39
|
+
gh extension install github/gh-actions-lock --pin v0.4.1
|
|
40
|
+
gh actions-lock --verify
|
|
41
|
+
- uses: ruby/setup-ruby@v1.338.0
|
|
42
|
+
with:
|
|
43
|
+
ruby-version: ${{ matrix.ruby }}
|
|
44
|
+
bundler: latest
|
|
45
|
+
- run: bundle install
|
|
46
|
+
- run: bundle exec rake test
|
|
47
|
+
|
|
48
|
+
publish:
|
|
49
|
+
name: Publish gem
|
|
50
|
+
needs: test
|
|
51
|
+
if: github.repository == 'knu/webrobots' && startsWith(github.ref, 'refs/tags/v')
|
|
52
|
+
runs-on: ubuntu-latest
|
|
53
|
+
timeout-minutes: 15
|
|
54
|
+
environment:
|
|
55
|
+
name: rubygems.org
|
|
56
|
+
url: https://rubygems.org/gems/webrobots/versions/${{ steps.gem.outputs.version }}
|
|
57
|
+
permissions:
|
|
58
|
+
contents: write
|
|
59
|
+
id-token: write
|
|
60
|
+
steps:
|
|
61
|
+
- name: Harden runner
|
|
62
|
+
uses: step-security/harden-runner@v2.22.4
|
|
63
|
+
with:
|
|
64
|
+
egress-policy: audit
|
|
65
|
+
- uses: actions/checkout@v7.0.1
|
|
66
|
+
with:
|
|
67
|
+
persist-credentials: false
|
|
68
|
+
- uses: ruby/setup-ruby@v1.338.0
|
|
69
|
+
with:
|
|
70
|
+
ruby-version: "4.0"
|
|
71
|
+
bundler: latest
|
|
72
|
+
- run: bundle install
|
|
73
|
+
- name: Check release version
|
|
74
|
+
id: gem
|
|
75
|
+
env:
|
|
76
|
+
RELEASE_TAG: ${{ github.ref_name }}
|
|
77
|
+
run: |
|
|
78
|
+
bundle exec ruby -e '
|
|
79
|
+
require "webrobots/version"
|
|
80
|
+
version = Webrobots::VERSION
|
|
81
|
+
abort "tag does not match gem version" unless ENV.fetch("RELEASE_TAG") == "v#{version}"
|
|
82
|
+
File.open(ENV.fetch("GITHUB_OUTPUT"), "a") { |f| f.puts "version=#{version}" }
|
|
83
|
+
'
|
|
84
|
+
- name: Build gem
|
|
85
|
+
run: bundle exec rake build
|
|
86
|
+
- name: Publish to RubyGems
|
|
87
|
+
uses: rubygems/release-gem@v1.4.1
|
|
88
|
+
with:
|
|
89
|
+
await-release: false
|
data/Gemfile
CHANGED
data/LICENSE.txt
CHANGED
data/README.rdoc
CHANGED
|
@@ -22,6 +22,8 @@ This is a library to help write robots.txt compliant web robots.
|
|
|
22
22
|
|
|
23
23
|
- Ruby 1.8.7 or 1.9.2+
|
|
24
24
|
|
|
25
|
+
Development and testing require Ruby 3.3 or later.
|
|
26
|
+
|
|
25
27
|
== Contributing to webrobots
|
|
26
28
|
|
|
27
29
|
* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
|
|
@@ -32,7 +34,16 @@ This is a library to help write robots.txt compliant web robots.
|
|
|
32
34
|
* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
|
|
33
35
|
* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
|
|
34
36
|
|
|
37
|
+
== Releasing
|
|
38
|
+
|
|
39
|
+
Update the version and push the release commit to master. After CI passes,
|
|
40
|
+
push the matching vVERSION tag. The CI workflow tests the tag and publishes
|
|
41
|
+
the gem to RubyGems.org through Trusted Publishing.
|
|
42
|
+
|
|
43
|
+
The trusted publisher uses repository knu/webrobots, workflow ci.yml, and
|
|
44
|
+
GitHub environment rubygems.org.
|
|
45
|
+
|
|
35
46
|
== Copyright
|
|
36
47
|
|
|
37
|
-
Copyright (c) 2010
|
|
48
|
+
Copyright (c) 2010-2016 Akinori MUSHA. See LICENSE.txt for
|
|
38
49
|
further details.
|
data/Rakefile
CHANGED
|
@@ -7,6 +7,7 @@ gemspec = Bundler::GemHelper.gemspec
|
|
|
7
7
|
require 'rake/testtask'
|
|
8
8
|
Rake::TestTask.new(:test) do |test|
|
|
9
9
|
test.libs << 'test'
|
|
10
|
+
test.ruby_opts << '-r./test/simplecov_start.rb' unless RUBY_VERSION < '1.9' || (RUBY_PLATFORM == 'java' && ENV['TRAVIS'])
|
|
10
11
|
test.test_files = gemspec.test_files
|
|
11
12
|
test.verbose = true
|
|
12
13
|
end
|
data/lib/webrobots/version.rb
CHANGED
data/lib/webrobots.rb
CHANGED
|
@@ -171,8 +171,9 @@ class WebRobots
|
|
|
171
171
|
end
|
|
172
172
|
|
|
173
173
|
def http_get(uri)
|
|
174
|
+
response = nil
|
|
174
175
|
referer = nil
|
|
175
|
-
|
|
176
|
+
5.times {
|
|
176
177
|
http = Net::HTTP.new(uri.host, uri.port)
|
|
177
178
|
if http.use_ssl = uri.is_a?(URI::HTTPS)
|
|
178
179
|
http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
|
@@ -190,13 +191,17 @@ class WebRobots
|
|
|
190
191
|
when Net::HTTPRedirection
|
|
191
192
|
referer = uri.to_s
|
|
192
193
|
uri = URI(response['location'])
|
|
193
|
-
when Net::
|
|
194
|
+
when Net::HTTPClientError
|
|
194
195
|
return ''
|
|
195
|
-
else
|
|
196
|
-
response.value
|
|
197
196
|
end
|
|
198
197
|
}
|
|
199
|
-
|
|
198
|
+
case response
|
|
199
|
+
when Net::HTTPRedirection
|
|
200
|
+
# Treat too many redirections as not found
|
|
201
|
+
''
|
|
202
|
+
else
|
|
203
|
+
raise "#{response.code} #{response.message}"
|
|
204
|
+
end
|
|
200
205
|
end
|
|
201
206
|
|
|
202
207
|
def crawl_delay_handler(delay, last_checked_at)
|
data/test/helper.rb
CHANGED
|
@@ -8,7 +8,14 @@ rescue Bundler::BundlerError => e
|
|
|
8
8
|
exit e.status_code
|
|
9
9
|
end
|
|
10
10
|
require 'test/unit'
|
|
11
|
-
require 'shoulda'
|
|
11
|
+
require 'shoulda-context'
|
|
12
|
+
require 'webmock/test_unit'
|
|
13
|
+
require 'vcr'
|
|
14
|
+
|
|
15
|
+
VCR.configure do |c|
|
|
16
|
+
c.cassette_library_dir = 'test/vcr_cassettes'
|
|
17
|
+
c.hook_into :webmock
|
|
18
|
+
end
|
|
12
19
|
|
|
13
20
|
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
|
14
21
|
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
data/test/test_webrobots.rb
CHANGED
|
@@ -31,7 +31,7 @@ class TestWebRobots < Test::Unit::TestCase
|
|
|
31
31
|
})
|
|
32
32
|
end
|
|
33
33
|
|
|
34
|
-
should "
|
|
34
|
+
should "be treated as full allow" do
|
|
35
35
|
assert @robots.allowed?('http://site1.example.org/index.html')
|
|
36
36
|
assert @robots.allowed?('http://site1.example.org/private/secret.txt')
|
|
37
37
|
assert @robots.allowed?('http://site2.example.org/index.html')
|
|
@@ -43,7 +43,69 @@ class TestWebRobots < Test::Unit::TestCase
|
|
|
43
43
|
end
|
|
44
44
|
end
|
|
45
45
|
|
|
46
|
-
context "
|
|
46
|
+
context "the default http_get proc" do
|
|
47
|
+
setup do
|
|
48
|
+
@robots = WebRobots.new('SomeBot')
|
|
49
|
+
@site_uri = URI('http://example.com/')
|
|
50
|
+
@txt_uri = @site_uri + '/robots.txt'
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
should "treat 201 as success" do
|
|
54
|
+
stub_request(:get, @txt_uri).to_return(:status => 201, :body => '# Just created!')
|
|
55
|
+
assert_equal '# Just created!', @robots.__send__(:http_get, @txt_uri)
|
|
56
|
+
assert_requested :get, @txt_uri
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
should "treat redirects more than 5 times as full allow" do
|
|
60
|
+
redirect_uri = @site_uri + '/redirect'
|
|
61
|
+
stub_request(:get, /\A#{Regexp.quote(@site_uri.to_s)}/).to_return(:status => 301, :headers => { 'Location' => redirect_uri.to_s })
|
|
62
|
+
assert_equal '', @robots.__send__(:http_get, @txt_uri)
|
|
63
|
+
assert_requested :get, @txt_uri, :times => 1
|
|
64
|
+
assert_requested :get, redirect_uri, :times => 4
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
should "treat 400 as full allow" do
|
|
68
|
+
stub_request(:get, @txt_uri).to_return(:status => 400, :body => 'Bad Request!')
|
|
69
|
+
assert_equal '', @robots.__send__(:http_get, @txt_uri)
|
|
70
|
+
assert_requested :get, @txt_uri
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
should "treat 401 as full allow" do
|
|
74
|
+
stub_request(:get, @txt_uri).to_return(:status => 401, :body => 'Unauthorized!')
|
|
75
|
+
assert_equal '', @robots.__send__(:http_get, @txt_uri)
|
|
76
|
+
assert_requested :get, @txt_uri
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
should "treat 403 as full allow" do
|
|
80
|
+
stub_request(:get, @txt_uri).to_return(:status => 403, :body => 'Forbidden!')
|
|
81
|
+
assert_equal '', @robots.__send__(:http_get, @txt_uri)
|
|
82
|
+
assert_requested :get, @txt_uri
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
should "treat 404 as full allow" do
|
|
86
|
+
stub_request(:get, @txt_uri).to_return(:status => 404, :body => 'Not Found!')
|
|
87
|
+
assert_equal '', @robots.__send__(:http_get, @txt_uri)
|
|
88
|
+
assert_requested :get, @txt_uri
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
should "treat 500 as error after retrying 5 times" do
|
|
92
|
+
stub_request(:get, @txt_uri).to_return(:status => 500, :body => 'Internal Server Error!')
|
|
93
|
+
assert_raise {
|
|
94
|
+
@robots.__send__(:http_get, @txt_uri)
|
|
95
|
+
}
|
|
96
|
+
assert_requested :get, @txt_uri, :times => 5
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
should "treat 503 as error after retrying 5 times" do
|
|
100
|
+
stub_request(:get, @txt_uri).to_return(:status => 503, :body => 'Service Unavailable!')
|
|
101
|
+
assert_raise {
|
|
102
|
+
@robots.__send__(:http_get, @txt_uri)
|
|
103
|
+
}
|
|
104
|
+
assert_requested :get, @txt_uri, :times => 5
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
context "nil or error from a custom http_get proc" do
|
|
47
109
|
setup do
|
|
48
110
|
@robots = WebRobots.new('RandomBot', :http_get => lambda { |uri|
|
|
49
111
|
case uri.to_s
|
|
@@ -52,9 +114,8 @@ class TestWebRobots < Test::Unit::TestCase
|
|
|
52
114
|
'Internal Server Error',
|
|
53
115
|
Net::HTTPInternalServerError.new('1.1', '500', 'Internal Server Error'))
|
|
54
116
|
when 'http://site2.example.org/robots.txt'
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
Net::HTTPFound.new('1.1', '302', 'Found'))
|
|
117
|
+
require 'timeout'
|
|
118
|
+
Timeout.timeout(0) {}
|
|
58
119
|
when 'http://site3.example.org/robots.txt'
|
|
59
120
|
raise Errno::ECONNREFUSED
|
|
60
121
|
when 'http://site4.example.org/robots.txt'
|
|
@@ -67,7 +128,7 @@ class TestWebRobots < Test::Unit::TestCase
|
|
|
67
128
|
})
|
|
68
129
|
end
|
|
69
130
|
|
|
70
|
-
should "
|
|
131
|
+
should "be treated as full disallow" do
|
|
71
132
|
assert @robots.disallowed?('http://site1.example.org/index.html')
|
|
72
133
|
assert @robots.disallowed?('http://site1.example.org/private/secret.txt')
|
|
73
134
|
assert @robots.disallowed?('http://site2.example.org/index.html')
|
|
@@ -412,7 +473,7 @@ Option3: Hi
|
|
|
412
473
|
@robots_hisbot = WebRobots.new('HisBot', :http_get => http_get)
|
|
413
474
|
end
|
|
414
475
|
|
|
415
|
-
should "read
|
|
476
|
+
should "be properly read" do
|
|
416
477
|
options = @robots_mybot.options('http://www.example.org/')
|
|
417
478
|
assert_equal 2, options.size
|
|
418
479
|
assert_equal 'Foo', @robots_mybot.option('http://www.example.org/', 'Option1')
|
|
@@ -531,28 +592,28 @@ Disallow: /
|
|
|
531
592
|
context "robots.txt in the real world" do
|
|
532
593
|
setup do
|
|
533
594
|
@testbot = WebRobots.new('TestBot')
|
|
534
|
-
@
|
|
595
|
+
@baidubot = WebRobots.new('TestBaiduspider') # matches Baiduspider
|
|
535
596
|
end
|
|
536
597
|
|
|
537
598
|
should "be parsed for major sites" do
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
599
|
+
VCR.use_cassette("major_sites") do
|
|
600
|
+
assert_nothing_raised {
|
|
601
|
+
assert !@testbot.allowed?("https://www.google.com/search")
|
|
602
|
+
assert !@testbot.allowed?("https://www.google.com/catalogs")
|
|
603
|
+
assert @testbot.allowed?("https://www.google.com/catalogs/about")
|
|
604
|
+
}
|
|
605
|
+
assert_nothing_raised {
|
|
606
|
+
assert @testbot.allowed?("https://www.yahoo.com/")
|
|
607
|
+
assert !@testbot.allowed?("https://www.yahoo.com/p/foo")
|
|
608
|
+
}
|
|
609
|
+
assert_nothing_raised {
|
|
610
|
+
assert @testbot.allowed?("http://www.apple.com/jp/foo")
|
|
611
|
+
assert !@baidubot.allowed?("http://www.apple.com/jp/foo")
|
|
612
|
+
}
|
|
613
|
+
assert_nothing_raised {
|
|
614
|
+
assert !@testbot.allowed?("https://github.com/login")
|
|
615
|
+
}
|
|
616
|
+
end
|
|
556
617
|
end
|
|
557
618
|
end
|
|
558
619
|
|