webrobots 0.1.1 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 897c27b67d9ebd1b6e4ab138725222489b1e7c27
4
- data.tar.gz: 23ea1e0155c11932841724b3379d8baf3d372e75
2
+ SHA256:
3
+ metadata.gz: 53157954049eb7dc8acdff6d7c6c8a7c60f45add22581eacf063c2dc18d11fd8
4
+ data.tar.gz: 81496d4a807cccd9fd43d87fe3e374b1e5985064612355f8ddafe9ef85ac8bdf
5
5
  SHA512:
6
- metadata.gz: 62b5bc7ece0fcef5d3c11f3c3305657c3862a63ea75e49fa57bc0b8dd0c4de0efbe24201124a74ab066f819776da9b35f434607b305b94175b31f80d0d4043cc
7
- data.tar.gz: 9a032d7d46e1b2b365a0ca7536d8e8cd2b777a617c5274b99b59afcaae429da6fa9f0df8c02468aeb71189a38c1d671020f985d45df3a6c60332d6a4af92ce70
6
+ metadata.gz: 96d3d17a236c342bc1e77fd458c4383ec1bb2584b0b63e968bebcb94484201e41a679f41601ddae5178f799c9c16756dff17e4fadbf6b43c66dc541f92ef22f6
7
+ data.tar.gz: 9736bc2a4c90b843c24e23c6ac71a9e98490ba39d275572d1465613565848e2b4d6a8bcb48cac1a3968ec34f57b55fa90617dd1a11969e07c38937ecc20fa878
@@ -0,0 +1,14 @@
1
+ version: 2
2
+ updates:
3
+ - package-ecosystem: github-actions
4
+ directory: /
5
+ schedule:
6
+ interval: weekly
7
+ day: friday
8
+ timezone: Asia/Tokyo
9
+ cooldown:
10
+ default-days: 7
11
+ groups:
12
+ github-actions:
13
+ patterns:
14
+ - "*"
@@ -0,0 +1,38 @@
1
+ # This file is machine-generated by `gh actions-lock`.
2
+ # Do not edit by hand; run `gh actions-lock` to update.
3
+ # Docs: https://gh.io/actions-lockfile
4
+ version: 'v0.0.3'
5
+ workflows:
6
+ '.github/workflows/ci.yml':
7
+ - 'actions/checkout@v7.0.1'
8
+ - 'ruby/setup-ruby@v1.338.0'
9
+ - 'rubygems/release-gem@v1.4.1'
10
+ - 'step-security/harden-runner@v2.22.4'
11
+ dependencies:
12
+ 'actions/checkout@v7.0.1':
13
+ ref: 'v7.0.1'
14
+ commit: 'sha1-3d3c42e5aac5ba805825da76410c181273ba90b1'
15
+ owner_id: 44036562
16
+ repo_id: 197814629
17
+ 'ruby/setup-ruby@v1.338.0':
18
+ ref: 'v1.338.0'
19
+ commit: 'sha1-b3aaceaef7e0ee231b499e7b5f49c03fc1d32199'
20
+ owner_id: 210414
21
+ repo_id: 231208785
22
+ 'rubygems/configure-rubygems-credentials@dc5a8d8553e6ee01fc26761a49e99e733d17954a':
23
+ ref: 'v2.1.0'
24
+ commit: 'sha1-dc5a8d8553e6ee01fc26761a49e99e733d17954a'
25
+ owner_id: 208761
26
+ repo_id: 631633510
27
+ 'rubygems/release-gem@v1.4.1':
28
+ ref: 'v1.4.1'
29
+ commit: 'sha1-7f9650160c1a4e7989fdc9855807bdbd421d8b6b'
30
+ owner_id: 208761
31
+ repo_id: 731435765
32
+ uses:
33
+ - 'rubygems/configure-rubygems-credentials@dc5a8d8553e6ee01fc26761a49e99e733d17954a'
34
+ 'step-security/harden-runner@v2.22.4':
35
+ ref: 'v2.22.4'
36
+ commit: 'sha1-1d88384fd173af6ecb012dfd0264a306b711241a'
37
+ owner_id: 88700172
38
+ repo_id: 422287306
@@ -0,0 +1,89 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [master]
6
+ tags: ["v*"]
7
+ pull_request:
8
+
9
+ permissions:
10
+ contents: read
11
+
12
+ concurrency:
13
+ group: ${{ github.workflow }}-${{ github.ref }}
14
+ cancel-in-progress: true
15
+
16
+ jobs:
17
+ test:
18
+ name: Ruby ${{ matrix.ruby }}
19
+ runs-on: ubuntu-latest
20
+ timeout-minutes: 15
21
+ continue-on-error: ${{ matrix.ruby == 'ruby-head' }}
22
+ strategy:
23
+ fail-fast: false
24
+ matrix:
25
+ ruby: ["3.3", "3.4", "4.0", ruby-head]
26
+ steps:
27
+ - name: Harden runner
28
+ uses: step-security/harden-runner@v2.22.4
29
+ with:
30
+ egress-policy: audit
31
+ - uses: actions/checkout@v7.0.1
32
+ with:
33
+ persist-credentials: false
34
+ - name: Verify action dependencies
35
+ if: matrix.ruby == '4.0'
36
+ env:
37
+ GH_TOKEN: ${{ github.token }}
38
+ run: |
39
+ gh extension install github/gh-actions-lock --pin v0.4.1
40
+ gh actions-lock --verify
41
+ - uses: ruby/setup-ruby@v1.338.0
42
+ with:
43
+ ruby-version: ${{ matrix.ruby }}
44
+ bundler: latest
45
+ - run: bundle install
46
+ - run: bundle exec rake test
47
+
48
+ publish:
49
+ name: Publish gem
50
+ needs: test
51
+ if: github.repository == 'knu/webrobots' && startsWith(github.ref, 'refs/tags/v')
52
+ runs-on: ubuntu-latest
53
+ timeout-minutes: 15
54
+ environment:
55
+ name: rubygems.org
56
+ url: https://rubygems.org/gems/webrobots/versions/${{ steps.gem.outputs.version }}
57
+ permissions:
58
+ contents: write
59
+ id-token: write
60
+ steps:
61
+ - name: Harden runner
62
+ uses: step-security/harden-runner@v2.22.4
63
+ with:
64
+ egress-policy: audit
65
+ - uses: actions/checkout@v7.0.1
66
+ with:
67
+ persist-credentials: false
68
+ - uses: ruby/setup-ruby@v1.338.0
69
+ with:
70
+ ruby-version: "4.0"
71
+ bundler: latest
72
+ - run: bundle install
73
+ - name: Check release version
74
+ id: gem
75
+ env:
76
+ RELEASE_TAG: ${{ github.ref_name }}
77
+ run: |
78
+ bundle exec ruby -e '
79
+ require "webrobots/version"
80
+ version = Webrobots::VERSION
81
+ abort "tag does not match gem version" unless ENV.fetch("RELEASE_TAG") == "v#{version}"
82
+ File.open(ENV.fetch("GITHUB_OUTPUT"), "a") { |f| f.puts "version=#{version}" }
83
+ '
84
+ - name: Build gem
85
+ run: bundle exec rake build
86
+ - name: Publish to RubyGems
87
+ uses: rubygems/release-gem@v1.4.1
88
+ with:
89
+ await-release: false
data/Gemfile CHANGED
@@ -1,4 +1,6 @@
1
- source "http://rubygems.org"
1
+ source "https://rubygems.org", cooldown: 3
2
+
3
+ ruby ">= 3.3"
2
4
 
3
5
  # Specify your gem's dependencies in webrobots.gemspec
4
6
  gemspec
data/LICENSE.txt CHANGED
@@ -1,4 +1,4 @@
1
- Copyright (c) 2010, 2011, 2012, 2013 Akinori MUSHA
1
+ Copyright (c) 2010-2016 Akinori MUSHA
2
2
 
3
3
  All rights reserved.
4
4
 
data/README.rdoc CHANGED
@@ -22,6 +22,8 @@ This is a library to help write robots.txt compliant web robots.
22
22
 
23
23
  - Ruby 1.8.7 or 1.9.2+
24
24
 
25
+ Development and testing require Ruby 3.3 or later.
26
+
25
27
  == Contributing to webrobots
26
28
 
27
29
  * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
@@ -32,7 +34,16 @@ This is a library to help write robots.txt compliant web robots.
32
34
  * Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
33
35
  * Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
34
36
 
37
+ == Releasing
38
+
39
+ Update the version and push the release commit to master. After CI passes,
40
+ push the matching vVERSION tag. The CI workflow tests the tag and publishes
41
+ the gem to RubyGems.org through Trusted Publishing.
42
+
43
+ The trusted publisher uses repository knu/webrobots, workflow ci.yml, and
44
+ GitHub environment rubygems.org.
45
+
35
46
  == Copyright
36
47
 
37
- Copyright (c) 2010, 2011, 2012, 2013 Akinori MUSHA. See LICENSE.txt for
48
+ Copyright (c) 2010-2016 Akinori MUSHA. See LICENSE.txt for
38
49
  further details.
data/Rakefile CHANGED
@@ -7,6 +7,7 @@ gemspec = Bundler::GemHelper.gemspec
7
7
  require 'rake/testtask'
8
8
  Rake::TestTask.new(:test) do |test|
9
9
  test.libs << 'test'
10
+ test.ruby_opts << '-r./test/simplecov_start.rb' unless RUBY_VERSION < '1.9' || (RUBY_PLATFORM == 'java' && ENV['TRAVIS'])
10
11
  test.test_files = gemspec.test_files
11
12
  test.verbose = true
12
13
  end
@@ -1,3 +1,3 @@
1
1
  module Webrobots
2
- VERSION = "0.1.1"
2
+ VERSION = "0.1.3"
3
3
  end
data/lib/webrobots.rb CHANGED
@@ -171,8 +171,9 @@ class WebRobots
171
171
  end
172
172
 
173
173
  def http_get(uri)
174
+ response = nil
174
175
  referer = nil
175
- 10.times {
176
+ 5.times {
176
177
  http = Net::HTTP.new(uri.host, uri.port)
177
178
  if http.use_ssl = uri.is_a?(URI::HTTPS)
178
179
  http.verify_mode = OpenSSL::SSL::VERIFY_PEER
@@ -190,13 +191,17 @@ class WebRobots
190
191
  when Net::HTTPRedirection
191
192
  referer = uri.to_s
192
193
  uri = URI(response['location'])
193
- when Net::HTTPNotFound
194
+ when Net::HTTPClientError
194
195
  return ''
195
- else
196
- response.value
197
196
  end
198
197
  }
199
- raise 'too many HTTP redirects'
198
+ case response
199
+ when Net::HTTPRedirection
200
+ # Treat too many redirections as not found
201
+ ''
202
+ else
203
+ raise "#{response.code} #{response.message}"
204
+ end
200
205
  end
201
206
 
202
207
  def crawl_delay_handler(delay, last_checked_at)
data/test/helper.rb CHANGED
@@ -8,7 +8,14 @@ rescue Bundler::BundlerError => e
8
8
  exit e.status_code
9
9
  end
10
10
  require 'test/unit'
11
- require 'shoulda'
11
+ require 'shoulda-context'
12
+ require 'webmock/test_unit'
13
+ require 'vcr'
14
+
15
+ VCR.configure do |c|
16
+ c.cassette_library_dir = 'test/vcr_cassettes'
17
+ c.hook_into :webmock
18
+ end
12
19
 
13
20
  $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
14
21
  $LOAD_PATH.unshift(File.dirname(__FILE__))
@@ -0,0 +1,3 @@
1
+ require 'simplecov'
2
+
3
+ SimpleCov.start
@@ -31,7 +31,7 @@ class TestWebRobots < Test::Unit::TestCase
31
31
  })
32
32
  end
33
33
 
34
- should "allow any robot" do
34
+ should "be treated as full allow" do
35
35
  assert @robots.allowed?('http://site1.example.org/index.html')
36
36
  assert @robots.allowed?('http://site1.example.org/private/secret.txt')
37
37
  assert @robots.allowed?('http://site2.example.org/index.html')
@@ -43,7 +43,69 @@ class TestWebRobots < Test::Unit::TestCase
43
43
  end
44
44
  end
45
45
 
46
- context "robots.txt that cannot be fetched" do
46
+ context "the default http_get proc" do
47
+ setup do
48
+ @robots = WebRobots.new('SomeBot')
49
+ @site_uri = URI('http://example.com/')
50
+ @txt_uri = @site_uri + '/robots.txt'
51
+ end
52
+
53
+ should "treat 201 as success" do
54
+ stub_request(:get, @txt_uri).to_return(:status => 201, :body => '# Just created!')
55
+ assert_equal '# Just created!', @robots.__send__(:http_get, @txt_uri)
56
+ assert_requested :get, @txt_uri
57
+ end
58
+
59
+ should "treat redirects more than 5 times as full allow" do
60
+ redirect_uri = @site_uri + '/redirect'
61
+ stub_request(:get, /\A#{Regexp.quote(@site_uri.to_s)}/).to_return(:status => 301, :headers => { 'Location' => redirect_uri.to_s })
62
+ assert_equal '', @robots.__send__(:http_get, @txt_uri)
63
+ assert_requested :get, @txt_uri, :times => 1
64
+ assert_requested :get, redirect_uri, :times => 4
65
+ end
66
+
67
+ should "treat 400 as full allow" do
68
+ stub_request(:get, @txt_uri).to_return(:status => 400, :body => 'Bad Request!')
69
+ assert_equal '', @robots.__send__(:http_get, @txt_uri)
70
+ assert_requested :get, @txt_uri
71
+ end
72
+
73
+ should "treat 401 as full allow" do
74
+ stub_request(:get, @txt_uri).to_return(:status => 401, :body => 'Unauthorized!')
75
+ assert_equal '', @robots.__send__(:http_get, @txt_uri)
76
+ assert_requested :get, @txt_uri
77
+ end
78
+
79
+ should "treat 403 as full allow" do
80
+ stub_request(:get, @txt_uri).to_return(:status => 403, :body => 'Forbidden!')
81
+ assert_equal '', @robots.__send__(:http_get, @txt_uri)
82
+ assert_requested :get, @txt_uri
83
+ end
84
+
85
+ should "treat 404 as full allow" do
86
+ stub_request(:get, @txt_uri).to_return(:status => 404, :body => 'Not Found!')
87
+ assert_equal '', @robots.__send__(:http_get, @txt_uri)
88
+ assert_requested :get, @txt_uri
89
+ end
90
+
91
+ should "treat 500 as error after retrying 5 times" do
92
+ stub_request(:get, @txt_uri).to_return(:status => 500, :body => 'Internal Server Error!')
93
+ assert_raise {
94
+ @robots.__send__(:http_get, @txt_uri)
95
+ }
96
+ assert_requested :get, @txt_uri, :times => 5
97
+ end
98
+
99
+ should "treat 503 as error after retrying 5 times" do
100
+ stub_request(:get, @txt_uri).to_return(:status => 503, :body => 'Service Unavailable!')
101
+ assert_raise {
102
+ @robots.__send__(:http_get, @txt_uri)
103
+ }
104
+ assert_requested :get, @txt_uri, :times => 5
105
+ end
106
+ end
107
+
108
+ context "nil or error from a custom http_get proc" do
47
109
  setup do
48
110
  @robots = WebRobots.new('RandomBot', :http_get => lambda { |uri|
49
111
  case uri.to_s
@@ -52,9 +114,8 @@ class TestWebRobots < Test::Unit::TestCase
52
114
  'Internal Server Error',
53
115
  Net::HTTPInternalServerError.new('1.1', '500', 'Internal Server Error'))
54
116
  when 'http://site2.example.org/robots.txt'
55
- raise Net::HTTPRetriableError.new(
56
- 'Found',
57
- Net::HTTPFound.new('1.1', '302', 'Found'))
117
+ require 'timeout'
118
+ Timeout.timeout(0) {}
58
119
  when 'http://site3.example.org/robots.txt'
59
120
  raise Errno::ECONNREFUSED
60
121
  when 'http://site4.example.org/robots.txt'
@@ -67,7 +128,7 @@ class TestWebRobots < Test::Unit::TestCase
67
128
  })
68
129
  end
69
130
 
70
- should "disallow any robot" do
131
+ should "be treated as full disallow" do
71
132
  assert @robots.disallowed?('http://site1.example.org/index.html')
72
133
  assert @robots.disallowed?('http://site1.example.org/private/secret.txt')
73
134
  assert @robots.disallowed?('http://site2.example.org/index.html')
@@ -412,7 +473,7 @@ Option3: Hi
412
473
  @robots_hisbot = WebRobots.new('HisBot', :http_get => http_get)
413
474
  end
414
475
 
415
- should "read options" do
476
+ should "be properly read" do
416
477
  options = @robots_mybot.options('http://www.example.org/')
417
478
  assert_equal 2, options.size
418
479
  assert_equal 'Foo', @robots_mybot.option('http://www.example.org/', 'Option1')
@@ -531,28 +592,28 @@ Disallow: /
531
592
  context "robots.txt in the real world" do
532
593
  setup do
533
594
  @testbot = WebRobots.new('TestBot')
534
- @msnbot = WebRobots.new('TestMSNBot') # matches msnbot
595
+ @baidubot = WebRobots.new('TestBaiduspider') # matches Baiduspider
535
596
  end
536
597
 
537
598
  should "be parsed for major sites" do
538
- assert_nothing_raised {
539
- assert !@testbot.allowed?("http://www.google.com/search")
540
- assert !@testbot.allowed?("https://www.google.com/search")
541
- assert !@testbot.allowed?("http://www.google.com/news/section?pz=1&cf=all&ned=jp&topic=y&ict=ln")
542
- assert @testbot.allowed?("http://www.google.com/news/directory?pz=1&cf=all&ned=us&hl=en&sort=users&category=6")
543
- }
544
- assert_nothing_raised {
545
- assert @testbot.allowed?("http://www.yahoo.com/")
546
- assert !@testbot.allowed?("http://www.yahoo.com/?")
547
- assert !@testbot.allowed?("http://www.yahoo.com/p/foo")
548
- }
549
- assert_nothing_raised {
550
- assert !@testbot.allowed?("http://store.apple.com/vieworder")
551
- assert @msnbot.allowed?("http://store.apple.com/vieworder")
552
- }
553
- assert_nothing_raised {
554
- assert !@testbot.allowed?("http://github.com/login")
555
- }
599
+ VCR.use_cassette("major_sites") do
600
+ assert_nothing_raised {
601
+ assert !@testbot.allowed?("https://www.google.com/search")
602
+ assert !@testbot.allowed?("https://www.google.com/catalogs")
603
+ assert @testbot.allowed?("https://www.google.com/catalogs/about")
604
+ }
605
+ assert_nothing_raised {
606
+ assert @testbot.allowed?("https://www.yahoo.com/")
607
+ assert !@testbot.allowed?("https://www.yahoo.com/p/foo")
608
+ }
609
+ assert_nothing_raised {
610
+ assert @testbot.allowed?("http://www.apple.com/jp/foo")
611
+ assert !@baidubot.allowed?("http://www.apple.com/jp/foo")
612
+ }
613
+ assert_nothing_raised {
614
+ assert !@testbot.allowed?("https://github.com/login")
615
+ }
616
+ end
556
617
  end
557
618
  end
558
619