mechanize 2.14.0 → 2.14.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +27 -1
- data/README.md +7 -0
- data/SECURITY.md +11 -0
- data/lib/mechanize/directory_saver.rb +2 -1
- data/lib/mechanize/file_saver.rb +2 -0
- data/lib/mechanize/http/agent.rb +83 -29
- data/lib/mechanize/parser.rb +33 -3
- data/lib/mechanize/test_case.rb +2 -9
- data/lib/mechanize/version.rb +1 -1
- data/lib/mechanize.rb +12 -2
- data/test/htdocs/frame_test.html +8 -6
- data/test/htdocs/relative/tc_relative_links.html +1 -1
- metadata +7 -174
- data/.autotest +0 -6
- data/.github/dependabot.yml +0 -11
- data/.github/workflows/ci.yml +0 -43
- data/.github/workflows/upstream.yml +0 -51
- data/.gitignore +0 -15
- data/.yardopts +0 -8
- data/Gemfile +0 -12
- data/Rakefile +0 -56
- data/examples/flickr_upload.rb +0 -22
- data/examples/latest_user_agents.rb +0 -100
- data/examples/mech-dump.rb +0 -5
- data/examples/proxy_req.rb +0 -7
- data/examples/rubygems.rb +0 -23
- data/examples/spider.rb +0 -22
- data/examples/wikipedia_links_to_philosophy.rb +0 -158
- data/mechanize.gemspec +0 -74
- data/test/data/htpasswd +0 -1
- data/test/data/server.crt +0 -16
- data/test/data/server.csr +0 -12
- data/test/data/server.key +0 -15
- data/test/data/server.pem +0 -15
- data/test/test_mechanize.rb +0 -1382
- data/test/test_mechanize_cookie.rb +0 -559
- data/test/test_mechanize_cookie_jar.rb +0 -615
- data/test/test_mechanize_directory_saver.rb +0 -60
- data/test/test_mechanize_download.rb +0 -102
- data/test/test_mechanize_element_not_found_error.rb +0 -16
- data/test/test_mechanize_file.rb +0 -118
- data/test/test_mechanize_file_connection.rb +0 -40
- data/test/test_mechanize_file_request.rb +0 -26
- data/test/test_mechanize_file_response.rb +0 -48
- data/test/test_mechanize_file_saver.rb +0 -22
- data/test/test_mechanize_form.rb +0 -1025
- data/test/test_mechanize_form_check_box.rb +0 -49
- data/test/test_mechanize_form_encoding.rb +0 -110
- data/test/test_mechanize_form_field.rb +0 -78
- data/test/test_mechanize_form_file_upload.rb +0 -21
- data/test/test_mechanize_form_image_button.rb +0 -13
- data/test/test_mechanize_form_keygen.rb +0 -34
- data/test/test_mechanize_form_multi_select_list.rb +0 -98
- data/test/test_mechanize_form_option.rb +0 -60
- data/test/test_mechanize_form_radio_button.rb +0 -96
- data/test/test_mechanize_form_select_list.rb +0 -81
- data/test/test_mechanize_form_textarea.rb +0 -53
- data/test/test_mechanize_headers.rb +0 -36
- data/test/test_mechanize_history.rb +0 -104
- data/test/test_mechanize_http_agent.rb +0 -1915
- data/test/test_mechanize_http_auth_challenge.rb +0 -64
- data/test/test_mechanize_http_auth_realm.rb +0 -56
- data/test/test_mechanize_http_auth_store.rb +0 -233
- data/test/test_mechanize_http_content_disposition_parser.rb +0 -167
- data/test/test_mechanize_http_www_authenticate_parser.rb +0 -204
- data/test/test_mechanize_image.rb +0 -9
- data/test/test_mechanize_link.rb +0 -171
- data/test/test_mechanize_page.rb +0 -295
- data/test/test_mechanize_page_encoding.rb +0 -215
- data/test/test_mechanize_page_frame.rb +0 -17
- data/test/test_mechanize_page_image.rb +0 -191
- data/test/test_mechanize_page_link.rb +0 -377
- data/test/test_mechanize_page_meta_refresh.rb +0 -151
- data/test/test_mechanize_parser.rb +0 -316
- data/test/test_mechanize_pluggable_parser.rb +0 -68
- data/test/test_mechanize_redirect_limit_reached_error.rb +0 -25
- data/test/test_mechanize_redirect_not_get_or_head_error.rb +0 -15
- data/test/test_mechanize_response_read_error.rb +0 -29
- data/test/test_mechanize_subclass.rb +0 -23
- data/test/test_mechanize_util.rb +0 -144
- data/test/test_mechanize_xml_file.rb +0 -30
- data/test/test_multi_select.rb +0 -120
data/.github/dependabot.yml
DELETED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
# To get started with Dependabot version updates, you'll need to specify which
|
|
2
|
-
# package ecosystems to update and where the package manifests are located.
|
|
3
|
-
# Please see the documentation for all configuration options:
|
|
4
|
-
# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
|
|
5
|
-
|
|
6
|
-
version: 2
|
|
7
|
-
updates:
|
|
8
|
-
- package-ecosystem: "bundler"
|
|
9
|
-
directory: "/"
|
|
10
|
-
schedule:
|
|
11
|
-
interval: "weekly"
|
data/.github/workflows/ci.yml
DELETED
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
name: "ci"
|
|
2
|
-
|
|
3
|
-
concurrency:
|
|
4
|
-
group: "${{github.workflow}}-${{github.ref}}"
|
|
5
|
-
cancel-in-progress: true
|
|
6
|
-
|
|
7
|
-
on:
|
|
8
|
-
workflow_dispatch:
|
|
9
|
-
schedule:
|
|
10
|
-
- cron: "0 8 * * 5" # At 08:00 on Friday # https://crontab.guru/#0_8_*_*_5
|
|
11
|
-
push:
|
|
12
|
-
branches:
|
|
13
|
-
- main
|
|
14
|
-
pull_request:
|
|
15
|
-
types: [opened, synchronize]
|
|
16
|
-
branches:
|
|
17
|
-
- main
|
|
18
|
-
|
|
19
|
-
jobs:
|
|
20
|
-
rubocop:
|
|
21
|
-
runs-on: ubuntu-latest
|
|
22
|
-
steps:
|
|
23
|
-
- uses: actions/checkout@v4
|
|
24
|
-
- uses: ruby/setup-ruby@v1
|
|
25
|
-
with:
|
|
26
|
-
ruby-version: "3.3"
|
|
27
|
-
bundler-cache: true
|
|
28
|
-
- run: bundle exec rake rubocop
|
|
29
|
-
|
|
30
|
-
test:
|
|
31
|
-
needs: ["rubocop"]
|
|
32
|
-
strategy:
|
|
33
|
-
fail-fast: false
|
|
34
|
-
matrix:
|
|
35
|
-
ruby-version: ["2.6", "2.7", "3.0", "3.1", "3.2", "3.3", "3.4", "head"]
|
|
36
|
-
runs-on: ubuntu-latest
|
|
37
|
-
steps:
|
|
38
|
-
- uses: actions/checkout@v4
|
|
39
|
-
- uses: ruby/setup-ruby@v1
|
|
40
|
-
with:
|
|
41
|
-
ruby-version: ${{matrix.ruby-version}}
|
|
42
|
-
bundler-cache: true
|
|
43
|
-
- run: bundle exec rake test
|
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
name: "upstream"
|
|
2
|
-
|
|
3
|
-
concurrency:
|
|
4
|
-
group: "${{github.workflow}}-${{github.ref}}"
|
|
5
|
-
cancel-in-progress: true
|
|
6
|
-
|
|
7
|
-
on:
|
|
8
|
-
workflow_dispatch:
|
|
9
|
-
schedule:
|
|
10
|
-
- cron: "0 8 * * 5" # At 08:00 on Friday # https://crontab.guru/#0_8_*_*_5
|
|
11
|
-
push:
|
|
12
|
-
branches:
|
|
13
|
-
- main
|
|
14
|
-
pull_request:
|
|
15
|
-
types: [opened, synchronize]
|
|
16
|
-
branches:
|
|
17
|
-
- main
|
|
18
|
-
paths:
|
|
19
|
-
- .github/workflows/upstream.yml # this file
|
|
20
|
-
|
|
21
|
-
jobs:
|
|
22
|
-
test:
|
|
23
|
-
strategy:
|
|
24
|
-
fail-fast: false
|
|
25
|
-
matrix:
|
|
26
|
-
ruby-version: ["head"]
|
|
27
|
-
runs-on: ubuntu-latest
|
|
28
|
-
steps:
|
|
29
|
-
- uses: actions/checkout@v4
|
|
30
|
-
- uses: ruby/setup-ruby@v1
|
|
31
|
-
with:
|
|
32
|
-
ruby-version: ${{matrix.ruby-version}}
|
|
33
|
-
bundler-cache: true
|
|
34
|
-
- run: bundle exec rake test
|
|
35
|
-
|
|
36
|
-
upstream:
|
|
37
|
-
name: "upstream (${{matrix.name}})"
|
|
38
|
-
strategy:
|
|
39
|
-
fail-fast: false
|
|
40
|
-
matrix:
|
|
41
|
-
include:
|
|
42
|
-
- { name: "nokogiri", git: "https://github.com/sparklemotion/nokogiri" }
|
|
43
|
-
runs-on: ubuntu-latest
|
|
44
|
-
steps:
|
|
45
|
-
- uses: actions/checkout@v4
|
|
46
|
-
- uses: ruby/setup-ruby@v1
|
|
47
|
-
with: { ruby-version: "3.4" }
|
|
48
|
-
- run: |
|
|
49
|
-
bundle add ${{matrix.name}} --git="${{matrix.git}}"
|
|
50
|
-
bundle show
|
|
51
|
-
- run: bundle exec rake test
|
data/.gitignore
DELETED
data/.yardopts
DELETED
data/Gemfile
DELETED
data/Rakefile
DELETED
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
require 'rubygems'
|
|
2
|
-
|
|
3
|
-
begin
|
|
4
|
-
require "bundler/gem_tasks"
|
|
5
|
-
rescue LoadError
|
|
6
|
-
end
|
|
7
|
-
|
|
8
|
-
require 'rdoc/task'
|
|
9
|
-
require 'rake/testtask'
|
|
10
|
-
|
|
11
|
-
task :prerelease => [:clobber_rdoc, :test]
|
|
12
|
-
|
|
13
|
-
desc "Update SSL Certificate"
|
|
14
|
-
task('ssl_cert') do |p|
|
|
15
|
-
sh "openssl genrsa -des3 -out server.key 1024"
|
|
16
|
-
sh "openssl req -new -key server.key -out server.csr"
|
|
17
|
-
sh "cp server.key server.key.org"
|
|
18
|
-
sh "openssl rsa -in server.key.org -out server.key"
|
|
19
|
-
sh "openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt"
|
|
20
|
-
sh "cp server.key server.pem"
|
|
21
|
-
sh "mv server.key server.csr server.crt server.pem test/data/"
|
|
22
|
-
sh "rm server.key.org"
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
RDoc::Task.new do |rdoc|
|
|
26
|
-
rdoc.main = "README.md"
|
|
27
|
-
rdoc.rdoc_dir = 'doc'
|
|
28
|
-
rdoc.rdoc_files.include( "CHANGELOG.md", "EXAMPLES.rdoc", "GUIDE.rdoc", "LICENSE.txt", "README.md", "lib/**/*.rb")
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
desc "Run tests"
|
|
32
|
-
Rake::TestTask.new { |t|
|
|
33
|
-
t.test_files = Dir['test/**/test*.rb']
|
|
34
|
-
t.verbose = true
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
task publish_docs: %w[rdoc] do
|
|
38
|
-
sh 'rsync', '-avzO', '--delete', 'doc/', 'docs-push.seattlerb.org:/data/www/docs.seattlerb.org/mechanize/'
|
|
39
|
-
end
|
|
40
|
-
|
|
41
|
-
desc "Run rubocop checks"
|
|
42
|
-
task :rubocop => ["rubocop:security", "rubocop:frozen_string_literals"]
|
|
43
|
-
|
|
44
|
-
namespace "rubocop" do
|
|
45
|
-
desc "Run rubocop security check"
|
|
46
|
-
task :security do
|
|
47
|
-
sh "rubocop lib --only Security"
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
desc "Run rubocop string literals check"
|
|
51
|
-
task :frozen_string_literals do
|
|
52
|
-
sh "rubocop lib test --auto-correct-all --only Style/FrozenStringLiteralComment"
|
|
53
|
-
end
|
|
54
|
-
end
|
|
55
|
-
|
|
56
|
-
task default: [:rubocop, :test]
|
data/examples/flickr_upload.rb
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
require 'rubygems'
|
|
2
|
-
require 'mechanize'
|
|
3
|
-
|
|
4
|
-
agent = Mechanize.new
|
|
5
|
-
|
|
6
|
-
# Get the flickr sign in page
|
|
7
|
-
page = agent.get 'http://flickr.com/signin/flickr/'
|
|
8
|
-
|
|
9
|
-
# Fill out the login form
|
|
10
|
-
form = page.form_with :name => 'flickrloginform'
|
|
11
|
-
form.email = ARGV[0]
|
|
12
|
-
form.password = ARGV[1]
|
|
13
|
-
form.submit
|
|
14
|
-
|
|
15
|
-
# Go to the upload page
|
|
16
|
-
page = page.link_with(:text => 'Upload').click
|
|
17
|
-
|
|
18
|
-
# Fill out the form
|
|
19
|
-
form = page.forms.action('/photos_upload_process.gne').first
|
|
20
|
-
form.file_uploads.name('file1').first.file_name = ARGV[2]
|
|
21
|
-
form.submit
|
|
22
|
-
|
|
@@ -1,100 +0,0 @@
|
|
|
1
|
-
require 'mechanize'
|
|
2
|
-
require 'ostruct'
|
|
3
|
-
|
|
4
|
-
# LatestUAFetcher fetches latest user agents from `WhatIsMyBrowser.com`.
|
|
5
|
-
# It can use to update `Mechanize::AGENT_ALIASES`.
|
|
6
|
-
class LatestUAFetcher
|
|
7
|
-
attr_reader :user_agents
|
|
8
|
-
|
|
9
|
-
USER_AGENT_TYPES = OpenStruct.new(
|
|
10
|
-
linux_firefox: "Linux Firefox",
|
|
11
|
-
mac_firefox: "Mac Firefox",
|
|
12
|
-
mac_safari: "Mac Safari",
|
|
13
|
-
windows_chrome: "Windows Chrome",
|
|
14
|
-
windows_edge: "Windows Edge",
|
|
15
|
-
windows_firefox: "Windows Firefox",
|
|
16
|
-
android: "Android",
|
|
17
|
-
iphone: "iPhone",
|
|
18
|
-
ipad: "iPad",
|
|
19
|
-
)
|
|
20
|
-
|
|
21
|
-
BASE_URL = 'https://www.whatismybrowser.com/guides/the-latest-user-agent'
|
|
22
|
-
|
|
23
|
-
def initialize
|
|
24
|
-
@agent = Mechanize.new.tap { |a| a.user_agent_alias = 'Mac Firefox' }
|
|
25
|
-
@user_agents = {}
|
|
26
|
-
end
|
|
27
|
-
|
|
28
|
-
def run
|
|
29
|
-
return unless user_agents.empty?
|
|
30
|
-
|
|
31
|
-
sleep_time = 1
|
|
32
|
-
|
|
33
|
-
fetch_user_agents('chrome')
|
|
34
|
-
fetch_user_agents('firefox')
|
|
35
|
-
fetch_user_agents('safari')
|
|
36
|
-
fetch_user_agents('edge')
|
|
37
|
-
end
|
|
38
|
-
|
|
39
|
-
def ordered_user_agents
|
|
40
|
-
USER_AGENT_TYPES.to_h.values.each_with_object({}) do |type, ordered_user_agents|
|
|
41
|
-
ordered_user_agents[type] = user_agents[type]
|
|
42
|
-
end
|
|
43
|
-
end
|
|
44
|
-
|
|
45
|
-
private
|
|
46
|
-
|
|
47
|
-
def fetch_user_agents(browser_name, sleep_time = 1)
|
|
48
|
-
puts "fetch #{browser_name} UA..."
|
|
49
|
-
send(browser_name)
|
|
50
|
-
puts "sleeping... (#{sleep_time}s)"
|
|
51
|
-
sleep sleep_time
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
def edge
|
|
55
|
-
page = @agent.get("#{BASE_URL}/edge")
|
|
56
|
-
|
|
57
|
-
windows_dom = page.css("h2:contains('Latest Edge on Windows User Agents')")
|
|
58
|
-
|
|
59
|
-
@user_agents[USER_AGENT_TYPES.windows_edge] = windows_dom.css('+ .listing-of-useragents .code').first.text
|
|
60
|
-
end
|
|
61
|
-
|
|
62
|
-
def firefox
|
|
63
|
-
page = @agent.get("#{BASE_URL}/firefox")
|
|
64
|
-
|
|
65
|
-
desktop_dom = page.css("h2:contains('Latest Firefox on Desktop User Agents')")
|
|
66
|
-
table_dom = desktop_dom.css('+ .listing-of-useragents')
|
|
67
|
-
|
|
68
|
-
@user_agents[USER_AGENT_TYPES.linux_firefox] = table_dom.css('td:contains("Linux")').css("+ td .code:contains('Ubuntu; Linux x86_64')").text
|
|
69
|
-
@user_agents[USER_AGENT_TYPES.windows_firefox] = table_dom.css('td:contains("Windows")').css('+ td .code').text
|
|
70
|
-
@user_agents[USER_AGENT_TYPES.mac_firefox] = table_dom.css('td:contains("Macos")').css('+ td .code').text
|
|
71
|
-
end
|
|
72
|
-
|
|
73
|
-
def safari
|
|
74
|
-
page = @agent.get("#{BASE_URL}/safari")
|
|
75
|
-
|
|
76
|
-
macos_dom = page.css("h2:contains('Latest Safari on macOS User Agents')")
|
|
77
|
-
ios_dom = page.css("h2:contains('Latest Safari on iOS User Agents')")
|
|
78
|
-
|
|
79
|
-
@user_agents[USER_AGENT_TYPES.mac_safari] = macos_dom.css('+ .listing-of-useragents .code').first.text
|
|
80
|
-
@user_agents[USER_AGENT_TYPES.iphone] = ios_dom.css('+ .listing-of-useragents').css("tr:contains('Iphone') .code").text
|
|
81
|
-
@user_agents[USER_AGENT_TYPES.ipad] = ios_dom.css('+ .listing-of-useragents').css("tr:contains('Ipad') .code").text
|
|
82
|
-
end
|
|
83
|
-
|
|
84
|
-
def chrome
|
|
85
|
-
page = @agent.get("#{BASE_URL}/chrome")
|
|
86
|
-
|
|
87
|
-
windows_dom = page.css("h2:contains('Latest Chrome on Windows 10 User Agents')")
|
|
88
|
-
android_dom = page.css("h2:contains('Latest Chrome on Android User Agents')")
|
|
89
|
-
|
|
90
|
-
@user_agents[USER_AGENT_TYPES.windows_chrome] = windows_dom.css('+ .listing-of-useragents .code').first.text
|
|
91
|
-
@user_agents[USER_AGENT_TYPES.android] = android_dom.css('+ .listing-of-useragents .code').first.text
|
|
92
|
-
end
|
|
93
|
-
end
|
|
94
|
-
|
|
95
|
-
if $0 == __FILE__
|
|
96
|
-
agent = LatestUAFetcher.new
|
|
97
|
-
agent.run
|
|
98
|
-
|
|
99
|
-
pp agent.ordered_user_agents
|
|
100
|
-
end
|
data/examples/mech-dump.rb
DELETED
data/examples/proxy_req.rb
DELETED
data/examples/rubygems.rb
DELETED
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
# This example logs a user in to rubygems and prints out the body of the
|
|
2
|
-
# page after logging the user in.
|
|
3
|
-
require 'rubygems'
|
|
4
|
-
require 'mechanize'
|
|
5
|
-
require 'logger'
|
|
6
|
-
|
|
7
|
-
# Create a new mechanize object
|
|
8
|
-
mech = Mechanize.new
|
|
9
|
-
mech.log = Logger.new $stderr
|
|
10
|
-
mech.agent.http.debug_output = $stderr
|
|
11
|
-
|
|
12
|
-
# Load the rubygems website
|
|
13
|
-
page = mech.get('https://rubygems.org/')
|
|
14
|
-
page = mech.click page.link_with(:text => /Sign in/) # Click the login link
|
|
15
|
-
form = page.forms[1] # Select the first form
|
|
16
|
-
form["session[who]"] = ARGV[0]
|
|
17
|
-
form["session[password]"] = ARGV[1]
|
|
18
|
-
form["commit"] = "Sign in"
|
|
19
|
-
|
|
20
|
-
# Submit the form
|
|
21
|
-
page = form.submit form.buttons.first
|
|
22
|
-
|
|
23
|
-
puts page.body # Print out the body
|
data/examples/spider.rb
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
require 'rubygems'
|
|
2
|
-
require 'mechanize'
|
|
3
|
-
|
|
4
|
-
agent = Mechanize.new
|
|
5
|
-
agent.max_history = nil # unlimited history
|
|
6
|
-
stack = agent.get(ARGV[0]).links
|
|
7
|
-
|
|
8
|
-
while l = stack.pop
|
|
9
|
-
next unless l.uri
|
|
10
|
-
host = l.uri.host
|
|
11
|
-
next unless host.nil? or host == agent.history.first.uri.host
|
|
12
|
-
next if agent.visited? l.href
|
|
13
|
-
|
|
14
|
-
puts "crawling #{l.uri}"
|
|
15
|
-
begin
|
|
16
|
-
page = l.click
|
|
17
|
-
next unless Mechanize::Page === page
|
|
18
|
-
stack.push(*page.links)
|
|
19
|
-
rescue Mechanize::ResponseCodeError
|
|
20
|
-
end
|
|
21
|
-
end
|
|
22
|
-
|
|
@@ -1,158 +0,0 @@
|
|
|
1
|
-
require 'mechanize'
|
|
2
|
-
require 'tsort'
|
|
3
|
-
|
|
4
|
-
##
|
|
5
|
-
# This example implements the alt-text of http://xkcd.com/903/ which states:
|
|
6
|
-
#
|
|
7
|
-
# Wikipedia trivia: if you take any article, click on the first link in the
|
|
8
|
-
# article text not in parentheses or italics, and then repeat, you will
|
|
9
|
-
# eventually end up at "Philosophy".
|
|
10
|
-
|
|
11
|
-
class WikipediaLinksToPhilosophy
|
|
12
|
-
|
|
13
|
-
def initialize
|
|
14
|
-
@agent = Mechanize.new
|
|
15
|
-
@agent.user_agent_alias = 'Mac Safari' # Wikipedia blocks "mechanize"
|
|
16
|
-
|
|
17
|
-
@history = @agent.history
|
|
18
|
-
|
|
19
|
-
@wiki_url = URI 'http://en.wikipedia.org'
|
|
20
|
-
@search_url = @wiki_url + '/w/index.php'
|
|
21
|
-
@random_url = @wiki_url + '/wiki/Special:Random'
|
|
22
|
-
|
|
23
|
-
@title = nil
|
|
24
|
-
@seen = nil
|
|
25
|
-
end
|
|
26
|
-
|
|
27
|
-
##
|
|
28
|
-
# Retrieves the title of the current page
|
|
29
|
-
|
|
30
|
-
def extract_title
|
|
31
|
-
@page.title =~ /(.*) - Wikipedia/
|
|
32
|
-
|
|
33
|
-
@title = $1
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
##
|
|
37
|
-
# Retrieves the initial page. If +query+ is not given a random page is
|
|
38
|
-
# chosen
|
|
39
|
-
|
|
40
|
-
def fetch_first_page query
|
|
41
|
-
if query then
|
|
42
|
-
search query
|
|
43
|
-
else
|
|
44
|
-
random
|
|
45
|
-
end
|
|
46
|
-
end
|
|
47
|
-
|
|
48
|
-
##
|
|
49
|
-
# The search is finished if we've seen the page before or we've reached
|
|
50
|
-
# Philosophy
|
|
51
|
-
|
|
52
|
-
def finished?
|
|
53
|
-
@seen or @title == 'Philosophy'
|
|
54
|
-
end
|
|
55
|
-
|
|
56
|
-
##
|
|
57
|
-
# Follows the first non-parenthetical, non-italic link in the main body of
|
|
58
|
-
# the article.
|
|
59
|
-
|
|
60
|
-
def follow_first_link
|
|
61
|
-
puts "#{@title} (#{@page.uri})"
|
|
62
|
-
|
|
63
|
-
# > p > a rejects italics
|
|
64
|
-
links = @page.root.css('.mw-content-ltr p > a[href^="/wiki/"]')
|
|
65
|
-
|
|
66
|
-
# reject disambiguation and special pages, images and files
|
|
67
|
-
links = links.reject do |link_node|
|
|
68
|
-
link_node['href'] =~ %r%/wiki/\w+:|\(disambiguation\)%
|
|
69
|
-
end
|
|
70
|
-
|
|
71
|
-
links = links.reject do |link_node|
|
|
72
|
-
in_parenthetical? link_node
|
|
73
|
-
end
|
|
74
|
-
|
|
75
|
-
link = links.first
|
|
76
|
-
|
|
77
|
-
if link.nil?
|
|
78
|
-
puts "Could not parse #{@page.uri}"
|
|
79
|
-
exit 1
|
|
80
|
-
end
|
|
81
|
-
|
|
82
|
-
# convert a Nokogiri HTML element back to a mechanize link
|
|
83
|
-
link = Mechanize::Page::Link.new link, @agent, @page
|
|
84
|
-
|
|
85
|
-
return if @seen = @agent.visited?(link)
|
|
86
|
-
|
|
87
|
-
@page = link.click
|
|
88
|
-
|
|
89
|
-
extract_title
|
|
90
|
-
end
|
|
91
|
-
|
|
92
|
-
##
|
|
93
|
-
# Is +link_node+ in an open parenthetical section?
|
|
94
|
-
|
|
95
|
-
def in_parenthetical? link_node
|
|
96
|
-
siblings = link_node.parent.children
|
|
97
|
-
|
|
98
|
-
seen = false
|
|
99
|
-
|
|
100
|
-
before = siblings.reject do |node|
|
|
101
|
-
seen or (seen = node == link_node)
|
|
102
|
-
end
|
|
103
|
-
|
|
104
|
-
preceding_text = before.map { |node| node.text }.join
|
|
105
|
-
|
|
106
|
-
open = preceding_text.count '('
|
|
107
|
-
close = preceding_text.count ')'
|
|
108
|
-
|
|
109
|
-
open > close
|
|
110
|
-
end
|
|
111
|
-
|
|
112
|
-
##
|
|
113
|
-
# Prints the result of the search
|
|
114
|
-
|
|
115
|
-
def print_result
|
|
116
|
-
if @seen then
|
|
117
|
-
puts "[Loop detected]"
|
|
118
|
-
else
|
|
119
|
-
puts @title
|
|
120
|
-
end
|
|
121
|
-
puts
|
|
122
|
-
# subtract initial search or Special:Random
|
|
123
|
-
puts "After #{@agent.history.length - 1} pages"
|
|
124
|
-
end
|
|
125
|
-
|
|
126
|
-
##
|
|
127
|
-
# Retrieves a random page from wikipedia
|
|
128
|
-
|
|
129
|
-
def random
|
|
130
|
-
@page = @agent.get @random_url
|
|
131
|
-
|
|
132
|
-
extract_title
|
|
133
|
-
end
|
|
134
|
-
|
|
135
|
-
##
|
|
136
|
-
# Entry point
|
|
137
|
-
|
|
138
|
-
def run query = nil
|
|
139
|
-
fetch_first_page query
|
|
140
|
-
|
|
141
|
-
follow_first_link until finished?
|
|
142
|
-
|
|
143
|
-
print_result
|
|
144
|
-
end
|
|
145
|
-
|
|
146
|
-
##
|
|
147
|
-
# Searches for +query+ on wikipedia
|
|
148
|
-
|
|
149
|
-
def search query
|
|
150
|
-
@page = @agent.get @search_url, search: query
|
|
151
|
-
|
|
152
|
-
extract_title
|
|
153
|
-
end
|
|
154
|
-
|
|
155
|
-
end
|
|
156
|
-
|
|
157
|
-
WikipediaLinksToPhilosophy.new.run ARGV.shift if $0 == __FILE__
|
|
158
|
-
|
data/mechanize.gemspec
DELETED
|
@@ -1,74 +0,0 @@
|
|
|
1
|
-
# coding: utf-8
|
|
2
|
-
# frozen_string_literal: true
|
|
3
|
-
require_relative 'lib/mechanize/version'
|
|
4
|
-
|
|
5
|
-
Gem::Specification.new do |spec|
|
|
6
|
-
spec.name = "mechanize"
|
|
7
|
-
spec.version = Mechanize::VERSION
|
|
8
|
-
spec.homepage = "https://github.com/sparklemotion/mechanize"
|
|
9
|
-
spec.summary = 'The Mechanize library is used for automating interaction with websites'
|
|
10
|
-
spec.description =
|
|
11
|
-
[
|
|
12
|
-
"The Mechanize library is used for automating interaction with websites.",
|
|
13
|
-
"Mechanize automatically stores and sends cookies, follows redirects,",
|
|
14
|
-
"and can follow links and submit forms. Form fields can be populated and",
|
|
15
|
-
"submitted. Mechanize also keeps track of the sites that you have visited as",
|
|
16
|
-
"a history.",
|
|
17
|
-
].join("\n")
|
|
18
|
-
|
|
19
|
-
spec.authors =
|
|
20
|
-
[
|
|
21
|
-
'Eric Hodel',
|
|
22
|
-
'Aaron Patterson',
|
|
23
|
-
'Mike Dalessio',
|
|
24
|
-
'Akinori MUSHA',
|
|
25
|
-
'Lee Jarvis',
|
|
26
|
-
]
|
|
27
|
-
spec.email =
|
|
28
|
-
[
|
|
29
|
-
'drbrain@segment7.net',
|
|
30
|
-
'aaron.patterson@gmail.com',
|
|
31
|
-
'mike.dalessio@gmail.com',
|
|
32
|
-
'knu@idaemons.org',
|
|
33
|
-
'ljjarvis@gmail.com',
|
|
34
|
-
]
|
|
35
|
-
|
|
36
|
-
spec.metadata = {
|
|
37
|
-
'yard.run' => 'yard',
|
|
38
|
-
'bug_tracker_uri' => 'https://github.com/sparklemotion/mechanize/issues',
|
|
39
|
-
'changelog_uri' => 'https://github.com/sparklemotion/mechanize/blob/main/CHANGELOG.md',
|
|
40
|
-
'documentation_uri' => 'https://www.rubydoc.info/gems/mechanize',
|
|
41
|
-
'homepage_uri' => 'https://github.com/sparklemotion/mechanize',
|
|
42
|
-
'source_code_uri' => 'https://github.com/sparklemotion/mechanize'
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
spec.license = "MIT"
|
|
46
|
-
|
|
47
|
-
spec.require_paths = ["lib"]
|
|
48
|
-
spec.files = %x(git ls-files).split($/)
|
|
49
|
-
spec.test_files = spec.files.grep(%r{^test/})
|
|
50
|
-
|
|
51
|
-
spec.extra_rdoc_files += Dir['*.rdoc', '*.md']
|
|
52
|
-
spec.rdoc_options = ["--main", "README.md"]
|
|
53
|
-
|
|
54
|
-
spec.required_ruby_version = ">= 2.6.0"
|
|
55
|
-
|
|
56
|
-
spec.add_runtime_dependency("addressable", "~> 2.8")
|
|
57
|
-
spec.add_runtime_dependency("domain_name", ">= 0.5.20190701", "~> 0.5")
|
|
58
|
-
spec.add_runtime_dependency("http-cookie", ">= 1.0.3", "~> 1.0")
|
|
59
|
-
spec.add_runtime_dependency("mime-types", "~> 3.3")
|
|
60
|
-
spec.add_runtime_dependency("net-http-digest_auth", ">= 1.4.1", "~> 1.4")
|
|
61
|
-
|
|
62
|
-
# careful! some folks are relying on older versions of net-http-persistent
|
|
63
|
-
# - see the socks proxy patch in use at #507 and #464
|
|
64
|
-
# - see use of retry_change_requests that was removed at #558
|
|
65
|
-
spec.add_runtime_dependency("net-http-persistent", ">= 2.5.2", "< 5.0.dev")
|
|
66
|
-
|
|
67
|
-
spec.add_runtime_dependency("nokogiri", ">= 1.11.2", "~> 1.11")
|
|
68
|
-
spec.add_runtime_dependency("webrick", "~> 1.7")
|
|
69
|
-
spec.add_runtime_dependency("webrobots", "~> 0.1.2")
|
|
70
|
-
|
|
71
|
-
spec.add_runtime_dependency("rubyntlm", ">= 0.6.3", "~> 0.6")
|
|
72
|
-
spec.add_runtime_dependency("base64") # removed from bundled gems in 3.4, and needed by rubyntlm (which doesn't declare this dependency)
|
|
73
|
-
spec.add_runtime_dependency("nkf") # removed from bundled gems in 3.4
|
|
74
|
-
end
|
data/test/data/htpasswd
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
mech:44E/qORekFV0E
|
data/test/data/server.crt
DELETED
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
-----BEGIN CERTIFICATE-----
|
|
2
|
-
MIICmzCCAgQCCQDq2kM3TCIM0DANBgkqhkiG9w0BAQQFADCBkTELMAkGA1UEBhMC
|
|
3
|
-
VVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1NlYXR0bGUxEjAQBgNV
|
|
4
|
-
BAoTCU1lY2hhbml6ZTESMBAGA1UECxMJTWVjaGFuaXplMQ4wDAYDVQQDEwVBYXJv
|
|
5
|
-
bjEjMCEGCSqGSIb3DQEJARYUYWFyb25wQHJ1Ynlmb3JnZS5vcmcwHhcNMDYwODIz
|
|
6
|
-
MDU0NTMwWhcNMDcwODIzMDU0NTMwWjCBkTELMAkGA1UEBhMCVVMxEzARBgNVBAgT
|
|
7
|
-
Cldhc2hpbmd0b24xEDAOBgNVBAcTB1NlYXR0bGUxEjAQBgNVBAoTCU1lY2hhbml6
|
|
8
|
-
ZTESMBAGA1UECxMJTWVjaGFuaXplMQ4wDAYDVQQDEwVBYXJvbjEjMCEGCSqGSIb3
|
|
9
|
-
DQEJARYUYWFyb25wQHJ1Ynlmb3JnZS5vcmcwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
|
|
10
|
-
MIGJAoGBAKpnmI4Y4tBz3SJsHR28QoUr7IYxkqbD8qjqgIN0LXOslyFyiGzNKca5
|
|
11
|
-
Ln2Au8CZLyCugt/vutjfV+Eq0bl0HwbKdo6HjP6RxmGX6EKWX7NOrsof+s96DhLm
|
|
12
|
-
JaWWbtvqP8eyy9PNInKjX6n/7wsVetQutjhNy/cAkrh6UOsjyCGvAgMBAAEwDQYJ
|
|
13
|
-
KoZIhvcNAQEEBQADgYEAGtqgxn1fh0X5MxDG1yMp5aGcZ6HhtEtlm5S0ZsRnMsqU
|
|
14
|
-
Hh6Bd57+zUQ66XnLCbQN2cwNeeSoqtI16Ccc1I5cAhQnIZESMsPG21i1BnpEhKph
|
|
15
|
-
HfNFNpWI/upT2EXNUM6Vx2Kk2aCw2ysrD2pHpsTo5bCOly00uK1ZkoJVQMTL4gU=
|
|
16
|
-
-----END CERTIFICATE-----
|
data/test/data/server.csr
DELETED
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
-----BEGIN CERTIFICATE REQUEST-----
|
|
2
|
-
MIIB0jCCATsCAQAwgZExCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9u
|
|
3
|
-
MRAwDgYDVQQHEwdTZWF0dGxlMRIwEAYDVQQKEwlNZWNoYW5pemUxEjAQBgNVBAsT
|
|
4
|
-
CU1lY2hhbml6ZTEOMAwGA1UEAxMFQWFyb24xIzAhBgkqhkiG9w0BCQEWFGFhcm9u
|
|
5
|
-
cEBydWJ5Zm9yZ2Uub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCqZ5iO
|
|
6
|
-
GOLQc90ibB0dvEKFK+yGMZKmw/Ko6oCDdC1zrJchcohszSnGuS59gLvAmS8groLf
|
|
7
|
-
77rY31fhKtG5dB8GynaOh4z+kcZhl+hCll+zTq7KH/rPeg4S5iWllm7b6j/HssvT
|
|
8
|
-
zSJyo1+p/+8LFXrULrY4Tcv3AJK4elDrI8ghrwIDAQABoAAwDQYJKoZIhvcNAQEE
|
|
9
|
-
BQADgYEAT7SPe71NQvT2BYGEmbWb7FlSQrPh+rDQMHt/Akb8+r91NLkxZtbD1e/F
|
|
10
|
-
iyI9JloPCEwJXxHBl0VVRpFCRuJNN0z0E/G4NUWu6n+ZkihtnmV6uazzAQmD4pTl
|
|
11
|
-
SjoiyVLWU+r4Q4yXWXtJ9GR8Attv32fL3PcP+GGLeurXJAn0MNU=
|
|
12
|
-
-----END CERTIFICATE REQUEST-----
|
data/test/data/server.key
DELETED
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
-----BEGIN RSA PRIVATE KEY-----
|
|
2
|
-
MIICXAIBAAKBgQCqZ5iOGOLQc90ibB0dvEKFK+yGMZKmw/Ko6oCDdC1zrJchcohs
|
|
3
|
-
zSnGuS59gLvAmS8groLf77rY31fhKtG5dB8GynaOh4z+kcZhl+hCll+zTq7KH/rP
|
|
4
|
-
eg4S5iWllm7b6j/HssvTzSJyo1+p/+8LFXrULrY4Tcv3AJK4elDrI8ghrwIDAQAB
|
|
5
|
-
AoGAC+iZfLS4hSDTv2gW0NErROtA6E/mk8j12GArAwTHeGIDXc8HQbNEzCJ84UBx
|
|
6
|
-
3o/V/06yzruOL0HMfmvjpDY9RLsH02xZb2F/lruw4MJLu50i/Zu8Sjmb1YPSfCh/
|
|
7
|
-
3+8lREA3Uznlq+wHC3yPxQzMBy5jaEdH4IKxT0Bq8TeF0AECQQDSpL47YpRVRsLn
|
|
8
|
-
sS00ndEgQQmT5AJWJJtPpbHk6AA0a+zdNeuDRbdF42zG483YEqU7meZbPKR8QbkK
|
|
9
|
-
ZQPEBuevAkEAzxjGcz6NZesmN/NQOtOpylewEs1bdIJyBIBmcnmkimLBtdxd0t34
|
|
10
|
-
wUKVHLDSj2aemuAHHwsyn/BNXs6F+obmAQJBALpbkAXAAFW1xefvo3vih8sOXyfd
|
|
11
|
-
WIfX2SRNBqbq7otyVFudQaChBDUrsOgBUPLyBAdH8DoV27wm9UuR9RPvu/cCQFRr
|
|
12
|
-
WgICXqtMFtE56tuACreD1S9k7MHqpsW0/Y3ujicnKKWUhd5+Q3esR5JhdgOkpkSl
|
|
13
|
-
y+FYtDNERpW+BBliwgECQA+Vc7pnxwDIOP8kFumdAUmRmhEZjuwArFcywPzrCUn9
|
|
14
|
-
4/KBOp5wDN7kanBwNGZCZ/eQtkb6thAS8C9pufHD1lw=
|
|
15
|
-
-----END RSA PRIVATE KEY-----
|