omni_scrape 0.1.8.1 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/omni_scrape.rb +6 -6
- data/lib/omni_scrape/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f44e8258cf10e287c2c9f089796197b111d98855
|
|
4
|
+
data.tar.gz: b34419efbbeef99c68e365dcbf7d162d9fe525e1
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9f8fcca1689244e860ad4ffb6639a0daf1b6f9e02ef59362a0a5f03deb987e49029ca354571b5b183842be6bc2389a60c52d61d74c010c521b66566880d2afcc
|
|
7
|
+
data.tar.gz: 2ebec8762d455b02c54fb1ce7770284f9a87d1c9ec5cd4b643420301a4dca88961ab160cb7e26f72ee640e4a446d3f8ed9b7030b7d7f38069dbdb85f9310d879
|
data/lib/omni_scrape.rb
CHANGED
|
@@ -9,7 +9,7 @@ def CrawlScrape(url, depth, sub_url)
|
|
|
9
9
|
end#if
|
|
10
10
|
s_depth = depth #true
|
|
11
11
|
#open the starting page
|
|
12
|
-
page = Nokogiri::HTML(open(url)) #good
|
|
12
|
+
page = Nokogiri::HTML(open(url,{ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE})) #good
|
|
13
13
|
#collect all of the links from the page
|
|
14
14
|
links= page.css('a') #good
|
|
15
15
|
|
|
@@ -53,7 +53,7 @@ for i in 0..check
|
|
|
53
53
|
|
|
54
54
|
begin
|
|
55
55
|
if(fourofour==false)
|
|
56
|
-
pagina = Nokogiri::HTML(open(url))
|
|
56
|
+
pagina = Nokogiri::HTML(open(url,{ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}))
|
|
57
57
|
end#if
|
|
58
58
|
#test for a 404
|
|
59
59
|
rescue Exception =>ex
|
|
@@ -87,7 +87,7 @@ def Localize(url, depth, sub_url)
|
|
|
87
87
|
depth=0
|
|
88
88
|
end
|
|
89
89
|
#open the starting page
|
|
90
|
-
page = Nokogiri::HTML(open(url))
|
|
90
|
+
page = Nokogiri::HTML(open(url,{ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}))
|
|
91
91
|
#collect all of the links from the page
|
|
92
92
|
links= page.css('a')
|
|
93
93
|
title = page.css('title')
|
|
@@ -144,7 +144,7 @@ end
|
|
|
144
144
|
|
|
145
145
|
begin
|
|
146
146
|
if(fourofour==false)
|
|
147
|
-
pagina = Nokogiri::HTML(open(url))
|
|
147
|
+
pagina = Nokogiri::HTML(open(url,{ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}))
|
|
148
148
|
end
|
|
149
149
|
#test for a 404
|
|
150
150
|
rescue Exception =>ex
|
|
@@ -270,7 +270,7 @@ def FLocalize(url, depth, sub_url, s_depth, i_page, prev_ipage, link_to_add)
|
|
|
270
270
|
if (depth<0)
|
|
271
271
|
depth=0
|
|
272
272
|
end
|
|
273
|
-
page = Nokogiri::HTML(open(url))
|
|
273
|
+
page = Nokogiri::HTML(open(url,{ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}))
|
|
274
274
|
#collect all of the links from the page
|
|
275
275
|
links= page.css('a')
|
|
276
276
|
title = page.css('title')
|
|
@@ -327,7 +327,7 @@ end
|
|
|
327
327
|
|
|
328
328
|
begin
|
|
329
329
|
if(fourofour==false)
|
|
330
|
-
pagina = Nokogiri::HTML(open(url))
|
|
330
|
+
pagina = Nokogiri::HTML(open(url,{ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}))
|
|
331
331
|
end
|
|
332
332
|
#test for a 404
|
|
333
333
|
rescue Exception =>ex
|
data/lib/omni_scrape/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: omni_scrape
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.9
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Bradley Maynard
|
|
@@ -88,7 +88,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
88
88
|
version: '0'
|
|
89
89
|
requirements: []
|
|
90
90
|
rubyforge_project:
|
|
91
|
-
rubygems_version: 2.
|
|
91
|
+
rubygems_version: 2.4.8
|
|
92
92
|
signing_key:
|
|
93
93
|
specification_version: 4
|
|
94
94
|
summary: This is an all-purpose web scraper
|