grubby 1.2.1 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +23 -0
- data/Gemfile +3 -0
- data/README.md +165 -121
- data/Rakefile +0 -13
- data/grubby.gemspec +14 -18
- data/lib/grubby/core_ext/uri.rb +14 -11
- data/lib/grubby/json_parser.rb +3 -27
- data/lib/grubby/json_scraper.rb +9 -2
- data/lib/grubby/mechanize/download.rb +1 -1
- data/lib/grubby/mechanize/fetch_with_retry.rb +2 -2
- data/lib/grubby/mechanize/file.rb +3 -2
- data/lib/grubby/mechanize/link.rb +9 -6
- data/lib/grubby/mechanize/page.rb +4 -2
- data/lib/grubby/mechanize/parser.rb +9 -9
- data/lib/grubby/page_scraper.rb +9 -2
- data/lib/grubby/scraper.rb +95 -69
- data/lib/grubby/version.rb +1 -1
- data/lib/grubby.rb +93 -58
- metadata +19 -108
- data/.gitignore +0 -9
- data/.travis.yml +0 -5
- data/lib/grubby/log.rb +0 -5
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8c66a9a4e4d35bdf31269bf55d42641aaef58d3309d67e4521706c4acc10b618
|
|
4
|
+
data.tar.gz: 7d097cef49f6e4fe22c914e236afcfee97b079a7731be3a1ef897382bc0882d8
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 428f5c6b4bf4e8fe53b8499b21f7e9963dbdbce987906c56dcc3167597510822b0ae9288beb5ccc82fdf6e627d950663de4a5e06a58e70da9f63d01aa53bafd9
|
|
7
|
+
data.tar.gz: 241ef2f45729fb3cb3da8f5b9def7e925b546b34281db070385643733d4061dda07474b53eef8f92602d24f75658f223785deb71aa84da266a84b35b1804ded0
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,26 @@
|
|
|
1
|
+
## 3.0.0
|
|
2
|
+
|
|
3
|
+
* [BREAKING] Drop support for Ruby < 3.4
|
|
4
|
+
* [BREAKING] Remove Active Support, `casual_support`, `mini_sanity`, and
|
|
5
|
+
`pleasant_path` as runtime dependencies
|
|
6
|
+
* Add and require those gems directly if your code relies on their
|
|
7
|
+
extension methods
|
|
8
|
+
* [BREAKING] Replace `$log` with `Grubby.logger`
|
|
9
|
+
* Use `Grubby.logger = ...` to customize logging
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
## 2.0.0
|
|
13
|
+
|
|
14
|
+
* [BREAKING] Drop support for Active Support < 6.0
|
|
15
|
+
* [BREAKING] Require casual_support ~> 4.0
|
|
16
|
+
* [BREAKING] Require mini_sanity ~> 2.0
|
|
17
|
+
* [BREAKING] Require pleasant_path ~> 2.0
|
|
18
|
+
* [BREAKING] Remove `JsonParser.json_parse_options`
|
|
19
|
+
* Use `::JSON.load_default_options` instead
|
|
20
|
+
* [BREAKING] Rename `Grubby#singleton` to `Grubby#fulfill`
|
|
21
|
+
* [BREAKING] Change `Grubby#fulfill` to return block's result
|
|
22
|
+
|
|
23
|
+
|
|
1
24
|
## 1.2.1
|
|
2
25
|
|
|
3
26
|
* Add `JsonParser#mech` attribute for parity with `Mechanize::Page#mech`
|
data/Gemfile
CHANGED
data/README.md
CHANGED
|
@@ -1,174 +1,218 @@
|
|
|
1
1
|
# grubby
|
|
2
2
|
|
|
3
|
-
[Fail-fast] web scraping.
|
|
4
|
-
error-checking atop the marvelous [Mechanize gem]. See API
|
|
5
|
-
below, or browse the [full documentation].
|
|
3
|
+
[Fail-fast][] web scraping. `grubby` adds a layer of utility and
|
|
4
|
+
error-checking atop the marvelous [Mechanize gem][]. See API listing
|
|
5
|
+
below, or browse the [full documentation][].
|
|
6
6
|
|
|
7
7
|
[Fail-fast]: https://en.wikipedia.org/wiki/Fail-fast
|
|
8
8
|
[Mechanize gem]: https://rubygems.org/gems/mechanize
|
|
9
|
-
[full documentation]:
|
|
9
|
+
[full documentation]: https://www.rubydoc.info/gems/grubby/
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
## Examples
|
|
13
13
|
|
|
14
|
-
The following
|
|
14
|
+
The following code scrapes stories from the [Hacker News](
|
|
15
|
+
https://news.ycombinator.com/news) front page:
|
|
15
16
|
|
|
16
17
|
```ruby
|
|
17
18
|
require "grubby"
|
|
18
19
|
|
|
19
20
|
class HackerNews < Grubby::PageScraper
|
|
20
21
|
scrapes(:items) do
|
|
21
|
-
page.search!(".athing").map{|
|
|
22
|
+
page.search!(".athing").map{|element| Item.new(element) }
|
|
22
23
|
end
|
|
23
24
|
|
|
24
25
|
class Item < Grubby::Scraper
|
|
25
|
-
scrapes(:story_link){ source.at!("a
|
|
26
|
-
|
|
26
|
+
scrapes(:story_link){ source.at!(".titleline > a") }
|
|
27
|
+
|
|
28
|
+
scrapes(:story_url){ expand_url(story_link["href"]) }
|
|
29
|
+
|
|
27
30
|
scrapes(:title){ story_link.text }
|
|
31
|
+
|
|
32
|
+
scrapes(:comments_link, optional: true) do
|
|
33
|
+
source.next_sibling.search!(".subtext a").find do |link|
|
|
34
|
+
link.text.match?(/comment|discuss/)
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
scrapes(:comments_url, if: :comments_link) do
|
|
39
|
+
expand_url(comments_link["href"])
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
scrapes(:comment_count, if: :comments_link) do
|
|
43
|
+
comments_link.text.to_i
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def expand_url(url)
|
|
47
|
+
url.include?("://") ? url : source.document.uri.merge(url).to_s
|
|
48
|
+
end
|
|
28
49
|
end
|
|
29
50
|
end
|
|
30
51
|
|
|
31
52
|
# The following line will raise an exception if anything goes wrong
|
|
32
53
|
# during the scraping process. For example, if the structure of the
|
|
33
|
-
# HTML does not match expectations
|
|
34
|
-
#
|
|
35
|
-
#
|
|
36
|
-
# hard-to-trace errors.
|
|
54
|
+
# HTML does not match expectations due to a site change, the script will
|
|
55
|
+
# terminate immediately with a helpful error message. This prevents bad
|
|
56
|
+
# data from propagating and causing hard-to-trace errors.
|
|
37
57
|
hn = HackerNews.scrape("https://news.ycombinator.com/news")
|
|
38
58
|
|
|
39
59
|
# Your processing logic goes here:
|
|
40
60
|
hn.items.take(10).each do |item|
|
|
41
61
|
puts "* #{item.title}"
|
|
42
|
-
puts " #{item.
|
|
62
|
+
puts " #{item.story_url}"
|
|
63
|
+
puts " #{item.comment_count} comments: #{item.comments_url}" if item.comments_url
|
|
43
64
|
puts
|
|
44
65
|
end
|
|
45
66
|
```
|
|
46
67
|
|
|
47
|
-
|
|
68
|
+
Hacker News also offers a [JSON API](https://github.com/HackerNews/API),
|
|
69
|
+
which may be more robust for scraping purposes. `grubby` can scrape
|
|
70
|
+
JSON just as well:
|
|
71
|
+
|
|
72
|
+
```ruby
|
|
73
|
+
require "grubby"
|
|
74
|
+
|
|
75
|
+
class HackerNews < Grubby::JsonScraper
|
|
76
|
+
scrapes(:items) do
|
|
77
|
+
# API returns array of top 500 item IDs, so limit as necessary
|
|
78
|
+
json.take(10).map do |item_id|
|
|
79
|
+
Item.scrape("https://hacker-news.firebaseio.com/v0/item/#{item_id}.json")
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
class Item < Grubby::JsonScraper
|
|
84
|
+
scrapes(:story_url){ json["url"] || hn_url }
|
|
85
|
+
|
|
86
|
+
scrapes(:title){ json["title"] }
|
|
87
|
+
|
|
88
|
+
scrapes(:comments_url, optional: true) do
|
|
89
|
+
hn_url if json["descendants"]
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
scrapes(:comment_count, optional: true) do
|
|
93
|
+
json["descendants"]&.to_i
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def hn_url
|
|
97
|
+
"https://news.ycombinator.com/item?id=#{json["id"]}"
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
hn = HackerNews.scrape("https://hacker-news.firebaseio.com/v0/topstories.json")
|
|
103
|
+
|
|
104
|
+
# Your processing logic goes here:
|
|
105
|
+
hn.items.each do |item|
|
|
106
|
+
puts "* #{item.title}"
|
|
107
|
+
puts " #{item.story_url}"
|
|
108
|
+
puts " #{item.comment_count} comments: #{item.comments_url}" if item.comments_url
|
|
109
|
+
puts
|
|
110
|
+
end
|
|
111
|
+
```
|
|
48
112
|
|
|
49
113
|
|
|
50
114
|
## Core API
|
|
51
115
|
|
|
52
|
-
- [Grubby](
|
|
53
|
-
- [
|
|
54
|
-
- [
|
|
55
|
-
- [
|
|
56
|
-
- [
|
|
57
|
-
- [Scraper](
|
|
58
|
-
- [
|
|
59
|
-
- [
|
|
60
|
-
- [
|
|
61
|
-
- [
|
|
62
|
-
- [
|
|
63
|
-
|
|
64
|
-
- [
|
|
65
|
-
- [
|
|
66
|
-
|
|
67
|
-
- [
|
|
68
|
-
- [
|
|
69
|
-
|
|
70
|
-
- [
|
|
71
|
-
- Mechanize
|
|
72
|
-
|
|
73
|
-
- [
|
|
74
|
-
- Mechanize
|
|
75
|
-
|
|
76
|
-
- [
|
|
77
|
-
-
|
|
78
|
-
- [
|
|
79
|
-
- [
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
## Supplemental API
|
|
88
|
-
|
|
89
|
-
*grubby* includes several gems which extend Ruby objects with
|
|
90
|
-
convenience methods. When you load *grubby* you automatically make
|
|
91
|
-
these methods available. The included gems are listed below, along with
|
|
92
|
-
**a few** of the methods each provides. See each gem's documentation
|
|
116
|
+
- [`Grubby`](https://www.rubydoc.info/gems/grubby/Grubby)
|
|
117
|
+
- [`#fulfill`](https://www.rubydoc.info/gems/grubby/Grubby:fulfill)
|
|
118
|
+
- [`#get_mirrored`](https://www.rubydoc.info/gems/grubby/Grubby:get_mirrored)
|
|
119
|
+
- [`#ok?`](https://www.rubydoc.info/gems/grubby/Grubby:ok%3F)
|
|
120
|
+
- [`#time_between_requests`](https://www.rubydoc.info/gems/grubby/Grubby:time_between_requests)
|
|
121
|
+
- [`Scraper`](https://www.rubydoc.info/gems/grubby/Grubby/Scraper)
|
|
122
|
+
- [`.each`](https://www.rubydoc.info/gems/grubby/Grubby/Scraper.each)
|
|
123
|
+
- [`.scrape`](https://www.rubydoc.info/gems/grubby/Grubby/Scraper.scrape)
|
|
124
|
+
- [`.scrapes`](https://www.rubydoc.info/gems/grubby/Grubby/Scraper.scrapes)
|
|
125
|
+
- [`#[]`](https://www.rubydoc.info/gems/grubby/Grubby/Scraper:[])
|
|
126
|
+
- [`#to_h`](https://www.rubydoc.info/gems/grubby/Grubby/Scraper:to_h)
|
|
127
|
+
- [`PageScraper`](https://www.rubydoc.info/gems/grubby/Grubby/PageScraper)
|
|
128
|
+
- [`.scrape_file`](https://www.rubydoc.info/gems/grubby/Grubby/PageScraper.scrape_file)
|
|
129
|
+
- [`#page`](https://www.rubydoc.info/gems/grubby/Grubby/PageScraper:page)
|
|
130
|
+
- [`JsonScraper`](https://www.rubydoc.info/gems/grubby/Grubby/JsonScraper)
|
|
131
|
+
- [`.scrape_file`](https://www.rubydoc.info/gems/grubby/Grubby/JsonScraper.scrape_file)
|
|
132
|
+
- [`#json`](https://www.rubydoc.info/gems/grubby/Grubby/JsonScraper:json)
|
|
133
|
+
- `Mechanize::File`
|
|
134
|
+
- [`#save_to`](https://www.rubydoc.info/gems/grubby/Mechanize/Parser:save_to)
|
|
135
|
+
- [`#save_to!`](https://www.rubydoc.info/gems/grubby/Mechanize/Parser:save_to%21)
|
|
136
|
+
- `Mechanize::Page`
|
|
137
|
+
- [`#at!`](https://www.rubydoc.info/gems/grubby/Mechanize/Page:at%21)
|
|
138
|
+
- [`#search!`](https://www.rubydoc.info/gems/grubby/Mechanize/Page:search%21)
|
|
139
|
+
- `Mechanize::Page::Link`
|
|
140
|
+
- [`#to_absolute_uri`](https://www.rubydoc.info/gems/grubby/Mechanize/Page/Link#to_absolute_uri)
|
|
141
|
+
- `URI`
|
|
142
|
+
- [`#basename`](https://www.rubydoc.info/gems/grubby/URI:basename)
|
|
143
|
+
- [`#query_param`](https://www.rubydoc.info/gems/grubby/URI:query_param)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
## Auxiliary API
|
|
147
|
+
|
|
148
|
+
`grubby` loads a few gems that extend Ruby objects with utility methods.
|
|
149
|
+
Some of those methods are listed below. See each gem's documentation
|
|
93
150
|
for a complete API listing.
|
|
94
151
|
|
|
95
|
-
- [
|
|
96
|
-
([docs](
|
|
97
|
-
- [
|
|
98
|
-
- [
|
|
99
|
-
- [
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
- [
|
|
103
|
-
- [
|
|
104
|
-
|
|
105
|
-
- [
|
|
106
|
-
- [
|
|
107
|
-
- [
|
|
108
|
-
- [
|
|
109
|
-
- [String#before_last](http://www.rubydoc.info/gems/casual_support/String:before_last)
|
|
110
|
-
- [String#between](http://www.rubydoc.info/gems/casual_support/String:between)
|
|
111
|
-
- [Time#to_hms](http://www.rubydoc.info/gems/casual_support/Time:to_hms)
|
|
112
|
-
- [Time#to_ymd](http://www.rubydoc.info/gems/casual_support/Time:to_ymd)
|
|
113
|
-
- [gorge](https://rubygems.org/gems/gorge)
|
|
114
|
-
([docs](http://www.rubydoc.info/gems/gorge/))
|
|
115
|
-
- [Pathname#file_crc32](http://www.rubydoc.info/gems/gorge/Pathname:file_crc32)
|
|
116
|
-
- [Pathname#file_md5](http://www.rubydoc.info/gems/gorge/Pathname:file_md5)
|
|
117
|
-
- [Pathname#file_sha1](http://www.rubydoc.info/gems/gorge/Pathname:file_sha1)
|
|
118
|
-
- [String#crc32](http://www.rubydoc.info/gems/gorge/String:crc32)
|
|
119
|
-
- [String#md5](http://www.rubydoc.info/gems/gorge/String:md5)
|
|
120
|
-
- [String#sha1](http://www.rubydoc.info/gems/gorge/String:sha1)
|
|
121
|
-
- [mini_sanity](https://rubygems.org/gems/mini_sanity)
|
|
122
|
-
([docs](http://www.rubydoc.info/gems/mini_sanity/))
|
|
123
|
-
- [Array#assert_length!](http://www.rubydoc.info/gems/mini_sanity/Array:assert_length%21)
|
|
124
|
-
- [Enumerable#refute_empty!](http://www.rubydoc.info/gems/mini_sanity/Enumerable:refute_empty%21)
|
|
125
|
-
- [Object#assert_equal!](http://www.rubydoc.info/gems/mini_sanity/Object:assert_equal%21)
|
|
126
|
-
- [Object#assert_in!](http://www.rubydoc.info/gems/mini_sanity/Object:assert_in%21)
|
|
127
|
-
- [Object#refute_nil!](http://www.rubydoc.info/gems/mini_sanity/Object:refute_nil%21)
|
|
128
|
-
- [Pathname#assert_exist!](http://www.rubydoc.info/gems/mini_sanity/Pathname:assert_exist%21)
|
|
129
|
-
- [String#assert_match!](http://www.rubydoc.info/gems/mini_sanity/String:assert_match%21)
|
|
130
|
-
- [pleasant_path](https://rubygems.org/gems/pleasant_path)
|
|
131
|
-
([docs](http://www.rubydoc.info/gems/pleasant_path/))
|
|
132
|
-
- [Pathname#available_name](http://www.rubydoc.info/gems/pleasant_path/Pathname:available_name)
|
|
133
|
-
- [Pathname#dirs](http://www.rubydoc.info/gems/pleasant_path/Pathname:dirs)
|
|
134
|
-
- [Pathname#files](http://www.rubydoc.info/gems/pleasant_path/Pathname:files)
|
|
135
|
-
- [Pathname#make_dirname](http://www.rubydoc.info/gems/pleasant_path/Pathname:make_dirname)
|
|
136
|
-
- [Pathname#make_file](http://www.rubydoc.info/gems/pleasant_path/Pathname:make_file)
|
|
137
|
-
- [Pathname#move_as](http://www.rubydoc.info/gems/pleasant_path/Pathname:move_as)
|
|
138
|
-
- [Pathname#rename_basename](http://www.rubydoc.info/gems/pleasant_path/Pathname:rename_basename)
|
|
139
|
-
- [Pathname#rename_extname](http://www.rubydoc.info/gems/pleasant_path/Pathname:rename_extname)
|
|
140
|
-
- [ryoba](https://rubygems.org/gems/ryoba)
|
|
141
|
-
([docs](http://www.rubydoc.info/gems/ryoba/))
|
|
142
|
-
- [Nokogiri::XML::Node#matches!](http://www.rubydoc.info/gems/ryoba/Nokogiri/XML/Node:matches%21)
|
|
143
|
-
- [Nokogiri::XML::Node#text!](http://www.rubydoc.info/gems/ryoba/Nokogiri/XML/Node:text%21)
|
|
144
|
-
- [Nokogiri::XML::Node#uri](http://www.rubydoc.info/gems/ryoba/Nokogiri/XML/Node:uri)
|
|
145
|
-
- [Nokogiri::XML::Searchable#ancestor!](http://www.rubydoc.info/gems/ryoba/Nokogiri/XML/Searchable:ancestor%21)
|
|
146
|
-
- [Nokogiri::XML::Searchable#ancestors!](http://www.rubydoc.info/gems/ryoba/Nokogiri/XML/Searchable:ancestors%21)
|
|
147
|
-
- [Nokogiri::XML::Searchable#at!](http://www.rubydoc.info/gems/ryoba/Nokogiri/XML/Searchable:at%21)
|
|
148
|
-
- [Nokogiri::XML::Searchable#search!](http://www.rubydoc.info/gems/ryoba/Nokogiri/XML/Searchable:search%21)
|
|
152
|
+
- [`gorge`](https://rubygems.org/gems/gorge)
|
|
153
|
+
([docs](https://www.rubydoc.info/gems/gorge/))
|
|
154
|
+
- [`Pathname#file_crc32`](https://www.rubydoc.info/gems/gorge/Pathname:file_crc32)
|
|
155
|
+
- [`Pathname#file_md5`](https://www.rubydoc.info/gems/gorge/Pathname:file_md5)
|
|
156
|
+
- [`Pathname#file_sha1`](https://www.rubydoc.info/gems/gorge/Pathname:file_sha1)
|
|
157
|
+
- [`ryoba`](https://rubygems.org/gems/ryoba)
|
|
158
|
+
([docs](https://www.rubydoc.info/gems/ryoba/))
|
|
159
|
+
- [`Nokogiri::XML::Node#matches!`](https://www.rubydoc.info/gems/ryoba/Nokogiri/XML/Node:matches%21)
|
|
160
|
+
- [`Nokogiri::XML::Node#text!`](https://www.rubydoc.info/gems/ryoba/Nokogiri/XML/Node:text%21)
|
|
161
|
+
- [`Nokogiri::XML::Node#uri`](https://www.rubydoc.info/gems/ryoba/Nokogiri/XML/Node:uri)
|
|
162
|
+
- [`Nokogiri::XML::Searchable#ancestor!`](https://www.rubydoc.info/gems/ryoba/Nokogiri/XML/Searchable:ancestor%21)
|
|
163
|
+
- [`Nokogiri::XML::Searchable#ancestors!`](https://www.rubydoc.info/gems/ryoba/Nokogiri/XML/Searchable:ancestors%21)
|
|
164
|
+
- [`Nokogiri::XML::Searchable#at!`](https://www.rubydoc.info/gems/ryoba/Nokogiri/XML/Searchable:at%21)
|
|
165
|
+
- [`Nokogiri::XML::Searchable#search!`](https://www.rubydoc.info/gems/ryoba/Nokogiri/XML/Searchable:search%21)
|
|
149
166
|
|
|
150
167
|
|
|
151
|
-
##
|
|
168
|
+
## Recommended Gems
|
|
152
169
|
|
|
153
|
-
|
|
170
|
+
The following gems will extend Ruby objects with utility methods that
|
|
171
|
+
can be useful when web scraping. Example methods are listed; see each
|
|
172
|
+
gem's documentation for a complete API listing.
|
|
154
173
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
174
|
+
- [Active Support](https://rubygems.org/gems/activesupport)
|
|
175
|
+
([docs](https://www.rubydoc.info/gems/activesupport/))
|
|
176
|
+
- [`Enumerable#index_by`](https://www.rubydoc.info/gems/activesupport/Enumerable:index_by)
|
|
177
|
+
- [`Enumerable#index_with`](https://www.rubydoc.info/gems/activesupport/Enumerable:index_with)
|
|
178
|
+
- [`File.atomic_write`](https://www.rubydoc.info/gems/activesupport/File:atomic_write)
|
|
179
|
+
- [`Object#presence`](https://www.rubydoc.info/gems/activesupport/Object:presence)
|
|
180
|
+
- [`String#blank?`](https://www.rubydoc.info/gems/activesupport/String:blank%3F)
|
|
181
|
+
- [`String#squish`](https://www.rubydoc.info/gems/activesupport/String:squish)
|
|
182
|
+
- [`casual_support`](https://rubygems.org/gems/casual_support)
|
|
183
|
+
([docs](https://www.rubydoc.info/gems/casual_support/))
|
|
184
|
+
- [`String#after`](https://www.rubydoc.info/gems/casual_support/String:after)
|
|
185
|
+
- [`String#after_last`](https://www.rubydoc.info/gems/casual_support/String:after_last)
|
|
186
|
+
- [`String#before`](https://www.rubydoc.info/gems/casual_support/String:before)
|
|
187
|
+
- [`String#before_last`](https://www.rubydoc.info/gems/casual_support/String:before_last)
|
|
188
|
+
- [`String#between`](https://www.rubydoc.info/gems/casual_support/String:between)
|
|
189
|
+
- [`mini_sanity`](https://rubygems.org/gems/mini_sanity)
|
|
190
|
+
([docs](https://www.rubydoc.info/gems/mini_sanity/))
|
|
191
|
+
- [`Enumerator#result!`](https://www.rubydoc.info/gems/mini_sanity/Enumerator:result%21)
|
|
192
|
+
- [`Enumerator#results!`](https://www.rubydoc.info/gems/mini_sanity/Enumerator:results%21)
|
|
193
|
+
- [`Object#assert!`](https://www.rubydoc.info/gems/mini_sanity/Object:assert%21)
|
|
194
|
+
- [`Object#refute!`](https://www.rubydoc.info/gems/mini_sanity/Object:refute%21)
|
|
195
|
+
- [`String#match!`](https://www.rubydoc.info/gems/mini_sanity/String:match%21)
|
|
196
|
+
- [`pleasant_path`](https://rubygems.org/gems/pleasant_path)
|
|
197
|
+
([docs](https://www.rubydoc.info/gems/pleasant_path/))
|
|
198
|
+
- [`Pathname#available_name`](https://www.rubydoc.info/gems/pleasant_path/Pathname:available_name)
|
|
199
|
+
- [`Pathname#existence`](https://www.rubydoc.info/gems/pleasant_path/Pathname:existence)
|
|
200
|
+
- [`Pathname#make_dirname`](https://www.rubydoc.info/gems/pleasant_path/Pathname:make_dirname)
|
|
201
|
+
- [`Pathname#move_as`](https://www.rubydoc.info/gems/pleasant_path/Pathname:move_as)
|
|
202
|
+
- [`Pathname#rename_basename`](https://www.rubydoc.info/gems/pleasant_path/Pathname:rename_basename)
|
|
203
|
+
- [`Pathname#rename_extname`](https://www.rubydoc.info/gems/pleasant_path/Pathname:rename_extname)
|
|
158
204
|
|
|
159
|
-
Then require in your Ruby script:
|
|
160
205
|
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
206
|
+
## Installation
|
|
207
|
+
|
|
208
|
+
Install the [`grubby` gem](https://rubygems.org/gems/grubby).
|
|
164
209
|
|
|
165
210
|
|
|
166
211
|
## Contributing
|
|
167
212
|
|
|
168
|
-
Run `rake test` to run the tests.
|
|
169
|
-
interactive prompt that pre-loads the project code.
|
|
213
|
+
Run `rake test` to run the tests.
|
|
170
214
|
|
|
171
215
|
|
|
172
216
|
## License
|
|
173
217
|
|
|
174
|
-
[MIT License](
|
|
218
|
+
[MIT License](LICENSE.txt)
|
data/Rakefile
CHANGED
|
@@ -1,18 +1,5 @@
|
|
|
1
1
|
require "bundler/gem_tasks"
|
|
2
2
|
require "rake/testtask"
|
|
3
|
-
require "yard"
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
YARD::Rake::YardocTask.new(:doc) do |t|
|
|
7
|
-
end
|
|
8
|
-
|
|
9
|
-
desc "Launch IRB with this gem pre-loaded"
|
|
10
|
-
task :irb do
|
|
11
|
-
require "grubby"
|
|
12
|
-
require "irb"
|
|
13
|
-
ARGV.clear
|
|
14
|
-
IRB.start
|
|
15
|
-
end
|
|
16
3
|
|
|
17
4
|
Rake::TestTask.new(:test) do |t|
|
|
18
5
|
t.libs << "test"
|
data/grubby.gemspec
CHANGED
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
lib = File.expand_path("../lib", __FILE__)
|
|
3
|
-
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
|
-
require "grubby/version"
|
|
1
|
+
require_relative "lib/grubby/version"
|
|
5
2
|
|
|
6
3
|
Gem::Specification.new do |spec|
|
|
7
4
|
spec.name = "grubby"
|
|
@@ -12,24 +9,23 @@ Gem::Specification.new do |spec|
|
|
|
12
9
|
spec.summary = %q{Fail-fast web scraping}
|
|
13
10
|
spec.homepage = "https://github.com/jonathanhefner/grubby"
|
|
14
11
|
spec.license = "MIT"
|
|
12
|
+
spec.required_ruby_version = ">= 3.4"
|
|
15
13
|
|
|
16
|
-
spec.
|
|
17
|
-
|
|
14
|
+
spec.metadata["source_code_uri"] = spec.homepage
|
|
15
|
+
spec.metadata["changelog_uri"] = spec.metadata["source_code_uri"] + "/blob/master/CHANGELOG.md"
|
|
16
|
+
|
|
17
|
+
# Specify which files should be added to the gem when it is released.
|
|
18
|
+
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
|
19
|
+
spec.files = Dir.chdir(__dir__) do
|
|
20
|
+
`git ls-files -z`.split("\x0").reject { |f| f.start_with?("test/", ".git") }
|
|
18
21
|
end
|
|
19
22
|
spec.bindir = "exe"
|
|
20
23
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
21
24
|
spec.require_paths = ["lib"]
|
|
22
25
|
|
|
23
|
-
spec.
|
|
24
|
-
spec.
|
|
25
|
-
spec.
|
|
26
|
-
spec.
|
|
27
|
-
spec.
|
|
28
|
-
spec.add_runtime_dependency "pleasant_path", "~> 1.1"
|
|
29
|
-
spec.add_runtime_dependency "ryoba", "~> 1.0"
|
|
30
|
-
|
|
31
|
-
spec.add_development_dependency "bundler", "~> 1.15"
|
|
32
|
-
spec.add_development_dependency "rake", "~> 10.0"
|
|
33
|
-
spec.add_development_dependency "minitest", "~> 5.0"
|
|
34
|
-
spec.add_development_dependency "yard", "~> 0.9"
|
|
26
|
+
spec.add_dependency "cgi"
|
|
27
|
+
spec.add_dependency "csv"
|
|
28
|
+
spec.add_dependency "gorge", ">= 1.0"
|
|
29
|
+
spec.add_dependency "mechanize", "~> 2.7"
|
|
30
|
+
spec.add_dependency "ryoba", ">= 1.0"
|
|
35
31
|
end
|
data/lib/grubby/core_ext/uri.rb
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
|
+
require "cgi"
|
|
2
|
+
|
|
1
3
|
module URI
|
|
2
4
|
|
|
3
5
|
# Returns the basename of the URI's +path+, a la +File.basename+.
|
|
4
6
|
#
|
|
5
7
|
# @example
|
|
6
|
-
# URI("
|
|
7
|
-
# URI("
|
|
8
|
-
# URI("
|
|
8
|
+
# URI("https://example.com/foo/bar").basename # == "bar"
|
|
9
|
+
# URI("https://example.com/foo").basename # == "foo"
|
|
10
|
+
# URI("https://example.com/").basename # == ""
|
|
9
11
|
#
|
|
10
12
|
# @return [String]
|
|
11
13
|
def basename
|
|
@@ -20,16 +22,16 @@ module URI
|
|
|
20
22
|
# Otherwise, only the last occurrence is returned.
|
|
21
23
|
#
|
|
22
24
|
# @example
|
|
23
|
-
# URI("
|
|
25
|
+
# URI("https://example.com/?foo=a").query_param("foo") # == "a"
|
|
24
26
|
#
|
|
25
|
-
# URI("
|
|
26
|
-
# URI("
|
|
27
|
+
# URI("https://example.com/?foo=a&foo=b").query_param("foo") # == "b"
|
|
28
|
+
# URI("https://example.com/?foo=a&foo=b").query_param("foo[]") # == nil
|
|
27
29
|
#
|
|
28
|
-
# URI("
|
|
29
|
-
# URI("
|
|
30
|
+
# URI("https://example.com/?foo[]=a&foo[]=b").query_param("foo") # == nil
|
|
31
|
+
# URI("https://example.com/?foo[]=a&foo[]=b").query_param("foo[]") # == ["a", "b"]
|
|
30
32
|
#
|
|
31
|
-
# URI("
|
|
32
|
-
# URI("
|
|
33
|
+
# URI("https://example.com/?foo[][x]=a&foo[][y]=b").query_param("foo[]") # == nil
|
|
34
|
+
# URI("https://example.com/?foo[][x]=a&foo[][y]=b").query_param("foo[][x]") # == ["a"]
|
|
33
35
|
#
|
|
34
36
|
# @param name [String]
|
|
35
37
|
# @return [String, Array<String>, nil]
|
|
@@ -38,7 +40,8 @@ module URI
|
|
|
38
40
|
(values.nil? || name.include?("[]")) ? values : values.last
|
|
39
41
|
end
|
|
40
42
|
|
|
41
|
-
# Raises an exception if the URI is not +absolute?+.
|
|
43
|
+
# Raises an exception if the URI is not +absolute?+. Otherwise,
|
|
44
|
+
# returns the URI.
|
|
42
45
|
#
|
|
43
46
|
# @return [self]
|
|
44
47
|
# @raise [RuntimeError]
|
data/lib/grubby/json_parser.rb
CHANGED
|
@@ -1,30 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
# Returns the options to use when parsing JSON. The returned options
|
|
4
|
-
# Hash is not +dup+ed and can be modified directly. Any modifications
|
|
5
|
-
# will be applied to all future parsing.
|
|
6
|
-
#
|
|
7
|
-
# For information about available options, see
|
|
8
|
-
# {https://docs.ruby-lang.org/en/trunk/JSON.html#method-i-parse
|
|
9
|
-
# +JSON.parse+}.
|
|
10
|
-
#
|
|
11
|
-
# @return [Hash]
|
|
12
|
-
def self.json_parse_options
|
|
13
|
-
@json_parse_options ||= JSON.load_default_options.merge(create_additions: false)
|
|
14
|
-
end
|
|
1
|
+
require "json"
|
|
15
2
|
|
|
16
|
-
|
|
17
|
-
# is replaced, and the new value will be applied to all future
|
|
18
|
-
# parsing. To set options individually, see {json_parse_options}.
|
|
19
|
-
#
|
|
20
|
-
# For information about available options, see
|
|
21
|
-
# {https://docs.ruby-lang.org/en/trunk/JSON.html#method-i-parse
|
|
22
|
-
# +JSON.parse+}.
|
|
23
|
-
#
|
|
24
|
-
# @param options [Hash]
|
|
25
|
-
def self.json_parse_options=(options)
|
|
26
|
-
@json_parse_options = options
|
|
27
|
-
end
|
|
3
|
+
class Grubby::JsonParser < Mechanize::File
|
|
28
4
|
|
|
29
5
|
# The parsed JSON data.
|
|
30
6
|
#
|
|
@@ -37,7 +13,7 @@ class Grubby::JsonParser < Mechanize::File
|
|
|
37
13
|
attr_accessor :mech
|
|
38
14
|
|
|
39
15
|
def initialize(uri = nil, response = nil, body = nil, code = nil, mech = nil)
|
|
40
|
-
@json =
|
|
16
|
+
@json = JSON.load(body, nil, create_additions: false)
|
|
41
17
|
@mech = mech
|
|
42
18
|
super(uri, response, body, code)
|
|
43
19
|
end
|
data/lib/grubby/json_scraper.rb
CHANGED
|
@@ -6,8 +6,13 @@ class Grubby::JsonScraper < Grubby::Scraper
|
|
|
6
6
|
attr_reader :json
|
|
7
7
|
|
|
8
8
|
# @param source [Grubby::JsonParser]
|
|
9
|
+
# @raise [Grubby::Scraper::Error]
|
|
10
|
+
# if any {Scraper.scrapes} blocks fail
|
|
9
11
|
def initialize(source)
|
|
10
|
-
|
|
12
|
+
unless source.is_a?(Grubby::JsonParser)
|
|
13
|
+
raise ArgumentError, "source must be a Grubby::JsonParser object"
|
|
14
|
+
end
|
|
15
|
+
@json = source.json
|
|
11
16
|
super
|
|
12
17
|
end
|
|
13
18
|
|
|
@@ -19,11 +24,13 @@ class Grubby::JsonScraper < Grubby::Scraper
|
|
|
19
24
|
# # ...
|
|
20
25
|
# end
|
|
21
26
|
#
|
|
22
|
-
# MyScraper.scrape_file("path/to/local_file.json")
|
|
27
|
+
# MyScraper.scrape_file("path/to/local_file.json") # === MyScraper
|
|
23
28
|
#
|
|
24
29
|
# @param path [String]
|
|
25
30
|
# @param agent [Mechanize]
|
|
26
31
|
# @return [Grubby::JsonScraper]
|
|
32
|
+
# @raise [Grubby::Scraper::Error]
|
|
33
|
+
# if any {Scraper.scrapes} blocks fail
|
|
27
34
|
def self.scrape_file(path, agent = $grubby)
|
|
28
35
|
self.new(Grubby::JsonParser.read_local(path).tap{|parser| parser.mech = agent })
|
|
29
36
|
end
|
|
@@ -25,9 +25,9 @@ class Mechanize::HTTP::Agent
|
|
|
25
25
|
|
|
26
26
|
# otherwise, shutdown the persistent HTTP connection and try again
|
|
27
27
|
retry_count += 1
|
|
28
|
-
|
|
28
|
+
Grubby.logger.warn("#{e.message} (#{e.class}). Retry in #{retry_count} seconds.")
|
|
29
29
|
sleep(retry_count) # incremental backoff to allow server to self-correct
|
|
30
|
-
|
|
30
|
+
Grubby.logger.warn("Retry #{http_method.to_s.upcase} #{uri}")
|
|
31
31
|
retry
|
|
32
32
|
end
|
|
33
33
|
end
|
|
@@ -1,12 +1,13 @@
|
|
|
1
|
+
require "cgi"
|
|
2
|
+
|
|
3
|
+
# @!visibility private
|
|
1
4
|
class Mechanize::File
|
|
2
5
|
|
|
3
|
-
# @!visibility private
|
|
4
6
|
def self.read_local(path)
|
|
5
7
|
uri_path = File.expand_path(path).gsub(%r"[^/\\]+"){|component| CGI.escape(component) }
|
|
6
8
|
self.new(URI::File.build(path: uri_path), nil, File.read(path), "200")
|
|
7
9
|
end
|
|
8
10
|
|
|
9
|
-
# @!visibility private
|
|
10
11
|
def content_hash
|
|
11
12
|
@content_hash ||= self.body.to_s.sha1
|
|
12
13
|
end
|