scraper_rb 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.bumpversion.cfg +1 -1
- data/Gemfile.lock +1 -1
- data/README.md +16 -3
- data/lib/scraper_rb.rb +10 -4
- data/lib/scraper_rb/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a828454fe71dc2b90bfd9fde3e859d149ca3a2c0c2f38fee1d82d7b78537746c
|
4
|
+
data.tar.gz: 98b66eef645c08144966f86a8e6fbf0d61ee8ff26eb5de53906680dee846c6b1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d3a3ab8786817af29be63cae500c04987f83418ced89779a4ee21dd296322f0817e85a0cb45491960ec0a8d36c34cff7a2a50af3b0855adf57ae8feedf30e421
|
7
|
+
data.tar.gz: 89a39dffbaea50064fcd944f89a5b02bae78cc5130f6f16fe58429f965e590f2ae15e41ddf1d9617bf9477be3f3467d6b700ff87c9f1367df81052ddae47dc83
|
data/.bumpversion.cfg
CHANGED
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -21,7 +21,7 @@ $ gem install scraper_rb
|
|
21
21
|
or; install from GitHub:
|
22
22
|
|
23
23
|
```bash
|
24
|
-
$ gem install scraper_rb --version "0.1.
|
24
|
+
$ gem install scraper_rb --version "0.1.2" --source "https://rubygems.pkg.github.com/promptapi"
|
25
25
|
```
|
26
26
|
|
27
27
|
---
|
@@ -86,13 +86,26 @@ Default **timeout** value is set to `10` seconds. You can change this while
|
|
86
86
|
initializing the instance:
|
87
87
|
|
88
88
|
```ruby
|
89
|
-
s = ScraperRb.new('https://pypi.org/classifiers/', {}, timeout=50)
|
89
|
+
s = ScraperRb.new('https://pypi.org/classifiers/', params={}, timeout=50)
|
90
90
|
# => 50 seconds timeout w/o params
|
91
91
|
|
92
|
-
s = ScraperRb.new('https://pypi.org/classifiers/', {country: 'EE'}, timeout=50)
|
92
|
+
s = ScraperRb.new('https://pypi.org/classifiers/', params={country: 'EE'}, timeout=50)
|
93
93
|
# => 50 seconds timeout
|
94
94
|
```
|
95
95
|
|
96
|
+
You can add extra `X-` headers:
|
97
|
+
|
98
|
+
```ruby
|
99
|
+
s = ScraperRb.new('https://pypi.org/classifiers/', headers={'X-Referer': 'https://www.google.com'})
|
100
|
+
|
101
|
+
# or
|
102
|
+
s = ScraperRb.new('https://pypi.org/classifiers/', params={country: 'EE'}, headers={'X-Referer': 'https://www.google.com'}, timeout=50)
|
103
|
+
# => 50 seconds timeout
|
104
|
+
```
|
105
|
+
|
106
|
+
`headers` param is a `Hash`, you can add key/value data. Header keys must star
|
107
|
+
with `X-` prefix. More detail can found at [Mozilla](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers) site.
|
108
|
+
|
96
109
|
---
|
97
110
|
|
98
111
|
## Development
|
data/lib/scraper_rb.rb
CHANGED
@@ -20,8 +20,8 @@ module ScraperRb
|
|
20
20
|
end
|
21
21
|
|
22
22
|
class << self
|
23
|
-
def new(url, params={}, timeout=10)
|
24
|
-
ScraperRb::Scraper.new(url, params, timeout)
|
23
|
+
def new(url, params={}, headers={}, timeout=10)
|
24
|
+
ScraperRb::Scraper.new(url, params, headers, timeout)
|
25
25
|
end
|
26
26
|
end
|
27
27
|
|
@@ -30,13 +30,19 @@ module ScraperRb
|
|
30
30
|
|
31
31
|
attr_accessor :options, :response
|
32
32
|
|
33
|
-
def initialize(url, params, timeout)
|
33
|
+
def initialize(url, params, extra_headers, timeout)
|
34
34
|
params = {} if params == nil
|
35
|
+
default_headers = {
|
36
|
+
'Accept' => 'application/json',
|
37
|
+
'apikey' => ENV['PROMPTAPI_TOKEN'],
|
38
|
+
}
|
39
|
+
default_headers.merge!(extra_headers) if extra_headers
|
40
|
+
|
35
41
|
@options = {
|
36
42
|
url: ENV['PROMPTAPI_TEST_ENDPOINT'] || 'https://api.promptapi.com/scraper',
|
37
43
|
params: {url: url},
|
38
44
|
request: {timeout: timeout},
|
39
|
-
headers:
|
45
|
+
headers: default_headers,
|
40
46
|
}
|
41
47
|
params.each do |key, value|
|
42
48
|
@options[:params][key] = value if VALID_PARAMS.map(&:to_sym).include?(key)
|
data/lib/scraper_rb/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: scraper_rb
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Prompt API
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-10-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: wirble
|