scraper_rb 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 879c8d4f6057bdb06c36073d71063a9ef40ad0544484bab1546093621e24c319
4
- data.tar.gz: f75e6f6e2105d8cc9dbd4e7808c061a0148115904e29fee6f93ab39ac54b270b
3
+ metadata.gz: a828454fe71dc2b90bfd9fde3e859d149ca3a2c0c2f38fee1d82d7b78537746c
4
+ data.tar.gz: 98b66eef645c08144966f86a8e6fbf0d61ee8ff26eb5de53906680dee846c6b1
5
5
  SHA512:
6
- metadata.gz: 54764b1ddda207a57b1a2c6ae12e62bdc5c823f855e04aeed18e3f9fd3064545c19e162a3bc3f51b53b545d891cb4fa74d98c0766cc1cc762a07308a9d3716e5
7
- data.tar.gz: bdc1300679693f8b8f4956f502256ae481b8959171785041f2fb42cef554c4dee9d82787c5b1454fe2274ce6189544d48731f2ba851974c3e9ae5c61ba5ef28a
6
+ metadata.gz: d3a3ab8786817af29be63cae500c04987f83418ced89779a4ee21dd296322f0817e85a0cb45491960ec0a8d36c34cff7a2a50af3b0855adf57ae8feedf30e421
7
+ data.tar.gz: 89a39dffbaea50064fcd944f89a5b02bae78cc5130f6f16fe58429f965e590f2ae15e41ddf1d9617bf9477be3f3467d6b700ff87c9f1367df81052ddae47dc83
@@ -1,5 +1,5 @@
1
1
  [bumpversion]
2
- current_version = 0.1.1
2
+ current_version = 0.1.2
3
3
  commit = True
4
4
 
5
5
  [bumpversion:file:README.md]
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- scraper_rb (0.1.1)
4
+ scraper_rb (0.1.2)
5
5
  faraday (~> 1.0, >= 1.0.1)
6
6
 
7
7
  GEM
data/README.md CHANGED
@@ -21,7 +21,7 @@ $ gem install scraper_rb
21
21
  or; install from GitHub:
22
22
 
23
23
  ```bash
24
- $ gem install scraper_rb --version "0.1.1" --source "https://rubygems.pkg.github.com/promptapi"
24
+ $ gem install scraper_rb --version "0.1.2" --source "https://rubygems.pkg.github.com/promptapi"
25
25
  ```
26
26
 
27
27
  ---
@@ -86,13 +86,26 @@ Default **timeout** value is set to `10` seconds. You can change this while
86
86
  initializing the instance:
87
87
 
88
88
  ```ruby
89
- s = ScraperRb.new('https://pypi.org/classifiers/', {}, timeout=50)
89
+ s = ScraperRb.new('https://pypi.org/classifiers/', params={}, timeout=50)
90
90
  # => 50 seconds timeout w/o params
91
91
 
92
- s = ScraperRb.new('https://pypi.org/classifiers/', {country: 'EE'}, timeout=50)
92
+ s = ScraperRb.new('https://pypi.org/classifiers/', params={country: 'EE'}, timeout=50)
93
93
  # => 50 seconds timeout
94
94
  ```
95
95
 
96
+ You can add extra `X-` headers:
97
+
98
+ ```ruby
99
+ s = ScraperRb.new('https://pypi.org/classifiers/', headers={'X-Referer': 'https://www.google.com'})
100
+
101
+ # or
102
+ s = ScraperRb.new('https://pypi.org/classifiers/', params={country: 'EE'}, headers={'X-Referer': 'https://www.google.com'}, timeout=50)
103
+ # => 50 seconds timeout
104
+ ```
105
+
106
+ `headers` param is a `Hash`, you can add key/value data. Header keys must star
107
+ with `X-` prefix. More detail can found at [Mozilla](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers) site.
108
+
96
109
  ---
97
110
 
98
111
  ## Development
@@ -20,8 +20,8 @@ module ScraperRb
20
20
  end
21
21
 
22
22
  class << self
23
- def new(url, params={}, timeout=10)
24
- ScraperRb::Scraper.new(url, params, timeout)
23
+ def new(url, params={}, headers={}, timeout=10)
24
+ ScraperRb::Scraper.new(url, params, headers, timeout)
25
25
  end
26
26
  end
27
27
 
@@ -30,13 +30,19 @@ module ScraperRb
30
30
 
31
31
  attr_accessor :options, :response
32
32
 
33
- def initialize(url, params, timeout)
33
+ def initialize(url, params, extra_headers, timeout)
34
34
  params = {} if params == nil
35
+ default_headers = {
36
+ 'Accept' => 'application/json',
37
+ 'apikey' => ENV['PROMPTAPI_TOKEN'],
38
+ }
39
+ default_headers.merge!(extra_headers) if extra_headers
40
+
35
41
  @options = {
36
42
  url: ENV['PROMPTAPI_TEST_ENDPOINT'] || 'https://api.promptapi.com/scraper',
37
43
  params: {url: url},
38
44
  request: {timeout: timeout},
39
- headers: {'Accept' => 'application/json', 'apikey' => ENV['PROMPTAPI_TOKEN']},
45
+ headers: default_headers,
40
46
  }
41
47
  params.each do |key, value|
42
48
  @options[:params][key] = value if VALID_PARAMS.map(&:to_sym).include?(key)
@@ -1,3 +1,3 @@
1
1
  module ScraperRb
2
- VERSION = "0.1.1"
2
+ VERSION = "0.1.2"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: scraper_rb
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.1
4
+ version: 0.1.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Prompt API
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-09-14 00:00:00.000000000 Z
11
+ date: 2020-10-06 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: wirble