datahen 0.14.19 → 0.14.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/datahen/cli/scraper.rb +1 -0
- data/lib/datahen/client/scraper_job.rb +7 -0
- data/lib/datahen/version.rb +1 -1
- metadata +5 -5
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 4a2f75a84728b7e8c228c578e4a4af6253dcd19db445f509ba44f9866847936c
|
|
4
|
+
data.tar.gz: c37744322cc3e3035a31dd69bca340397ac906d27a3497604e3d98321187d972
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: fcf774827b35bf23048b47241da29b1a17e2b1d691a360d2afcb5b34c206c26b7ae316c457e26b946ae3144b4d9075dda5db90a2fd0fb7e56b9f53b0cd612d13
|
|
7
|
+
data.tar.gz: 56fe24f850c0b695b87c61629ab6939f856ecf8f0ea6b6557d59646214b6ffd87557a6ade6bf59e03b993626663cc1af3071ca10858802272d75d47cf34de56a
|
data/lib/datahen/cli/scraper.rb
CHANGED
|
@@ -93,6 +93,7 @@ module Datahen
|
|
|
93
93
|
option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Default: 1'
|
|
94
94
|
option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
|
|
95
95
|
option :proxy_type, desc: 'Set the Proxy type. Default: standard'
|
|
96
|
+
option :vars, type: :string, banner: :JSON, desc: 'Set input vars. Must be in json format. i.e: {"name":"foo", "value":"bar", "secret":false} '
|
|
96
97
|
def start(scraper_name)
|
|
97
98
|
client = Client::ScraperJob.new(options)
|
|
98
99
|
puts "Starting a scrape job..."
|
|
@@ -11,6 +11,13 @@ module Datahen
|
|
|
11
11
|
body[:standard_worker_count] = opts[:workers] if opts[:workers]
|
|
12
12
|
body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
|
|
13
13
|
body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
|
|
14
|
+
if opts[:vars]
|
|
15
|
+
if opts[:vars].is_a?(Array)
|
|
16
|
+
body[:vars] = opts[:vars]
|
|
17
|
+
elsif opts[:vars].is_a?(String)
|
|
18
|
+
body[:vars] = JSON.parse(opts[:vars])
|
|
19
|
+
end
|
|
20
|
+
end
|
|
14
21
|
params = @options.merge({body: body.to_json})
|
|
15
22
|
self.class.post("/scrapers/#{scraper_name}/jobs", params)
|
|
16
23
|
end
|
data/lib/datahen/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: datahen
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.14.
|
|
4
|
+
version: 0.14.20
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Parama Danoesubroto
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2020-
|
|
11
|
+
date: 2020-12-14 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: thor
|
|
@@ -253,7 +253,7 @@ metadata:
|
|
|
253
253
|
allowed_push_host: https://rubygems.org
|
|
254
254
|
homepage_uri: https://datahen.com
|
|
255
255
|
source_code_uri: https://github.com/DataHenOfficial/datahen-ruby
|
|
256
|
-
post_install_message:
|
|
256
|
+
post_install_message:
|
|
257
257
|
rdoc_options: []
|
|
258
258
|
require_paths:
|
|
259
259
|
- lib
|
|
@@ -269,7 +269,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
269
269
|
version: '0'
|
|
270
270
|
requirements: []
|
|
271
271
|
rubygems_version: 3.0.3
|
|
272
|
-
signing_key:
|
|
272
|
+
signing_key:
|
|
273
273
|
specification_version: 4
|
|
274
274
|
summary: DataHen toolbelt for developers
|
|
275
275
|
test_files: []
|