datahen 0.14.1 → 0.14.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/datahen/cli/scraper_job.rb +8 -2
- data/lib/datahen/cli/scraper_page.rb +1 -1
- data/lib/datahen/scraper/executor.rb +9 -3
- data/lib/datahen/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 10e4e646ff403fbdb91e0be43e0a78a879c48f3686be251f26185cae961e4b4f
|
4
|
+
data.tar.gz: bc8a4f11cb3929e2cf1655d7463f02c6dd9166a1994a7cbdb69cddf327099f95
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7d57fc498d3f3b6bcf08a31a663a48f5ace6fff5f9fdeeb1b8f396710e6d7e96f31b1a0dc460aeab312d468d19098b759cc19d6c60ecee9286a409934cfd5872
|
7
|
+
data.tar.gz: ff1585a9b3be081898c597d5e696f0927b90f54d420de3bab39451b16dbd55d622a1aa7c9b631dac22cc9f84d1c5e60b86bf5284bde1cb0abb0a24ed69a5834e
|
@@ -7,10 +7,16 @@ module Datahen
|
|
7
7
|
end
|
8
8
|
|
9
9
|
desc "show <scraper_name>", "Show a scraper's current job (Defaults to showing data from cached job)"
|
10
|
+
option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
|
10
11
|
option :live, type: :boolean, desc: 'Get data from the live job, not cached job.'
|
11
12
|
def show(scraper_name)
|
12
|
-
|
13
|
-
|
13
|
+
if options[:job]
|
14
|
+
client = Client::Job.new(options)
|
15
|
+
puts "#{client.find(options[:job], options)}"
|
16
|
+
else
|
17
|
+
client = Client::ScraperJob.new(options)
|
18
|
+
puts "#{client.find(scraper_name, options)}"
|
19
|
+
end
|
14
20
|
end
|
15
21
|
|
16
22
|
|
@@ -230,7 +230,7 @@ module Datahen
|
|
230
230
|
end
|
231
231
|
end
|
232
232
|
|
233
|
-
desc "failedcontent <gid>", "Show a page's failed content in scraper's current job"
|
233
|
+
desc "failedcontent <scraper_name> <gid>", "Show a page's failed content in scraper's current job"
|
234
234
|
option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
|
235
235
|
def failedcontent(scraper_name, gid)
|
236
236
|
result = nil
|
@@ -60,7 +60,12 @@ module Datahen
|
|
60
60
|
|
61
61
|
def init_global_page()
|
62
62
|
client = Client::GlobalPage.new()
|
63
|
-
client.find(gid)
|
63
|
+
global_page = client.find(gid)
|
64
|
+
unless global_page.code == 200
|
65
|
+
raise "GID #{gid} not found. Aborting execution!"
|
66
|
+
else
|
67
|
+
global_page
|
68
|
+
end
|
64
69
|
end
|
65
70
|
|
66
71
|
def get_content(job_id, gid)
|
@@ -287,11 +292,12 @@ module Datahen
|
|
287
292
|
end
|
288
293
|
|
289
294
|
# behave differently if it is a real save
|
295
|
+
save_status = status
|
290
296
|
if save
|
291
297
|
log_msg = "Saving #{log_msgs.join(' and ')}."
|
292
298
|
puts "#{log_msg}"
|
293
299
|
else
|
294
|
-
|
300
|
+
save_status = "#{status}_try"
|
295
301
|
end
|
296
302
|
|
297
303
|
# saving to server
|
@@ -300,7 +306,7 @@ module Datahen
|
|
300
306
|
gid: gid,
|
301
307
|
pages: pages_slice,
|
302
308
|
outputs: outputs_slice,
|
303
|
-
status:
|
309
|
+
status: save_status)
|
304
310
|
|
305
311
|
if response.code == 200
|
306
312
|
if save
|
data/lib/datahen/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: datahen
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.14.
|
4
|
+
version: 0.14.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Parama Danoesubroto
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-07-
|
11
|
+
date: 2020-07-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|
@@ -264,7 +264,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
264
264
|
- !ruby/object:Gem::Version
|
265
265
|
version: '0'
|
266
266
|
requirements: []
|
267
|
-
rubygems_version: 3.
|
267
|
+
rubygems_version: 3.0.3
|
268
268
|
signing_key:
|
269
269
|
specification_version: 4
|
270
270
|
summary: DataHen toolbelt for developers
|