datahen 0.12.0 → 0.13.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/datahen/cli/scraper.rb +2 -0
- data/lib/datahen/scraper/executor.rb +2 -2
- data/lib/datahen/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c3d1584f235873f6d22ae107c60e7b50a6d8ab5918f45a4052bef80e58b7cbf7
|
4
|
+
data.tar.gz: b6b6ce4871017eddb70fd279c92246e498e57c3d7ca2c0bd72c225bdcdb5e119
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 713f5907efc90be21ba04b83c060f0f656d1f72176191399e1ff575952cad2b54c834ddfd6722441e933857d12539f55553adf2b4d71adead952465f0ccf1005
|
7
|
+
data.tar.gz: bd7809ed5fbd6d12dc8680f31f194ecfb51ba0138765a88c9dde4745be2bd46533da97885f63524edca09cee1ab2d4cc3d7fd96bf04a2b6a6930a046646a8093
|
data/lib/datahen/cli/scraper.rb
CHANGED
@@ -102,6 +102,7 @@ module Datahen
|
|
102
102
|
option :head, :aliases => :H, desc: 'Show the oldest log entries. If not set, newest entries is shown'
|
103
103
|
option :parsing, :aliases => :p, type: :boolean, desc: 'Show only log entries related to parsing errors'
|
104
104
|
option :seeding, :aliases => :s, type: :boolean, desc: 'Show only log entries related to seeding errors'
|
105
|
+
option :finisher, :aliases => :f, type: :boolean, desc: 'Show only log entries related to finisher errors'
|
105
106
|
option :more, :aliases => :m, desc: 'Show next set of log entries. Enter the `More token`'
|
106
107
|
option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 5000 per page.'
|
107
108
|
def log(scraper_name)
|
@@ -111,6 +112,7 @@ module Datahen
|
|
111
112
|
query["order"] = options.delete(:head) if options[:head]
|
112
113
|
query["job_type"] = "parsing" if options[:parsing]
|
113
114
|
query["job_type"] = "seeding" if options[:seeding]
|
115
|
+
query["job_type"] = "finisher executing" if options[:finisher]
|
114
116
|
query["page_token"] = options.delete(:more) if options[:more]
|
115
117
|
query["per_page"] = options.delete(:per_page) if options[:per_page]
|
116
118
|
|
@@ -76,8 +76,8 @@ module Datahen
|
|
76
76
|
end
|
77
77
|
|
78
78
|
def get_failed_content(gid)
|
79
|
-
client = Client::
|
80
|
-
content_json = client.find_failed_content(gid)
|
79
|
+
client = Client::JobPage.new()
|
80
|
+
content_json = client.find_failed_content(job_id, gid)
|
81
81
|
|
82
82
|
if content_json['available']
|
83
83
|
signed_url = content_json['signed_url']
|
data/lib/datahen/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: datahen
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.13.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Parama Danoesubroto
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-02
|
11
|
+
date: 2020-03-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|