datahen 0.14.16 → 0.14.17

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 206ac0cf7cc5dc64deecff24f1d731f353951390c4c96d3cc6721a4dd0ecb258
4
- data.tar.gz: 00d8e528649b2296d585d5008ebbf2de62d100449e5841f4f220147fcebc2484
3
+ metadata.gz: a3cf1bf50610ab9ba523ca8f9ae1fdb307831cb56018036076f31353b357edfa
4
+ data.tar.gz: '0199ea98a2f171675168699adc523932418b9822ff156c71786458c6362b6cdb'
5
5
  SHA512:
6
- metadata.gz: 2c66c9527f38db20a7f0ed589c560a2dbafdd038e1834b8552305cd665fec8063fab3c6dcd31eab9df07f2bc04ce5682272bea61772f0dfac20cf84f05b4be35
7
- data.tar.gz: 6c327375e3e1eba2235194f0b58ab280b5ecbbb36fd0cf4808d9cf89b539240d0fead54ee8a4019bd0f196ddd2c7621b2ace8c1dd2da80ba6b665b1ca41a015f
6
+ metadata.gz: 8d66226573dbd9bd3ef795ce021eb9ee202b21cce1d0b79211093bb9abda6f1982a721bf8ae1fbc9bf84aae6e50d1878bf9d389ee67d685e4d1c440fd88cceeb
7
+ data.tar.gz: 6ea1a7748cf77ae1cb8f750da71df86120f68e671c84e535e8536ee3324ed71026034a042662d081108ff7b72ccb3f0ef3ab3f049bdb45c92fc0e86ba2e10d46
@@ -111,6 +111,7 @@ module Datahen
111
111
  puts "Must specify either a --gid, --fetch-fail, --parse-fail or --status"
112
112
  return
113
113
  end
114
+
114
115
  if options[:job]
115
116
  client = Client::JobPage.new(options)
116
117
  puts "#{client.refetch(options[:job])}"
@@ -129,26 +130,39 @@ module Datahen
129
130
  option :status, type: :string, desc: 'Reparse only pages with a specific status.'
130
131
  option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
131
132
  def reparse(scraper_name)
132
- begin
133
- options[:vars] = JSON.parse(options[:vars]) if options[:vars]
133
+ if !options.key?(:gid) && !options.key?(:parse_fail) && !options.key?(:status)
134
+ puts "Must specify either a --gid, --parse-fail or --status"
135
+ return
136
+ end
134
137
 
135
- if !options.key?(:gid) && !options.key?(:parse_fail) && !options.key?(:status)
136
- puts "Must specify either a --gid, --parse-fail or --status"
137
- return
138
- end
138
+ if options[:job]
139
+ client = Client::JobPage.new(options)
140
+ puts "#{client.reparse(options[:job])}"
141
+ else
142
+ client = Client::ScraperJobPage.new(options)
143
+ puts "#{client.reparse(scraper_name)}"
144
+ end
145
+ end
139
146
 
140
- if options[:job]
141
- client = Client::JobPage.new(options)
142
- puts "#{client.reparse(options[:job])}"
143
- else
144
- client = Client::ScraperJobPage.new(options)
145
- puts "#{client.reparse(scraper_name)}"
146
- end
147
+ desc "limbo <scraper_name>", "Move pages on a scraper's current job to limbo"
148
+ long_desc <<-LONGDESC
149
+ Move pages in a scraper's current job to limbo. You need to specify either a --gid or --status.\x5
150
+ LONGDESC
151
+ option :gid, :aliases => :g, type: :string, desc: 'Move a specific GID to limbo'
152
+ option :status, type: :string, desc: 'Move pages with a specific status to limbo.'
153
+ option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
154
+ def limbo(scraper_name)
155
+ if !options.key?(:gid) && !options.key?(:status)
156
+ puts "Must specify either a --gid or --status"
157
+ return
158
+ end
147
159
 
148
- rescue JSON::ParserError
149
- if options[:vars]
150
- puts "Error: #{options[:vars]} on vars is not a valid JSON"
151
- end
160
+ if options[:job]
161
+ client = Client::JobPage.new(options)
162
+ puts "#{client.limbo(options[:job])}"
163
+ else
164
+ client = Client::ScraperJobPage.new(options)
165
+ puts "#{client.limbo(scraper_name)}"
152
166
  end
153
167
  end
154
168
 
@@ -72,6 +72,11 @@ module Datahen
72
72
  params = @options.merge(opts)
73
73
  self.class.put("/jobs/#{job_id}/pages/refetch", params)
74
74
  end
75
+
76
+ def limbo(job_id, opts={})
77
+ params = @options.merge(opts)
78
+ self.class.put("/jobs/#{job_id}/pages/limbo", params)
79
+ end
75
80
  end
76
81
  end
77
82
  end
@@ -26,7 +26,7 @@ module Datahen
26
26
  self.class.put("/scrapers/#{scraper_name}/current_job/pages/refetch", params)
27
27
  end
28
28
 
29
- # Deprecated, please use Datahen::Client::JobVar#refetch instead.
29
+ # Deprecated, please use Datahen::Client::JobPage#refetch instead.
30
30
  #
31
31
  # @note This method will be removed at some point in the future.
32
32
  def refetch_by_job(job_id, opts={})
@@ -39,6 +39,11 @@ module Datahen
39
39
  self.class.put("/scrapers/#{scraper_name}/current_job/pages/reparse", params)
40
40
  end
41
41
 
42
+ def limbo(scraper_name, opts={})
43
+ params = @options.merge(opts)
44
+ self.class.put("/scrapers/#{scraper_name}/current_job/pages/limbo", params)
45
+ end
46
+
42
47
  def enqueue(scraper_name, method, url, opts={})
43
48
  body = {}
44
49
  body[:method] = method != "" ? method : "GET"
@@ -1,3 +1,3 @@
1
1
  module Datahen
2
- VERSION = "0.14.16"
2
+ VERSION = "0.14.17"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: datahen
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.14.16
4
+ version: 0.14.17
5
5
  platform: ruby
6
6
  authors:
7
7
  - Parama Danoesubroto
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-10-14 00:00:00.000000000 Z
11
+ date: 2020-10-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: thor
@@ -266,7 +266,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
266
266
  - !ruby/object:Gem::Version
267
267
  version: '0'
268
268
  requirements: []
269
- rubygems_version: 3.1.2
269
+ rubygems_version: 3.0.3
270
270
  signing_key:
271
271
  specification_version: 4
272
272
  summary: DataHen toolbelt for developers