answersengine 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/lib/answersengine/cli.rb +1 -0
- data/lib/answersengine/cli/scraper.rb +8 -5
- data/lib/answersengine/cli/scraper_finisher.rb +20 -0
- data/lib/answersengine/client.rb +1 -0
- data/lib/answersengine/client/scraper_finisher.rb +16 -0
- data/lib/answersengine/version.rb +1 -1
- metadata +5 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 9c2102f889a975b955b5e8c93b87765a41f0a8d7170be8957c1c5bf9fb01df83
|
4
|
+
data.tar.gz: 323570c1356eba229120ef90c0ca62fdf7b57115f1a7ac60acb8142ebfe6bad0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1ac0f2b8dd121a39ef4f9dafd672ff48908067f4aa5eb390a67e76110e8de4b09aaf5458bef4623298ca37a879e614ef4556a485873a17603fd99584521db22a
|
7
|
+
data.tar.gz: 66ba95c54f2aee53c6bd34a1a9a5b50fb4b070c2755be8b98b37f57a24469551dae05ed2474d6ed0a2e5acdf7b045efbdfeafaea9ff0678c1951506341813d97
|
data/lib/answersengine/cli.rb
CHANGED
@@ -5,6 +5,7 @@ require 'answersengine/cli/scraper_exporter'
|
|
5
5
|
require 'answersengine/cli/scraper_export'
|
6
6
|
require 'answersengine/cli/scraper_job_var'
|
7
7
|
require 'answersengine/cli/scraper_job'
|
8
|
+
require 'answersengine/cli/scraper_finisher'
|
8
9
|
require 'answersengine/cli/global_page'
|
9
10
|
require 'answersengine/cli/scraper_page'
|
10
11
|
require 'answersengine/cli/job_output'
|
@@ -2,7 +2,7 @@ module AnswersEngine
|
|
2
2
|
class CLI < Thor
|
3
3
|
class Scraper < Thor
|
4
4
|
desc "list", "List scrapers"
|
5
|
-
|
5
|
+
|
6
6
|
long_desc <<-LONGDESC
|
7
7
|
List all scrapers.
|
8
8
|
LONGDESC
|
@@ -25,8 +25,8 @@ module AnswersEngine
|
|
25
25
|
option :force_fetch, :aliases => :f, type: :boolean, desc: 'Set true to force fetch page that is not within freshness criteria. Default: false'
|
26
26
|
option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Default: 1'
|
27
27
|
option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
|
28
|
-
option :disable_scheduler, type: :boolean, desc: 'Set true to disable scheduler. Default: false'
|
29
|
-
option :cancel_current_job, type: :boolean, desc: 'Set true to cancel currently active job if scheduler starts. Default: false'
|
28
|
+
option :disable_scheduler, type: :boolean, desc: 'Set true to disable scheduler. Default: false'
|
29
|
+
option :cancel_current_job, type: :boolean, desc: 'Set true to cancel currently active job if scheduler starts. Default: false'
|
30
30
|
option :schedule, type: :string, desc: 'Set the schedule of the scraper to run. Must be in CRON format.'
|
31
31
|
option :timezone, type: :string, desc: "Set the scheduler's timezone. Must be in IANA Timezone format. Defaults to \"America/Toronto\""
|
32
32
|
def create(scraper_name, git_repository)
|
@@ -47,8 +47,8 @@ module AnswersEngine
|
|
47
47
|
option :force_fetch, :aliases => :f, type: :boolean, desc: 'Set true to force fetch page that is not within freshness criteria. Default: false'
|
48
48
|
option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Default: 1'
|
49
49
|
option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
|
50
|
-
option :disable_scheduler, type: :boolean, desc: 'Set true to disable scheduler. Default: false'
|
51
|
-
option :cancel_current_job, type: :boolean, desc: 'Set true to cancel currently active job if scheduler starts. Default: false'
|
50
|
+
option :disable_scheduler, type: :boolean, desc: 'Set true to disable scheduler. Default: false'
|
51
|
+
option :cancel_current_job, type: :boolean, desc: 'Set true to cancel currently active job if scheduler starts. Default: false'
|
52
52
|
option :schedule, type: :string, desc: 'Set the schedule of the scraper to run. Must be in CRON format.'
|
53
53
|
option :timezone, type: :string, desc: "Set the scheduler's timezone. Must be in IANA Timezone format. Defaults to \"America/Toronto\""
|
54
54
|
def update(scraper_name)
|
@@ -160,6 +160,9 @@ module AnswersEngine
|
|
160
160
|
desc "deployment SUBCOMMAND ...ARGS", "manage scrapers deployments"
|
161
161
|
subcommand "deployment", ScraperDeployment
|
162
162
|
|
163
|
+
desc "finisher SUBCOMMAND ...ARGS", "manage scrapers finishers"
|
164
|
+
subcommand "finisher", ScraperFinisher
|
165
|
+
|
163
166
|
desc "output SUBCOMMAND ...ARGS", "view scraper outputs"
|
164
167
|
subcommand "output", JobOutput
|
165
168
|
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module AnswersEngine
|
2
|
+
class CLI < Thor
|
3
|
+
class ScraperFinisher < Thor
|
4
|
+
|
5
|
+
package_name "scraper finisher"
|
6
|
+
def self.banner(command, namespace = nil, subcommand = false)
|
7
|
+
"#{basename} #{@package_name} #{command.usage}"
|
8
|
+
end
|
9
|
+
|
10
|
+
desc "reset <scraper_name>", "Reset finisher on a scraper's current job"
|
11
|
+
long_desc <<-LONGDESC
|
12
|
+
Reset finisher on a scraper's current job.\x5
|
13
|
+
LONGDESC
|
14
|
+
def reset(scraper_name)
|
15
|
+
client = Client::ScraperFinisher.new(options)
|
16
|
+
puts "#{client.reset(scraper_name)}"
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
data/lib/answersengine/client.rb
CHANGED
@@ -9,6 +9,7 @@ require "answersengine/client/scraper_job_page"
|
|
9
9
|
require "answersengine/client/scraper_exporter"
|
10
10
|
require "answersengine/client/scraper_export"
|
11
11
|
require "answersengine/client/scraper_job"
|
12
|
+
require "answersengine/client/scraper_finisher"
|
12
13
|
require 'answersengine/client/job_export'
|
13
14
|
require "answersengine/client/job"
|
14
15
|
require "answersengine/client/job_log"
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module AnswersEngine
|
2
|
+
module Client
|
3
|
+
class ScraperFinisher < AnswersEngine::Client::Base
|
4
|
+
# Reset finisher on a scraper's current job.
|
5
|
+
#
|
6
|
+
# @param [String] scraper_name Scraper name.
|
7
|
+
# @param [Hash] opts ({}) API custom parameters.
|
8
|
+
#
|
9
|
+
# @return [HTTParty::Response]
|
10
|
+
def reset(scraper_name, opts={})
|
11
|
+
params = @options.merge(opts)
|
12
|
+
self.class.put("/scrapers/#{scraper_name}/current_job/finisher/reset", params)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: answersengine
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.9.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Parama Danoesubroto
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-09-
|
11
|
+
date: 2019-09-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|
@@ -199,6 +199,7 @@ files:
|
|
199
199
|
- lib/answersengine/cli/scraper_deployment.rb
|
200
200
|
- lib/answersengine/cli/scraper_export.rb
|
201
201
|
- lib/answersengine/cli/scraper_exporter.rb
|
202
|
+
- lib/answersengine/cli/scraper_finisher.rb
|
202
203
|
- lib/answersengine/cli/scraper_job.rb
|
203
204
|
- lib/answersengine/cli/scraper_job_var.rb
|
204
205
|
- lib/answersengine/cli/scraper_page.rb
|
@@ -222,6 +223,7 @@ files:
|
|
222
223
|
- lib/answersengine/client/scraper_deployment.rb
|
223
224
|
- lib/answersengine/client/scraper_export.rb
|
224
225
|
- lib/answersengine/client/scraper_exporter.rb
|
226
|
+
- lib/answersengine/client/scraper_finisher.rb
|
225
227
|
- lib/answersengine/client/scraper_job.rb
|
226
228
|
- lib/answersengine/client/scraper_job_output.rb
|
227
229
|
- lib/answersengine/client/scraper_job_page.rb
|
@@ -261,7 +263,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
261
263
|
version: '0'
|
262
264
|
requirements: []
|
263
265
|
rubyforge_project:
|
264
|
-
rubygems_version: 2.6
|
266
|
+
rubygems_version: 2.7.6
|
265
267
|
signing_key:
|
266
268
|
specification_version: 4
|
267
269
|
summary: AnswersEngine toolbelt for developers
|