answersengine 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f6abf0aa1f88309da00bf88ad024eed7b7a99b2e5c0a4f63311b42751985bca2
-  data.tar.gz: a3d8c454ed93e08bfadb67f4b4934055cf04025e08a8372617dc86b4b4feb1e7
+  metadata.gz: be8e1b2d4a1ca60a2b0f1ccc1fd5c00f33aa2f412214d9294e357ec2bfd353f1
+  data.tar.gz: ec3e082f0b8905a313f43a1d7d6018640c06baa6d06501f62126cab2dbe0af32
 SHA512:
-  metadata.gz: 3ca6e10e1e577f013bb8d72dd2df785362975c9619d2fb93d0ed4b69bac85466ad1a9d172b9c96438aea2682504a06b40e73595b59be3930ed54e9cbf9dbadbe
-  data.tar.gz: 3dccf75a92138fd9f879b128f48f7e1443eac668fa700e2bbbf74e3ae3f4bda93019cac0ee1c9626608594f063ecb5addb3bdb73328792e9d2ecbbeda9f72533
+  metadata.gz: 5dec73a502a3431a5471f02a2b4093d57ed0cd23528a5ea6ec3915ea67031e724471df15d36499e5217a37ad0f32a51a87e7329d095670baedde6f5186ee2f42
+  data.tar.gz: a2864beba98cdc6660e091649e89fb0756f0edd3528443ba8231ae17d079cd1efa8adab80329271f32b7dc80ab68b0f1ca9d12897dea82ab0284d66bda2566f3

data/lib/answersengine/cli/finisher.rb

@@ -0,0 +1,40 @@
+module AnswersEngine
+  class CLI < Thor
+    class Finisher < Thor
+      desc "try <scraper_name> <finisher_file>", "Tries a finisher file"
+      long_desc <<-LONGDESC
+            Takes a finisher script and tries to execute it without saving anything.\x5
+            <seeder_file>: Finisher script file will be executed.\x5
+          LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      def try_finisher(scraper_name, finisher_file)
+        if options[:job]
+          job_id = options[:job]
+        else
+          job = Client::ScraperJob.new(options).find(scraper_name)
+          job_id = job['id']
+        end
+
+        puts AnswersEngine::Scraper::Finisher.exec_finisher(finisher_file, job_id, false)
+      end
+
+      desc "exec <scraper_name> <finisher_file>", "Executes a finisher script onto a scraper's current job."
+      long_desc <<-LONGDESC
+            Takes a finisher script and execute it against a job and save outputs into the scraper's current job\x5
+            <finisher_file>: Finisher script file that will be executed on the scraper's current job.\x5
+          LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      def exec_parse(scraper_name, finisher_file)
+        if options[:job]
+          job_id = options[:job]
+        else
+          job = Client::ScraperJob.new(options).find(scraper_name)
+          job_id = job['id']
+        end
+
+        puts AnswersEngine::Scraper::Finisher.exec_finisher(finisher_file, job_id, true)
+      end
+    end
+  end
+
+end

data/lib/answersengine/cli.rb

@@ -11,6 +11,7 @@ require 'answersengine/cli/scraper_deployment'
 require 'answersengine/cli/scraper'
 require 'answersengine/cli/parser'
 require 'answersengine/cli/seeder'
+require 'answersengine/cli/finisher'
 
 
 module AnswersEngine
@@ -29,5 +30,8 @@ module AnswersEngine
 
     desc "seeder SUBCOMMAND ...ARGS", "for seeding related activities"
     subcommand "seeder", Seeder
+
+    desc "seeder SUBCOMMAND ...ARGS", "for seeding related activities"
+    subcommand "finisher", Finisher
   end
-end
+end

data/lib/answersengine/client/auth_token.rb

@@ -7,7 +7,8 @@ module AnswersEngine
       end
 
       def all(opts={})
-        self.class.get("/auth_tokens", @options)
+        params = @options.merge(opts)
+        self.class.get("/auth_tokens", params)
       end
 
       def create(role, description, opts={})
@@ -15,8 +16,8 @@ module AnswersEngine
             role: role,
             description: description}
 
-        @options.merge!({body: body.to_json})
-        self.class.post("/auth_tokens", @options)
+        params = @options.merge({body: body.to_json})
+        self.class.post("/auth_tokens", params)
       end
 
       def create_on_account(account_id, role, description)
@@ -24,8 +25,8 @@ module AnswersEngine
             role: role,
             description: description}
 
-        @options.merge!({body: body.to_json})
-        self.class.post("/accounts/#{account_id}/auth_tokens", @options)
+        params = @options.merge({body: body.to_json})
+        self.class.post("/accounts/#{account_id}/auth_tokens", params)
       end
 
       def update(token, role, description="", opts={})
@@ -33,18 +34,17 @@ module AnswersEngine
 
         body[:role] = role
         body[:description] = description if description.present?
-        @options.merge!({body: body.to_json})
+        params = @options.merge({body: body.to_json})
 
-        self.class.put("/auth_tokens/#{token}", @options)
+        self.class.put("/auth_tokens/#{token}", params)
       end
 
       def delete(token, opts={})
         body = {}
-        @options.merge!({body: body.to_json})
+        params = @options.merge({body: body.to_json})
 
-        self.class.delete("/auth_tokens/#{token}", @options)
+        self.class.delete("/auth_tokens/#{token}", params)
       end
     end
   end
 end
-

data/lib/answersengine/client/deploy_key.rb

@@ -3,17 +3,19 @@ module AnswersEngine
     class DeployKey < AnswersEngine::Client::Base
 
       def find(opts={})
-        self.class.get("/deploy_key", @options)
+        params = @options.merge(opts)
+        self.class.get("/deploy_key", params)
       end
 
       def create(opts={})
-        self.class.post("/deploy_key", @options)
+        params = @options.merge(opts)
+        self.class.post("/deploy_key", params)
       end
 
       def delete(opts={})
-        self.class.delete("/deploy_key", @options)
+        params = @options.merge(opts)
+        self.class.delete("/deploy_key", params)
       end
     end
   end
 end
-

data/lib/answersengine/client/export.rb

@@ -2,9 +2,9 @@ module AnswersEngine
   module Client
     class Export < AnswersEngine::Client::Base
       def all(opts={})
-        self.class.get("/scrapers/exports", @options)
+        params = @options.merge(opts)
+        self.class.get("/scrapers/exports", params)
       end
     end
   end
 end
-

data/lib/answersengine/client/job.rb

@@ -2,7 +2,8 @@ module AnswersEngine
   module Client
     class Job < AnswersEngine::Client::Base
       def all(opts={})
-        self.class.get("/jobs", @options)
+        params = @options.merge(opts)
+        self.class.get("/jobs", params)
       end
 
       def find(job_id)
@@ -14,9 +15,9 @@ module AnswersEngine
         body[:status] = opts[:status] if opts[:status]
         body[:standard_worker_count] = opts[:workers] if opts[:workers]
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
-        @options.merge!({body: body.to_json})
+        params = @options.merge({body: body.to_json})
 
-        self.class.put("/jobs/#{job_id}", @options)
+        self.class.put("/jobs/#{job_id}", params)
       end
 
       def cancel(job_id, opts={})
@@ -41,9 +42,20 @@ module AnswersEngine
         body[:seeding_status] = opts.fetch(:seeding_status){ nil }
         body[:log_error] = opts[:log_error] if opts[:log_error]
 
-        @options.merge!({body: body.to_json})
+        params = @options.merge({body: body.to_json})
 
-        self.class.put("/jobs/#{job_id}/seeding_update", @options)
+        self.class.put("/jobs/#{job_id}/seeding_update", params)
+      end
+
+      def finisher_update(job_id, opts={})
+        body = {}
+        body[:outputs] = opts.fetch(:outputs) {[]}
+        body[:finisher_status] = opts.fetch(:finisher_status){ nil }
+        body[:log_error] = opts[:log_error] if opts[:log_error]
+
+        params = @options.merge({body: body.to_json})
+
+        self.class.put("/jobs/#{job_id}/finisher_update", params)
       end
 
     end

data/lib/answersengine/client/job_log.rb

@@ -2,26 +2,25 @@ module AnswersEngine
   module Client
     class JobLog < AnswersEngine::Client::Base
       def all_job_page_log(job_id, gid, opts={})
-        @options.merge!(opts)
-        self.class.get("/jobs/#{job_id}/pages/#{gid}/log", @options)
+        params = @options.merge(opts)
+        self.class.get("/jobs/#{job_id}/pages/#{gid}/log", params)
       end
 
       def scraper_all_job_page_log(scraper_name, gid, opts={})
-        @options.merge!(opts)
-        self.class.get("/scrapers/#{scraper_name}/current_job/pages/#{gid}/log", @options)
+        params = @options.merge(opts)
+        self.class.get("/scrapers/#{scraper_name}/current_job/pages/#{gid}/log", params)
       end
 
       def all_job_log(job_id, opts={})
-        @options.merge!(opts)
-        self.class.get("/jobs/#{job_id}/log", @options)
+        params = @options.merge(opts)
+        self.class.get("/jobs/#{job_id}/log", params)
       end
 
       def scraper_all_job_log(scraper_name, opts={})
-        @options.merge!(opts)
-        self.class.get("/scrapers/#{scraper_name}/current_job/log", @options)
+        params = @options.merge(opts)
+        self.class.get("/scrapers/#{scraper_name}/current_job/log", params)
       end
 
     end
   end
 end
-

data/lib/answersengine/client/job_page.rb

@@ -6,7 +6,8 @@ module AnswersEngine
       end
 
       def all(job_id, opts={})
-        self.class.get("/jobs/#{job_id}/pages", @options)
+        params = @options.merge(opts)
+        self.class.get("/jobs/#{job_id}/pages", params)
       end
 
       def update(job_id, gid, opts={})
@@ -15,9 +16,9 @@ module AnswersEngine
         body[:priority] = opts[:priority] if opts[:priority]
         body[:vars] = opts[:vars] if opts[:vars]
 
-        @options.merge!({body: body.to_json})
+        params = @options.merge({body: body.to_json})
 
-        self.class.put("/jobs/#{job_id}/pages/#{gid}", @options)
+        self.class.put("/jobs/#{job_id}/pages/#{gid}", params)
       end
 
       def enqueue(job_id, method, url, opts={})
@@ -36,9 +37,9 @@ module AnswersEngine
         body[:no_redirect] = opts[:no_redirect] if opts[:no_redirect]
         body[:cookie] = opts[:cookie] if opts[:cookie]
 
-        @options.merge!({body: body.to_json})
+        params = @options.merge({body: body.to_json})
 
-        self.class.post("/jobs/#{job_id}/pages", @options)
+        self.class.post("/jobs/#{job_id}/pages", params)
       end
 
       def parsing_update(job_id, gid, opts={})
@@ -48,9 +49,9 @@ module AnswersEngine
         body[:parsing_status] = opts.fetch(:parsing_status){ nil }
         body[:log_error] = opts[:log_error] if opts[:log_error]
 
-        @options.merge!({body: body.to_json})
+        params = @options.merge({body: body.to_json})
 
-        self.class.put("/jobs/#{job_id}/pages/#{gid}/parsing_update", @options)
+        self.class.put("/jobs/#{job_id}/pages/#{gid}/parsing_update", params)
       end
     end
   end

data/lib/answersengine/client/scraper.rb

@@ -7,7 +7,8 @@ module AnswersEngine
       end
 
       def all(opts={})
-        self.class.get("/scrapers", @options)
+        params = @options.merge opts
+        self.class.get("/scrapers", params)
       end
 
       def create(scraper_name, git_repository, opts={})
@@ -24,8 +25,8 @@ module AnswersEngine
         body[:cancel_current_job] = opts[:cancel_current_job] if opts[:cancel_current_job]
         body[:schedule] = opts[:schedule] if opts[:schedule]
         body[:timezone] = opts[:timezone] if opts[:timezone]
-        @options.merge!({body: body.to_json})
-        self.class.post("/scrapers", @options)
+        params = @options.merge({body: body.to_json})
+        self.class.post("/scrapers", params)
       end
 
       def update(scraper_name, opts={})
@@ -42,15 +43,15 @@ module AnswersEngine
         body[:cancel_current_job] = opts[:cancel_current_job] if opts.has_key?("cancel_current_job") || opts.has_key?(:cancel_current_job)
         body[:schedule] = opts[:schedule] if opts[:schedule]
         body[:timezone] = opts[:timezone] if opts[:timezone]
-        @options.merge!({body: body.to_json})
+        params = @options.merge({body: body.to_json})
 
-        self.class.put("/scrapers/#{scraper_name}", @options)
+        self.class.put("/scrapers/#{scraper_name}", params)
       end
 
       def delete(scraper_name, opts={})
-        self.class.delete("/scrapers/#{scraper_name}", @options)
+        params = @options.merge(opts)
+        self.class.delete("/scrapers/#{scraper_name}", params)
       end
     end
   end
 end
-

data/lib/answersengine/client/scraper_deployment.rb

@@ -3,15 +3,16 @@ module AnswersEngine
     class ScraperDeployment < AnswersEngine::Client::Base
 
       def all(scraper_name, opts={})
-        self.class.get("/scrapers/#{scraper_name}/deployments", @options)
+        params = @options.merge(opts)
+        self.class.get("/scrapers/#{scraper_name}/deployments", params)
       end
 
 
       def deploy(scraper_name, opts={})
-        self.class.post("/scrapers/#{scraper_name}/deployments", @options)
+        params = @options.merge(opts)
+        self.class.post("/scrapers/#{scraper_name}/deployments", params)
       end
 
     end
   end
 end
-

data/lib/answersengine/client/scraper_export.rb

@@ -2,7 +2,8 @@ module AnswersEngine
   module Client
     class ScraperExport < AnswersEngine::Client::Base
       def all(scraper_name, opts={})
-        self.class.get("/scrapers/#{scraper_name}/exports", @options)
+        params = @options.merge(opts)
+        self.class.get("/scrapers/#{scraper_name}/exports", params)
       end
 
       def find(export_id)
@@ -19,4 +20,3 @@ module AnswersEngine
     end
   end
 end
-

data/lib/answersengine/client/scraper_exporter.rb

@@ -2,7 +2,8 @@ module AnswersEngine
   module Client
     class ScraperExporter < AnswersEngine::Client::Base
       def all(scraper_name, opts={})
-        self.class.get("/scrapers/#{scraper_name}/exporters", @options)
+        params = @options.merge(opts)
+        self.class.get("/scrapers/#{scraper_name}/exporters", params)
       end
 
       def find(scraper_name, exporter_name)
@@ -11,4 +12,3 @@ module AnswersEngine
     end
   end
 end
-

data/lib/answersengine/client/scraper_job.rb

@@ -2,7 +2,8 @@ module AnswersEngine
   module Client
     class ScraperJob < AnswersEngine::Client::Base
       def all(scraper_name, opts={})
-        self.class.get("/scrapers/#{scraper_name}/jobs", @options)
+        params = @options.merge(opts)
+        self.class.get("/scrapers/#{scraper_name}/jobs", params)
       end
 
       def create(scraper_name, opts={})
@@ -10,8 +11,8 @@ module AnswersEngine
         body[:standard_worker_count] = opts[:workers] if opts[:workers]
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
-        @options.merge!({body: body.to_json})
-        self.class.post("/scrapers/#{scraper_name}/jobs", @options)
+        params = @options.merge({body: body.to_json})
+        self.class.post("/scrapers/#{scraper_name}/jobs", params)
       end
 
       def find(scraper_name)
@@ -24,9 +25,9 @@ module AnswersEngine
         body[:standard_worker_count] = opts[:workers] if opts[:workers]
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
-        @options.merge!({body: body.to_json})
+        params = @options.merge({body: body.to_json})
 
-        self.class.put("/scrapers/#{scraper_name}/current_job", @options)
+        self.class.put("/scrapers/#{scraper_name}/current_job", params)
       end
 
       def cancel(scraper_name, opts={})
@@ -46,4 +47,3 @@ module AnswersEngine
     end
   end
 end
-

data/lib/answersengine/client/scraper_job_page.rb

@@ -6,7 +6,8 @@ module AnswersEngine
       end
 
       def all(scraper_name, opts={})
-        self.class.get("/scrapers/#{scraper_name}/current_job/pages", @options)
+        params = @options.merge(opts)
+        self.class.get("/scrapers/#{scraper_name}/current_job/pages", params)
       end
 
       def update(scraper_name, gid, opts={})
@@ -15,19 +16,19 @@ module AnswersEngine
         body[:priority] = opts[:priority] if opts[:priority]
         body[:vars] = opts[:vars] if opts[:vars]
 
-        @options.merge!({body: body.to_json})
+        params = @options.merge({body: body.to_json})
 
-        self.class.put("/scrapers/#{scraper_name}/current_job/pages/#{gid}", @options)
+        self.class.put("/scrapers/#{scraper_name}/current_job/pages/#{gid}", params)
       end
 
-      def refetch(scraper_name, opts = nil)
-        opts ||= @options
-        self.class.put("/scrapers/#{scraper_name}/current_job/pages/refetch", opts)
+      def refetch(scraper_name, opts={})
+        params = @options.merge(opts)
+        self.class.put("/scrapers/#{scraper_name}/current_job/pages/refetch", params)
       end
 
-      def reparse(scraper_name, opts = nil)
-        opts ||= @options
-        self.class.put("/scrapers/#{scraper_name}/current_job/pages/reparse", opts)
+      def reparse(scraper_name, opts={})
+        params = @options.merge(opts)
+        self.class.put("/scrapers/#{scraper_name}/current_job/pages/reparse", params)
       end
 
       def enqueue(scraper_name, method, url, opts={})
@@ -46,9 +47,9 @@ module AnswersEngine
         body[:no_redirect] = opts[:no_redirect] if opts[:no_redirect]
         body[:cookie] = opts[:cookie] if opts[:cookie]
 
-        @options.merge!({body: body.to_json})
+        params = @options.merge({body: body.to_json})
 
-        self.class.post("/scrapers/#{scraper_name}/current_job/pages", @options)
+        self.class.post("/scrapers/#{scraper_name}/current_job/pages", params)
       end
 
     end

data/lib/answersengine/scraper/executor.rb

@@ -51,6 +51,13 @@ module AnswersEngine
         client.seeding_update(job_id, options)
       end
 
+      def finisher_update(options={})
+        client = Client::Job.new()
+        job_id = options.fetch(:job_id)
+
+        client.finisher_update(job_id, options)
+      end
+
       def init_global_page()
         client = Client::GlobalPage.new()
         client.find(gid)

data/lib/answersengine/scraper/finisher.rb

@@ -0,0 +1,18 @@
+module AnswersEngine
+  module Scraper
+    class Finisher
+
+      def self.exec_finisher(filename, job_id=nil, save=false)
+        extname = File.extname(filename)
+        case extname
+        when '.rb'
+          executor = RubyFinisherExecutor.new(filename: filename, job_id: job_id)
+          executor.exec_finisher(save)
+        else
+          puts "Unable to find a finisher executor for file type \"#{extname}\""
+        end
+      end
+
+    end
+  end
+end

data/lib/answersengine/scraper/ruby_finisher_executor.rb

@@ -0,0 +1,110 @@
+module AnswersEngine
+  module Scraper
+    class RubyFinisherExecutor < Executor
+      attr_accessor :save
+
+      def initialize(options={})
+        @filename = options.fetch(:filename) { raise "Filename is required"}
+        @job_id = options[:job_id]
+      end
+
+      def self.exposed_methods
+        [
+          :outputs,
+          :save_outputs,
+          :find_output,
+          :find_outputs
+        ].freeze
+      end
+
+      def exec_finisher(save=false)
+        @save = save
+        if save
+          puts "Executing finisher script"
+        else
+          puts "Trying finisher script"
+        end
+
+        eval_finisher_script(save)
+      end
+
+      def eval_finisher_script(save=false)
+        update_finisher_starting_status
+
+        proc = Proc.new do
+          outputs = []
+
+          begin
+            context = isolated_binding({
+              outputs: outputs,
+              job_id: job_id
+            })
+            eval_with_context filename, context
+          rescue SyntaxError => e
+            handle_error(e) if save
+            raise e
+          rescue => e
+            handle_error(e) if save
+            raise e
+          end
+
+          puts "=========== Finisher Executed ==========="
+          save_outputs(outputs)
+          update_finisher_done_status
+        end
+        proc.call
+      end
+
+      def save_type
+        :executing
+      end
+
+      def update_to_server(opts = {})
+        finisher_update(
+          job_id: opts[:job_id],
+          outputs: opts[:outputs],
+          finisher_status: opts[:status])
+      end
+
+      def update_finisher_starting_status
+        return unless save
+
+        response = finisher_update(
+          job_id: job_id,
+          finisher_status: :starting)
+
+        if response.code == 200
+          puts "Finisher Status Updated."
+        else
+          puts "Error: Unable to save Finisher Status to server: #{response.body}"
+          raise "Unable to save Finisher Status to server: #{response.body}"
+        end
+      end
+
+      def update_finisher_done_status
+        return unless save
+
+        response = finisher_update(
+          job_id: job_id,
+          finisher_status: :done)
+
+        if response.code == 200
+          puts "Finisher Done."
+        else
+          puts "Error: Unable to save Finisher Done Status to server: #{response.body}"
+          raise "Unable to save Finisher Done Status to server: #{response.body}"
+        end
+      end
+
+      def handle_error(e)
+        error = ["Finisher #{e.class}: #{e.to_s} (Job:#{job_id}",clean_backtrace(e.backtrace)].join("\n")
+
+        finisher_update(
+          job_id: job_id,
+          finisher_status: :failed,
+          log_error: error)
+      end
+
+    end
+  end
+end

data/lib/answersengine/scraper.rb

@@ -1,9 +1,11 @@
 require "answersengine/plugin"
 require "answersengine/scraper/parser"
 require "answersengine/scraper/seeder"
+require "answersengine/scraper/finisher"
 require "answersengine/scraper/executor"
 require "answersengine/scraper/ruby_parser_executor"
 require "answersengine/scraper/ruby_seeder_executor"
+require "answersengine/scraper/ruby_finisher_executor"
 require "answersengine/client"
 
 module AnswersEngine

data/lib/answersengine/version.rb

@@ -1,3 +1,3 @@
 module AnswersEngine
-  VERSION = "0.4.1"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: answersengine
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-07-30 00:00:00.000000000 Z
+date: 2019-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -189,6 +189,7 @@ files:
 - exe/answersengine
 - lib/answersengine.rb
 - lib/answersengine/cli.rb
+- lib/answersengine/cli/finisher.rb
 - lib/answersengine/cli/global_page.rb
 - lib/answersengine/cli/job.rb
 - lib/answersengine/cli/job_output.rb
@@ -224,7 +225,9 @@ files:
 - lib/answersengine/plugin/context_exposer.rb
 - lib/answersengine/scraper.rb
 - lib/answersengine/scraper/executor.rb
+- lib/answersengine/scraper/finisher.rb
 - lib/answersengine/scraper/parser.rb
+- lib/answersengine/scraper/ruby_finisher_executor.rb
 - lib/answersengine/scraper/ruby_parser_executor.rb
 - lib/answersengine/scraper/ruby_seeder_executor.rb
 - lib/answersengine/scraper/seeder.rb