datahen 0.13.0 → 0.14.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c3d1584f235873f6d22ae107c60e7b50a6d8ab5918f45a4052bef80e58b7cbf7
-  data.tar.gz: b6b6ce4871017eddb70fd279c92246e498e57c3d7ca2c0bd72c225bdcdb5e119
+  metadata.gz: b00adfb4f357beeae276a130cc5e0ee1d34ddd8bdef4a0374f4b55f89f894460
+  data.tar.gz: 6363d591d93d99addcaaea6d964b2fb07e4d8222f873c3a8f6496f48c97b1483
 SHA512:
-  metadata.gz: 713f5907efc90be21ba04b83c060f0f656d1f72176191399e1ff575952cad2b54c834ddfd6722441e933857d12539f55553adf2b4d71adead952465f0ccf1005
-  data.tar.gz: bd7809ed5fbd6d12dc8680f31f194ecfb51ba0138765a88c9dde4745be2bd46533da97885f63524edca09cee1ab2d4cc3d7fd96bf04a2b6a6930a046646a8093
+  metadata.gz: 75e9f1b1e5ba61563c3ff9d12071cb76d77382353003776d88817940623e37aeb300df2114260f86a5a18a59e1dc9f74a33c00029e183c0681c616dce109d6c6
+  data.tar.gz: 8e2a07b11d20fe88c93aa0af0abf10ceb6593d5d2db63d9b5c3744c6f8b59754274a3949b6331d7d71a3018f7cf65e1a7d31e2ab6f4dbbcde494b33357dab40a

data/lib/datahen/cli/job.rb

@@ -18,12 +18,24 @@ module Datahen
         puts "#{client.all()}"
       end
 
-      desc "show <job_id>", "Show a job"
+      desc "show <job_id>", "Show a job (Defaults to showing data from cached job)"
+      option :live, type: :boolean, desc: 'Get data from the live job, not cached job.'
       def show(job_id)
         client = Client::Job.new(options)
-        puts "#{client.find(job_id)}"
+        puts "#{client.find(job_id, options)}"
       end
 
+      desc "stats <job_id>", "Get the stat for a job (Defaults to showing data from cached stats)"
+      long_desc <<-LONGDESC
+        Get stats for a scraper's current job\n
+      LONGDESC
+      option :live, type: :boolean, desc: 'Get data from the live stats, not cached stats.'
+      def stats(job_id)
+        client = Client::JobStat.new(options)
+        puts "#{client.job_current_stats(job_id, options)}"
+      end
+
+
     end
   end
 

data/lib/datahen/cli/parser.rb

@@ -10,12 +10,13 @@ module Datahen
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       option :global, :aliases => :g, type: :boolean, default: false, desc: 'Use globalpage instead of a job page'
       option :vars, :aliases => :v, type: :string, desc: 'Set user-defined page variables. Must be in json format. i.e: {"Foo":"bar"}'
+      option :"keep-outputs", :aliases => :ko, type: :boolean, default: false, desc: "Don't delete existing outputs"
       def try_parse(scraper_name, parser_file, gid)
-        begin 
-          
+        begin
+
             if options[:job]
               job_id = options[:job]
-            elsif options[:global] 
+            elsif options[:global]
               job_id = nil
             else
               job = Client::ScraperJob.new(options).find(scraper_name)
@@ -24,7 +25,7 @@ module Datahen
 
 
           vars = JSON.parse(options[:vars]) if options[:vars]
-          puts Datahen::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, false, vars)
+          puts Datahen::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, false, vars, options[:"keep-outputs"])
 
           rescue JSON::ParserError
           if options[:vars]
@@ -40,6 +41,8 @@ module Datahen
             <GID>: Global ID of the page.\x5
           LONGDESC
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :vars, :aliases => :v, type: :string, desc: 'Set user-defined page variables. Must be in json format. i.e: {"Foo":"bar"}'
+      option :"keep-outputs", :aliases => :ko, type: :boolean, default: false, desc: "Don't delete existing outputs"
       def exec_parse(scraper_name, parser_file, *gids)
         gids.each do |gid|
           begin
@@ -52,7 +55,8 @@ module Datahen
               job_id = job['id']
             end
 
-            puts Datahen::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, true)
+            vars = JSON.parse(options[:vars]) if options[:vars]
+            puts Datahen::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, true, vars, options[:"keep-outputs"])
           rescue => e
             puts e
           end

data/lib/datahen/cli/scraper.rb

@@ -140,17 +140,18 @@ module Datahen
         end
       end
 
-      desc "stats <scraper_name>", "Get the current stat for a job"
+      desc "stats <scraper_name>", "Get the stat for a current job (Defaults to showing data from cached stats)"
       long_desc <<-LONGDESC
         Get stats for a scraper's current job\n
       LONGDESC
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :live, type: :boolean, desc: 'Get data from the live stats, not cached stats.'
       def stats(scraper_name)
         client = Client::JobStat.new(options)
         if options[:job]
-          puts "#{client.job_current_stats(options[:job])}"
+          puts "#{client.job_current_stats(options[:job], options)}"
         else
-          puts "#{client.scraper_job_current_stats(scraper_name)}"
+          puts "#{client.scraper_job_current_stats(scraper_name, options)}"
         end
       end
 

data/lib/datahen/cli/scraper_export.rb

@@ -12,7 +12,6 @@ module Datahen
         puts "#{client.find(export_id)}"
       end
 
-
       desc "list", "Gets a list of exports"
       long_desc <<-LONGDESC
         List exports.
@@ -34,13 +33,13 @@ module Datahen
       def download(export_id)
         client = Client::ScraperExport.new(options)
         result = JSON.parse(client.download(export_id).to_s)
-        
+
         if result['signed_url']
           puts "Download url: \"#{result['signed_url']}\""
           `open "#{result['signed_url']}"`
         else
           puts "Exported file does not exist"
-        end        
+        end
       end
 
 

data/lib/datahen/cli/scraper_finisher.rb

@@ -11,9 +11,15 @@ module Datahen
       long_desc <<-LONGDESC
         Reset finisher on a scraper's current job.\x5
       LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def reset(scraper_name)
-        client = Client::ScraperFinisher.new(options)
-        puts "#{client.reset(scraper_name)}"
+        if options[:job]
+          client = Client::JobFinisher.new(options)
+          puts "#{client.reset(options[:job])}"
+        else
+          client = Client::ScraperFinisher.new(options)
+          puts "#{client.reset(scraper_name)}"
+        end
       end
     end
   end

data/lib/datahen/cli/scraper_job.rb

@@ -6,10 +6,17 @@ module Datahen
         "#{basename} #{@package_name} #{command.usage}"
       end
 
-      desc "show <scraper_name>", "Show a scraper's current job"
+      desc "show <scraper_name>", "Show a scraper's current job (Defaults to showing data from cached job)"
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :live, type: :boolean, desc: 'Get data from the live job, not cached job.'
       def show(scraper_name)
-        client = Client::ScraperJob.new(options)
-        puts "#{client.find(scraper_name)}"
+        if options[:job]
+          client = Client::Job.new(options)
+          puts "#{client.find(options[:job], options)}"
+        else
+          client = Client::ScraperJob.new(options)
+          puts "#{client.find(scraper_name, options)}"
+        end
       end
 
 
@@ -29,27 +36,45 @@ module Datahen
       long_desc <<-LONGDESC
         Cancels a scraper's current job
       LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def cancel(scraper_name)
-        client = Client::ScraperJob.new(options)
-        puts "#{client.cancel(scraper_name)}"
+        if options[:job]
+          client = Client::Job.new(options)
+          puts "#{client.cancel(options[:job])}"
+        else
+          client = Client::ScraperJob.new(options)
+          puts "#{client.cancel(scraper_name)}"
+        end
       end
 
       desc "resume <scraper_name>", "resumes a scraper's current job"
       long_desc <<-LONGDESC
         Resumes a scraper's current job
       LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def resume(scraper_name)
-        client = Client::ScraperJob.new(options)
-        puts "#{client.resume(scraper_name)}"
+        if options[:job]
+          client = Client::Job.new(options)
+          puts "#{client.resume(options[:job])}"
+        else
+          client = Client::ScraperJob.new(options)
+          puts "#{client.resume(scraper_name)}"
+        end
       end
 
       desc "pause <scraper_name>", "pauses a scraper's current job"
       long_desc <<-LONGDESC
         pauses a scraper's current job
       LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def pause(scraper_name)
-        client = Client::ScraperJob.new(options)
-        puts "#{client.pause(scraper_name)}"
+        if options[:job]
+          client = Client::Job.new(options)
+          puts "#{client.pause(options[:job])}"
+        else
+          client = Client::ScraperJob.new(options)
+          puts "#{client.pause(scraper_name)}"
+        end
       end
 
 
@@ -60,9 +85,15 @@ module Datahen
       option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Scraper job must be restarted(paused then resumed, or cancelled then resumed) for it to take effect. Default: 1. '
       option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Scraper job must be restarted(paused then resumed, or cancelled then resumed) for it to take effect. Default: 0. '
       option :proxy_type, desc: 'Set the Proxy type. Default: standard'
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def update(scraper_name)
-        client = Client::ScraperJob.new(options)
-        puts "#{client.update(scraper_name, options)}"
+        if options[:job]
+          client = Client::Job.new(options)
+          puts "#{client.update(options[:job], options)}"
+        else
+          client = Client::ScraperJob.new(options)
+          puts "#{client.update(scraper_name, options)}"
+        end
       end
 
       desc "var SUBCOMMAND ...ARGS", "for managing scraper's job variables"

data/lib/datahen/cli/scraper_job_var.rb

@@ -13,9 +13,15 @@ module Datahen
       LONGDESC
       option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
       option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def list(scraper_name)
-        client = Client::ScraperJobVar.new(options)
-        puts "#{client.all(scraper_name)}"
+        if options[:job]
+          client = Client::JobVar.new(options)
+          puts "#{client.all(options[:job])}"
+        else
+          client = Client::ScraperJobVar.new(options)
+          puts "#{client.all(scraper_name)}"
+        end
       end
 
       desc "set <scraper_name> <var_name> <value>", "Set an environment var on the scrape job"
@@ -24,23 +30,40 @@ module Datahen
           <var_name>: Var name can only consist of alphabets, numbers, underscores. Name must be unique to your scrape job, otherwise it will be overwritten.\x5
           <value>: Value of variable.\x5
           LONGDESC
-      option :secret, type: :boolean, desc: 'Set true to make it decrypt the value. Default: false' 
+      option :secret, type: :boolean, desc: 'Set true to make it decrypt the value. Default: false'
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def set(scraper_name, var_name, value)
-        # puts "options #{options}"
-        client = Client::ScraperJobVar.new(options)
-        puts "#{client.set(scraper_name, var_name, value, options)}"
+        if options[:job]
+          client = Client::JobVar.new(options)
+          puts "#{client.set(options[:job], var_name, value, options)}"
+        else
+          client = Client::ScraperJobVar.new(options)
+          puts "#{client.set(scraper_name, var_name, value, options)}"
+        end
       end
 
       desc "show <scraper_name> <var_name>", "Show an environment variable on the scrape job"
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def show(scraper_name, var_name)
-        client = Client::ScraperJobVar.new(options)
-        puts "#{client.find(scraper_name, var_name)}"
+        if options[:job]
+          client = Client::JobVar.new(options)
+          puts "#{client.find(options[:job], var_name)}"
+        else
+          client = Client::ScraperJobVar.new(options)
+          puts "#{client.find(scraper_name, var_name)}"
+        end
       end
 
       desc "unset <scraper_name> <var_name>", "Deletes an environment variable on the scrape job"
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def unset(scraper_name, var_name)
-        client = Client::ScraperJobVar.new(options)
-        puts "#{client.unset(scraper_name, var_name)}"
+        if options[:job]
+          client = Client::JobVar.new(options)
+          puts "#{client.unset(options[:job], var_name)}"
+        else
+          client = Client::ScraperJobVar.new(options)
+          puts "#{client.unset(scraper_name, var_name)}"
+        end
       end
     end
   end

data/lib/datahen/cli/scraper_page.rb

@@ -105,13 +105,19 @@ module Datahen
       option :fetch_fail, type: :boolean, desc: 'Refetches only pages that fails fetching.'
       option :parse_fail, type: :boolean, desc: 'Refetches only pages that fails parsing.'
       option :status, type: :string, desc: 'Refetches only pages with a specific status.'
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def refetch(scraper_name)
         if !options.key?(:gid) && !options.key?(:fetch_fail) && !options.key?(:parse_fail) && !options.key?(:status)
           puts "Must specify either a --gid, --fetch-fail, --parse-fail or --status"
           return
         end
-        client = Client::ScraperJobPage.new(options)
-        puts "#{client.refetch(scraper_name)}"
+        if options[:job]
+          client = Client::JobPage.new(options)
+          puts "#{client.refetch(options[:job])}"
+        else
+          client = Client::ScraperJobPage.new(options)
+          puts "#{client.refetch(scraper_name)}"
+        end
       end
 
       desc "reparse <scraper_name>", "Reparse Pages on a scraper's current job"
@@ -121,6 +127,7 @@ module Datahen
       option :gid, :aliases => :g, type: :string, desc: 'Reparse a specific GID'
       option :parse_fail, type: :boolean, desc: 'Reparse only pages that fails parsing.'
       option :status, type: :string, desc: 'Reparse only pages with a specific status.'
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def reparse(scraper_name)
         begin
           options[:vars] = JSON.parse(options[:vars]) if options[:vars]
@@ -130,8 +137,13 @@ module Datahen
             return
           end
 
-          client = Client::ScraperJobPage.new(options)
-          puts "#{client.reparse(scraper_name)}"
+          if options[:job]
+            client = Client::JobPage.new(options)
+            puts "#{client.reparse(options[:job])}"
+          else
+            client = Client::ScraperJobPage.new(options)
+            puts "#{client.reparse(scraper_name)}"
+          end
 
         rescue JSON::ParserError
           if options[:vars]
@@ -218,7 +230,7 @@ module Datahen
         end
       end
 
-      desc "failedcontent <gid>", "Show a page's failed content in scraper's current job"
+      desc "failedcontent <scraper_name> <gid>", "Show a page's failed content in scraper's current job"
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def failedcontent(scraper_name, gid)
         result = nil

data/lib/datahen/cli/seeder.rb

@@ -7,6 +7,7 @@ module Datahen
             <seeder_file>: Seeder script file will be executed.\x5
           LONGDESC
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :"keep-outputs", :aliases => :ko, type: :boolean, default: false, desc: "Don't delete existing outputs"
       def try_seed(scraper_name, seeder_file)
         if options[:job]
           job_id = options[:job]
@@ -14,8 +15,8 @@ module Datahen
           job = Client::ScraperJob.new(options).find(scraper_name)
           job_id = job['id']
         end
-        
-        puts Datahen::Scraper::Seeder.exec_seeder(seeder_file, job_id, false)
+
+        puts Datahen::Scraper::Seeder.exec_seeder(seeder_file, job_id, false, options[:"keep-outputs"])
       end
 
       desc "exec <scraper_name> <seeder_file>", "Executes a seeder script onto a scraper's current job."
@@ -24,6 +25,7 @@ module Datahen
             <seeder_file>: Seeder script file that will be executed on the scraper's current job.\x5
           LONGDESC
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :"keep-outputs", :aliases => :ko, type: :boolean, default: false, desc: "Don't delete existing outputs"
       def exec_parse(scraper_name, seeder_file)
         if options[:job]
           job_id = options[:job]

data/lib/datahen/client.rb

@@ -20,7 +20,9 @@ require "datahen/client/job_stat"
 require "datahen/client/backblaze_content"
 require "datahen/client/env_var"
 require "datahen/client/scraper_var"
+require "datahen/client/job_var"
 require "datahen/client/scraper_job_var"
+require "datahen/client/job_finisher"
 
 
 module Datahen

data/lib/datahen/client/base.rb

@@ -5,6 +5,8 @@ module Datahen
     class Base
       include HTTParty
 
+      default_timeout 60
+
       def self.env_auth_token
         ENV['DATAHEN_TOKEN']
       end

data/lib/datahen/client/job.rb

@@ -6,8 +6,12 @@ module Datahen
         self.class.get("/jobs", params)
       end
 
-      def find(job_id)
-        self.class.get("/jobs/#{job_id}", @options)
+      def find(job_id, opts={})
+        if opts[:live]
+          self.class.get("/jobs/#{job_id}", @options)
+        else
+          self.class.get("/cached/jobs/#{job_id}", @options)
+        end
       end
 
       def update(job_id, opts={})
@@ -15,6 +19,7 @@ module Datahen
         body[:status] = opts[:status] if opts[:status]
         body[:standard_worker_count] = opts[:workers] if opts[:workers]
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
+        body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
         params = @options.merge({body: body.to_json})
 
         self.class.put("/jobs/#{job_id}", params)
@@ -41,6 +46,7 @@ module Datahen
         body[:pages] = opts.fetch(:pages) {[]}
         body[:seeding_status] = opts.fetch(:seeding_status){ nil }
         body[:log_error] = opts[:log_error] if opts[:log_error]
+        body[:keep_outputs] = !!opts[:keep_outputs] if opts.has_key?(:keep_outputs)
 
         params = @options.merge({body: body.to_json})
 

data/lib/datahen/client/job_finisher.rb

@@ -0,0 +1,16 @@
+module Datahen
+  module Client
+    class JobFinisher < Datahen::Client::Base
+      # Reset finisher on a scraper's current job.
+      #
+      # @param [Integer] job_id Job ID
+      # @param [Hash] opts ({}) API custom parameters.
+      #
+      # @return [HTTParty::Response]
+      def reset(job_id, opts={})
+        params = @options.merge(opts)
+        self.class.put("/jobs/#{job_id}/finisher/reset", params)
+      end
+    end
+  end
+end

data/lib/datahen/client/job_page.rb

@@ -48,6 +48,7 @@ module Datahen
         body[:pages] = opts.fetch(:pages) {[]}
         body[:parsing_status] = opts.fetch(:parsing_status){ nil }
         body[:log_error] = opts[:log_error] if opts[:log_error]
+        body[:keep_outputs] = !!opts[:keep_outputs] if opts.has_key?(:keep_outputs)
 
         params = @options.merge({body: body.to_json})
 
@@ -61,6 +62,16 @@ module Datahen
       def find_failed_content(job_id, gid)
         self.class.get("/jobs/#{job_id}/pages/#{gid}/failed_content", @options)
       end
+
+      def reparse(job_id, opts={})
+        params = @options.merge(opts)
+        self.class.put("/jobs/#{job_id}/pages/reparse", params)
+      end
+
+      def refetch(job_id, opts={})
+        params = @options.merge(opts)
+        self.class.put("/jobs/#{job_id}/pages/refetch", params)
+      end
     end
   end
 end

data/lib/datahen/client/job_stat.rb

@@ -2,12 +2,20 @@ module Datahen
   module Client
     class JobStat < Datahen::Client::Base
 
-      def job_current_stats(job_id)
-        self.class.get("/jobs/#{job_id}/stats/current", @options)
+      def job_current_stats(job_id, opts={})
+        if opts[:live]
+          self.class.get("/jobs/#{job_id}/stats/current", @options)
+        else
+          self.class.get("/cached/jobs/#{job_id}/stats/current", @options)
+        end
       end
 
-      def scraper_job_current_stats(scraper_name)
-        self.class.get("/scrapers/#{scraper_name}/current_job/stats/current", @options)
+      def scraper_job_current_stats(scraper_name, opts={})
+        if opts[:live]
+          self.class.get("/scrapers/#{scraper_name}/current_job/stats/current", @options)
+        else
+          self.class.get("/cached/scrapers/#{scraper_name}/current_job/stats/current", @options)
+        end
       end
 
       def job_stats_history(job_id)

data/lib/datahen/client/job_var.rb

@@ -0,0 +1,28 @@
+module Datahen
+  module Client
+    class JobVar < Datahen::Client::Base
+
+      def find(job_id, var_name)
+        self.class.get("/jobs/#{job_id}/vars/#{var_name}", @options)
+      end
+
+      def all(job_id, opts={})
+        params = @options.merge opts
+        self.class.get("/jobs/#{job_id}/vars", params)
+      end
+
+      def set(job_id, var_name, value, opts={})
+        body = {}
+        body[:value] = value
+        body[:secret] = opts[:secret] if opts[:secret]
+        params = @options.merge({body: body.to_json})
+        self.class.put("/jobs/#{job_id}/vars/#{var_name}", params)
+      end
+
+      def unset(job_id, var_name, opts={})
+        params = @options.merge(opts)
+        self.class.delete("/jobs/#{job_id}/vars/#{var_name}", params)
+      end
+    end
+  end
+end

data/lib/datahen/client/scraper_job.rb

@@ -15,8 +15,12 @@ module Datahen
         self.class.post("/scrapers/#{scraper_name}/jobs", params)
       end
 
-      def find(scraper_name)
-        self.class.get("/scrapers/#{scraper_name}/current_job", @options)
+      def find(scraper_name, opts={})
+        if opts[:live]
+          self.class.get("/scrapers/#{scraper_name}/current_job", @options)
+        else
+          self.class.get("/cached/scrapers/#{scraper_name}/current_job", @options)
+        end
       end
 
       def update(scraper_name, opts={})

data/lib/datahen/client/scraper_job_page.rb

@@ -26,6 +26,9 @@ module Datahen
         self.class.put("/scrapers/#{scraper_name}/current_job/pages/refetch", params)
       end
 
+      # Deprecated, please use Datahen::Client::JobVar#refetch instead.
+      #
+      # @note This method will be removed at some point in the future.
       def refetch_by_job(job_id, opts={})
         params = @options.merge(opts)
         self.class.put("/jobs/#{job_id}/pages/refetch", params)
@@ -36,11 +39,6 @@ module Datahen
         self.class.put("/scrapers/#{scraper_name}/current_job/pages/reparse", params)
       end
 
-      def reparse_by_job(job_id, opts={})
-        params = @options.merge(opts)
-        self.class.put("/jobs/#{job_id}/pages/reparse", params)
-      end
-
       def enqueue(scraper_name, method, url, opts={})
         body = {}
         body[:method] =  method != "" ? method : "GET"

data/lib/datahen/scraper/executor.rb

@@ -60,12 +60,17 @@ module Datahen
 
       def init_global_page()
         client = Client::GlobalPage.new()
-        client.find(gid)
+        global_page = client.find(gid)
+        unless global_page.code == 200
+          raise "GID #{gid} not found. Aborting execution!"
+        else
+          global_page
+        end
       end
 
-      def get_content(gid)
-        client = Client::GlobalPage.new()
-        content_json = client.find_content(gid)
+      def get_content(job_id, gid)
+        client = Client::JobPage.new()
+        content_json = client.find_content(job_id, gid)
 
         if content_json['available']
           signed_url = content_json['signed_url']
@@ -75,7 +80,7 @@ module Datahen
         end
       end
 
-      def get_failed_content(gid)
+      def get_failed_content(job_id, gid)
         client = Client::JobPage.new()
         content_json = client.find_failed_content(job_id, gid)
 
@@ -287,11 +292,12 @@ module Datahen
           end
 
           # behave differently if it is a real save
+          save_status = status
           if save
             log_msg = "Saving #{log_msgs.join(' and ')}."
             puts "#{log_msg}"
           else
-            status = "#{status}_try"
+            save_status = "#{status}_try"
           end
 
           # saving to server
@@ -300,7 +306,7 @@ module Datahen
             gid: gid,
             pages: pages_slice,
             outputs: outputs_slice,
-            status: status)
+            status: save_status)
 
           if response.code == 200
             if save

data/lib/datahen/scraper/parser.rb

@@ -1,18 +1,24 @@
 module Datahen
   module Scraper
     class Parser
-      def self.exec_parser_page(filename, gid, job_id=nil, save=false, vars = {})
+      def self.exec_parser_page(filename, gid, job_id=nil, save=false, vars = {}, keep_outputs=false)
         extname = File.extname(filename)
         case extname
         when '.rb'
-          executor = RubyParserExecutor.new(filename: filename, gid: gid, job_id: job_id, vars: vars)
+          executor = RubyParserExecutor.new(
+            filename: filename,
+            gid: gid,
+            job_id: job_id,
+            vars: vars,
+            keep_outputs: keep_outputs
+          )
           executor.exec_parser(save)
         else
           puts "Unable to find a parser executor for file type \"#{extname}\""
         end
       end
 
-    
+
     end
   end
-end
+end

data/lib/datahen/scraper/ruby_parser_executor.rb

@@ -15,6 +15,7 @@ module Datahen
         @gid = options.fetch(:gid) { raise "GID is required"}
         @job_id = options.fetch(:job_id)
         @page_vars = options.fetch(:vars) { {} }
+        @keep_outputs = !!(options.fetch(:keep_outputs) { false })
       end
 
       def self.exposed_methods
@@ -66,7 +67,9 @@ module Datahen
         response = parsing_update(
           job_id: job_id,
           gid: gid,
-          parsing_status: :starting)
+          parsing_status: :starting,
+          keep_outputs: @keep_outputs
+        )
 
         if response.code == 200
           puts "Page Parsing Status Updated."
@@ -165,7 +168,7 @@ module Datahen
             handle_error(e) if save
             raise e
           end
-          
+
           if refetch_self
             refetch_page gid
           elsif reparse_self
@@ -178,11 +181,11 @@ module Datahen
       end
 
       def content
-        @content ||= get_content(gid)
+        @content ||= get_content(job_id, gid)
       end
 
       def failed_content
-        @failed_content ||= get_failed_content(gid)
+        @failed_content ||= get_failed_content(job_id, gid)
       end
 
       def handle_error(e)

data/lib/datahen/scraper/ruby_seeder_executor.rb

@@ -6,6 +6,7 @@ module Datahen
       def initialize(options={})
         @filename = options.fetch(:filename) { raise "Filename is required"}
         @job_id = options[:job_id]
+        @keep_outputs = !!(options.fetch(:keep_outputs) { false })
       end
 
       def self.exposed_methods
@@ -81,7 +82,9 @@ module Datahen
 
         response = seeding_update(
           job_id: job_id,
-          seeding_status: :starting)
+          seeding_status: :starting,
+          keep_outputs: @keep_outputs
+        )
 
         if response.code == 200
           puts "Seeding Status Updated."

data/lib/datahen/scraper/seeder.rb

@@ -2,11 +2,15 @@ module Datahen
   module Scraper
     class Seeder
 
-      def self.exec_seeder(filename, job_id=nil, save=false)
+      def self.exec_seeder(filename, job_id=nil, save=false, keep_outputs=false)
         extname = File.extname(filename)
         case extname
         when '.rb'
-          executor = RubySeederExecutor.new(filename: filename, job_id: job_id)
+          executor = RubySeederExecutor.new(
+            filename: filename,
+            job_id: job_id,
+            keep_outputs: keep_outputs
+          )
           executor.exec_seeder(save)
         else
           puts "Unable to find a seeder executor for file type \"#{extname}\""
@@ -15,4 +19,4 @@ module Datahen
 
     end
   end
-end
+end

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "0.13.0"
+  VERSION = "0.14.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 0.13.0
+  version: 0.14.3
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-02 00:00:00.000000000 Z
+date: 2020-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -215,10 +215,12 @@ files:
 - lib/datahen/client/global_page.rb
 - lib/datahen/client/job.rb
 - lib/datahen/client/job_export.rb
+- lib/datahen/client/job_finisher.rb
 - lib/datahen/client/job_log.rb
 - lib/datahen/client/job_output.rb
 - lib/datahen/client/job_page.rb
 - lib/datahen/client/job_stat.rb
+- lib/datahen/client/job_var.rb
 - lib/datahen/client/scraper.rb
 - lib/datahen/client/scraper_deployment.rb
 - lib/datahen/client/scraper_export.rb
@@ -262,7 +264,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: DataHen toolbelt for developers