datahen 0.11.1 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 80934b5ed446c59f749866cd5f497ef66f90e5af5509f3cd459d295352e537aa
-  data.tar.gz: 7d7c4ebae89f5cbcaa1d56950d6b4c36089c10c20d54ec2e8e11fee6cc81433e
+  metadata.gz: 3ff2ed2cd4772450c01e3e88248ae89441de709198fdd177d3e572bbc5f0e474
+  data.tar.gz: 5701717fcba8a05b6f3e027d9bce33a3830fa20dabe3413255779899478cb4ab
 SHA512:
-  metadata.gz: f1c68e2c7bb7f40e5c40c960d0dc412da03b7bc9f82e4d3b60f08e49fa1a796b34246b547bfb9ae67677aaf15944ff9d00f97dd97fa898309db0ee37864cd2b1
-  data.tar.gz: 65f4a182ecc8e9638c1c22c43d03b4487588c112f33e75d85f2ffbee1f29e2d55bf67f861c3b6b34a97ca1014c39a0570c93b8f704c19fe28e7b5266d5a1795a
+  metadata.gz: 949ad06a090a4ac8c2ef5b4e053ed4b7668c051be15b6959a2948614e771c25e18774d9ee97fe1f5c03c130986b671a8b26ac253f592a993fa4ad393bcad7673
+  data.tar.gz: b73cfc6c070314f97cbc7917d571de67031247aac42f3474b2e71d04e8b3d650fc380a0ce3ca65c1d8339bf8743d94b666ecccca4431f7b89df4e7485a03a382

data/lib/datahen/cli/global_page.rb

@@ -12,28 +12,15 @@ module Datahen
       def content(gid)
         client = Client::GlobalPage.new(options)
         result = JSON.parse(client.find_content(gid).to_s)
-        
+
         if result['available'] == true
           puts "Preview content url: \"#{result['preview_url']}\""
           `open "#{result['preview_url']}"`
         else
           puts "Content does not exist"
-        end        
+        end
       end
 
-      desc "failedcontent <gid>", "Show failed content of a globalpage"
-      def failedcontent(gid)
-        client = Client::GlobalPage.new(options)
-        result = JSON.parse(client.find_failed_content(gid).to_s)
-        
-        if result['available'] == true
-          puts "Preview failed content url: \"#{result['preview_url']}\""
-          `open "#{result['preview_url']}"`
-        else
-          puts "Failed Content does not exist"
-        end        
-      end
-      
     end
   end
 end

data/lib/datahen/cli/job.rb

@@ -18,12 +18,24 @@ module Datahen
         puts "#{client.all()}"
       end
 
-      desc "show <job_id>", "Show a job"
+      desc "show <job_id>", "Show a job (Defaults to showing data from cached job)"
+      option :live, type: :boolean, desc: 'Get data from the live job, not cached job.'
       def show(job_id)
         client = Client::Job.new(options)
-        puts "#{client.find(job_id)}"
+        puts "#{client.find(job_id, options)}"
       end
 
+      desc "stats <job_id>", "Get the stat for a job (Defaults to showing data from cached stats)"
+      long_desc <<-LONGDESC
+        Get stats for a scraper's current job\n
+      LONGDESC
+      option :live, type: :boolean, desc: 'Get data from the live stats, not cached stats.'
+      def stats(job_id)
+        client = Client::JobStat.new(options)
+        puts "#{client.job_current_stats(job_id, options)}"
+      end
+
+
     end
   end
 

data/lib/datahen/cli/parser.rb

@@ -10,12 +10,13 @@ module Datahen
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       option :global, :aliases => :g, type: :boolean, default: false, desc: 'Use globalpage instead of a job page'
       option :vars, :aliases => :v, type: :string, desc: 'Set user-defined page variables. Must be in json format. i.e: {"Foo":"bar"}'
+      option :"keep-outputs", :aliases => :ko, type: :boolean, default: false, desc: "Don't delete existing outputs"
       def try_parse(scraper_name, parser_file, gid)
-        begin 
-          
+        begin
+
             if options[:job]
               job_id = options[:job]
-            elsif options[:global] 
+            elsif options[:global]
               job_id = nil
             else
               job = Client::ScraperJob.new(options).find(scraper_name)
@@ -24,7 +25,7 @@ module Datahen
 
 
           vars = JSON.parse(options[:vars]) if options[:vars]
-          puts Datahen::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, false, vars)
+          puts Datahen::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, false, vars, options[:"keep-outputs"])
 
           rescue JSON::ParserError
           if options[:vars]
@@ -40,6 +41,8 @@ module Datahen
             <GID>: Global ID of the page.\x5
           LONGDESC
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :vars, :aliases => :v, type: :string, desc: 'Set user-defined page variables. Must be in json format. i.e: {"Foo":"bar"}'
+      option :"keep-outputs", :aliases => :ko, type: :boolean, default: false, desc: "Don't delete existing outputs"
       def exec_parse(scraper_name, parser_file, *gids)
         gids.each do |gid|
           begin
@@ -52,7 +55,8 @@ module Datahen
               job_id = job['id']
             end
 
-            puts Datahen::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, true)
+            vars = JSON.parse(options[:vars]) if options[:vars]
+            puts Datahen::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, true, vars, options[:"keep-outputs"])
           rescue => e
             puts e
           end

data/lib/datahen/cli/scraper.rb

@@ -60,7 +60,7 @@ module Datahen
       desc "show <scraper_name>", "Show a scraper"
       def show(scraper_name)
         client = Client::Scraper.new(options)
-        puts "#{client.find(scraper_name)}"
+        puts "#{client.find(scraper_name, options)}"
       end
 
       desc "delete <scraper_name>", "Delete a scraper and related records"
@@ -102,6 +102,7 @@ module Datahen
       option :head, :aliases => :H, desc: 'Show the oldest log entries. If not set, newest entries is shown'
       option :parsing, :aliases => :p, type: :boolean, desc: 'Show only log entries related to parsing errors'
       option :seeding, :aliases => :s, type: :boolean, desc: 'Show only log entries related to seeding errors'
+      option :finisher, :aliases => :f, type: :boolean, desc: 'Show only log entries related to finisher errors'
       option :more, :aliases => :m, desc: 'Show next set of log entries. Enter the `More token`'
       option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 5000 per page.'
       def log(scraper_name)
@@ -111,6 +112,7 @@ module Datahen
         query["order"] = options.delete(:head) if options[:head]
         query["job_type"] = "parsing" if options[:parsing]
         query["job_type"] = "seeding" if options[:seeding]
+        query["job_type"] = "finisher executing" if options[:finisher]
         query["page_token"] = options.delete(:more) if options[:more]
         query["per_page"] = options.delete(:per_page) if options[:per_page]
 
@@ -138,17 +140,18 @@ module Datahen
         end
       end
 
-      desc "stats <scraper_name>", "Get the current stat for a job"
+      desc "stats <scraper_name>", "Get the stat for a current job (Defaults to showing data from cached stats)"
       long_desc <<-LONGDESC
         Get stats for a scraper's current job\n
       LONGDESC
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :live, type: :boolean, desc: 'Get data from the live stats, not cached stats.'
       def stats(scraper_name)
         client = Client::JobStat.new(options)
         if options[:job]
-          puts "#{client.job_current_stats(options[:job])}"
+          puts "#{client.job_current_stats(options[:job], options)}"
         else
-          puts "#{client.scraper_job_current_stats(scraper_name)}"
+          puts "#{client.scraper_job_current_stats(scraper_name, options)}"
         end
       end
 

data/lib/datahen/cli/scraper_export.rb

@@ -12,7 +12,6 @@ module Datahen
         puts "#{client.find(export_id)}"
       end
 
-
       desc "list", "Gets a list of exports"
       long_desc <<-LONGDESC
         List exports.
@@ -34,13 +33,13 @@ module Datahen
       def download(export_id)
         client = Client::ScraperExport.new(options)
         result = JSON.parse(client.download(export_id).to_s)
-        
+
         if result['signed_url']
           puts "Download url: \"#{result['signed_url']}\""
           `open "#{result['signed_url']}"`
         else
           puts "Exported file does not exist"
-        end        
+        end
       end
 
 

data/lib/datahen/cli/scraper_finisher.rb

@@ -11,9 +11,15 @@ module Datahen
       long_desc <<-LONGDESC
         Reset finisher on a scraper's current job.\x5
       LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def reset(scraper_name)
-        client = Client::ScraperFinisher.new(options)
-        puts "#{client.reset(scraper_name)}"
+        if options[:job]
+          client = Client::JobFinisher.new(options)
+          puts "#{client.reset(options[:job])}"
+        else
+          client = Client::ScraperFinisher.new(options)
+          puts "#{client.reset(scraper_name)}"
+        end
       end
     end
   end

data/lib/datahen/cli/scraper_job.rb

@@ -6,10 +6,11 @@ module Datahen
         "#{basename} #{@package_name} #{command.usage}"
       end
 
-      desc "show <scraper_name>", "Show a scraper's current job"
+      desc "show <scraper_name>", "Show a scraper's current job (Defaults to showing data from cached job)"
+      option :live, type: :boolean, desc: 'Get data from the live job, not cached job.'
       def show(scraper_name)
         client = Client::ScraperJob.new(options)
-        puts "#{client.find(scraper_name)}"
+        puts "#{client.find(scraper_name, options)}"
       end
 
 
@@ -29,27 +30,45 @@ module Datahen
       long_desc <<-LONGDESC
         Cancels a scraper's current job
       LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def cancel(scraper_name)
-        client = Client::ScraperJob.new(options)
-        puts "#{client.cancel(scraper_name)}"
+        if options[:job]
+          client = Client::Job.new(options)
+          puts "#{client.cancel(options[:job])}"
+        else
+          client = Client::ScraperJob.new(options)
+          puts "#{client.cancel(scraper_name)}"
+        end
       end
 
       desc "resume <scraper_name>", "resumes a scraper's current job"
       long_desc <<-LONGDESC
         Resumes a scraper's current job
       LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def resume(scraper_name)
-        client = Client::ScraperJob.new(options)
-        puts "#{client.resume(scraper_name)}"
+        if options[:job]
+          client = Client::Job.new(options)
+          puts "#{client.resume(options[:job])}"
+        else
+          client = Client::ScraperJob.new(options)
+          puts "#{client.resume(scraper_name)}"
+        end
       end
 
       desc "pause <scraper_name>", "pauses a scraper's current job"
       long_desc <<-LONGDESC
         pauses a scraper's current job
       LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def pause(scraper_name)
-        client = Client::ScraperJob.new(options)
-        puts "#{client.pause(scraper_name)}"
+        if options[:job]
+          client = Client::Job.new(options)
+          puts "#{client.pause(options[:job])}"
+        else
+          client = Client::ScraperJob.new(options)
+          puts "#{client.pause(scraper_name)}"
+        end
       end
 
 
@@ -60,9 +79,15 @@ module Datahen
       option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Scraper job must be restarted(paused then resumed, or cancelled then resumed) for it to take effect. Default: 1. '
       option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Scraper job must be restarted(paused then resumed, or cancelled then resumed) for it to take effect. Default: 0. '
       option :proxy_type, desc: 'Set the Proxy type. Default: standard'
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def update(scraper_name)
-        client = Client::ScraperJob.new(options)
-        puts "#{client.update(scraper_name, options)}"
+        if options[:job]
+          client = Client::Job.new(options)
+          puts "#{client.update(options[:job], options)}"
+        else
+          client = Client::ScraperJob.new(options)
+          puts "#{client.update(scraper_name, options)}"
+        end
       end
 
       desc "var SUBCOMMAND ...ARGS", "for managing scraper's job variables"

data/lib/datahen/cli/scraper_job_var.rb

@@ -13,9 +13,15 @@ module Datahen
       LONGDESC
       option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
       option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def list(scraper_name)
-        client = Client::ScraperJobVar.new(options)
-        puts "#{client.all(scraper_name)}"
+        if options[:job]
+          client = Client::JobVar.new(options)
+          puts "#{client.all(options[:job])}"
+        else
+          client = Client::ScraperJobVar.new(options)
+          puts "#{client.all(scraper_name)}"
+        end
       end
 
       desc "set <scraper_name> <var_name> <value>", "Set an environment var on the scrape job"
@@ -24,23 +30,40 @@ module Datahen
           <var_name>: Var name can only consist of alphabets, numbers, underscores. Name must be unique to your scrape job, otherwise it will be overwritten.\x5
           <value>: Value of variable.\x5
           LONGDESC
-      option :secret, type: :boolean, desc: 'Set true to make it decrypt the value. Default: false' 
+      option :secret, type: :boolean, desc: 'Set true to make it decrypt the value. Default: false'
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def set(scraper_name, var_name, value)
-        # puts "options #{options}"
-        client = Client::ScraperJobVar.new(options)
-        puts "#{client.set(scraper_name, var_name, value, options)}"
+        if options[:job]
+          client = Client::JobVar.new(options)
+          puts "#{client.set(options[:job], var_name, value, options)}"
+        else
+          client = Client::ScraperJobVar.new(options)
+          puts "#{client.set(scraper_name, var_name, value, options)}"
+        end
       end
 
       desc "show <scraper_name> <var_name>", "Show an environment variable on the scrape job"
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def show(scraper_name, var_name)
-        client = Client::ScraperJobVar.new(options)
-        puts "#{client.find(scraper_name, var_name)}"
+        if options[:job]
+          client = Client::JobVar.new(options)
+          puts "#{client.find(options[:job], var_name)}"
+        else
+          client = Client::ScraperJobVar.new(options)
+          puts "#{client.find(scraper_name, var_name)}"
+        end
       end
 
       desc "unset <scraper_name> <var_name>", "Deletes an environment variable on the scrape job"
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def unset(scraper_name, var_name)
-        client = Client::ScraperJobVar.new(options)
-        puts "#{client.unset(scraper_name, var_name)}"
+        if options[:job]
+          client = Client::JobVar.new(options)
+          puts "#{client.unset(options[:job], var_name)}"
+        else
+          client = Client::ScraperJobVar.new(options)
+          puts "#{client.unset(scraper_name, var_name)}"
+        end
       end
     end
   end

data/lib/datahen/cli/scraper_page.rb

@@ -17,6 +17,7 @@ module Datahen
       option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
       option :fetch_fail, type: :boolean, desc: 'Returns only pages that fails fetching.'
       option :parse_fail, type: :boolean, desc: 'Returns only pages that fails parsing.'
+      option :status, type: :string, desc: 'Returns only pages with specific status.'
       def list(scraper_name)
         if options[:job]
           client = Client::JobPage.new(options)
@@ -104,13 +105,19 @@ module Datahen
       option :fetch_fail, type: :boolean, desc: 'Refetches only pages that fails fetching.'
       option :parse_fail, type: :boolean, desc: 'Refetches only pages that fails parsing.'
       option :status, type: :string, desc: 'Refetches only pages with a specific status.'
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def refetch(scraper_name)
         if !options.key?(:gid) && !options.key?(:fetch_fail) && !options.key?(:parse_fail) && !options.key?(:status)
           puts "Must specify either a --gid, --fetch-fail, --parse-fail or --status"
           return
         end
-        client = Client::ScraperJobPage.new(options)
-        puts "#{client.refetch(scraper_name)}"
+        if options[:job]
+          client = Client::JobPage.new(options)
+          puts "#{client.refetch(options[:job])}"
+        else
+          client = Client::ScraperJobPage.new(options)
+          puts "#{client.refetch(scraper_name)}"
+        end
       end
 
       desc "reparse <scraper_name>", "Reparse Pages on a scraper's current job"
@@ -120,6 +127,7 @@ module Datahen
       option :gid, :aliases => :g, type: :string, desc: 'Reparse a specific GID'
       option :parse_fail, type: :boolean, desc: 'Reparse only pages that fails parsing.'
       option :status, type: :string, desc: 'Reparse only pages with a specific status.'
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def reparse(scraper_name)
         begin
           options[:vars] = JSON.parse(options[:vars]) if options[:vars]
@@ -129,8 +137,13 @@ module Datahen
             return
           end
 
-          client = Client::ScraperJobPage.new(options)
-          puts "#{client.reparse(scraper_name)}"
+          if options[:job]
+            client = Client::JobPage.new(options)
+            puts "#{client.reparse(options[:job])}"
+          else
+            client = Client::ScraperJobPage.new(options)
+            puts "#{client.reparse(scraper_name)}"
+          end
 
         rescue JSON::ParserError
           if options[:vars]
@@ -197,6 +210,46 @@ module Datahen
         end
       end
 
+      desc "content <scraper_name> <gid>", "Show a page's content in scraper's current job"
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      def content(scraper_name, gid)
+        result = nil
+        if options[:job]
+          client = Client::JobPage.new(options)
+          result = JSON.parse(client.find_content(options[:job], gid).to_s)
+        else
+          client = Client::ScraperJobPage.new(options)
+          result = JSON.parse(client.find_content(scraper_name, gid).to_s)
+        end
+
+        if result['available'] == true
+          puts "Preview content url: \"#{result['preview_url']}\""
+          `open "#{result['preview_url']}"`
+        else
+          puts "Content does not exist"
+        end
+      end
+
+      desc "failedcontent <gid>", "Show a page's failed content in scraper's current job"
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      def failedcontent(scraper_name, gid)
+        result = nil
+        if options[:job]
+          client = Client::JobPage.new(options)
+          result = JSON.parse(client.find_failed_content(options[:job], gid).to_s)
+        else
+          client = Client::ScraperJobPage.new(options)
+          result = JSON.parse(client.find_failed_content(scraper_name, gid).to_s)
+        end
+
+        if result['available'] == true
+          puts "Preview failed content url: \"#{result['preview_url']}\""
+          `open "#{result['preview_url']}"`
+        else
+          puts "Failed Content does not exist"
+        end
+      end
+
     end
   end
 

data/lib/datahen/cli/seeder.rb

@@ -7,6 +7,7 @@ module Datahen
             <seeder_file>: Seeder script file will be executed.\x5
           LONGDESC
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :"keep-outputs", :aliases => :ko, type: :boolean, default: false, desc: "Don't delete existing outputs"
       def try_seed(scraper_name, seeder_file)
         if options[:job]
           job_id = options[:job]
@@ -14,8 +15,8 @@ module Datahen
           job = Client::ScraperJob.new(options).find(scraper_name)
           job_id = job['id']
         end
-        
-        puts Datahen::Scraper::Seeder.exec_seeder(seeder_file, job_id, false)
+
+        puts Datahen::Scraper::Seeder.exec_seeder(seeder_file, job_id, false, options[:"keep-outputs"])
       end
 
       desc "exec <scraper_name> <seeder_file>", "Executes a seeder script onto a scraper's current job."
@@ -24,6 +25,7 @@ module Datahen
             <seeder_file>: Seeder script file that will be executed on the scraper's current job.\x5
           LONGDESC
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :"keep-outputs", :aliases => :ko, type: :boolean, default: false, desc: "Don't delete existing outputs"
       def exec_parse(scraper_name, seeder_file)
         if options[:job]
           job_id = options[:job]

data/lib/datahen/client.rb

@@ -20,7 +20,9 @@ require "datahen/client/job_stat"
 require "datahen/client/backblaze_content"
 require "datahen/client/env_var"
 require "datahen/client/scraper_var"
+require "datahen/client/job_var"
 require "datahen/client/scraper_job_var"
+require "datahen/client/job_finisher"
 
 
 module Datahen

data/lib/datahen/client/base.rb

@@ -51,10 +51,10 @@ module Datahen
         query[:status] = opts[:status] if opts[:status]
         query[:page_type] = opts[:page_type] if opts[:page_type]
         query[:gid] = opts[:gid] if opts[:gid]
-        query[:"min-timestamp"] = opts[:"min-timestamp"]
-        query[:"max-timestamp"] = opts[:"max-timestamp"]
-        query[:limit] = opts[:limit]
-        query[:order] = opts[:order]
+        query[:"min-timestamp"] = opts[:"min-timestamp"] if opts[:"min-timestamp"]
+        query[:"max-timestamp"] = opts[:"max-timestamp"] if opts[:"max-timestamp"]
+        query[:limit] = opts[:limit] if opts[:limit]
+        query[:order] = opts[:order] if opts[:order]
 
         if opts[:query]
           if opts[:query].is_a?(Hash)

data/lib/datahen/client/global_page.rb

@@ -8,11 +8,6 @@ module Datahen
       def find_content(gid)
         self.class.get("/global_pages/#{gid}/content", @options)
       end
-
-      def find_failed_content(gid)
-        self.class.get("/global_pages/#{gid}/failed_content", @options)
-      end
     end
   end
 end
-

data/lib/datahen/client/job.rb

@@ -6,8 +6,12 @@ module Datahen
         self.class.get("/jobs", params)
       end
 
-      def find(job_id)
-        self.class.get("/jobs/#{job_id}", @options)
+      def find(job_id, opts={})
+        if opts[:live]
+          self.class.get("/jobs/#{job_id}", @options)
+        else
+          self.class.get("/cached/jobs/#{job_id}", @options)
+        end
       end
 
       def update(job_id, opts={})
@@ -15,6 +19,7 @@ module Datahen
         body[:status] = opts[:status] if opts[:status]
         body[:standard_worker_count] = opts[:workers] if opts[:workers]
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
+        body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
         params = @options.merge({body: body.to_json})
 
         self.class.put("/jobs/#{job_id}", params)
@@ -41,6 +46,7 @@ module Datahen
         body[:pages] = opts.fetch(:pages) {[]}
         body[:seeding_status] = opts.fetch(:seeding_status){ nil }
         body[:log_error] = opts[:log_error] if opts[:log_error]
+        body[:keep_outputs] = !!opts[:keep_outputs] if opts.has_key?(:keep_outputs)
 
         params = @options.merge({body: body.to_json})
 

data/lib/datahen/client/job_finisher.rb

@@ -0,0 +1,16 @@
+module Datahen
+  module Client
+    class JobFinisher < Datahen::Client::Base
+      # Reset finisher on a scraper's current job.
+      #
+      # @param [Integer] job_id Job ID
+      # @param [Hash] opts ({}) API custom parameters.
+      #
+      # @return [HTTParty::Response]
+      def reset(job_id, opts={})
+        params = @options.merge(opts)
+        self.class.put("/jobs/#{job_id}/finisher/reset", params)
+      end
+    end
+  end
+end

data/lib/datahen/client/job_page.rb

@@ -48,11 +48,30 @@ module Datahen
         body[:pages] = opts.fetch(:pages) {[]}
         body[:parsing_status] = opts.fetch(:parsing_status){ nil }
         body[:log_error] = opts[:log_error] if opts[:log_error]
+        body[:keep_outputs] = !!opts[:keep_outputs] if opts.has_key?(:keep_outputs)
 
         params = @options.merge({body: body.to_json})
 
         self.class.put("/jobs/#{job_id}/pages/#{gid}/parsing_update", params)
       end
+
+      def find_content(job_id, gid)
+        self.class.get("/jobs/#{job_id}/pages/#{gid}/content", @options)
+      end
+
+      def find_failed_content(job_id, gid)
+        self.class.get("/jobs/#{job_id}/pages/#{gid}/failed_content", @options)
+      end
+
+      def reparse(job_id, opts={})
+        params = @options.merge(opts)
+        self.class.put("/jobs/#{job_id}/pages/reparse", params)
+      end
+
+      def refetch(job_id, opts={})
+        params = @options.merge(opts)
+        self.class.put("/jobs/#{job_id}/pages/refetch", params)
+      end
     end
   end
 end

data/lib/datahen/client/job_stat.rb

@@ -2,12 +2,20 @@ module Datahen
   module Client
     class JobStat < Datahen::Client::Base
 
-      def job_current_stats(job_id)
-        self.class.get("/jobs/#{job_id}/stats/current", @options)
+      def job_current_stats(job_id, opts={})
+        if opts[:live]
+          self.class.get("/jobs/#{job_id}/stats/current", @options)
+        else
+          self.class.get("/cached/jobs/#{job_id}/stats/current", @options)
+        end
       end
 
-      def scraper_job_current_stats(scraper_name)
-        self.class.get("/scrapers/#{scraper_name}/current_job/stats/current", @options)
+      def scraper_job_current_stats(scraper_name, opts={})
+        if opts[:live]
+          self.class.get("/scrapers/#{scraper_name}/current_job/stats/current", @options)
+        else
+          self.class.get("/cached/scrapers/#{scraper_name}/current_job/stats/current", @options)
+        end
       end
 
       def job_stats_history(job_id)

data/lib/datahen/client/job_var.rb

@@ -0,0 +1,28 @@
+module Datahen
+  module Client
+    class JobVar < Datahen::Client::Base
+
+      def find(job_id, var_name)
+        self.class.get("/jobs/#{job_id}/vars/#{var_name}", @options)
+      end
+
+      def all(job_id, opts={})
+        params = @options.merge opts
+        self.class.get("/jobs/#{job_id}/vars", params)
+      end
+
+      def set(job_id, var_name, value, opts={})
+        body = {}
+        body[:value] = value
+        body[:secret] = opts[:secret] if opts[:secret]
+        params = @options.merge({body: body.to_json})
+        self.class.put("/jobs/#{job_id}/vars/#{var_name}", params)
+      end
+
+      def unset(job_id, var_name, opts={})
+        params = @options.merge(opts)
+        self.class.delete("/jobs/#{job_id}/vars/#{var_name}", params)
+      end
+    end
+  end
+end

data/lib/datahen/client/scraper_job.rb

@@ -15,8 +15,12 @@ module Datahen
         self.class.post("/scrapers/#{scraper_name}/jobs", params)
       end
 
-      def find(scraper_name)
-        self.class.get("/scrapers/#{scraper_name}/current_job", @options)
+      def find(scraper_name, opts={})
+        if opts[:live]
+          self.class.get("/scrapers/#{scraper_name}/current_job", @options)
+        else
+          self.class.get("/cached/scrapers/#{scraper_name}/current_job", @options)
+        end
       end
 
       def update(scraper_name, opts={})

data/lib/datahen/client/scraper_job_page.rb

@@ -26,6 +26,9 @@ module Datahen
         self.class.put("/scrapers/#{scraper_name}/current_job/pages/refetch", params)
       end
 
+      # Deprecated, please use Datahen::Client::JobVar#refetch instead.
+      #
+      # @note This method will be removed at some point in the future.
       def refetch_by_job(job_id, opts={})
         params = @options.merge(opts)
         self.class.put("/jobs/#{job_id}/pages/refetch", params)
@@ -36,11 +39,6 @@ module Datahen
         self.class.put("/scrapers/#{scraper_name}/current_job/pages/reparse", params)
       end
 
-      def reparse_by_job(job_id, opts={})
-        params = @options.merge(opts)
-        self.class.put("/jobs/#{job_id}/pages/reparse", params)
-      end
-
       def enqueue(scraper_name, method, url, opts={})
         body = {}
         body[:method] =  method != "" ? method : "GET"
@@ -62,6 +60,14 @@ module Datahen
         self.class.post("/scrapers/#{scraper_name}/current_job/pages", params)
       end
 
+      def find_content(scraper_name, gid)
+        self.class.get("/scrapers/#{scraper_name}/current_job/pages/#{gid}/content", @options)
+      end
+
+      def find_failed_content(scraper_name, gid)
+        self.class.get("/scrapers/#{scraper_name}/current_job/pages/#{gid}/failed_content", @options)
+      end
+
     end
   end
 end

data/lib/datahen/scraper/executor.rb

@@ -63,9 +63,9 @@ module Datahen
         client.find(gid)
       end
 
-      def get_content(gid)
-        client = Client::GlobalPage.new()
-        content_json = client.find_content(gid)
+      def get_content(job_id, gid)
+        client = Client::JobPage.new()
+        content_json = client.find_content(job_id, gid)
 
         if content_json['available']
           signed_url = content_json['signed_url']
@@ -75,9 +75,9 @@ module Datahen
         end
       end
 
-      def get_failed_content(gid)
-        client = Client::GlobalPage.new()
-        content_json = client.find_failed_content(gid)
+      def get_failed_content(job_id, gid)
+        client = Client::JobPage.new()
+        content_json = client.find_failed_content(job_id, gid)
 
         if content_json['available']
           signed_url = content_json['signed_url']

data/lib/datahen/scraper/parser.rb

@@ -1,18 +1,24 @@
 module Datahen
   module Scraper
     class Parser
-      def self.exec_parser_page(filename, gid, job_id=nil, save=false, vars = {})
+      def self.exec_parser_page(filename, gid, job_id=nil, save=false, vars = {}, keep_outputs=false)
         extname = File.extname(filename)
         case extname
         when '.rb'
-          executor = RubyParserExecutor.new(filename: filename, gid: gid, job_id: job_id, vars: vars)
+          executor = RubyParserExecutor.new(
+            filename: filename,
+            gid: gid,
+            job_id: job_id,
+            vars: vars,
+            keep_outputs: keep_outputs
+          )
           executor.exec_parser(save)
         else
           puts "Unable to find a parser executor for file type \"#{extname}\""
         end
       end
 
-    
+
     end
   end
-end
+end

data/lib/datahen/scraper/ruby_parser_executor.rb

@@ -15,6 +15,7 @@ module Datahen
         @gid = options.fetch(:gid) { raise "GID is required"}
         @job_id = options.fetch(:job_id)
         @page_vars = options.fetch(:vars) { {} }
+        @keep_outputs = !!(options.fetch(:keep_outputs) { false })
       end
 
       def self.exposed_methods
@@ -66,7 +67,9 @@ module Datahen
         response = parsing_update(
           job_id: job_id,
           gid: gid,
-          parsing_status: :starting)
+          parsing_status: :starting,
+          keep_outputs: @keep_outputs
+        )
 
         if response.code == 200
           puts "Page Parsing Status Updated."
@@ -165,7 +168,7 @@ module Datahen
             handle_error(e) if save
             raise e
           end
-          
+
           if refetch_self
             refetch_page gid
           elsif reparse_self
@@ -178,11 +181,11 @@ module Datahen
       end
 
       def content
-        @content ||= get_content(gid)
+        @content ||= get_content(job_id, gid)
       end
 
       def failed_content
-        @failed_content ||= get_failed_content(gid)
+        @failed_content ||= get_failed_content(job_id, gid)
       end
 
       def handle_error(e)

data/lib/datahen/scraper/ruby_seeder_executor.rb

@@ -6,6 +6,7 @@ module Datahen
       def initialize(options={})
         @filename = options.fetch(:filename) { raise "Filename is required"}
         @job_id = options[:job_id]
+        @keep_outputs = !!(options.fetch(:keep_outputs) { false })
       end
 
       def self.exposed_methods
@@ -81,7 +82,9 @@ module Datahen
 
         response = seeding_update(
           job_id: job_id,
-          seeding_status: :starting)
+          seeding_status: :starting,
+          keep_outputs: @keep_outputs
+        )
 
         if response.code == 200
           puts "Seeding Status Updated."

data/lib/datahen/scraper/seeder.rb

@@ -2,11 +2,15 @@ module Datahen
   module Scraper
     class Seeder
 
-      def self.exec_seeder(filename, job_id=nil, save=false)
+      def self.exec_seeder(filename, job_id=nil, save=false, keep_outputs=false)
         extname = File.extname(filename)
         case extname
         when '.rb'
-          executor = RubySeederExecutor.new(filename: filename, job_id: job_id)
+          executor = RubySeederExecutor.new(
+            filename: filename,
+            job_id: job_id,
+            keep_outputs: keep_outputs
+          )
           executor.exec_seeder(save)
         else
           puts "Unable to find a seeder executor for file type \"#{extname}\""
@@ -15,4 +19,4 @@ module Datahen
 
     end
   end
-end
+end

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "0.11.1"
+  VERSION = "0.14.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 0.11.1
+  version: 0.14.0
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-01-17 00:00:00.000000000 Z
+date: 2020-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -215,10 +215,12 @@ files:
 - lib/datahen/client/global_page.rb
 - lib/datahen/client/job.rb
 - lib/datahen/client/job_export.rb
+- lib/datahen/client/job_finisher.rb
 - lib/datahen/client/job_log.rb
 - lib/datahen/client/job_output.rb
 - lib/datahen/client/job_page.rb
 - lib/datahen/client/job_stat.rb
+- lib/datahen/client/job_var.rb
 - lib/datahen/client/scraper.rb
 - lib/datahen/client/scraper_deployment.rb
 - lib/datahen/client/scraper_export.rb
@@ -262,7 +264,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: DataHen toolbelt for developers