datahen 0.13.0 → 0.13.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c3d1584f235873f6d22ae107c60e7b50a6d8ab5918f45a4052bef80e58b7cbf7
-  data.tar.gz: b6b6ce4871017eddb70fd279c92246e498e57c3d7ca2c0bd72c225bdcdb5e119
+  metadata.gz: c05f6ab973fe74a21e7f68411a66b97444575cc75c5812e99c5e0da4ffe05d56
+  data.tar.gz: dac1d2be8f6281d3da328abd5f844036765991764bb1e3ffad4b72ee4c18eac6
 SHA512:
-  metadata.gz: 713f5907efc90be21ba04b83c060f0f656d1f72176191399e1ff575952cad2b54c834ddfd6722441e933857d12539f55553adf2b4d71adead952465f0ccf1005
-  data.tar.gz: bd7809ed5fbd6d12dc8680f31f194ecfb51ba0138765a88c9dde4745be2bd46533da97885f63524edca09cee1ab2d4cc3d7fd96bf04a2b6a6930a046646a8093
+  metadata.gz: 48c15f9830308488d434dce7b7cf4888795724d88d4ca9de63cc2deca8397798b815145430af2143eb646366bfc4e97f37e563315214258b434276919e724ac7
+  data.tar.gz: faee0369f81ab45dbb6c62258a6214d530d51d5dcb0f6b6ebd7028bda2c48ccbe630b41006ffcc96dba773a28e84e724b6b1f2266ef7866605e5423f1ced61d2

data/lib/datahen/cli/parser.rb

@@ -10,12 +10,13 @@ module Datahen
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       option :global, :aliases => :g, type: :boolean, default: false, desc: 'Use globalpage instead of a job page'
       option :vars, :aliases => :v, type: :string, desc: 'Set user-defined page variables. Must be in json format. i.e: {"Foo":"bar"}'
+      option :"keep-outputs", :aliases => :ko, type: :boolean, default: false, desc: "Don't delete existing outputs"
       def try_parse(scraper_name, parser_file, gid)
-        begin 
-          
+        begin
+
             if options[:job]
               job_id = options[:job]
-            elsif options[:global] 
+            elsif options[:global]
               job_id = nil
             else
               job = Client::ScraperJob.new(options).find(scraper_name)
@@ -24,7 +25,7 @@ module Datahen
 
 
           vars = JSON.parse(options[:vars]) if options[:vars]
-          puts Datahen::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, false, vars)
+          puts Datahen::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, false, vars, options[:"keep-outputs"])
 
           rescue JSON::ParserError
           if options[:vars]
@@ -40,6 +41,8 @@ module Datahen
             <GID>: Global ID of the page.\x5
           LONGDESC
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :vars, :aliases => :v, type: :string, desc: 'Set user-defined page variables. Must be in json format. i.e: {"Foo":"bar"}'
+      option :"keep-outputs", :aliases => :ko, type: :boolean, default: false, desc: "Don't delete existing outputs"
       def exec_parse(scraper_name, parser_file, *gids)
         gids.each do |gid|
           begin
@@ -52,7 +55,8 @@ module Datahen
               job_id = job['id']
             end
 
-            puts Datahen::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, true)
+            vars = JSON.parse(options[:vars]) if options[:vars]
+            puts Datahen::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, true, vars, options[:"keep-outputs"])
           rescue => e
             puts e
           end

data/lib/datahen/cli/scraper_export.rb

@@ -12,7 +12,6 @@ module Datahen
         puts "#{client.find(export_id)}"
       end
 
-
       desc "list", "Gets a list of exports"
       long_desc <<-LONGDESC
         List exports.
@@ -34,13 +33,13 @@ module Datahen
       def download(export_id)
         client = Client::ScraperExport.new(options)
         result = JSON.parse(client.download(export_id).to_s)
-        
+
         if result['signed_url']
           puts "Download url: \"#{result['signed_url']}\""
           `open "#{result['signed_url']}"`
         else
           puts "Exported file does not exist"
-        end        
+        end
       end
 
 

data/lib/datahen/cli/scraper_finisher.rb

@@ -11,9 +11,15 @@ module Datahen
       long_desc <<-LONGDESC
         Reset finisher on a scraper's current job.\x5
       LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def reset(scraper_name)
-        client = Client::ScraperFinisher.new(options)
-        puts "#{client.reset(scraper_name)}"
+        if options[:job]
+          client = Client::JobFinisher.new(options)
+          puts "#{client.reset(options[:job])}"
+        else
+          client = Client::ScraperFinisher.new(options)
+          puts "#{client.reset(scraper_name)}"
+        end
       end
     end
   end

data/lib/datahen/cli/scraper_job.rb

@@ -29,27 +29,45 @@ module Datahen
       long_desc <<-LONGDESC
         Cancels a scraper's current job
       LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def cancel(scraper_name)
-        client = Client::ScraperJob.new(options)
-        puts "#{client.cancel(scraper_name)}"
+        if options[:job]
+          client = Client::Job.new(options)
+          puts "#{client.cancel(options[:job])}"
+        else
+          client = Client::ScraperJob.new(options)
+          puts "#{client.cancel(scraper_name)}"
+        end
       end
 
       desc "resume <scraper_name>", "resumes a scraper's current job"
       long_desc <<-LONGDESC
         Resumes a scraper's current job
       LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def resume(scraper_name)
-        client = Client::ScraperJob.new(options)
-        puts "#{client.resume(scraper_name)}"
+        if options[:job]
+          client = Client::Job.new(options)
+          puts "#{client.resume(options[:job])}"
+        else
+          client = Client::ScraperJob.new(options)
+          puts "#{client.resume(scraper_name)}"
+        end
       end
 
       desc "pause <scraper_name>", "pauses a scraper's current job"
       long_desc <<-LONGDESC
         pauses a scraper's current job
       LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def pause(scraper_name)
-        client = Client::ScraperJob.new(options)
-        puts "#{client.pause(scraper_name)}"
+        if options[:job]
+          client = Client::Job.new(options)
+          puts "#{client.pause(options[:job])}"
+        else
+          client = Client::ScraperJob.new(options)
+          puts "#{client.pause(scraper_name)}"
+        end
       end
 
 
@@ -60,9 +78,15 @@ module Datahen
       option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Scraper job must be restarted(paused then resumed, or cancelled then resumed) for it to take effect. Default: 1. '
       option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Scraper job must be restarted(paused then resumed, or cancelled then resumed) for it to take effect. Default: 0. '
       option :proxy_type, desc: 'Set the Proxy type. Default: standard'
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def update(scraper_name)
-        client = Client::ScraperJob.new(options)
-        puts "#{client.update(scraper_name, options)}"
+        if options[:job]
+          client = Client::Job.new(options)
+          puts "#{client.update(options[:job], options)}"
+        else
+          client = Client::ScraperJob.new(options)
+          puts "#{client.update(scraper_name, options)}"
+        end
       end
 
       desc "var SUBCOMMAND ...ARGS", "for managing scraper's job variables"

data/lib/datahen/cli/scraper_job_var.rb

@@ -13,9 +13,15 @@ module Datahen
       LONGDESC
       option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
       option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def list(scraper_name)
-        client = Client::ScraperJobVar.new(options)
-        puts "#{client.all(scraper_name)}"
+        if options[:job]
+          client = Client::JobVar.new(options)
+          puts "#{client.all(options[:job])}"
+        else
+          client = Client::ScraperJobVar.new(options)
+          puts "#{client.all(scraper_name)}"
+        end
       end
 
       desc "set <scraper_name> <var_name> <value>", "Set an environment var on the scrape job"
@@ -24,23 +30,40 @@ module Datahen
           <var_name>: Var name can only consist of alphabets, numbers, underscores. Name must be unique to your scrape job, otherwise it will be overwritten.\x5
           <value>: Value of variable.\x5
           LONGDESC
-      option :secret, type: :boolean, desc: 'Set true to make it decrypt the value. Default: false' 
+      option :secret, type: :boolean, desc: 'Set true to make it decrypt the value. Default: false'
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def set(scraper_name, var_name, value)
-        # puts "options #{options}"
-        client = Client::ScraperJobVar.new(options)
-        puts "#{client.set(scraper_name, var_name, value, options)}"
+        if options[:job]
+          client = Client::JobVar.new(options)
+          puts "#{client.set(options[:job], var_name, value, options)}"
+        else
+          client = Client::ScraperJobVar.new(options)
+          puts "#{client.set(scraper_name, var_name, value, options)}"
+        end
       end
 
       desc "show <scraper_name> <var_name>", "Show an environment variable on the scrape job"
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def show(scraper_name, var_name)
-        client = Client::ScraperJobVar.new(options)
-        puts "#{client.find(scraper_name, var_name)}"
+        if options[:job]
+          client = Client::JobVar.new(options)
+          puts "#{client.find(options[:job], var_name)}"
+        else
+          client = Client::ScraperJobVar.new(options)
+          puts "#{client.find(scraper_name, var_name)}"
+        end
       end
 
       desc "unset <scraper_name> <var_name>", "Deletes an environment variable on the scrape job"
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def unset(scraper_name, var_name)
-        client = Client::ScraperJobVar.new(options)
-        puts "#{client.unset(scraper_name, var_name)}"
+        if options[:job]
+          client = Client::JobVar.new(options)
+          puts "#{client.unset(options[:job], var_name)}"
+        else
+          client = Client::ScraperJobVar.new(options)
+          puts "#{client.unset(scraper_name, var_name)}"
+        end
       end
     end
   end

data/lib/datahen/cli/scraper_page.rb

@@ -105,13 +105,19 @@ module Datahen
       option :fetch_fail, type: :boolean, desc: 'Refetches only pages that fails fetching.'
       option :parse_fail, type: :boolean, desc: 'Refetches only pages that fails parsing.'
       option :status, type: :string, desc: 'Refetches only pages with a specific status.'
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def refetch(scraper_name)
         if !options.key?(:gid) && !options.key?(:fetch_fail) && !options.key?(:parse_fail) && !options.key?(:status)
           puts "Must specify either a --gid, --fetch-fail, --parse-fail or --status"
           return
         end
-        client = Client::ScraperJobPage.new(options)
-        puts "#{client.refetch(scraper_name)}"
+        if options[:job]
+          client = Client::JobPage.new(options)
+          puts "#{client.refetch(options[:job])}"
+        else
+          client = Client::ScraperJobPage.new(options)
+          puts "#{client.refetch(scraper_name)}"
+        end
       end
 
       desc "reparse <scraper_name>", "Reparse Pages on a scraper's current job"
@@ -121,6 +127,7 @@ module Datahen
       option :gid, :aliases => :g, type: :string, desc: 'Reparse a specific GID'
       option :parse_fail, type: :boolean, desc: 'Reparse only pages that fails parsing.'
       option :status, type: :string, desc: 'Reparse only pages with a specific status.'
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def reparse(scraper_name)
         begin
           options[:vars] = JSON.parse(options[:vars]) if options[:vars]
@@ -130,8 +137,13 @@ module Datahen
             return
           end
 
-          client = Client::ScraperJobPage.new(options)
-          puts "#{client.reparse(scraper_name)}"
+          if options[:job]
+            client = Client::JobPage.new(options)
+            puts "#{client.reparse(options[:job])}"
+          else
+            client = Client::ScraperJobPage.new(options)
+            puts "#{client.reparse(scraper_name)}"
+          end
 
         rescue JSON::ParserError
           if options[:vars]

data/lib/datahen/cli/seeder.rb

@@ -7,6 +7,7 @@ module Datahen
             <seeder_file>: Seeder script file will be executed.\x5
           LONGDESC
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :"keep-outputs", :aliases => :ko, type: :boolean, default: false, desc: "Don't delete existing outputs"
       def try_seed(scraper_name, seeder_file)
         if options[:job]
           job_id = options[:job]
@@ -14,8 +15,8 @@ module Datahen
           job = Client::ScraperJob.new(options).find(scraper_name)
           job_id = job['id']
         end
-        
-        puts Datahen::Scraper::Seeder.exec_seeder(seeder_file, job_id, false)
+
+        puts Datahen::Scraper::Seeder.exec_seeder(seeder_file, job_id, false, options[:"keep-outputs"])
       end
 
       desc "exec <scraper_name> <seeder_file>", "Executes a seeder script onto a scraper's current job."
@@ -24,6 +25,7 @@ module Datahen
             <seeder_file>: Seeder script file that will be executed on the scraper's current job.\x5
           LONGDESC
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :"keep-outputs", :aliases => :ko, type: :boolean, default: false, desc: "Don't delete existing outputs"
       def exec_parse(scraper_name, seeder_file)
         if options[:job]
           job_id = options[:job]

data/lib/datahen/client.rb

@@ -20,7 +20,9 @@ require "datahen/client/job_stat"
 require "datahen/client/backblaze_content"
 require "datahen/client/env_var"
 require "datahen/client/scraper_var"
+require "datahen/client/job_var"
 require "datahen/client/scraper_job_var"
+require "datahen/client/job_finisher"
 
 
 module Datahen

data/lib/datahen/client/job.rb

@@ -15,6 +15,7 @@ module Datahen
         body[:status] = opts[:status] if opts[:status]
         body[:standard_worker_count] = opts[:workers] if opts[:workers]
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
+        body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
         params = @options.merge({body: body.to_json})
 
         self.class.put("/jobs/#{job_id}", params)
@@ -41,6 +42,7 @@ module Datahen
         body[:pages] = opts.fetch(:pages) {[]}
         body[:seeding_status] = opts.fetch(:seeding_status){ nil }
         body[:log_error] = opts[:log_error] if opts[:log_error]
+        body[:keep_outputs] = !!opts[:keep_outputs] if opts.has_key?(:keep_outputs)
 
         params = @options.merge({body: body.to_json})
 

data/lib/datahen/client/job_finisher.rb

@@ -0,0 +1,16 @@
+module Datahen
+  module Client
+    class JobFinisher < Datahen::Client::Base
+      # Reset finisher on a scraper's current job.
+      #
+      # @param [Integer] job_id Job ID
+      # @param [Hash] opts ({}) API custom parameters.
+      #
+      # @return [HTTParty::Response]
+      def reset(job_id, opts={})
+        params = @options.merge(opts)
+        self.class.put("/jobs/#{job_id}/finisher/reset", params)
+      end
+    end
+  end
+end

data/lib/datahen/client/job_page.rb

@@ -48,6 +48,7 @@ module Datahen
         body[:pages] = opts.fetch(:pages) {[]}
         body[:parsing_status] = opts.fetch(:parsing_status){ nil }
         body[:log_error] = opts[:log_error] if opts[:log_error]
+        body[:keep_outputs] = !!opts[:keep_outputs] if opts.has_key?(:keep_outputs)
 
         params = @options.merge({body: body.to_json})
 
@@ -61,6 +62,16 @@ module Datahen
       def find_failed_content(job_id, gid)
         self.class.get("/jobs/#{job_id}/pages/#{gid}/failed_content", @options)
       end
+
+      def reparse(job_id, opts={})
+        params = @options.merge(opts)
+        self.class.put("/jobs/#{job_id}/pages/reparse", params)
+      end
+
+      def refetch(job_id, opts={})
+        params = @options.merge(opts)
+        self.class.put("/jobs/#{job_id}/pages/refetch", params)
+      end
     end
   end
 end

data/lib/datahen/client/job_var.rb

@@ -0,0 +1,28 @@
+module Datahen
+  module Client
+    class JobVar < Datahen::Client::Base
+
+      def find(job_id, var_name)
+        self.class.get("/jobs/#{job_id}/vars/#{var_name}", @options)
+      end
+
+      def all(job_id, opts={})
+        params = @options.merge opts
+        self.class.get("/jobs/#{job_id}/vars", params)
+      end
+
+      def set(job_id, var_name, value, opts={})
+        body = {}
+        body[:value] = value
+        body[:secret] = opts[:secret] if opts[:secret]
+        params = @options.merge({body: body.to_json})
+        self.class.put("/jobs/#{job_id}/vars/#{var_name}", params)
+      end
+
+      def unset(job_id, var_name, opts={})
+        params = @options.merge(opts)
+        self.class.delete("/jobs/#{job_id}/vars/#{var_name}", params)
+      end
+    end
+  end
+end

data/lib/datahen/client/scraper_job_page.rb

@@ -26,6 +26,9 @@ module Datahen
         self.class.put("/scrapers/#{scraper_name}/current_job/pages/refetch", params)
       end
 
+      # Deprecated, please use Datahen::Client::JobVar#refetch instead.
+      #
+      # @note This method will be removed at some point in the future.
       def refetch_by_job(job_id, opts={})
         params = @options.merge(opts)
         self.class.put("/jobs/#{job_id}/pages/refetch", params)
@@ -36,11 +39,6 @@ module Datahen
         self.class.put("/scrapers/#{scraper_name}/current_job/pages/reparse", params)
       end
 
-      def reparse_by_job(job_id, opts={})
-        params = @options.merge(opts)
-        self.class.put("/jobs/#{job_id}/pages/reparse", params)
-      end
-
       def enqueue(scraper_name, method, url, opts={})
         body = {}
         body[:method] =  method != "" ? method : "GET"

data/lib/datahen/scraper/executor.rb

@@ -63,9 +63,9 @@ module Datahen
         client.find(gid)
       end
 
-      def get_content(gid)
-        client = Client::GlobalPage.new()
-        content_json = client.find_content(gid)
+      def get_content(job_id, gid)
+        client = Client::JobPage.new()
+        content_json = client.find_content(job_id, gid)
 
         if content_json['available']
           signed_url = content_json['signed_url']
@@ -75,7 +75,7 @@ module Datahen
         end
       end
 
-      def get_failed_content(gid)
+      def get_failed_content(job_id, gid)
         client = Client::JobPage.new()
         content_json = client.find_failed_content(job_id, gid)
 

data/lib/datahen/scraper/parser.rb

@@ -1,18 +1,24 @@
 module Datahen
   module Scraper
     class Parser
-      def self.exec_parser_page(filename, gid, job_id=nil, save=false, vars = {})
+      def self.exec_parser_page(filename, gid, job_id=nil, save=false, vars = {}, keep_outputs=false)
         extname = File.extname(filename)
         case extname
         when '.rb'
-          executor = RubyParserExecutor.new(filename: filename, gid: gid, job_id: job_id, vars: vars)
+          executor = RubyParserExecutor.new(
+            filename: filename,
+            gid: gid,
+            job_id: job_id,
+            vars: vars,
+            keep_outputs: keep_outputs
+          )
           executor.exec_parser(save)
         else
           puts "Unable to find a parser executor for file type \"#{extname}\""
         end
       end
 
-    
+
     end
   end
-end
+end

data/lib/datahen/scraper/ruby_parser_executor.rb

@@ -15,6 +15,7 @@ module Datahen
         @gid = options.fetch(:gid) { raise "GID is required"}
         @job_id = options.fetch(:job_id)
         @page_vars = options.fetch(:vars) { {} }
+        @keep_outputs = !!(options.fetch(:keep_outputs) { false })
       end
 
       def self.exposed_methods
@@ -66,7 +67,9 @@ module Datahen
         response = parsing_update(
           job_id: job_id,
           gid: gid,
-          parsing_status: :starting)
+          parsing_status: :starting,
+          keep_outputs: @keep_outputs
+        )
 
         if response.code == 200
           puts "Page Parsing Status Updated."
@@ -165,7 +168,7 @@ module Datahen
             handle_error(e) if save
             raise e
           end
-          
+
           if refetch_self
             refetch_page gid
           elsif reparse_self
@@ -178,11 +181,11 @@ module Datahen
       end
 
       def content
-        @content ||= get_content(gid)
+        @content ||= get_content(job_id, gid)
       end
 
       def failed_content
-        @failed_content ||= get_failed_content(gid)
+        @failed_content ||= get_failed_content(job_id, gid)
       end
 
       def handle_error(e)

data/lib/datahen/scraper/ruby_seeder_executor.rb

@@ -6,6 +6,7 @@ module Datahen
       def initialize(options={})
         @filename = options.fetch(:filename) { raise "Filename is required"}
         @job_id = options[:job_id]
+        @keep_outputs = !!(options.fetch(:keep_outputs) { false })
       end
 
       def self.exposed_methods
@@ -81,7 +82,9 @@ module Datahen
 
         response = seeding_update(
           job_id: job_id,
-          seeding_status: :starting)
+          seeding_status: :starting,
+          keep_outputs: @keep_outputs
+        )
 
         if response.code == 200
           puts "Seeding Status Updated."

data/lib/datahen/scraper/seeder.rb

@@ -2,11 +2,15 @@ module Datahen
   module Scraper
     class Seeder
 
-      def self.exec_seeder(filename, job_id=nil, save=false)
+      def self.exec_seeder(filename, job_id=nil, save=false, keep_outputs=false)
         extname = File.extname(filename)
         case extname
         when '.rb'
-          executor = RubySeederExecutor.new(filename: filename, job_id: job_id)
+          executor = RubySeederExecutor.new(
+            filename: filename,
+            job_id: job_id,
+            keep_outputs: keep_outputs
+          )
           executor.exec_seeder(save)
         else
           puts "Unable to find a seeder executor for file type \"#{extname}\""
@@ -15,4 +19,4 @@ module Datahen
 
     end
   end
-end
+end

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "0.13.0"
+  VERSION = "0.13.7"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 0.13.0
+  version: 0.13.7
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-02 00:00:00.000000000 Z
+date: 2020-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -215,10 +215,12 @@ files:
 - lib/datahen/client/global_page.rb
 - lib/datahen/client/job.rb
 - lib/datahen/client/job_export.rb
+- lib/datahen/client/job_finisher.rb
 - lib/datahen/client/job_log.rb
 - lib/datahen/client/job_output.rb
 - lib/datahen/client/job_page.rb
 - lib/datahen/client/job_stat.rb
+- lib/datahen/client/job_var.rb
 - lib/datahen/client/scraper.rb
 - lib/datahen/client/scraper_deployment.rb
 - lib/datahen/client/scraper_export.rb