datahen 0.14.1 → 0.14.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f53bb8631bee37890dddae7045b89342fcb01611f693fe454a2d001ea9cdfe30
-  data.tar.gz: fe2ab54f12f5865ea60bb7d2f447e09e7eb9cf5dff8a74bfbdced103f8bd03e1
+  metadata.gz: 63ae3d8aba6bb70a89033d592db3242905d70d7e0ea6a90455c370cf3b8a2dde
+  data.tar.gz: 3f4644be8702eb789f8d4c3e099750dade2e9d9f83ab6a359f222972d11938bc
 SHA512:
-  metadata.gz: 59a642fe8ea6274bfb378e60ae4e8e730bb76cc58a188f0c267636d063afa45eb202c635b05b3a906a67f278e03e9f2e5637750f79ef5b8edf8e5ad3a42668d7
-  data.tar.gz: b86f95465887321c090724a65db613a57350050ccdd5c1fb4c407701ac560d6bf6a2ef2964b7c44722578205672483d713b8358e5e3e68c29f59076602391f84
+  metadata.gz: 20f50d8b4a52d360fe07f32aaa0350a190b80c30157d0d2c1e33ca745013ed8ab1a9a97d81acfc400fa6ff3250394ea44e69bfe894318d4798ec8d99567ab736
+  data.tar.gz: 2724495723be6e2d249e8697e28102674c3685ce4cb384bcefaeca03d606361bf2e0e24c098954ad5b872b7e067a0cc5d0c1ed84a10b7b703144ad3a2ba16b4f

data/lib/datahen/cli.rb

@@ -16,6 +16,7 @@ require 'datahen/cli/parser'
 require 'datahen/cli/seeder'
 require 'datahen/cli/finisher'
 require 'datahen/cli/env_var'
+require 'datahen/cli/account'
 
 
 
@@ -41,5 +42,8 @@ module Datahen
 
     desc "var SUBCOMMAND ...ARGS", "for environment variable related activities"
     subcommand "var", EnvVar
+
+    desc "account SUBCOMMAND ...ARGS", "for account related activities"
+    subcommand "account", Account
   end
 end

data/lib/datahen/cli/account.rb

@@ -0,0 +1,17 @@
+module Datahen
+  class CLI < Thor
+    class Account < Thor
+
+      desc "profile", "displays the account applied profile"
+      long_desc <<-LONGDESC
+        Displays the account applied profile
+      LONGDESC
+      def profile()
+        client = Client::Account.new(options)
+        puts "#{client.profile()}"
+      end
+
+    end
+  end
+
+end

data/lib/datahen/cli/env_var.rb

@@ -2,7 +2,7 @@ module Datahen
   class CLI < Thor
     class EnvVar < Thor
       desc "list", "List environment variables on the account"
-      
+
       long_desc <<-LONGDESC
         List all environment variables on the account.
       LONGDESC
@@ -19,7 +19,7 @@ module Datahen
           <name>: Var name can only consist of alphabets, numbers, underscores. Name must be unique to your account, otherwise it will be overwritten.\x5
           <value>: Value of variable.\x5
           LONGDESC
-      option :secret, type: :boolean, desc: 'Set true to make it decrypt the value. Default: false' 
+      option :secret, type: :boolean, desc: 'Set true to make it decrypt the value. Default: false'
       def set(name, value)
         # puts "options #{options}"
         client = Client::EnvVar.new(options)
@@ -38,10 +38,6 @@ module Datahen
         puts "#{client.unset(name)}"
       end
 
-
-      
-
-
     end
   end
 

data/lib/datahen/cli/scraper.rb

@@ -1,8 +1,8 @@
 module Datahen
   class CLI < Thor
     class Scraper < Thor
-      desc "list", "List scrapers"
 
+      desc "list", "List scrapers"
       long_desc <<-LONGDESC
         List all scrapers.
       LONGDESC
@@ -29,6 +29,7 @@ module Datahen
       option :cancel_current_job, type: :boolean, desc: 'Set true to cancel currently active job if scheduler starts. Default: false'
       option :schedule, type: :string, desc: 'Set the schedule of the scraper to run. Must be in CRON format.'
       option :timezone, type: :string, desc: "Set the scheduler's timezone. Must be in IANA Timezone format. Defaults to \"America/Toronto\""
+      option :profile, type: :string, desc: 'Set the profiles (comma separated) to apply to the job. Default: default'
       def create(scraper_name, git_repository)
         # puts "options #{options}"
         client = Client::Scraper.new(options)
@@ -51,6 +52,7 @@ module Datahen
       option :cancel_current_job, type: :boolean, desc: 'Set true to cancel currently active job if scheduler starts. Default: false'
       option :schedule, type: :string, desc: 'Set the schedule of the scraper to run. Must be in CRON format.'
       option :timezone, type: :string, desc: "Set the scheduler's timezone. Must be in IANA Timezone format. Defaults to \"America/Toronto\""
+      option :profile, type: :string, desc: 'Set the profiles (comma separated) to apply to the job. Default: default'
       def update(scraper_name)
         client = Client::Scraper.new(options)
         puts "#{client.update(scraper_name, options)}"
@@ -164,15 +166,25 @@ module Datahen
       option :"max-timestamp", type: :string, desc: 'Ending timestamp point in time to query historic stats (inclusive)'
       option :"limit", type: :numeric, desc: 'Limit stats retrieved'
       option :"order", type: :numeric, desc: 'Order stats by timestamp [DESC]'
+      option :live, type: :boolean, desc: 'Get data from the live stats history, not cached stats history.'
+      option :filter, type: :string, desc: 'Filter results on `day` or `hour`, if not specified will return all records.'
       def history(scraper_name)
         client = Client::JobStat.new(options)
         if options[:job]
-          puts "#{client.job_stats_history(options[:job])}"
+          puts "#{client.job_stats_history(options[:job], options)}"
         else
-          puts "#{client.scraper_job_stats_history(scraper_name)}"
+          puts "#{client.scraper_job_stats_history(scraper_name, options)}"
         end
       end
 
+      desc "profile <scraper_name>", "displays the scraper applied profile"
+      long_desc <<-LONGDESC
+        Displays the account applied profile
+      LONGDESC
+      def profile(scraper_name)
+        client = Client::Scraper.new(options)
+        puts "#{client.profile(scraper_name)}"
+      end
 
       desc "job SUBCOMMAND ...ARGS", "manage scrapers jobs"
       subcommand "job", ScraperJob

data/lib/datahen/cli/scraper_job.rb

@@ -7,10 +7,16 @@ module Datahen
       end
 
       desc "show <scraper_name>", "Show a scraper's current job (Defaults to showing data from cached job)"
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       option :live, type: :boolean, desc: 'Get data from the live job, not cached job.'
       def show(scraper_name)
-        client = Client::ScraperJob.new(options)
-        puts "#{client.find(scraper_name, options)}"
+        if options[:job]
+          client = Client::Job.new(options)
+          puts "#{client.find(options[:job], options)}"
+        else
+          client = Client::ScraperJob.new(options)
+          puts "#{client.find(scraper_name, options)}"
+        end
       end
 
 
@@ -58,7 +64,7 @@ module Datahen
 
       desc "pause <scraper_name>", "pauses a scraper's current job"
       long_desc <<-LONGDESC
-        pauses a scraper's current job
+        Pauses a scraper's current job
       LONGDESC
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def pause(scraper_name)
@@ -79,6 +85,7 @@ module Datahen
       option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Scraper job must be restarted(paused then resumed, or cancelled then resumed) for it to take effect. Default: 1. '
       option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Scraper job must be restarted(paused then resumed, or cancelled then resumed) for it to take effect. Default: 0. '
       option :proxy_type, desc: 'Set the Proxy type. Default: standard'
+      option :profile, type: :string, desc: 'Set the profiles (comma separated) to apply to the job. Default: default'
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def update(scraper_name)
         if options[:job]
@@ -90,6 +97,21 @@ module Datahen
         end
       end
 
+      desc "profile <scraper_name>", "displays a scraper's current job applied profile"
+      long_desc <<-LONGDESC
+        Displays a scraper's current job applied profile
+      LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      def profile(scraper_name)
+        if options[:job]
+          client = Client::Job.new(options)
+          puts "#{client.profile(options[:job])}"
+        else
+          client = Client::ScraperJob.new(options)
+          puts "#{client.profile(scraper_name)}"
+        end
+      end
+
       desc "var SUBCOMMAND ...ARGS", "for managing scraper's job variables"
       subcommand "var", ScraperJobVar
 

data/lib/datahen/cli/scraper_page.rb

@@ -230,7 +230,7 @@ module Datahen
         end
       end
 
-      desc "failedcontent <gid>", "Show a page's failed content in scraper's current job"
+      desc "failedcontent <scraper_name> <gid>", "Show a page's failed content in scraper's current job"
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def failedcontent(scraper_name, gid)
         result = nil

data/lib/datahen/client.rb

@@ -1,4 +1,5 @@
 require "datahen/client/base"
+require "datahen/client/account"
 require "datahen/client/auth_token"
 require "datahen/client/deploy_key"
 require 'datahen/client/export'
@@ -24,7 +25,6 @@ require "datahen/client/job_var"
 require "datahen/client/scraper_job_var"
 require "datahen/client/job_finisher"
 
-
 module Datahen
   module Client
   end

data/lib/datahen/client/account.rb

@@ -0,0 +1,13 @@
+module Datahen
+  module Client
+    class Account < Datahen::Client::Base
+
+      def profile(opts={})
+        params = @options.merge(opts)
+
+        self.class.get("/profile", params)
+      end
+
+    end
+  end
+end

data/lib/datahen/client/base.rb

@@ -5,6 +5,8 @@ module Datahen
     class Base
       include HTTParty
 
+      default_timeout 60
+
       def self.env_auth_token
         ENV['DATAHEN_TOKEN']
       end
@@ -55,6 +57,7 @@ module Datahen
         query[:"max-timestamp"] = opts[:"max-timestamp"] if opts[:"max-timestamp"]
         query[:limit] = opts[:limit] if opts[:limit]
         query[:order] = opts[:order] if opts[:order]
+        query[:filter] = opts[:filter] if opts[:filter]
 
         if opts[:query]
           if opts[:query].is_a?(Hash)

data/lib/datahen/client/job.rb

@@ -20,6 +20,7 @@ module Datahen
         body[:standard_worker_count] = opts[:workers] if opts[:workers]
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
+        body[:profile] = opts[:profile] if opts[:profile]
         params = @options.merge({body: body.to_json})
 
         self.class.put("/jobs/#{job_id}", params)
@@ -64,6 +65,12 @@ module Datahen
         self.class.put("/jobs/#{job_id}/finisher_update", params)
       end
 
+      def profile(job_id, opts={})
+        params = @options.merge(opts)
+
+        self.class.get("/jobs/#{job_id}/profile", params)
+      end
+
     end
 
   end

data/lib/datahen/client/job_stat.rb

@@ -18,12 +18,20 @@ module Datahen
         end
       end
 
-      def job_stats_history(job_id)
-        self.class.get("/jobs/#{job_id}/stats/history", @options)
+      def job_stats_history(job_id, opts={})
+        if opts[:live]
+          self.class.get("/jobs/#{job_id}/stats/history", @options)
+        else
+          self.class.get("/cached/jobs/#{job_id}/stats/history", @options)
+        end
       end
 
-      def scraper_job_stats_history(scraper_name)
-        self.class.get("/scrapers/#{scraper_name}/current_job/stats/history", @options)
+      def scraper_job_stats_history(scraper_name, opts={})
+        if opts[:live]
+          self.class.get("/scrapers/#{scraper_name}/current_job/stats/history", @options)
+        else
+          self.class.get("/cached/scrapers/#{scraper_name}/current_job/stats/history", @options)
+        end
       end
 
     end

data/lib/datahen/client/scraper.rb

@@ -25,6 +25,7 @@ module Datahen
         body[:cancel_current_job] = opts[:cancel_current_job] if opts[:cancel_current_job]
         body[:schedule] = opts[:schedule] if opts[:schedule]
         body[:timezone] = opts[:timezone] if opts[:timezone]
+        body[:profile] = opts[:profile] if opts[:profile]
         params = @options.merge({body: body.to_json})
         self.class.post("/scrapers", params)
       end
@@ -43,6 +44,7 @@ module Datahen
         body[:cancel_current_job] = opts[:cancel_current_job] if opts.has_key?("cancel_current_job") || opts.has_key?(:cancel_current_job)
         body[:schedule] = opts[:schedule] if opts[:schedule]
         body[:timezone] = opts[:timezone] if opts[:timezone]
+        body[:profile] = opts[:profile] if opts[:profile]
         params = @options.merge({body: body.to_json})
 
         self.class.put("/scrapers/#{scraper_name}", params)
@@ -52,6 +54,12 @@ module Datahen
         params = @options.merge(opts)
         self.class.delete("/scrapers/#{scraper_name}", params)
       end
+
+      def profile(scraper_name, opts={})
+        params = @options.merge(opts)
+
+        self.class.get("/scrapers/#{scraper_name}/profile", params)
+      end
     end
   end
 end

data/lib/datahen/client/scraper_job.rb

@@ -29,6 +29,7 @@ module Datahen
         body[:standard_worker_count] = opts[:workers] if opts[:workers]
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
+        body[:profile] = opts[:profile] if opts[:profile]
         params = @options.merge({body: body.to_json})
 
         self.class.put("/scrapers/#{scraper_name}/current_job", params)
@@ -48,6 +49,12 @@ module Datahen
         opts[:status] = 'paused'
         update(scraper_name, opts)
       end
+
+      def profile(scraper_name, opts={})
+        params = @options.merge(opts)
+
+        self.class.get("/scrapers/#{scraper_name}/current_job/profile", params)
+      end
     end
   end
 end

data/lib/datahen/scraper/executor.rb

@@ -60,7 +60,12 @@ module Datahen
 
       def init_global_page()
         client = Client::GlobalPage.new()
-        client.find(gid)
+        global_page = client.find(gid)
+        unless global_page.code == 200
+          raise "GID #{gid} not found. Aborting execution!"
+        else
+          global_page
+        end
       end
 
       def get_content(job_id, gid)
@@ -287,11 +292,12 @@ module Datahen
           end
 
           # behave differently if it is a real save
+          save_status = status
           if save
             log_msg = "Saving #{log_msgs.join(' and ')}."
             puts "#{log_msg}"
           else
-            status = "#{status}_try"
+            save_status = "#{status}_try"
           end
 
           # saving to server
@@ -300,7 +306,7 @@ module Datahen
             gid: gid,
             pages: pages_slice,
             outputs: outputs_slice,
-            status: status)
+            status: save_status)
 
           if response.code == 200
             if save

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "0.14.1"
+  VERSION = "0.14.9"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 0.14.1
+  version: 0.14.9
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-07-26 00:00:00.000000000 Z
+date: 2020-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -189,6 +189,7 @@ files:
 - exe/hen
 - lib/datahen.rb
 - lib/datahen/cli.rb
+- lib/datahen/cli/account.rb
 - lib/datahen/cli/env_var.rb
 - lib/datahen/cli/finisher.rb
 - lib/datahen/cli/global_page.rb
@@ -206,6 +207,7 @@ files:
 - lib/datahen/cli/scraper_var.rb
 - lib/datahen/cli/seeder.rb
 - lib/datahen/client.rb
+- lib/datahen/client/account.rb
 - lib/datahen/client/auth_token.rb
 - lib/datahen/client/backblaze_content.rb
 - lib/datahen/client/base.rb
@@ -264,7 +266,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: DataHen toolbelt for developers