datahen 0.13.7 → 0.14.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c05f6ab973fe74a21e7f68411a66b97444575cc75c5812e99c5e0da4ffe05d56
-  data.tar.gz: dac1d2be8f6281d3da328abd5f844036765991764bb1e3ffad4b72ee4c18eac6
+  metadata.gz: b5f93e47c85c172722c4517fe4af0805d81387a440fb06922fb5d6e941994f66
+  data.tar.gz: 1c895d1e6ec3e8415202d581ed9fe48e3c0443f53355b0ea34c0a3418b4e306d
 SHA512:
-  metadata.gz: 48c15f9830308488d434dce7b7cf4888795724d88d4ca9de63cc2deca8397798b815145430af2143eb646366bfc4e97f37e563315214258b434276919e724ac7
-  data.tar.gz: faee0369f81ab45dbb6c62258a6214d530d51d5dcb0f6b6ebd7028bda2c48ccbe630b41006ffcc96dba773a28e84e724b6b1f2266ef7866605e5423f1ced61d2
+  metadata.gz: 0bc52173785501b7fe3ad3bd1de448d818033441eefb9a0526f68cb6a17595c4dda91e7f231501c4212892aaccda68e28ea2ac72c90ce5c89a7e78d06db5bf4b
+  data.tar.gz: d9fa5425007496e404258504127bedb1a7b786d609d9db15f6cdc485a2922aea5c7e8f25ca124142e0e41daadf22039d79c666f507029c3fc0c8388ca4a221df

data/lib/datahen/cli/job.rb

@@ -18,12 +18,24 @@ module Datahen
         puts "#{client.all()}"
       end
 
-      desc "show <job_id>", "Show a job"
+      desc "show <job_id>", "Show a job (Defaults to showing data from cached job)"
+      option :live, type: :boolean, desc: 'Get data from the live job, not cached job.'
       def show(job_id)
         client = Client::Job.new(options)
-        puts "#{client.find(job_id)}"
+        puts "#{client.find(job_id, options)}"
       end
 
+      desc "stats <job_id>", "Get the stat for a job (Defaults to showing data from cached stats)"
+      long_desc <<-LONGDESC
+        Get stats for a scraper's current job\n
+      LONGDESC
+      option :live, type: :boolean, desc: 'Get data from the live stats, not cached stats.'
+      def stats(job_id)
+        client = Client::JobStat.new(options)
+        puts "#{client.job_current_stats(job_id, options)}"
+      end
+
+
     end
   end
 

data/lib/datahen/cli/scraper.rb

@@ -140,17 +140,18 @@ module Datahen
         end
       end
 
-      desc "stats <scraper_name>", "Get the current stat for a job"
+      desc "stats <scraper_name>", "Get the stat for a current job (Defaults to showing data from cached stats)"
       long_desc <<-LONGDESC
         Get stats for a scraper's current job\n
       LONGDESC
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :live, type: :boolean, desc: 'Get data from the live stats, not cached stats.'
       def stats(scraper_name)
         client = Client::JobStat.new(options)
         if options[:job]
-          puts "#{client.job_current_stats(options[:job])}"
+          puts "#{client.job_current_stats(options[:job], options)}"
         else
-          puts "#{client.scraper_job_current_stats(scraper_name)}"
+          puts "#{client.scraper_job_current_stats(scraper_name, options)}"
         end
       end
 
@@ -163,12 +164,13 @@ module Datahen
       option :"max-timestamp", type: :string, desc: 'Ending timestamp point in time to query historic stats (inclusive)'
       option :"limit", type: :numeric, desc: 'Limit stats retrieved'
       option :"order", type: :numeric, desc: 'Order stats by timestamp [DESC]'
+      option :live, type: :boolean, desc: 'Get data from the live stats history, not cached stats history.'
       def history(scraper_name)
         client = Client::JobStat.new(options)
         if options[:job]
-          puts "#{client.job_stats_history(options[:job])}"
+          puts "#{client.job_stats_history(options[:job], options)}"
         else
-          puts "#{client.scraper_job_stats_history(scraper_name)}"
+          puts "#{client.scraper_job_stats_history(scraper_name, options)}"
         end
       end
 

data/lib/datahen/cli/scraper_job.rb

@@ -6,10 +6,17 @@ module Datahen
         "#{basename} #{@package_name} #{command.usage}"
       end
 
-      desc "show <scraper_name>", "Show a scraper's current job"
+      desc "show <scraper_name>", "Show a scraper's current job (Defaults to showing data from cached job)"
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :live, type: :boolean, desc: 'Get data from the live job, not cached job.'
       def show(scraper_name)
-        client = Client::ScraperJob.new(options)
-        puts "#{client.find(scraper_name)}"
+        if options[:job]
+          client = Client::Job.new(options)
+          puts "#{client.find(options[:job], options)}"
+        else
+          client = Client::ScraperJob.new(options)
+          puts "#{client.find(scraper_name, options)}"
+        end
       end
 
 

data/lib/datahen/cli/scraper_page.rb

@@ -230,7 +230,7 @@ module Datahen
         end
       end
 
-      desc "failedcontent <gid>", "Show a page's failed content in scraper's current job"
+      desc "failedcontent <scraper_name> <gid>", "Show a page's failed content in scraper's current job"
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def failedcontent(scraper_name, gid)
         result = nil

data/lib/datahen/client/base.rb

@@ -5,6 +5,8 @@ module Datahen
     class Base
       include HTTParty
 
+      default_timeout 60
+
       def self.env_auth_token
         ENV['DATAHEN_TOKEN']
       end

data/lib/datahen/client/job.rb

@@ -6,8 +6,12 @@ module Datahen
         self.class.get("/jobs", params)
       end
 
-      def find(job_id)
-        self.class.get("/jobs/#{job_id}", @options)
+      def find(job_id, opts={})
+        if opts[:live]
+          self.class.get("/jobs/#{job_id}", @options)
+        else
+          self.class.get("/cached/jobs/#{job_id}", @options)
+        end
       end
 
       def update(job_id, opts={})

data/lib/datahen/client/job_stat.rb

@@ -2,20 +2,36 @@ module Datahen
   module Client
     class JobStat < Datahen::Client::Base
 
-      def job_current_stats(job_id)
-        self.class.get("/jobs/#{job_id}/stats/current", @options)
+      def job_current_stats(job_id, opts={})
+        if opts[:live]
+          self.class.get("/jobs/#{job_id}/stats/current", @options)
+        else
+          self.class.get("/cached/jobs/#{job_id}/stats/current", @options)
+        end
       end
 
-      def scraper_job_current_stats(scraper_name)
-        self.class.get("/scrapers/#{scraper_name}/current_job/stats/current", @options)
+      def scraper_job_current_stats(scraper_name, opts={})
+        if opts[:live]
+          self.class.get("/scrapers/#{scraper_name}/current_job/stats/current", @options)
+        else
+          self.class.get("/cached/scrapers/#{scraper_name}/current_job/stats/current", @options)
+        end
       end
 
-      def job_stats_history(job_id)
-        self.class.get("/jobs/#{job_id}/stats/history", @options)
+      def job_stats_history(job_id, opts={})
+        if opts[:live]
+          self.class.get("/jobs/#{job_id}/stats/history", @options)
+        else
+          self.class.get("/cached/jobs/#{job_id}/stats/history", @options)
+        end
       end
 
-      def scraper_job_stats_history(scraper_name)
-        self.class.get("/scrapers/#{scraper_name}/current_job/stats/history", @options)
+      def scraper_job_stats_history(scraper_name, opts={})
+        if opts[:live]
+          self.class.get("/scrapers/#{scraper_name}/current_job/stats/history", @options)
+        else
+          self.class.get("/cached/scrapers/#{scraper_name}/current_job/stats/history", @options)
+        end
       end
 
     end

data/lib/datahen/client/scraper_job.rb

@@ -15,8 +15,12 @@ module Datahen
         self.class.post("/scrapers/#{scraper_name}/jobs", params)
       end
 
-      def find(scraper_name)
-        self.class.get("/scrapers/#{scraper_name}/current_job", @options)
+      def find(scraper_name, opts={})
+        if opts[:live]
+          self.class.get("/scrapers/#{scraper_name}/current_job", @options)
+        else
+          self.class.get("/cached/scrapers/#{scraper_name}/current_job", @options)
+        end
       end
 
       def update(scraper_name, opts={})

data/lib/datahen/scraper/executor.rb

@@ -60,7 +60,12 @@ module Datahen
 
       def init_global_page()
         client = Client::GlobalPage.new()
-        client.find(gid)
+        global_page = client.find(gid)
+        unless global_page.code == 200
+          raise "GID #{gid} not found. Aborting execution!"
+        else
+          global_page
+        end
       end
 
       def get_content(job_id, gid)
@@ -287,11 +292,12 @@ module Datahen
           end
 
           # behave differently if it is a real save
+          save_status = status
           if save
             log_msg = "Saving #{log_msgs.join(' and ')}."
             puts "#{log_msg}"
           else
-            status = "#{status}_try"
+            save_status = "#{status}_try"
           end
 
           # saving to server
@@ -300,7 +306,7 @@ module Datahen
             gid: gid,
             pages: pages_slice,
             outputs: outputs_slice,
-            status: status)
+            status: save_status)
 
           if response.code == 200
             if save

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "0.13.7"
+  VERSION = "0.14.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 0.13.7
+  version: 0.14.4
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-05-06 00:00:00.000000000 Z
+date: 2020-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -264,7 +264,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: DataHen toolbelt for developers