datahen 0.14.0 → 0.14.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3ff2ed2cd4772450c01e3e88248ae89441de709198fdd177d3e572bbc5f0e474
-  data.tar.gz: 5701717fcba8a05b6f3e027d9bce33a3830fa20dabe3413255779899478cb4ab
+  metadata.gz: e81f515b9874f6b9cd2943873e7ce54249d0c4662833e4eb9133bb8827901a3f
+  data.tar.gz: 02f860429555067ab0d034c7e46164ee3537844c34f40ea0949300a2de294187
 SHA512:
-  metadata.gz: 949ad06a090a4ac8c2ef5b4e053ed4b7668c051be15b6959a2948614e771c25e18774d9ee97fe1f5c03c130986b671a8b26ac253f592a993fa4ad393bcad7673
-  data.tar.gz: b73cfc6c070314f97cbc7917d571de67031247aac42f3474b2e71d04e8b3d650fc380a0ce3ca65c1d8339bf8743d94b666ecccca4431f7b89df4e7485a03a382
+  metadata.gz: 8cffdec9fa56f0d5ee22215b0eb64c0f2de45b1a3f031729f9929625ccf69e270fea2e554a34d583de71e02c7bbc4f0acc82ca6ad5c0f3ea496176277afd8818
+  data.tar.gz: 66b108bc587b3381a6202877f7915653a4ee1c1e058eeeab192aa1ba3614d7843c8dbcb67a7f08d57c6131ac03398a20a09ac7f309acdd0c244f06c46aa84d30

data/lib/datahen/cli/scraper.rb

@@ -60,7 +60,7 @@ module Datahen
       desc "show <scraper_name>", "Show a scraper"
       def show(scraper_name)
         client = Client::Scraper.new(options)
-        puts "#{client.find(scraper_name, options)}"
+        puts "#{client.find(scraper_name)}"
       end
 
       desc "delete <scraper_name>", "Delete a scraper and related records"
@@ -164,12 +164,14 @@ module Datahen
       option :"max-timestamp", type: :string, desc: 'Ending timestamp point in time to query historic stats (inclusive)'
       option :"limit", type: :numeric, desc: 'Limit stats retrieved'
       option :"order", type: :numeric, desc: 'Order stats by timestamp [DESC]'
+      option :live, type: :boolean, desc: 'Get data from the live stats history, not cached stats history.'
+      option :filter, type: :string, desc: 'Filter results on `day` or `hour`, if not specified will return all records.'
       def history(scraper_name)
         client = Client::JobStat.new(options)
         if options[:job]
-          puts "#{client.job_stats_history(options[:job])}"
+          puts "#{client.job_stats_history(options[:job], options)}"
         else
-          puts "#{client.scraper_job_stats_history(scraper_name)}"
+          puts "#{client.scraper_job_stats_history(scraper_name, options)}"
         end
       end
 

data/lib/datahen/cli/scraper_job.rb

@@ -7,10 +7,16 @@ module Datahen
       end
 
       desc "show <scraper_name>", "Show a scraper's current job (Defaults to showing data from cached job)"
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       option :live, type: :boolean, desc: 'Get data from the live job, not cached job.'
       def show(scraper_name)
-        client = Client::ScraperJob.new(options)
-        puts "#{client.find(scraper_name, options)}"
+        if options[:job]
+          client = Client::Job.new(options)
+          puts "#{client.find(options[:job], options)}"
+        else
+          client = Client::ScraperJob.new(options)
+          puts "#{client.find(scraper_name, options)}"
+        end
       end
 
 

data/lib/datahen/cli/scraper_page.rb

@@ -230,7 +230,7 @@ module Datahen
         end
       end
 
-      desc "failedcontent <gid>", "Show a page's failed content in scraper's current job"
+      desc "failedcontent <scraper_name> <gid>", "Show a page's failed content in scraper's current job"
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def failedcontent(scraper_name, gid)
         result = nil

data/lib/datahen/client/base.rb

@@ -5,6 +5,8 @@ module Datahen
     class Base
       include HTTParty
 
+      default_timeout 60
+
       def self.env_auth_token
         ENV['DATAHEN_TOKEN']
       end
@@ -55,6 +57,7 @@ module Datahen
         query[:"max-timestamp"] = opts[:"max-timestamp"] if opts[:"max-timestamp"]
         query[:limit] = opts[:limit] if opts[:limit]
         query[:order] = opts[:order] if opts[:order]
+        query[:filter] = opts[:filter] if opts[:filter]
 
         if opts[:query]
           if opts[:query].is_a?(Hash)

data/lib/datahen/client/job_stat.rb

@@ -18,12 +18,20 @@ module Datahen
         end
       end
 
-      def job_stats_history(job_id)
-        self.class.get("/jobs/#{job_id}/stats/history", @options)
+      def job_stats_history(job_id, opts={})
+        if opts[:live]
+          self.class.get("/jobs/#{job_id}/stats/history", @options)
+        else
+          self.class.get("/cached/jobs/#{job_id}/stats/history", @options)
+        end
       end
 
-      def scraper_job_stats_history(scraper_name)
-        self.class.get("/scrapers/#{scraper_name}/current_job/stats/history", @options)
+      def scraper_job_stats_history(scraper_name, opts={})
+        if opts[:live]
+          self.class.get("/scrapers/#{scraper_name}/current_job/stats/history", @options)
+        else
+          self.class.get("/cached/scrapers/#{scraper_name}/current_job/stats/history", @options)
+        end
       end
 
     end

data/lib/datahen/scraper/executor.rb

@@ -60,7 +60,12 @@ module Datahen
 
       def init_global_page()
         client = Client::GlobalPage.new()
-        client.find(gid)
+        global_page = client.find(gid)
+        unless global_page.code == 200
+          raise "GID #{gid} not found. Aborting execution!"
+        else
+          global_page
+        end
       end
 
       def get_content(job_id, gid)
@@ -287,11 +292,12 @@ module Datahen
           end
 
           # behave differently if it is a real save
+          save_status = status
           if save
             log_msg = "Saving #{log_msgs.join(' and ')}."
             puts "#{log_msg}"
           else
-            status = "#{status}_try"
+            save_status = "#{status}_try"
           end
 
           # saving to server
@@ -300,7 +306,7 @@ module Datahen
             gid: gid,
             pages: pages_slice,
             outputs: outputs_slice,
-            status: status)
+            status: save_status)
 
           if response.code == 200
             if save

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "0.14.0"
+  VERSION = "0.14.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 0.14.0
+  version: 0.14.5
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-07-26 00:00:00.000000000 Z
+date: 2020-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor