datahen 1.5.2 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dc47c55d814a573f9de29c4725aba0cad212dceae5dfa2d330dc980ad4f64253
-  data.tar.gz: 508fe6249ef13f07c835297758f31eb8459080b13c4256c9393d0c8c6ea2b171
+  metadata.gz: 94937cbe6614336d283523740d1bd237b154a7e084dd3c0757dda96a068cc4c2
+  data.tar.gz: 51552871510232be116af8d0f108e5ee51d6733657abb681b1561d78eb68efff
 SHA512:
-  metadata.gz: 23c817ae6f20698c95fdae1503bfe9fc6072f617389e979a1389eab746de07632fab432b2a3a5a824c56c9ad3b6a254a1508ecfd7bb691013843e7b93831daf5
-  data.tar.gz: c3a1df12099bc6bf159ac7689d7c7be36dc15416b6ecc3c347e7ef1ba1e8e844a58d9600f00803c7fd73837915b88674aa8e470e504acc6ac89d6fd0897df632
+  metadata.gz: a37ba23e5e4915540ee5ad656d7c93dbd4d78ab36ad9e7f05ee2019d51061793b5a914e9e5527ce40bdf42ca5f6cb758124503293fb229ccf112b8d1e2f70193
+  data.tar.gz: 10f1cf69238cf4e8fae71e9ae892bc771d392b426d98b7168bbe9fbc9985bfcc3f0cf62dff8f9ba6fbca3484b13f29a4e2513259e0659a1a33b6708b764dccfa

data/lib/datahen/cli/scraper.rb

@@ -252,6 +252,9 @@ module Datahen
       desc "task SUBCOMMAND ...ARGS", "manage task on a job"
       subcommand "task", ScraperTask
 
+      desc "resource SUBCOMMAND ...ARGS", "manage resource on a job"
+      subcommand "resource", ScraperResource
+
 
     end
   end

data/lib/datahen/cli/scraper_resource.rb

@@ -0,0 +1,35 @@
+module Datahen
+  class CLI < Thor
+    class ScraperResource < Thor
+      package_name "scraper resource"
+      def self.banner(command, namespace = nil, subcommand = false)
+        "#{basename} #{@package_name} #{command.usage}"
+      end
+
+      desc "list", "List resources on a scraper's current job"
+      long_desc <<-LONGDESC
+        List all resources in a scraper's current job or given job ID.\x5
+      LONGDESC
+      option :scraper_name, :aliases => :s, type: :string, desc: 'Filter by a specific scraper_name'
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :pod, type: :string, desc: 'Returns only tasks with specific pod.'
+      option :container, type: :string, desc: 'Returns only tasks with specific container.'
+      option :executor, type: :string, desc: 'Returns only tasks with specific executor.'
+      def list()
+        if options[:job]
+          client = Client::JobResource.new(options)
+          puts "#{client.all(options[:job])}"
+        else
+          if options[:scraper_name]
+            client = Client::ScraperResource.new(options)
+            puts "#{client.all(options[:scraper_name])}"
+          else
+            puts 'Must specify either a job ID or a scraper name'
+          end
+        end
+      end
+      
+    end
+  end
+
+end

data/lib/datahen/cli.rb

@@ -7,6 +7,7 @@ require 'datahen/cli/scraper_job_var'
 require 'datahen/cli/scraper_job'
 require 'datahen/cli/scraper_finisher'
 require 'datahen/cli/global_page'
+require 'datahen/cli/scraper_resource'
 require 'datahen/cli/scraper_page'
 require 'datahen/cli/job_output'
 require 'datahen/cli/job'

data/lib/datahen/client/base.rb

@@ -13,6 +13,9 @@ module Datahen
         finisher: nil
       }
 
+      CHECK_NIL = lambda{|v|v.nil?}
+      CHECK_EMPTY_BODY = lambda{|v|v.body.nil? || v.body.empty?}
+
       def self.env_auth_token
         ENV['DATAHEN_TOKEN']
       end
@@ -56,14 +59,14 @@ module Datahen
         target.merge(source.select{|k,v|target.has_key?(k)})
       end
 
-      def retry times, delay = nil, err_msg = nil, stream = false
+      def retry(times, delay = nil, err_msg = nil, stream = false, check_nil = CHECK_NIL)
         limit = times.nil? ? nil : times.to_i
         delay = delay.nil? ? 5 : delay.to_i
         count = 0
         begin
           val = yield
           if stream
-            return if val.nil?
+            return if check_nil.call(val)
             if val['error'] != ""
               raise StandardError.new(val['error'])
             end
@@ -125,6 +128,10 @@ module Datahen
         query[:force] = opts[:force] if opts[:force]
         query[:action] = opts[:action] if opts[:action]
         query[:"include-system"] = opts[:"include-system"] if opts[:"include-system"]
+        query[:"pod"] = opts[:"pod"] if opts[:"pod"]
+        query[:"container"] = opts[:"container"] if opts[:"container"]
+        query[:"executor"] = opts[:"executor"] if opts[:"executor"]
+        
 
         if opts[:query]
           if opts[:query].is_a?(Hash)

data/lib/datahen/client/job.rb

@@ -60,7 +60,7 @@ module Datahen
         params = @options.merge({body: body.to_json})
 
         limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : self.default_retry_limit[:seeder]
-        self.retry(limit, 5, "Error while updating the seeder.") do
+        self.retry(limit, 5, "Error while updating the seeder.", false, CHECK_EMPTY_BODY) do
           response = self.class.put("/jobs/#{job_id}/seeding_update", params)
           if response.code == 422 && response.body.to_s =~ /pq:\s*deadlock/i
             raise CustomRetryError.new(self.class.random_delay(5), response.body.to_s)
@@ -78,7 +78,7 @@ module Datahen
         params = @options.merge({body: body.to_json})
 
         limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : self.default_retry_limit[:finisher]
-        self.retry(limit, 5, "Error while updating the finisher.") do
+        self.retry(limit, 5, "Error while updating the finisher.", false, CHECK_EMPTY_BODY) do
           response = self.class.put("/jobs/#{job_id}/finisher_update", params)
           if response.code == 422 && response.body.to_s =~ /pq:\s*deadlock/
             raise CustomRetryError.new(self.class.random_delay(5), response.body.to_s)

data/lib/datahen/client/job_output.rb

@@ -7,7 +7,7 @@ module Datahen
 
       def all(job_id, collection = 'default', opts = {})
         limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : 0
-        self.retry(limit, 10, "Error while updating the seeder.", true) do
+        self.retry(limit, 10, "Error while updating the seeder.", true, CHECK_EMPTY_BODY) do
           self.class.get("/jobs/#{job_id}/output/collections/#{collection}/records", @options)
         end
       end

data/lib/datahen/client/job_page.rb

@@ -63,7 +63,7 @@ module Datahen
         params = @options.merge({body: body.to_json})
 
         limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : self.default_retry_limit[:parser]
-        self.retry(limit, 5, "Error while updating the parser.") do
+        self.retry(limit, 5, "Error while updating the parser.", false, CHECK_EMPTY_BODY) do
           response = self.class.put("/jobs/#{job_id}/pages/#{gid}/parsing_update", params)
           if response.code == 422 && response.body.to_s =~ /pq:\s*deadlock/i
             raise Error::CustomRetryError.new(self.class.random_delay(5), response.body.to_s)

data/lib/datahen/client/job_resource.rb

@@ -0,0 +1,11 @@
+module Datahen
+  module Client
+    class JobResource < Datahen::Client::Base
+      def all(job_id, opts={})
+        params = @options.merge(opts)
+        self.class.get("/jobs/#{job_id}/resources", params)
+      end
+    end
+
+  end
+end

data/lib/datahen/client/scraper_resource.rb

@@ -0,0 +1,11 @@
+module Datahen
+  module Client
+    class ScraperResource < Datahen::Client::Base
+      def all(scraper_name, opts={})
+        params = @options.merge(opts)
+        self.class.get("/scrapers/#{scraper_name}/resources", params)
+      end
+    end
+
+  end
+end

data/lib/datahen/client.rb

@@ -26,6 +26,8 @@ require "datahen/client/scraper_job_var"
 require "datahen/client/job_finisher"
 require "datahen/client/job_task"
 require "datahen/client/scraper_task"
+require "datahen/client/job_resource"
+require "datahen/client/scraper_resource"
 
 module Datahen
   module Client

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "1.5.2"
+  VERSION = "1.6.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 1.5.2
+  version: 1.6.0
 platform: ruby
 authors:
 - Parama Danoesubroto
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-01-09 00:00:00.000000000 Z
+date: 2024-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -227,6 +227,7 @@ files:
 - lib/datahen/cli/scraper_job.rb
 - lib/datahen/cli/scraper_job_var.rb
 - lib/datahen/cli/scraper_page.rb
+- lib/datahen/cli/scraper_resource.rb
 - lib/datahen/cli/scraper_task.rb
 - lib/datahen/cli/scraper_var.rb
 - lib/datahen/cli/seeder.rb
@@ -245,6 +246,7 @@ files:
 - lib/datahen/client/job_log.rb
 - lib/datahen/client/job_output.rb
 - lib/datahen/client/job_page.rb
+- lib/datahen/client/job_resource.rb
 - lib/datahen/client/job_stat.rb
 - lib/datahen/client/job_task.rb
 - lib/datahen/client/job_var.rb
@@ -257,6 +259,7 @@ files:
 - lib/datahen/client/scraper_job_output.rb
 - lib/datahen/client/scraper_job_page.rb
 - lib/datahen/client/scraper_job_var.rb
+- lib/datahen/client/scraper_resource.rb
 - lib/datahen/client/scraper_task.rb
 - lib/datahen/client/scraper_var.rb
 - lib/datahen/error.rb
@@ -281,7 +284,7 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://datahen.com
   source_code_uri: https://github.com/DataHenOfficial/datahen-ruby
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -296,8 +299,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.2.15
+signing_key:
 specification_version: 4
 summary: DataHen toolbelt for developers
 test_files: []