datahen 1.5.3 → 1.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: beadb0e5b371134e96d8e1888250de66411eb287a30cfc5f717e525c19343d6c
-  data.tar.gz: 298daaf6f56f1fcfb4e842b9e82d0e4f8896315d264b9481aa2ed4ec55bc8a9b
+  metadata.gz: 0aeab36656b35d48d158c46e90cf504007727d60000352503468c743b8f519f3
+  data.tar.gz: 62dc78ad8d8388f62db06217507e4ff8f22d004501f264d899dfb14afd9d5df1
 SHA512:
-  metadata.gz: 8cba2e599b72315d65cd20440c6a8804d7298bd577d5d24f9408bc71e35f47ee5abb8e7d580b7b42cc95c10608f852339a1d44ecfedaedd82acb565f5ffb05d2
-  data.tar.gz: 6a8c1c8f7d8deca7663699f7f20ea0e9cf7e7608b2d93150fb2fd4da438780fecff010332e43675800f94012739bfcda6a85dd27df1055958e6de0bbd4c158ab
+  metadata.gz: f83d2cf14b3b1f00ff2f7b8f319c52bb408301e318a390b419f1b87e7b4a79a0ba7b40468079c8b8b22562d24a8eef6c6bde60d40a11930dbc3af44c16c3acd7
+  data.tar.gz: 8a05fc8d0419c13f0849fdeb53f1c91455d359539431623138e31895c192994a786fc271df7dc2ae223f1d7a6cdaa46eeb81939e093cb6d850aded106809c66c

data/lib/datahen/cli/scraper.rb

@@ -252,6 +252,9 @@ module Datahen
       desc "task SUBCOMMAND ...ARGS", "manage task on a job"
       subcommand "task", ScraperTask
 
+      desc "resource SUBCOMMAND ...ARGS", "manage resource on a job"
+      subcommand "resource", ScraperResource
+
 
     end
   end

data/lib/datahen/cli/scraper_job.rb

@@ -112,6 +112,7 @@ module Datahen
       option :soft_refetch_limit, type: :numeric, desc: 'Set the soft refetch limit value.'
       option :parsing_try_limit, type: :numeric, desc: 'Set the parsing try limit value.'
       option :prevent_kb_autoscaler, type: :boolean, desc: 'Set true to prevent the autoscaler from restarting the job. Default: false'
+      option :deletion_protected, type: :boolean, desc: 'Set true to prevent the job to be deleted from max job count or job older than X days rules. Default: false'
       def update(scraper_name)
         if options[:job]
           client = Client::Job.new(options)

data/lib/datahen/cli/scraper_resource.rb

@@ -0,0 +1,35 @@
+module Datahen
+  class CLI < Thor
+    class ScraperResource < Thor
+      package_name "scraper resource"
+      def self.banner(command, namespace = nil, subcommand = false)
+        "#{basename} #{@package_name} #{command.usage}"
+      end
+
+      desc "list", "List resources on a scraper's current job"
+      long_desc <<-LONGDESC
+        List all resources in a scraper's current job or given job ID.\x5
+      LONGDESC
+      option :scraper_name, :aliases => :s, type: :string, desc: 'Filter by a specific scraper_name'
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :pod, type: :string, desc: 'Returns only tasks with specific pod.'
+      option :container, type: :string, desc: 'Returns only tasks with specific container.'
+      option :executor, type: :string, desc: 'Returns only tasks with specific executor.'
+      def list()
+        if options[:job]
+          client = Client::JobResource.new(options)
+          puts "#{client.all(options[:job])}"
+        else
+          if options[:scraper_name]
+            client = Client::ScraperResource.new(options)
+            puts "#{client.all(options[:scraper_name])}"
+          else
+            puts 'Must specify either a job ID or a scraper name'
+          end
+        end
+      end
+      
+    end
+  end
+
+end

data/lib/datahen/cli.rb

@@ -7,6 +7,7 @@ require 'datahen/cli/scraper_job_var'
 require 'datahen/cli/scraper_job'
 require 'datahen/cli/scraper_finisher'
 require 'datahen/cli/global_page'
+require 'datahen/cli/scraper_resource'
 require 'datahen/cli/scraper_page'
 require 'datahen/cli/job_output'
 require 'datahen/cli/job'

data/lib/datahen/client/base.rb

@@ -128,6 +128,10 @@ module Datahen
         query[:force] = opts[:force] if opts[:force]
         query[:action] = opts[:action] if opts[:action]
         query[:"include-system"] = opts[:"include-system"] if opts[:"include-system"]
+        query[:"pod"] = opts[:"pod"] if opts[:"pod"]
+        query[:"container"] = opts[:"container"] if opts[:"container"]
+        query[:"executor"] = opts[:"executor"] if opts[:"executor"]
+        
 
         if opts[:query]
           if opts[:query].is_a?(Hash)

data/lib/datahen/client/job.rb

@@ -29,6 +29,7 @@ module Datahen
         body[:soft_refetch_limit] = opts[:soft_refetch_limit] if opts[:soft_refetch_limit]
         body[:parsing_try_limit] = opts[:parsing_try_limit] if opts[:parsing_try_limit]
         body[:prevent_kb_autoscaler] = opts[:prevent_kb_autoscaler] if opts.has_key?("prevent_kb_autoscaler") || opts.has_key?(:prevent_kb_autoscaler)
+        body[:deletion_protected] = opts[:deletion_protected] if opts.has_key?("deletion_protected") || opts.has_key?(:deletion_protected)
         params = @options.merge({body: body.to_json})
 
         self.class.put("/jobs/#{job_id}", params)

data/lib/datahen/client/job_resource.rb

@@ -0,0 +1,11 @@
+module Datahen
+  module Client
+    class JobResource < Datahen::Client::Base
+      def all(job_id, opts={})
+        params = @options.merge(opts)
+        self.class.get("/jobs/#{job_id}/resources", params)
+      end
+    end
+
+  end
+end

data/lib/datahen/client/scraper_job.rb

@@ -19,6 +19,7 @@ module Datahen
         body[:soft_refetch_limit] = opts[:soft_refetch_limit] if opts[:soft_refetch_limit]
         body[:parsing_try_limit] = opts[:parsing_try_limit] if opts[:parsing_try_limit]
         body[:prevent_kb_autoscaler] = opts[:prevent_kb_autoscaler] if opts.has_key?("prevent_kb_autoscaler") || opts.has_key?(:prevent_kb_autoscaler)
+        body[:deletion_protected] = opts[:deletion_protected] if opts.has_key?("deletion_protected") || opts.has_key?(:deletion_protected)
         if opts[:vars]
           if opts[:vars].is_a?(Array)
             body[:vars] = opts[:vars]
@@ -53,6 +54,7 @@ module Datahen
         body[:soft_refetch_limit] = opts[:soft_refetch_limit] if opts[:soft_refetch_limit]
         body[:parsing_try_limit] = opts[:parsing_try_limit] if opts[:parsing_try_limit]
         body[:prevent_kb_autoscaler] = opts[:prevent_kb_autoscaler] if opts.has_key?("prevent_kb_autoscaler") || opts.has_key?(:prevent_kb_autoscaler)
+        body[:deletion_protected] = opts[:deletion_protected] if opts.has_key?("deletion_protected") || opts.has_key?(:deletion_protected)
         params = @options.merge({body: body.to_json})
 
         self.class.put("/scrapers/#{scraper_name}/current_job", params)

data/lib/datahen/client/scraper_resource.rb

@@ -0,0 +1,11 @@
+module Datahen
+  module Client
+    class ScraperResource < Datahen::Client::Base
+      def all(scraper_name, opts={})
+        params = @options.merge(opts)
+        self.class.get("/scrapers/#{scraper_name}/resources", params)
+      end
+    end
+
+  end
+end

data/lib/datahen/client.rb

@@ -26,6 +26,8 @@ require "datahen/client/scraper_job_var"
 require "datahen/client/job_finisher"
 require "datahen/client/job_task"
 require "datahen/client/scraper_task"
+require "datahen/client/job_resource"
+require "datahen/client/scraper_resource"
 
 module Datahen
   module Client

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "1.5.3"
+  VERSION = "1.6.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 1.5.3
+  version: 1.6.1
 platform: ruby
 authors:
 - Parama Danoesubroto
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-01-25 00:00:00.000000000 Z
+date: 2024-03-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -227,6 +227,7 @@ files:
 - lib/datahen/cli/scraper_job.rb
 - lib/datahen/cli/scraper_job_var.rb
 - lib/datahen/cli/scraper_page.rb
+- lib/datahen/cli/scraper_resource.rb
 - lib/datahen/cli/scraper_task.rb
 - lib/datahen/cli/scraper_var.rb
 - lib/datahen/cli/seeder.rb
@@ -245,6 +246,7 @@ files:
 - lib/datahen/client/job_log.rb
 - lib/datahen/client/job_output.rb
 - lib/datahen/client/job_page.rb
+- lib/datahen/client/job_resource.rb
 - lib/datahen/client/job_stat.rb
 - lib/datahen/client/job_task.rb
 - lib/datahen/client/job_var.rb
@@ -257,6 +259,7 @@ files:
 - lib/datahen/client/scraper_job_output.rb
 - lib/datahen/client/scraper_job_page.rb
 - lib/datahen/client/scraper_job_var.rb
+- lib/datahen/client/scraper_resource.rb
 - lib/datahen/client/scraper_task.rb
 - lib/datahen/client/scraper_var.rb
 - lib/datahen/error.rb
@@ -281,7 +284,7 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://datahen.com
   source_code_uri: https://github.com/DataHenOfficial/datahen-ruby
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -296,8 +299,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.5.6
+signing_key:
 specification_version: 4
 summary: DataHen toolbelt for developers
 test_files: []