datahen 1.3.2 → 1.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 30a28a5de1830e60d44942ff9605b99557e15804bc9ea3a00939859000cd1169
-  data.tar.gz: 6d2c4bff4b795e02def479f6e37f8535045de4ab02e4b1a4fe4f8fd7ca4284ef
+  metadata.gz: 96d2bc30d1c96ce684d83efa54b6dff5966db2a1bba7ab4856b11caba2803086
+  data.tar.gz: 985712d5d7e6559ac64b76669241f56d704c754deb06a164e1f449aad10ef29e
 SHA512:
-  metadata.gz: 0642ad5e85396000ad9e493d0f8da0963d29b45b52f845b3555d792e427abdee8e5a80f7a58b57e75929f0b45887209d682e38bfc4087e4f9f0f0133da9259d5
-  data.tar.gz: 24f9166b668a755602e63307dbd88c7b8d385b44a697c2b634b161cd1c54bc73b8ef084c59004996df2e9f3770bb63c02bb0fb6846a5326f3d2fe1b0b7537393
+  metadata.gz: d9c6bd3e60034339a8354fe4bda365b91f21b6ec68da8f384d7380abcafa5ccce2c2aacd6cc7a8da37378b8681afe58765bcc461211812c623a8958eac7a5f72
+  data.tar.gz: ac5eb5c8de4e4b0a6d28d96179bab4bf347662247b94e775ed0a25e0f0ef00a542f01f8a1a06525b565e7bd1055d5cd30b480a28d28c7ebf5de893b89b9f5e3a

data/lib/datahen/cli/job_output.rb

@@ -20,10 +20,20 @@ module Datahen
         collection = options.fetch(:collection) { 'default' }
         if options[:job]
           client = Client::JobOutput.new(options)
-          puts "#{client.all(options[:job], collection)}"
+          json = JSON.parse(client.all(options[:job], collection).body)
+          if json['error'] == ""
+            puts "#{JSON.pretty_generate(json['data'])}"
+          else 
+            puts "#{JSON.pretty_generate(json['error'])}"
+          end
         else
           client = Client::ScraperJobOutput.new(options)
-          puts "#{client.all(scraper_name, collection)}"
+          json = JSON.parse(client.all(scraper_name, collection).body)
+          if json['error'] == ""
+            puts "#{JSON.pretty_generate(json['data'])}"
+          else 
+            puts "#{JSON.pretty_generate(json['error'])}"
+          end
         end
       end
 
@@ -38,10 +48,20 @@ module Datahen
         collection = options.fetch(:collection) { 'default' }
         if options[:job]
           client = Client::JobOutput.new(options)
-          puts "#{client.find(options[:job], collection, id)}"
+          json = JSON.parse(client.find(options[:job], collection, id).body)
+          if json['error'] == ""
+            puts "#{JSON.pretty_generate(json['data'])}"
+          else 
+            puts "#{JSON.pretty_generate(json['error'])}"
+          end
         else
           client = Client::ScraperJobOutput.new(options)
-          puts "#{client.find(scraper_name, collection, id)}"
+          json = JSON.parse(client.find(scraper_name, collection, id).body)
+          if json['error'] == ""
+            puts "#{JSON.pretty_generate(json['data'])}"
+          else 
+            puts "#{JSON.pretty_generate(json['error'])}"
+          end
         end
       end
 
@@ -56,10 +76,20 @@ module Datahen
 
         if options[:job]
           client = Client::JobOutput.new(options)
-          puts "#{client.collections(options[:job])}"
+          json = JSON.parse(client.collections(options[:job]).body)
+          if json['error'] == ""
+            puts "#{JSON.pretty_generate(json['data'])}"
+          else 
+            puts "#{JSON.pretty_generate(json['error'])}"
+          end
         else
           client = Client::ScraperJobOutput.new(options)
-          puts "#{client.collections(scraper_name)}"
+          json = JSON.parse(client.collections(scraper_name).body)
+          if json['error'] == ""
+            puts "#{JSON.pretty_generate(json['data'])}"
+          else 
+            puts "#{JSON.pretty_generate(json['error'])}"
+          end
         end
       end
 

data/lib/datahen/cli/scraper.rb

@@ -37,6 +37,10 @@ module Datahen
       option :max_page_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
       option :enable_global_cache, type: :boolean, desc: 'Set true to enable page cache. Default: false'
       option :retry_interval, type: :numeric, desc: 'Set a value to set retry time interval on seconds when refetching a page. Set a value grather than 0 to set it as new time to refetch, 0 means default time. Default: 0'
+      option :soft_fetching_try_limit, type: :numeric, desc: 'Set the soft fetching try limit value.'
+      option :soft_refetch_limit, type: :numeric, desc: 'Set the soft refetch limit value.'
+      option :parsing_try_limit, type: :numeric, desc: 'Set the parsing try limit value.'
+      option :prevent_kb_autoscaler, type: :boolean, desc: 'Set true to prevent the autoscaler from restarting the job. Default: false'
       def create(scraper_name, git_repository)
         # puts "options #{options}"
         client = Client::Scraper.new(options)
@@ -66,6 +70,10 @@ module Datahen
       option :max_page_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
       option :enable_global_cache, type: :boolean, desc: 'Set true to enable page cache. Default: false'
       option :retry_interval, type: :numeric, desc: 'Set a value to set retry time interval on seconds when refetching a page. Set a value grather than 0 to set it as new time to refetch, 0 means default time. Default: 0'
+      option :soft_fetching_try_limit, type: :numeric, desc: 'Set the soft fetching try limit value.'
+      option :soft_refetch_limit, type: :numeric, desc: 'Set the soft refetch limit value.'
+      option :parsing_try_limit, type: :numeric, desc: 'Set the parsing try limit value.'
+      option :prevent_kb_autoscaler, type: :boolean, desc: 'Set true to prevent the autoscaler from restarting the job. Default: false'
       def update(scraper_name)
         client = Client::Scraper.new(options)
         puts "#{client.update(scraper_name, options)}"
@@ -106,6 +114,10 @@ module Datahen
       option :vars, type: :string, banner: :JSON, desc: 'Set input vars. Must be in json format. i.e: [{"name":"foo", "value":"bar", "secret":false}] '
       option :max_page_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
       option :retry_interval, type: :numeric, desc: 'Set a value to set retry time interval on seconds when refetching a page. Set a value grather than 0 to set it as new time to refetch, 0 means default time. Default: 0'
+      option :soft_fetching_try_limit, type: :numeric, desc: 'Set the soft fetching try limit value.'
+      option :soft_refetch_limit, type: :numeric, desc: 'Set the soft refetch limit value.'
+      option :parsing_try_limit, type: :numeric, desc: 'Set the parsing try limit value.'
+      option :prevent_kb_autoscaler, type: :boolean, desc: 'Set true to prevent the autoscaler from restarting the job. Default: false'
       def start(scraper_name)
         client = Client::ScraperJob.new(options)
         puts "Starting a scrape job..."
@@ -188,9 +200,19 @@ module Datahen
       def history(scraper_name)
         client = Client::JobStat.new(options)
         if options[:job]
-          puts "#{client.job_stats_history(options[:job], options)}"
+          json = JSON.parse(client.job_stats_history(options[:job], options).body)
+          if json['error'] == ""
+            puts "#{JSON.pretty_generate(json['data'])}"
+          else 
+            puts "#{JSON.pretty_generate(json['error'])}"
+          end
         else
-          puts "#{client.scraper_job_stats_history(scraper_name, options)}"
+          json = JSON.parse(client.scraper_job_stats_history(scraper_name, options).body)
+          if json['error'] == ""
+            puts "#{JSON.pretty_generate(json['data'])}"
+          else 
+            puts "#{JSON.pretty_generate(json['error'])}"
+          end
         end
       end
 
@@ -227,6 +249,9 @@ module Datahen
       desc "var SUBCOMMAND ...ARGS", "for managing scraper's variables"
       subcommand "var", ScraperVar
 
+      desc "task SUBCOMMAND ...ARGS", "manage task on a job"
+      subcommand "task", ScraperTask
+
 
     end
   end

data/lib/datahen/cli/scraper_job.rb

@@ -108,6 +108,10 @@ module Datahen
       option :max_page_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
       option :enable_global_cache, type: :boolean, desc: 'Set true to enable page cache. Default: false'
       option :retry_interval, type: :numeric, desc: 'Set a value to set retry time interval on seconds when refetching a page. Set a value grather than 0 to set it as new time to refetch, 0 means default time. Default: 0'
+      option :soft_fetching_try_limit, type: :numeric, desc: 'Set the soft fetching try limit value.'
+      option :soft_refetch_limit, type: :numeric, desc: 'Set the soft refetch limit value.'
+      option :parsing_try_limit, type: :numeric, desc: 'Set the parsing try limit value.'
+      option :prevent_kb_autoscaler, type: :boolean, desc: 'Set true to prevent the autoscaler from restarting the job. Default: false'
       def update(scraper_name)
         if options[:job]
           client = Client::Job.new(options)

data/lib/datahen/cli/scraper_page.rb

@@ -13,6 +13,10 @@ module Datahen
       LONGDESC
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       option :page_type, :aliases => :t, type: :string, desc: 'Filter by page_type'
+      option :url, :aliases => :u, type: :string, desc: 'Filter by url'
+      option :effective_url, :aliases => :U, type: :string, desc: 'Filter by effective_url'
+      option :body, :aliases => :b, type: :string, desc: 'Filter by body'
+      option :parent_gid, :aliases => :G, type: :string, desc: 'Filter by parent_gid'
       option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
       option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
       option :fetch_fail, type: :boolean, desc: 'Returns only pages that fails fetching.'
@@ -21,53 +25,65 @@ module Datahen
       def list(scraper_name)
         if options[:job]
           client = Client::JobPage.new(options)
-          puts "#{client.all(options[:job])}"
+          json = JSON.parse(client.all(options[:job]).body)
+          if json['error'] == ""
+            puts "#{JSON.pretty_generate(json['data'])}"
+          else 
+            puts "#{JSON.pretty_generate(json['error'])}"
+          end
         else
           client = Client::ScraperJobPage.new(options)
-          puts "#{client.all(scraper_name)}"
+          json = JSON.parse(client.all(scraper_name).body)
+          if json['error'] == ""
+            puts "#{JSON.pretty_generate(json['data'])}"
+          else 
+            puts "#{JSON.pretty_generate(json['error'])}"
+          end
         end
       end
 
-      desc "add <scraper_name> <url>", "Enqueues a page to a scraper's current job"
+      desc "add <scraper_name> <page_json>", "Enqueues a page to a scraper's current job"
       long_desc <<-LONGDESC
           Enqueues a page to a scraper's current job\x5
           LONGDESC
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
-      option :method, :aliases => :m, type: :string, desc: 'Set request method. Default: GET'
-      option :headers, :aliases => :H, type: :string, banner: :JSON, desc: 'Set request headers. Must be in json format. i.e: {"Foo":"bar"} '
-      option :cookie, :aliases => :c, type: :string, desc: 'Set request cookie.'
-      option :vars, :aliases => :v, type: :string, banner: :JSON, desc: 'Set user-defined page variables. Must be in json format. i.e: {"Foo":"bar"}'
-      option :page_type, :aliases => :t, desc: 'Set page type'
-      option :priority, type: :numeric, desc: 'Set fetch priority. The higher the value, the sooner the page gets fetched. Default: 0'
-      option :fetch_type, :aliases => :F, desc: 'Set fetch type. Default: http'
-      option :body, :aliases => :b, desc: 'Set request body'
-      option :force_fetch, :aliases => :f, type: :boolean, desc: 'Set true to force fetch page that is not within freshness criteria. Default: false'
-      option :freshness, :aliases => :s, desc: 'Set how fresh the page cache is. Accepts timestap format.'
-      option :ua_type, :aliases => :u, desc: 'Set user agent type. Default: desktop'
-      option :no_redirect, :aliases => :n, type: :boolean, desc: 'Set true to not follow redirect. Default: false'
-      option :max_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
-      option :retry_interval, type: :numeric, desc: 'Set a value to set retry time interval on seconds when refetching a page. Set a value grather than 0 to set it as new time to refetch, 0 means default time. Default: 0'
-      def add(scraper_name, url)
+      def add(scraper_name, page_json)
         begin
-          options[:headers] = JSON.parse(options[:headers]) if options[:headers]
-          options[:vars] = JSON.parse(options[:vars]) if options[:vars]
-          method = options[:method]
+          page = JSON.parse(page_json)
 
           if options[:job]
             client = Client::JobPage.new(options)
-            puts "#{client.enqueue(options[:job], method, url, options)}"
+            puts "#{client.enqueue(options[:job], page, options)}"
           else
             client = Client::ScraperJobPage.new(options)
-            puts "#{client.enqueue(scraper_name, method, url, options)}"
+            puts "#{client.enqueue(scraper_name, page, options)}"
           end
 
         rescue JSON::ParserError
-          if options[:headers]
-            puts "Error: #{options[:headers]} on headers is not a valid JSON"
-          end
-          if options[:vars]
-            puts "Error: #{options[:vars]} on vars is not a valid JSON"
+            puts "Error: Invalid JSON"
+        end
+      end
+
+
+      desc "getgid <scraper_name> <page_json>", "Get the generated GID for a scraper's current job"
+      long_desc <<-LONGDESC
+          Get the generated GID for a scraper's current job.\x5
+          LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      def getgid(scraper_name, page_json)
+        begin
+          page = JSON.parse(page_json)
+          
+          if options[:job]
+            client = Client::JobPage.new(options)
+            puts "#{client.get_gid(options[:job], page,  options)}"
+          else
+            client = Client::ScraperJobPage.new(options)
+            puts "#{client.get_gid(scraper_name, page, options)}"
           end
+
+        rescue JSON::ParserError
+          puts "Error: Invalid JSON"
         end
       end
 
@@ -82,6 +98,9 @@ module Datahen
       option :vars, :aliases => :v, type: :string, desc: 'Set user-defined page variables. Must be in json format. i.e: {"Foo":"bar"}'
       option :max_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
       option :retry_interval, type: :numeric, desc: 'Set a value to set retry time interval on seconds when refetching a page. Set a value grather than 0 to set it as new time to refetch, 0 means default time. Default: 0'
+      option :soft_fetching_try_limit, type: :numeric, desc: 'Set the soft fetching try limit value.'
+      option :soft_refetch_limit, type: :numeric, desc: 'Set the soft refetch limit value.'
+      option :parsing_try_limit, type: :numeric, desc: 'Set the parsing try limit value.'
       def update(scraper_name, gid)
         begin
           options[:vars] = JSON.parse(options[:vars]) if options[:vars]

data/lib/datahen/cli/scraper_task.rb

@@ -0,0 +1,48 @@
+module Datahen
+  class CLI < Thor
+    class ScraperTask < Thor
+      package_name "scraper task"
+      def self.banner(command, namespace = nil, subcommand = false)
+        "#{basename} #{@package_name} #{command.usage}"
+      end
+
+      desc "list <scraper_name>", "List Tasks on a scraper's current job"
+      long_desc <<-LONGDESC
+        List all tasks in a scraper's current job or given job ID.\x5
+      LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
+      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
+      option :status, type: :array, desc: 'Returns only tasks with specific status.'
+      option :action, type: :array, desc: 'Returns only tasks with specific action.'
+      option :"include-system", type: :boolean, desc: 'If it is true, will returns all actions. If it is false only tasks with specific action ["refetch", "reparse", "terminate"].'
+      def list(scraper_name)
+        if options[:job]
+          client = Client::JobTask.new(options)
+          puts "#{client.all(options[:job])}"
+        else
+          client = Client::ScraperTask.new(options)
+          puts "#{client.all(scraper_name)}"
+        end
+      end
+
+
+      desc "show <scraper_name> <task_id>", "Show task in scraper's current job"
+      long_desc <<-LONGDESC
+          Shows a task in a scraper's current job or given job ID.\x5
+          LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      def show(scraper_name, task_id)
+        if options[:job]
+          client = Client::JobTask.new(options)
+          puts "#{client.find(options[:job], task_id)}"
+        else
+          client = Client::ScraperTask.new(options)
+          puts "#{client.find(scraper_name, task_id)}"
+        end
+      end
+      
+    end
+  end
+
+end

data/lib/datahen/cli.rb

@@ -11,6 +11,7 @@ require 'datahen/cli/scraper_page'
 require 'datahen/cli/job_output'
 require 'datahen/cli/job'
 require 'datahen/cli/scraper_deployment'
+require 'datahen/cli/scraper_task'
 require 'datahen/cli/scraper'
 require 'datahen/cli/parser'
 require 'datahen/cli/seeder'

data/lib/datahen/client/base.rb

@@ -56,12 +56,18 @@ module Datahen
         target.merge(source.select{|k,v|target.has_key?(k)})
       end
 
-      def retry times, delay = nil, err_msg = nil
+      def retry times, delay = nil, err_msg = nil, stream = false
         limit = times.nil? ? nil : times.to_i
         delay = delay.nil? ? 5 : delay.to_i
         count = 0
         begin
-          yield
+          val = yield
+          if stream
+            return if val.nil?
+            if val['error'] != ""
+              raise StandardError.new(val['error'])
+            end
+          end
         rescue Error::CustomRetryError, StandardError => e
           is_custom_retry = e.is_a? Error::CustomRetryError
           real_delay = is_custom_retry ? e.delay : delay
@@ -81,6 +87,7 @@ module Datahen
           puts "#{err_msg.nil? ? '' : "#{err_msg} "}Retry \##{count}#{should_aprox ? '+' : ''}..."
           retry
         end
+        val
       end
 
       def initialize(opts={})
@@ -105,6 +112,10 @@ module Datahen
         query[:parsefail] = opts[:parse_fail] if opts[:parse_fail]
         query[:status] = opts[:status] if opts[:status]
         query[:page_type] = opts[:page_type] if opts[:page_type]
+        query[:url] = opts[:url] if opts[:url]
+        query[:effective_url] = opts[:effective_url] if opts[:effective_url]
+        query[:body] = opts[:body] if opts[:body]
+        query[:parent_gid] = opts[:parent_gid] if opts[:parent_gid]
         query[:gid] = opts[:gid] if opts[:gid]
         query[:"min-timestamp"] = opts[:"min-timestamp"] if opts[:"min-timestamp"]
         query[:"max-timestamp"] = opts[:"max-timestamp"] if opts[:"max-timestamp"]
@@ -112,6 +123,8 @@ module Datahen
         query[:order] = opts[:order] if opts[:order]
         query[:filter] = opts[:filter] if opts[:filter]
         query[:force] = opts[:force] if opts[:force]
+        query[:action] = opts[:action] if opts[:action]
+        query[:"include-system"] = opts[:"include-system"] if opts[:"include-system"]
 
         if opts[:query]
           if opts[:query].is_a?(Hash)

data/lib/datahen/client/job.rb

@@ -25,6 +25,10 @@ module Datahen
         body[:max_page_size] = opts[:max_page_size] if opts[:max_page_size]
         body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
         body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]
+        body[:soft_fetching_try_limit] = opts[:soft_fetching_try_limit] if opts[:soft_fetching_try_limit]
+        body[:soft_refetch_limit] = opts[:soft_refetch_limit] if opts[:soft_refetch_limit]
+        body[:parsing_try_limit] = opts[:parsing_try_limit] if opts[:parsing_try_limit]
+        body[:prevent_kb_autoscaler] = opts[:prevent_kb_autoscaler] if opts.has_key?("prevent_kb_autoscaler") || opts.has_key?(:prevent_kb_autoscaler)
         params = @options.merge({body: body.to_json})
 
         self.class.put("/jobs/#{job_id}", params)
@@ -97,7 +101,7 @@ module Datahen
       def sync_schema(job_id, opts={})
         params = @options.merge(opts)
 
-        self.class.put("/sync/jobs/#{job_id}/schema", params)
+        self.class.put("/jobs/#{job_id}/sync/schema", params)
       end
 
     end

data/lib/datahen/client/job_output.rb

@@ -7,7 +7,7 @@ module Datahen
 
       def all(job_id, collection = 'default', opts = {})
         limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : 0
-        self.retry(limit, 10, "Error while updating the seeder.") do
+        self.retry(limit, 10, "Error while updating the seeder.", true) do
           self.class.get("/jobs/#{job_id}/output/collections/#{collection}/records", @options)
         end
       end

data/lib/datahen/client/job_page.rb

@@ -18,34 +18,27 @@ module Datahen
         body[:max_size] = opts[:max_size] if opts[:max_size]
         body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
         body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]
+        body[:soft_fetching_try_limit] = opts[:soft_fetching_try_limit] if opts[:soft_fetching_try_limit]
+        body[:soft_refetch_limit] = opts[:soft_refetch_limit] if opts[:soft_refetch_limit]
+        body[:parsing_try_limit] = opts[:parsing_try_limit] if opts[:parsing_try_limit]
 
         params = @options.merge({body: body.to_json})
 
         self.class.put("/jobs/#{job_id}/pages/#{gid}", params)
       end
 
-      def enqueue(job_id, method, url, opts={})
-        body = {}
-        body[:method] =  method != "" ? method : "GET"
-        body[:url] =  url
-        body[:page_type] = opts[:page_type] if opts[:page_type]
-        body[:priority] = opts[:priority] if opts[:priority]
-        body[:fetch_type] = opts[:fetch_type] if opts[:fetch_type]
-        body[:body] = opts[:body] if opts[:body]
-        body[:headers] = opts[:headers] if opts[:headers]
-        body[:vars] = opts[:vars] if opts[:vars]
-        body[:force_fetch] = opts[:force_fetch] if opts[:force_fetch]
-        body[:freshness] = opts[:freshness] if opts[:freshness]
-        body[:ua_type] = opts[:ua_type] if opts[:ua_type]
-        body[:no_redirect] = opts[:no_redirect] if opts[:no_redirect]
-        body[:cookie] = opts[:cookie] if opts[:cookie]
-        body[:max_size] = opts[:max_size] if opts[:max_size]
-        body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
-        body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]
-
-        params = @options.merge({body: body.to_json})
+      def enqueue(job_id, page, opts={})
+        params = @options.merge(opts).merge({body: page.to_json})
 
         self.class.post("/jobs/#{job_id}/pages", params)
+        
+      end
+
+      def get_gid(job_id, page, opts={})
+      
+        params = @options.merge(opts).merge({body: page.to_json})
+
+        self.class.post("/jobs/#{job_id}/generate_gid", params)
       end
 
       def dequeue(job_id, limit, page_types, parse_fetching_failed, opts = {})
@@ -65,6 +58,7 @@ module Datahen
         body[:parsing_status] = opts.fetch(:parsing_status){ nil }
         body[:log_error] = opts[:log_error] if opts[:log_error]
         body[:keep_outputs] = !!opts[:keep_outputs] if opts.has_key?(:keep_outputs)
+        body[:parsing_try_limit] = opts[:parsing_try_limit] if opts.fetch(:parsing_try_limit){ nil }
 
         params = @options.merge({body: body.to_json})
 
@@ -100,6 +94,11 @@ module Datahen
         params = @options.merge(opts)
         self.class.put("/jobs/#{job_id}/pages/limbo", params)
       end
+
+      def still_alive(job_id, gid, opts={})
+        params = @options.merge(opts)
+        self.class.put("/jobs/#{job_id}/pages/#{gid}/still_alive", params)
+      end
     end
   end
 end

data/lib/datahen/client/job_task.rb

@@ -0,0 +1,17 @@
+module Datahen
+  module Client
+    class JobTask < Datahen::Client::Base
+      def all(job_id, opts={})
+        params = @options.merge(opts)
+        self.class.get("/jobs/#{job_id}/tasks", params)
+      end
+
+      def find(job_id, task_id, opts={})
+        params = @options.merge(opts)
+        self.class.get("/jobs/#{job_id}/tasks/#{task_id}", params)
+      end
+
+    end
+
+  end
+end

data/lib/datahen/client/scraper.rb

@@ -32,6 +32,10 @@ module Datahen
         body[:max_page_size] = opts[:max_page_size] if opts[:max_page_size]
         body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
         body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]
+        body[:soft_fetching_try_limit] = opts[:soft_fetching_try_limit] if opts[:soft_fetching_try_limit]
+        body[:soft_refetch_limit] = opts[:soft_refetch_limit] if opts[:soft_refetch_limit]
+        body[:parsing_try_limit] = opts[:parsing_try_limit] if opts[:parsing_try_limit]
+        body[:prevent_kb_autoscaler] = opts[:prevent_kb_autoscaler] if opts.has_key?("prevent_kb_autoscaler") || opts.has_key?(:prevent_kb_autoscaler)
         params = @options.merge({body: body.to_json})
         self.class.post("/scrapers", params)
       end
@@ -57,6 +61,10 @@ module Datahen
         body[:max_page_size] = opts[:max_page_size] if opts.has_key?("max_page_size") || opts.has_key?(:max_page_size)
         body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
         body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]
+        body[:soft_fetching_try_limit] = opts[:soft_fetching_try_limit] if opts[:soft_fetching_try_limit]
+        body[:soft_refetch_limit] = opts[:soft_refetch_limit] if opts[:soft_refetch_limit]
+        body[:parsing_try_limit] = opts[:parsing_try_limit] if opts[:parsing_try_limit]
+        body[:prevent_kb_autoscaler] = opts[:prevent_kb_autoscaler] if opts.has_key?("prevent_kb_autoscaler") || opts.has_key?(:prevent_kb_autoscaler)
         params = @options.merge({body: body.to_json})
 
         self.class.put("/scrapers/#{scraper_name}", params)

data/lib/datahen/client/scraper_job.rb

@@ -15,6 +15,10 @@ module Datahen
         body[:max_page_size] = opts[:max_page_size] if opts[:max_page_size]
         body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
         body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]
+        body[:soft_fetching_try_limit] = opts[:soft_fetching_try_limit] if opts[:soft_fetching_try_limit]
+        body[:soft_refetch_limit] = opts[:soft_refetch_limit] if opts[:soft_refetch_limit]
+        body[:parsing_try_limit] = opts[:parsing_try_limit] if opts[:parsing_try_limit]
+        body[:prevent_kb_autoscaler] = opts[:prevent_kb_autoscaler] if opts.has_key?("prevent_kb_autoscaler") || opts.has_key?(:prevent_kb_autoscaler)
         if opts[:vars]
           if opts[:vars].is_a?(Array)
             body[:vars] = opts[:vars]
@@ -45,6 +49,10 @@ module Datahen
         body[:max_page_size] = opts[:max_page_size] if opts[:max_page_size]
         body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
         body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]
+        body[:soft_fetching_try_limit] = opts[:soft_fetching_try_limit] if opts[:soft_fetching_try_limit]
+        body[:soft_refetch_limit] = opts[:soft_refetch_limit] if opts[:soft_refetch_limit]
+        body[:parsing_try_limit] = opts[:parsing_try_limit] if opts[:parsing_try_limit]
+        body[:prevent_kb_autoscaler] = opts[:prevent_kb_autoscaler] if opts.has_key?("prevent_kb_autoscaler") || opts.has_key?(:prevent_kb_autoscaler)
         params = @options.merge({body: body.to_json})
 
         self.class.put("/scrapers/#{scraper_name}/current_job", params)

data/lib/datahen/client/scraper_job_page.rb

@@ -47,30 +47,19 @@ module Datahen
         self.class.put("/scrapers/#{scraper_name}/current_job/pages/limbo", params)
       end
 
-      def enqueue(scraper_name, method, url, opts={})
-        body = {}
-        body[:method] =  method != "" ? method : "GET"
-        body[:url] =  url
-        body[:page_type] = opts[:page_type] if opts[:page_type]
-        body[:priority] = opts[:priority] if opts[:priority]
-        body[:fetch_type] = opts[:fetch_type] if opts[:fetch_type]
-        body[:body] = opts[:body] if opts[:body]
-        body[:headers] = opts[:headers] if opts[:headers]
-        body[:vars] = opts[:vars] if opts[:vars]
-        body[:force_fetch] = opts[:force_fetch] if opts[:force_fetch]
-        body[:freshness] = opts[:freshness] if opts[:freshness]
-        body[:ua_type] = opts[:ua_type] if opts[:ua_type]
-        body[:no_redirect] = opts[:no_redirect] if opts[:no_redirect]
-        body[:cookie] = opts[:cookie] if opts[:cookie]
-        body[:max_size] = opts[:max_size] if opts[:max_size]
-        body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
-        body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]
-
-        params = @options.merge({body: body.to_json})
+      def enqueue(scraper_name, page, opts={})
+      params = @options.merge(opts).merge({body: page.to_json})
 
         self.class.post("/scrapers/#{scraper_name}/current_job/pages", params)
       end
 
+      def get_gid(scraper_name, page, opts={})
+      
+        params = @options.merge(opts).merge({body: page.to_json})
+
+        self.class.post("/scrapers/#{scraper_name}/current_job/generate_gid", params)
+      end
+
       def find_content(scraper_name, gid)
         self.class.get("/scrapers/#{scraper_name}/current_job/pages/#{gid}/content", @options)
       end

data/lib/datahen/client/scraper_task.rb

@@ -0,0 +1,17 @@
+module Datahen
+  module Client
+    class ScraperTask < Datahen::Client::Base
+      def all(scraper_name, opts={})
+        params = @options.merge(opts)
+        self.class.get("/scrapers/#{scraper_name}/current_job/tasks", params)
+      end
+
+      def find(scraper_name, task_id, opts={})
+        params = @options.merge(opts)
+        self.class.get("/scrapers/#{scraper_name}/current_job/tasks/#{task_id}", params)
+      end
+
+    end
+
+  end
+end

data/lib/datahen/client.rb

@@ -24,6 +24,8 @@ require "datahen/client/scraper_var"
 require "datahen/client/job_var"
 require "datahen/client/scraper_job_var"
 require "datahen/client/job_finisher"
+require "datahen/client/job_task"
+require "datahen/client/scraper_task"
 
 module Datahen
   module Client

data/lib/datahen/scraper/batch_parser.rb

@@ -227,7 +227,11 @@ module Datahen
 
         # add pages
         count = 0
-        (JSON.parse(response.body) || []).each do |page|
+        json = JSON.parse(response.body)
+          if json['error'] != ""
+            return 0
+          end
+        (json['data'] || []).each do |page|
           count += 1
           next if self.loaded_pages.has_key? page['gid']
           self.pages << (self.loaded_pages[page['gid']] = page)
@@ -307,7 +311,7 @@ module Datahen
             is_waiting = true
             puts "[Worker #{Parallel.worker_number}]: Is waiting for a page..."
             if self.second_dequeue_count > 1 && !self.not_found
-              puts "\nWARNING: Your job is not optimized, increase your job's \"parser_dequeue_scale\"\n"
+              puts "\nWARNING: Your job might not be optimized. Consider increasing your job's \"parser_dequeue_scale\" if the `to_parse` queue is not empty or near empty \n"
             end
           end
           self.class.wait 1

data/lib/datahen/scraper/executor.rb

@@ -172,11 +172,16 @@ module Datahen
         response = client.all(query_job_id, collection, {
           retry_limit: retry_limit
         })
-
         if response.code != 200
           raise "response_code: #{response.code}|#{response.parsed_response}"
         end
-        (response.body != 'null') ? response.parsed_response : []
+
+        # check stream error
+        json_data = response.body != 'null' ? response.parsed_response : {}
+        if json_data['error'] != ""
+          raise "response_code: #{response.code}|Stream error: #{json_data['error']}"
+        end
+        json_data['data'].nil? ? [] : json_data['data']
       end
 
       # Find one output by collection and query with pagination.

data/lib/datahen/scraper/ruby_parser_executor.rb

@@ -33,7 +33,6 @@ module Datahen
           :failed_content,
           :outputs,
           :pages,
-          :page,
           :save_pages,
           :save_outputs,
           :find_output,
@@ -41,7 +40,8 @@ module Datahen
           :refetch,
           :reparse,
           :limbo,
-          :finish
+          :finish,
+          :still_alive
         ].freeze
       end
 
@@ -240,6 +240,12 @@ module Datahen
         @failed_content ||= get_failed_content(job_id, gid)
       end
 
+      def still_alive page_gid = nil
+        page_gid = gid if page_gid.nil?
+        client = Client::JobPage.new()
+        client.still_alive(job_id, page_gid)
+      end
+
       def handle_error(e)
         error = ["Parsing #{e.class}: #{e.to_s} (Job:#{job_id} GID:#{gid})",clean_backtrace(e.backtrace)].join("\n")
 
@@ -247,7 +253,8 @@ module Datahen
           job_id: job_id,
           gid: gid,
           parsing_status: :failed,
-          log_error: error)
+          log_error: error,
+          parsing_try_limit: (page || {})['parsing_try_limit'])
       end
 
     end

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "1.3.2"
+  VERSION = "1.5.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 1.3.2
+  version: 1.5.1
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-06-08 00:00:00.000000000 Z
+date: 2024-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -227,6 +227,7 @@ files:
 - lib/datahen/cli/scraper_job.rb
 - lib/datahen/cli/scraper_job_var.rb
 - lib/datahen/cli/scraper_page.rb
+- lib/datahen/cli/scraper_task.rb
 - lib/datahen/cli/scraper_var.rb
 - lib/datahen/cli/seeder.rb
 - lib/datahen/client.rb
@@ -245,6 +246,7 @@ files:
 - lib/datahen/client/job_output.rb
 - lib/datahen/client/job_page.rb
 - lib/datahen/client/job_stat.rb
+- lib/datahen/client/job_task.rb
 - lib/datahen/client/job_var.rb
 - lib/datahen/client/scraper.rb
 - lib/datahen/client/scraper_deployment.rb
@@ -255,6 +257,7 @@ files:
 - lib/datahen/client/scraper_job_output.rb
 - lib/datahen/client/scraper_job_page.rb
 - lib/datahen/client/scraper_job_var.rb
+- lib/datahen/client/scraper_task.rb
 - lib/datahen/client/scraper_var.rb
 - lib/datahen/error.rb
 - lib/datahen/error/custom_retry_error.rb