datahen 0.15.10 → 0.16.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 84e5a734ca1b827228db1550d14f4e9b21e1369d88259a7067dc9209c9a5605b
-  data.tar.gz: 629b472f401b88dc4caabdb87264f9ab1d39d492ff27ac673a79d8ec2c411928
+  metadata.gz: 8945724e5d11f40eba22a9ffca7ca7d024b8565dae4ba1d3da9e486eac262575
+  data.tar.gz: 867319a0c6358c593951e6241b24d9c3fd9dcb759ce1287eeb640ade0c23e69a
 SHA512:
-  metadata.gz: 320bd2aded5b02fa14a0e4f4cb1c1d4a7d64fe6f4fedcffc62bb04a4c71eab215174f874dc2408dc4416ebf357017f5e743e3f1120fd0e944daeca51deee5311
-  data.tar.gz: 8a86dbc795fae177e4bbd7bde44892ced937cd0fe74383ad962528f25d7d1ef9118897ae24219379299fdd7650fa433ffd436faca71af925c723679f8780db14
+  metadata.gz: bad8dea41951df061c84934fa63a0594bc8caf22b8665c3d6746bb29ca2821103ecd3814a9a9eba020009e8165390900b1c45a1fd0a7175fb7ea52f7a077fdab
+  data.tar.gz: 74342e01eaa21a590ef998219282fb40342cde4bf8db58617d24583bfa00b967df3f8c38e41c9b92868e90113b7cc5f6346fb2062b446edd3bb4612e053c5da7

data/lib/datahen/cli/parser.rb

@@ -72,6 +72,7 @@ module Datahen
       option :"max-garbage", type: :numeric, default: 5, desc: "Pages processed before calling the garbage collector"
       option :"dequeue-interval", type: :numeric, default: 3, desc: "Seconds to wait between dequeueing"
       option :"dequeue-scale", type: :numeric, default: 2, desc: "Scale vs worker count describing how many pages to dequeue"
+      option :"dequeue-timeout", type: :numeric, default: 30, desc: "Dequeue pages API request timeout"
       def batch_exec_parse(scraper_name, config_file)
         if options[:job]
           job_id = options[:job]

data/lib/datahen/cli/scraper.rb

@@ -32,6 +32,7 @@ module Datahen
       option :profile, type: :string, desc: 'Set the profiles (comma separated) to apply to the job. Default: default'
       option :multiple_jobs, type: :boolean, desc: 'Set true to enable multiple jobs. Default: false'
       option :max_job_count, type: :numeric, desc: 'Set a value to set max number of jobs available. Set -1 for unlimited. Default: 3'
+      option :max_page_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
       def create(scraper_name, git_repository)
         # puts "options #{options}"
         client = Client::Scraper.new(options)
@@ -57,6 +58,7 @@ module Datahen
       option :profile, type: :string, desc: 'Set the profiles (comma separated) to apply to the job. Default: default'
       option :multiple_jobs, type: :boolean, desc: 'Set true to enable multiple jobs. Default: false'
       option :max_job_count, type: :numeric, desc: 'Set a value to set max number of jobs available. Set -1 for unlimited. Default: 3'
+      option :max_page_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
       def update(scraper_name)
         client = Client::Scraper.new(options)
         puts "#{client.update(scraper_name, options)}"
@@ -94,6 +96,7 @@ module Datahen
       option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
       option :proxy_type, desc: 'Set the Proxy type. Default: standard'
       option :vars, type: :string, banner: :JSON, desc: 'Set input vars. Must be in json format. i.e: [{"name":"foo", "value":"bar", "secret":false}] '
+      option :max_page_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
       def start(scraper_name)
         client = Client::ScraperJob.new(options)
         puts "Starting a scrape job..."

data/lib/datahen/cli/scraper_job.rb

@@ -104,6 +104,7 @@ module Datahen
       option :proxy_type, desc: 'Set the Proxy type. Default: standard'
       option :profile, type: :string, desc: 'Set the profiles (comma separated) to apply to the job. Default: default'
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :max_page_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
       def update(scraper_name)
         if options[:job]
           client = Client::Job.new(options)

data/lib/datahen/cli/scraper_page.rb

@@ -45,6 +45,7 @@ module Datahen
       option :freshness, :aliases => :s, desc: 'Set how fresh the page cache is. Accepts timestap format.'
       option :ua_type, :aliases => :u, desc: 'Set user agent type. Default: desktop'
       option :no_redirect, :aliases => :n, type: :boolean, desc: 'Set true to not follow redirect. Default: false'
+      option :max_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
       def add(scraper_name, url)
         begin
           options[:headers] = JSON.parse(options[:headers]) if options[:headers]
@@ -78,6 +79,7 @@ module Datahen
       option :page_type, :aliases => :t, desc: 'Set page type'
       option :priority, type: :numeric, desc: 'Set fetch priority. The higher the value, the sooner the page gets fetched. Default: 0'
       option :vars, :aliases => :v, type: :string, desc: 'Set user-defined page variables. Must be in json format. i.e: {"Foo":"bar"}'
+      option :max_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
       def update(scraper_name, gid)
         begin
           options[:vars] = JSON.parse(options[:vars]) if options[:vars]

data/lib/datahen/client/job.rb

@@ -21,6 +21,7 @@ module Datahen
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
         body[:profile] = opts[:profile] if opts[:profile]
+        body[:max_page_size] = opts[:max_page_size] if opts[:max_page_size]
         params = @options.merge({body: body.to_json})
 
         self.class.put("/jobs/#{job_id}", params)

data/lib/datahen/client/job_page.rb

@@ -15,6 +15,7 @@ module Datahen
         body[:page_type] = opts[:page_type] if opts[:page_type]
         body[:priority] = opts[:priority] if opts[:priority]
         body[:vars] = opts[:vars] if opts[:vars]
+        body[:max_size] = opts[:max_size] if opts[:max_size]
 
         params = @options.merge({body: body.to_json})
 
@@ -36,6 +37,7 @@ module Datahen
         body[:ua_type] = opts[:ua_type] if opts[:ua_type]
         body[:no_redirect] = opts[:no_redirect] if opts[:no_redirect]
         body[:cookie] = opts[:cookie] if opts[:cookie]
+        body[:max_size] = opts[:max_size] if opts[:max_size]
 
         params = @options.merge({body: body.to_json})
 
@@ -48,7 +50,7 @@ module Datahen
           page_types: page_types,
           parse_fetching_failed: parse_fetching_failed
         }
-        params = @options.merge({body: body.to_json, timeout: 30})
+        params = @options.merge(opts).merge({body: body.to_json})
         self.class.put("/jobs/#{job_id}/pages/parse_dequeue", params)
       end
 

data/lib/datahen/client/scraper.rb

@@ -28,6 +28,7 @@ module Datahen
         body[:profile] = opts[:profile] if opts[:profile]
         body[:multiple_jobs] = opts[:multiple_jobs] if opts[:multiple_jobs]
         body[:max_job_count] = opts[:max_job_count] if opts[:max_job_count]
+        body[:max_page_size] = opts[:max_page_size] if opts[:max_page_size]
         params = @options.merge({body: body.to_json})
         self.class.post("/scrapers", params)
       end
@@ -49,6 +50,7 @@ module Datahen
         body[:profile] = opts[:profile] if opts[:profile]
         body[:multiple_jobs] = opts[:multiple_jobs] if opts.has_key?("multiple_jobs") || opts.has_key?(:multiple_jobs)
         body[:max_job_count] = opts[:max_job_count] if opts.has_key?("max_job_count") || opts.has_key?(:max_job_count)
+        body[:max_page_size] = opts[:max_page_size] if opts.has_key?("max_page_size") || opts.has_key?(:max_page_size)
         params = @options.merge({body: body.to_json})
 
         self.class.put("/scrapers/#{scraper_name}", params)

data/lib/datahen/client/scraper_job.rb

@@ -11,6 +11,7 @@ module Datahen
         body[:standard_worker_count] = opts[:workers] if opts[:workers]
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
+        body[:max_page_size] = opts[:max_page_size] if opts[:max_page_size]
         if opts[:vars]
           if opts[:vars].is_a?(Array)
             body[:vars] = opts[:vars]
@@ -37,6 +38,7 @@ module Datahen
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
         body[:profile] = opts[:profile] if opts[:profile]
+        body[:max_page_size] = opts[:max_page_size] if opts[:max_page_size]
         params = @options.merge({body: body.to_json})
 
         self.class.put("/scrapers/#{scraper_name}/current_job", params)

data/lib/datahen/client/scraper_job_page.rb

@@ -15,6 +15,7 @@ module Datahen
         body[:page_type] = opts[:page_type] if opts[:page_type]
         body[:priority] = opts[:priority] if opts[:priority]
         body[:vars] = opts[:vars] if opts[:vars]
+        body[:max_size] = opts[:max_size] if opts[:max_size]
 
         params = @options.merge({body: body.to_json})
 
@@ -59,6 +60,7 @@ module Datahen
         body[:ua_type] = opts[:ua_type] if opts[:ua_type]
         body[:no_redirect] = opts[:no_redirect] if opts[:no_redirect]
         body[:cookie] = opts[:cookie] if opts[:cookie]
+        body[:max_size] = opts[:max_size] if opts[:max_size]
 
         params = @options.merge({body: body.to_json})
 

data/lib/datahen/scraper/batch_parser.rb

@@ -8,22 +8,107 @@ module Datahen
       NO_DEQUEUE_COUNT_MSG = "\nWarning: Max page to parse dequeue count is 0, check pages to parse scale\n"
       NO_WORKERS_MSG = "\nWarning: There are no parser workers\n"
 
-      attr_accessor :config_file, :garbage_count, :last_message, :second_dequeue_count
-      attr_reader :job_id, :worker_count, :pages, :max_garbage
-      attr_reader :dequeue_interval, :dequeue_scale
-      attr_reader :page_types, :parsers
-      attr_reader :config, :client, :garbage_mutex
+      # Configuration file path.
+      # @return [String] config file path
+      attr_accessor :config_file
+      # Garbage collector request counter.
+      # @return [Integer] garbage collector counter
+      attr_accessor :garbage_count
+      # Last printed message, useful to prevent duplicated log messages.
+      # @return [String] last printed message
+      attr_accessor :last_message
+      # Second dequeue counter used to prevent false negative warning messages.
+      # @return [Integer] second dequeue counter
+      attr_accessor :second_dequeue_count
+      # Dequeue API request timeout in seconds.
+      # @return [Integer] dequeue API request timeout in seconds
+      attr_accessor :dequeue_timeout
+      # Job id to be executed.
+      # @return [Integer] job id
+      attr_reader :job_id
+      # Parallel worker quantity.
+      # @return [Integer] parallel worker quantity
+      attr_reader :worker_count
+      # Loaded pages array.
+      # @return [Concurrent::Array<Hash>] loaded pages as an array
+      attr_reader :pages
+      # Loaded pages hash, useful to avoid duplicates on the loaded pages array.
+      # @return [Concurrent::Hash<String, Hash>] loaded pages as a concurrent hash
+      attr_reader :loaded_pages
+      # Max garbage collector requests before actually executing the garbage
+      #   collector.
+      # @return [Integer] max garbage request quantity before actually executing
+      #   it
+      attr_reader :max_garbage
+      # Dequeue interval in seconds.
+      # @return [Integer] dequeue interval in seconds
+      attr_reader :dequeue_interval
+      # Dequeue scale used to calculate the ideal dequeue size.
+      # @return [Numeric] dequeue scale
+      attr_reader :dequeue_scale
+      # Known page types extracted from the config file.
+      # @return [Array<String>] known page types
+      attr_reader :page_types
+      # Known parsers extracted from the config file.
+      # @return [Concurrent::Hash<String, String>] known parsers
+      attr_reader :parsers
+      # Current config file loaded.
+      # @return [Hash] current loaded configuration
+      attr_reader :config
+      # Datahen job pages client used for API pages dequeuing.
+      # @return [Datahen::Client::JobPage] datahen job pages API client
+      attr_reader :client
+      # Garbage collector mutex used to synchronize garbage collector requests.
+      # @return [Mutex] garbage collector mutex
+      attr_reader :garbage_mutex
+      # Current dequeuer thread.
+      # @return [Thread] dequeuer thread
+      attr_reader :dequeuer_thread
+      # Dequeuer mutext used to synchronize page dequeuing.
+      # @return [Mutex] dequeuer mutex
+      attr_reader :dequeue_mutex
+      # Dequeuer last run unix timestamp.
+      # @return [Integer] dequeuer last run unix timestamp
+      attr_reader :dequeuer_still_alive
+      # Indicates whenever the wait time is because there are no more pages.
+      # @return [Boolean] `true` when wait time is due to no more pages,
+      #   else `false`
+      attr_reader :not_found
 
+      # Wait a specific amount of seconds.
+      # @param [Integer] time_in_seconds Seconds to wait.
       def self.wait time_in_seconds
         Kernel.sleep time_in_seconds
       end
 
+      # Get a unix timestamp.
+      # @return [Integer] unix timestamp
+      def self.timestamp
+        Time.new.utc.to_i
+      end
+
+      # Initialize a batch parser object.
+      # @param [Integer] job_id Job id.
+      # @param [String] config_file Config file path.
+      # @param [Hash] opts ({}) Configuration options
+        # @option opts [Integer] :worker_count (1) Parallel worker quantity.
+        # @option opts [Integer] :max_garbage (5) Max amount of times the garbage
+        #   collector can be requested before actually executing.
+        # @option opts [Integer] :dequeue_interval (3) Time in seconds to wait
+        #   between page dequeuing.
+        # @option opts [Numeric] :dequeue_scale (2) Scaling factor to used to
+        #   calculate page dequeue size.
+        # @option opts [Numeric] :dequeue_timeout (30) Page dequeue API request
+        #   timeout in seconds.
+        # @option opts [Hash] :client_options ({}) Datahen client gem additional
+        #   options (see Datahen::Client::Base#initialize method).
       def initialize(job_id, config_file, opts = {})
         opts = {
           worker_count: 1,
           max_garbage: 5,
           dequeue_interval: 3,
           dequeue_scale: 2,
+          dequeue_timeout: 30,
           client_options: {}
         }.merge opts
 
@@ -32,16 +117,23 @@ module Datahen
         @dequeue_interval = opts[:dequeue_interval]
         @dequeue_scale = opts[:dequeue_scale]
         @max_garbage = opts[:max_garbage]
-        @pages = Concurrent::Hash.new
+        @pages = Concurrent::Array.new
+        @loaded_pages = Concurrent::Hash.new
         @garbage_mutex = Mutex.new
+        @dequeue_mutex = Mutex.new
+        @not_found = false
+        self.dequeue_timeout = opts[:dequeue_timeout]
         self.second_dequeue_count = 0
         self.garbage_count = 0
         self.config_file = config_file
         self.load_config
 
         @client = Datahen::Client::JobPage.new(opts[:client_options])
+        nil
       end
 
+      # Execute garbage collector after it is requested as many times as
+      #   described by #max_garbage.
       def recollect_garbage
         self.garbage_mutex.synchronize do
           self.garbage_count += 1
@@ -51,8 +143,10 @@ module Datahen
             self.garbage_count = 0
           end
         end
+        nil
       end
 
+      # Loads the config file into a Hash.
       def load_config
         # build page type to script file map
         @page_types = []
@@ -64,20 +158,40 @@ module Datahen
           self.parsers[v['page_type']] = v['file']
         end
         self.recollect_garbage
+        nil
       end
 
+      # Print the message regardless of it being the same as the last message.
+      # @param [String] message Message to display.
       def repeat_puts message
         puts message
-        self.last_message = ''
+        self.last_message = message
+        nil
       end
 
+      # Print the message only when it is different from the last recorded
+      #   message.
+      # @param [String] message Message to display.
       def no_repeat_puts message
         return if message == self.last_message
         puts message
         self.last_message = message
+        nil
+      end
+
+      # Refresh dequeuer's still alive timestamp
+      def dequeuer_is_alive!
+        self.dequeue_mutex.synchronize do
+          @dequeuer_still_alive = self.class.timestamp
+        end
+        nil
       end
 
+      # Load new pages by dequeuing from the API.
+      # @return [Integer] amount of pages loaded
       def load_pages
+        self.dequeuer_is_alive!
+
         # calculate dequeue size
         max_dequeue_size = (self.worker_count * self.dequeue_scale).ceil
         current_size = self.pages.length
@@ -93,13 +207,16 @@ module Datahen
           response = client.dequeue self.job_id,
             dequeue_size,
             self.page_types,
-            config['parse_fetching_failed']
+            config['parse_fetching_failed'],
+            timeout: self.dequeue_timeout
         rescue Net::ReadTimeout, Net::OpenTimeout => e
-          self.no_repeat_puts "Dequeue API call timeout! Contact infra team, your job needs a profile change"
+          self.repeat_puts "Dequeue API call timeout! Contact infra team, your job needs a profile change"
+          self.dequeuer_is_alive!
           return 0
         rescue => e
           raise e
         end
+        self.dequeuer_is_alive!
 
         # ensure a valid response or try again
         if response.nil? || response.response.code.to_i != 200
@@ -112,17 +229,20 @@ module Datahen
         count = 0
         (JSON.parse(response.body) || []).each do |page|
           count += 1
-          next if self.pages.has_key? page['gid']
-          self.pages[page['gid']] = page
+          next if self.loaded_pages.has_key? page['gid']
+          self.pages << (self.loaded_pages[page['gid']] = page)
         end
         response = nil
+        self.dequeuer_is_alive!
 
         # recolect garbage to free some memory before parsing
         if count > 0
+          @not_found = false
           self.recollect_garbage
           self.repeat_puts "Found #{count} page(s) to parse"
           self.second_dequeue_count += 1 unless self.second_dequeue_count > 1
         else
+          @not_found = true
           self.no_repeat_puts NOT_FOUND_MSG
         end
 
@@ -130,6 +250,44 @@ module Datahen
         count
       end
 
+      # Ensures that the dequeuer thread exists and is running.
+      # @return [Boolean] `true` if thread was alive, or `false` if had to
+      #   create a new thread
+      def ensure_dequeuer_thread
+        self.dequeue_mutex.synchronize do
+          # check if dequeuer thread is alive and healthy
+          if !self.dequeuer_thread.nil? && self.dequeuer_thread.alive?
+            still_alive_timeout = (self.dequeue_timeout + self.dequeue_interval) * 2 + self.dequeuer_still_alive
+            return true if self.class.timestamp < still_alive_timeout
+
+            # kill dequeuer thread
+            self.repeat_puts "Dequeuer isn't healthy, will restart it..."
+            self.dequeuer_thread.kill
+            @dequeuer_thread = nil
+            self.recollect_garbage
+            self.no_repeat_puts "Dequeuer thread was killed!"
+          end
+
+          # dequeuing on parallel (the ride never ends :D)
+          @dequeuer_thread = Thread.new do
+            while true
+              begin
+                self.load_pages
+                self.class.wait self.dequeue_interval
+              rescue => e
+                puts [e.message] + e.backtrace rescue 'error'
+              end
+            end
+            puts "Error: dequeuer died! D:"
+          end
+          self.repeat_puts "Dequeuer thread was started!"
+        end
+        false
+      end
+
+      # Dequeue one page from the previously loaded pages, and waits until there
+      #   are new pages whenever there are no loaded pages.
+      # @return [Hash] dequeued page
       def dequeue_pages
         # collect garbage
         self.recollect_garbage
@@ -137,24 +295,29 @@ module Datahen
         # return page if there are loeaded pages
         is_waiting = false
         while true do
-          key_value = self.pages.shift
-          unless key_value.nil?
+          page = self.pages.shift
+          unless page.nil?
             puts "[Worker #{Parallel.worker_number}]: Finish waiting" if is_waiting
-            return key_value[1]
+            loaded_pages.delete(page['gid'])
+            return page
           end
 
           # be more verbose on worker waiting
           unless is_waiting
             is_waiting = true
             puts "[Worker #{Parallel.worker_number}]: Is waiting for a page..."
-            if self.second_dequeue_count > 1
+            if self.second_dequeue_count > 1 && !self.not_found
               puts "\nWARNING: Your job is not optimized, increase your job's \"parser_dequeue_scale\"\n"
             end
           end
           self.class.wait 1
+
+          # ensure the dequeuer thread is alive and healthy
+          self.ensure_dequeuer_thread
         end
       end
 
+      # Dequeue pages and execute the parsers associated to them on parallel.
       def exec_parse save = false, keep_outputs = false
         if self.worker_count < 1
           self.no_repeat_puts NO_WORKERS_MSG
@@ -163,24 +326,15 @@ module Datahen
           self.no_repeat_puts "Spawing #{self.worker_count} workers"
         end
 
-        # dequeuing on parallel (the ride never ends :D)
-        Thread.new do
-          while true
-            begin
-              self.load_pages
-              self.class.wait self.dequeue_interval
-            rescue => e
-              puts [e.message] + e.backtrace rescue 'error'
-            end
-          end
-          puts "Error: dequeuer died! D:"
-        end
+        # start dequeuer
+        self.ensure_dequeuer_thread
 
         # process the pages
         dequeue = lambda{ self.dequeue_pages }
         Parallel.each(dequeue, in_threads: (worker_count)) do |page|
           parser_file = self.parsers[page['page_type']]
           begin
+            self.repeat_puts("Parsing page with GID #{page['gid']}")
             puts Datahen::Scraper::Parser.exec_parser_by_page(
               parser_file,
               page,
@@ -189,6 +343,7 @@ module Datahen
               nil,
               keep_outputs
             )
+            self.repeat_puts("Finish parsing page with GID #{page['gid']}")
           rescue Parallel::Kill => e
             puts "[Worker #{Parallel.worker_number}]: Someone tried to kill Parallel!!!"
           rescue Parallel::Break => e
@@ -197,6 +352,8 @@ module Datahen
             puts [e.message] + e.backtrace rescue 'error'
           end
         end
+
+        nil
       end
     end
   end

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "0.15.10"
+  VERSION = "0.16.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 0.15.10
+  version: 0.16.2
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-05-28 00:00:00.000000000 Z
+date: 2021-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor