datahen 0.14.26 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dac57d98132102aa9ae8244b6528394473b2bdeb9992c7ea15d6979eaf87d4af
-  data.tar.gz: e68858d2f088b2d7b8538411dd59cf2ae2de7866416fc213c6a6fa009d93c556
+  metadata.gz: 39397d5cb4e60a6d24cdec5bd979f543a23019b7c9b9dffe6140a204d330465c
+  data.tar.gz: 1db7c2b448179c2bc4b56e99428dfb4303cbb1451df032ec43cb5264f58935ec
 SHA512:
-  metadata.gz: 857126b2f7ec4fa058aaa8d5b4a7095108224bdf3f6ece690dbfc930e0527a294853705227f0e63be5af3524982fff21f7d3c9d940c22b31caade5139a3d607b
-  data.tar.gz: 81ecf95378e6f4aa31a87e39a82bc815216fce1b84aa65d8f7f2aa8ee8b19b871f08eb8c86025d9dc8d84617f20864f5f39c21d7b8ac4900a739599c0aa6283c
+  metadata.gz: 7058506211d537c8ea3c9a521625fd339b255f41188a70341cc04683ca1abc1fa7f19ed796026b5e07679bb2fd7e57f096d319fbe8a75ae6fb7fd59a704a9824
+  data.tar.gz: b8b60607cd27acbd654afe0816b0b7738871ca61c370fbafa747985d3723fec64f9b07c6078b34c1192db1d7b160682522a4901a7047c3d067860bc2b745b0b0

data/datahen.gemspec

@@ -33,10 +33,12 @@ Gem::Specification.new do |spec|
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-  spec.required_ruby_version = '>= 2.2.2'
+  spec.required_ruby_version = '>= 2.4.4'
   spec.add_dependency "thor", "~> 0.20.3"
   spec.add_dependency 'httparty', '~> 0.16.2'
   spec.add_dependency 'nokogiri', '~> 1.6'
+  spec.add_dependency 'concurrent-ruby', '~> 1.1'
+  spec.add_dependency 'parallel', '~> 1.20'
   spec.add_development_dependency 'bundler', '>= 1.16'
   spec.add_development_dependency 'rake', '>= 10.0'
   spec.add_development_dependency 'minitest', '>= 5.11'

data/lib/datahen/cli/parser.rb

@@ -43,17 +43,17 @@ module Datahen
       option :vars, :aliases => :v, type: :string, desc: 'Set user-defined page variables. Must be in json format. i.e: {"Foo":"bar"}'
       option :"keep-outputs", :aliases => :ko, type: :boolean, default: false, desc: "Don't delete existing outputs"
       def exec_parse(scraper_name, parser_file, *gids)
+        if options[:job]
+          job_id = options[:job]
+        else
+          job = Client::ScraperJob.new(options).find(scraper_name)
+          job_id = job['id']
+        end
+
         gids.each do |gid|
           begin
             puts "Parsing #{gid}"
 
-            if options[:job]
-              job_id = options[:job]
-            else
-              job = Client::ScraperJob.new(options).find(scraper_name)
-              job_id = job['id']
-            end
-
             vars = JSON.parse(options[:vars]) if options[:vars]
             puts Datahen::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, true, vars, options[:"keep-outputs"])
           rescue => e
@@ -61,6 +61,47 @@ module Datahen
           end
         end
       end
+
+      desc "batch <scraper_name> <config_file>", "Dequeue and execute Job Pages within a scraper's current job"
+      long_desc <<-LONGDESC
+            Dequeue pending job page(s) to execute their scripts and save the output to the scraper's current job\x5
+          LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :"keep-outputs", :aliases => :ko, type: :boolean, default: false, desc: "Don't delete existing outputs"
+      option :"workers", type: :numeric, default: 1, desc: "Worker count"
+      option :"max-garbage", type: :numeric, default: 5, desc: "Pages processed before calling the garbage collector"
+      option :"dequeue-interval", type: :numeric, default: 3, desc: "Seconds to wait between dequeueing"
+      option :"dequeue-scale", type: :numeric, default: 2, desc: "Scale vs worker count describing how many pages to dequeue"
+      option :"dequeue-timeout", type: :numeric, default: 30, desc: "Dequeue pages API request timeout"
+      def batch_exec_parse(scraper_name, config_file)
+        if options[:job]
+          job_id = options[:job]
+        else
+          job = Client::ScraperJob.new(options).find(scraper_name)
+          job_id = job['id']
+        end
+
+        # make the stdout and stderr sync to prevent buffering
+        old_stdout_sync = $stdout.sync
+        old_stderr_sync = $stderr.sync
+        $stdout.sync = true
+        $stderr.sync = true
+
+        begin
+          batch = Datahen::Scraper::BatchParser.new job_id, config_file,
+            worker_count: options[:"workers"],
+            max_garbage: options[:"max-garbage"],
+            dequeue_interval: options[:"dequeue-interval"],
+            dequeue_scale: options[:"dequeue-scale"]
+          batch.exec_parse true, options[:"keep-outputs"]
+        rescue => e
+          puts [e.message] + e.backtrace
+        end
+
+        # resume whatever state the stdout and stderr sync were
+        $stdout.sync = old_stdout_sync
+        $stderr.sync = old_stderr_sync
+      end
     end
   end
 

data/lib/datahen/cli/scraper.rb

@@ -32,6 +32,7 @@ module Datahen
       option :profile, type: :string, desc: 'Set the profiles (comma separated) to apply to the job. Default: default'
       option :multiple_jobs, type: :boolean, desc: 'Set true to enable multiple jobs. Default: false'
       option :max_job_count, type: :numeric, desc: 'Set a value to set max number of jobs available. Set -1 for unlimited. Default: 3'
+      option :max_page_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
       def create(scraper_name, git_repository)
         # puts "options #{options}"
         client = Client::Scraper.new(options)
@@ -57,6 +58,7 @@ module Datahen
       option :profile, type: :string, desc: 'Set the profiles (comma separated) to apply to the job. Default: default'
       option :multiple_jobs, type: :boolean, desc: 'Set true to enable multiple jobs. Default: false'
       option :max_job_count, type: :numeric, desc: 'Set a value to set max number of jobs available. Set -1 for unlimited. Default: 3'
+      option :max_page_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
       def update(scraper_name)
         client = Client::Scraper.new(options)
         puts "#{client.update(scraper_name, options)}"
@@ -94,6 +96,7 @@ module Datahen
       option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
       option :proxy_type, desc: 'Set the Proxy type. Default: standard'
       option :vars, type: :string, banner: :JSON, desc: 'Set input vars. Must be in json format. i.e: [{"name":"foo", "value":"bar", "secret":false}] '
+      option :max_page_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
       def start(scraper_name)
         client = Client::ScraperJob.new(options)
         puts "Starting a scrape job..."

data/lib/datahen/cli/scraper_job.rb

@@ -104,6 +104,7 @@ module Datahen
       option :proxy_type, desc: 'Set the Proxy type. Default: standard'
       option :profile, type: :string, desc: 'Set the profiles (comma separated) to apply to the job. Default: default'
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :max_page_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
       def update(scraper_name)
         if options[:job]
           client = Client::Job.new(options)

data/lib/datahen/cli/scraper_page.rb

@@ -45,6 +45,7 @@ module Datahen
       option :freshness, :aliases => :s, desc: 'Set how fresh the page cache is. Accepts timestap format.'
       option :ua_type, :aliases => :u, desc: 'Set user agent type. Default: desktop'
       option :no_redirect, :aliases => :n, type: :boolean, desc: 'Set true to not follow redirect. Default: false'
+      option :max_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
       def add(scraper_name, url)
         begin
           options[:headers] = JSON.parse(options[:headers]) if options[:headers]
@@ -78,6 +79,7 @@ module Datahen
       option :page_type, :aliases => :t, desc: 'Set page type'
       option :priority, type: :numeric, desc: 'Set fetch priority. The higher the value, the sooner the page gets fetched. Default: 0'
       option :vars, :aliases => :v, type: :string, desc: 'Set user-defined page variables. Must be in json format. i.e: {"Foo":"bar"}'
+      option :max_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
       def update(scraper_name, gid)
         begin
           options[:vars] = JSON.parse(options[:vars]) if options[:vars]

data/lib/datahen/client/job.rb

@@ -21,6 +21,7 @@ module Datahen
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
         body[:profile] = opts[:profile] if opts[:profile]
+        body[:max_page_size] = opts[:max_page_size] if opts[:max_page_size]
         params = @options.merge({body: body.to_json})
 
         self.class.put("/jobs/#{job_id}", params)

data/lib/datahen/client/job_page.rb

@@ -15,6 +15,7 @@ module Datahen
         body[:page_type] = opts[:page_type] if opts[:page_type]
         body[:priority] = opts[:priority] if opts[:priority]
         body[:vars] = opts[:vars] if opts[:vars]
+        body[:max_size] = opts[:max_size] if opts[:max_size]
 
         params = @options.merge({body: body.to_json})
 
@@ -36,12 +37,23 @@ module Datahen
         body[:ua_type] = opts[:ua_type] if opts[:ua_type]
         body[:no_redirect] = opts[:no_redirect] if opts[:no_redirect]
         body[:cookie] = opts[:cookie] if opts[:cookie]
+        body[:max_size] = opts[:max_size] if opts[:max_size]
 
         params = @options.merge({body: body.to_json})
 
         self.class.post("/jobs/#{job_id}/pages", params)
       end
 
+      def dequeue(job_id, limit, page_types, parse_fetching_failed, opts = {})
+        body = {
+          limit: limit,
+          page_types: page_types,
+          parse_fetching_failed: parse_fetching_failed
+        }
+        params = @options.merge(opts).merge({body: body.to_json})
+        self.class.put("/jobs/#{job_id}/pages/parse_dequeue", params)
+      end
+
       def parsing_update(job_id, gid, opts={})
         body = {}
         body[:outputs] = opts.fetch(:outputs) {[]}

data/lib/datahen/client/scraper.rb

@@ -28,6 +28,7 @@ module Datahen
         body[:profile] = opts[:profile] if opts[:profile]
         body[:multiple_jobs] = opts[:multiple_jobs] if opts[:multiple_jobs]
         body[:max_job_count] = opts[:max_job_count] if opts[:max_job_count]
+        body[:max_page_size] = opts[:max_page_size] if opts[:max_page_size]
         params = @options.merge({body: body.to_json})
         self.class.post("/scrapers", params)
       end
@@ -49,6 +50,7 @@ module Datahen
         body[:profile] = opts[:profile] if opts[:profile]
         body[:multiple_jobs] = opts[:multiple_jobs] if opts.has_key?("multiple_jobs") || opts.has_key?(:multiple_jobs)
         body[:max_job_count] = opts[:max_job_count] if opts.has_key?("max_job_count") || opts.has_key?(:max_job_count)
+        body[:max_page_size] = opts[:max_page_size] if opts.has_key?("max_page_size") || opts.has_key?(:max_page_size)
         params = @options.merge({body: body.to_json})
 
         self.class.put("/scrapers/#{scraper_name}", params)

data/lib/datahen/client/scraper_job.rb

@@ -11,6 +11,7 @@ module Datahen
         body[:standard_worker_count] = opts[:workers] if opts[:workers]
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
+        body[:max_page_size] = opts[:max_page_size] if opts[:max_page_size]
         if opts[:vars]
           if opts[:vars].is_a?(Array)
             body[:vars] = opts[:vars]
@@ -37,6 +38,7 @@ module Datahen
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
         body[:profile] = opts[:profile] if opts[:profile]
+        body[:max_page_size] = opts[:max_page_size] if opts[:max_page_size]
         params = @options.merge({body: body.to_json})
 
         self.class.put("/scrapers/#{scraper_name}/current_job", params)

data/lib/datahen/scraper.rb

@@ -1,6 +1,7 @@
 require "datahen/error"
 require "datahen/plugin"
 require "datahen/scraper/parser"
+require "datahen/scraper/batch_parser"
 require "datahen/scraper/seeder"
 require "datahen/scraper/finisher"
 require "datahen/scraper/executor"

data/lib/datahen/scraper/batch_parser.rb

@@ -0,0 +1,358 @@
+require 'concurrent'
+require 'parallel'
+
+module Datahen
+  module Scraper
+    class BatchParser
+      NOT_FOUND_MSG = "No more pages to parse found"
+      NO_DEQUEUE_COUNT_MSG = "\nWarning: Max page to parse dequeue count is 0, check pages to parse scale\n"
+      NO_WORKERS_MSG = "\nWarning: There are no parser workers\n"
+
+      # Configuration file path.
+      # @return [String] config file path
+      attr_accessor :config_file
+      # Garbage collector request counter.
+      # @return [Integer] garbage collector counter
+      attr_accessor :garbage_count
+      # Last printed message, useful to prevent duplicated log messages.
+      # @return [String] last printed message
+      attr_accessor :last_message
+      # Second dequeue counter used to prevent false negative warning messages.
+      # @return [Integer] second dequeue counter
+      attr_accessor :second_dequeue_count
+      # Dequeue API request timeout in seconds.
+      # @return [Integer] dequeue API request timeout in seconds
+      attr_accessor :dequeue_timeout
+      # Job id to be executed.
+      # @return [Integer] job id
+      attr_reader :job_id
+      # Parallel worker quantity.
+      # @return [Integer] parallel worker quantity
+      attr_reader :worker_count
+      # Loaded pages array.
+      # @return [Concurrent::Array<Hash>] loaded pages as an array
+      attr_reader :pages
+      # Loaded pages hash, useful to avoid duplicates on the loaded pages array.
+      # @return [Concurrent::Hash<String, Hash>] loaded pages as a concurrent hash
+      attr_reader :loaded_pages
+      # Max garbage collector requests before actually executing the garbage
+      #   collector.
+      # @return [Integer] max garbage request quantity before actually executing
+      #   it
+      attr_reader :max_garbage
+      # Dequeue interval in seconds.
+      # @return [Integer] dequeue interval in seconds
+      attr_reader :dequeue_interval
+      # Dequeue scale used to calculate the ideal dequeue size.
+      # @return [Numeric] dequeue scale
+      attr_reader :dequeue_scale
+      # Known page types extracted from the config file.
+      # @return [Array<String>] known page types
+      attr_reader :page_types
+      # Known parsers extracted from the config file.
+      # @return [Concurrent::Hash<String, String>] known parsers
+      attr_reader :parsers
+      # Current config file loaded.
+      # @return [Hash] current loaded configuration
+      attr_reader :config
+      # Datahen job pages client used for API pages dequeuing.
+      # @return [Datahen::Client::JobPage] datahen job pages API client
+      attr_reader :client
+      # Garbage collector mutex used to synchronize garbage collector requests.
+      # @return [Mutex] garbage collector mutex
+      attr_reader :garbage_mutex
+      # Current dequeuer thread.
+      # @return [Thread] dequeuer thread
+      attr_reader :dequeuer_thread
+      # Dequeuer mutext used to synchronize page dequeuing.
+      # @return [Mutex] dequeuer mutex
+      attr_reader :dequeue_mutex
+      # Dequeuer last run unix timestamp.
+      # @return [Integer] dequeuer last run unix timestamp
+      attr_reader :dequeuer_still_alive
+      # Indicates whenever the wait time is because there are no more pages.
+      # @return [Boolean] `true` when wait time is due to no more pages,
+      #   else `false`
+      attr_reader :not_found
+
+      # Wait a specific amount of seconds.
+      # @param [Integer] time_in_seconds Seconds to wait.
+      def self.wait time_in_seconds
+        Kernel.sleep time_in_seconds
+      end
+
+      # Get a unix timestamp.
+      # @return [Integer] unix timestamp
+      def self.timestamp
+        Time.new.utc.to_i
+      end
+
+      # Initialize a batch parser object.
+      # @param [Integer] job_id Job id.
+      # @param [String] config_file Config file path.
+      # @param [Hash] opts ({}) Configuration options
+        # @option opts [Integer] :worker_count (1) Parallel worker quantity.
+        # @option opts [Integer] :max_garbage (5) Max amount of times the garbage
+        #   collector can be requested before actually executing.
+        # @option opts [Integer] :dequeue_interval (3) Time in seconds to wait
+        #   between page dequeuing.
+        # @option opts [Numeric] :dequeue_scale (2) Scaling factor to used to
+        #   calculate page dequeue size.
+        # @option opts [Numeric] :dequeue_timeout (30) Page dequeue API request
+        #   timeout in seconds.
+        # @option opts [Hash] :client_options ({}) Datahen client gem additional
+        #   options (see Datahen::Client::Base#initialize method).
+      def initialize(job_id, config_file, opts = {})
+        opts = {
+          worker_count: 1,
+          max_garbage: 5,
+          dequeue_interval: 3,
+          dequeue_scale: 2,
+          dequeue_timeout: 30,
+          client_options: {}
+        }.merge opts
+
+        @job_id = job_id
+        @worker_count = opts[:worker_count]
+        @dequeue_interval = opts[:dequeue_interval]
+        @dequeue_scale = opts[:dequeue_scale]
+        @max_garbage = opts[:max_garbage]
+        @pages = Concurrent::Array.new
+        @loaded_pages = Concurrent::Hash.new
+        @garbage_mutex = Mutex.new
+        @dequeue_mutex = Mutex.new
+        @not_found = false
+        self.dequeue_timeout = opts[:dequeue_timeout]
+        self.second_dequeue_count = 0
+        self.garbage_count = 0
+        self.config_file = config_file
+        self.load_config
+
+        @client = Datahen::Client::JobPage.new(opts[:client_options])
+        nil
+      end
+
+      # Execute garbage collector after it is requested as many times as
+      #   described by #max_garbage.
+      def recollect_garbage
+        self.garbage_mutex.synchronize do
+          self.garbage_count += 1
+          if self.garbage_count > self.max_garbage
+            puts "Recollect garbage"
+            GC.start
+            self.garbage_count = 0
+          end
+        end
+        nil
+      end
+
+      # Loads the config file into a Hash.
+      def load_config
+        # build page type to script file map
+        @page_types = []
+        @parsers = Concurrent::Hash.new
+        @config = YAML.load_file(config_file)
+        self.config['parsers'].each do |v|
+          next if !v['disabled'].nil? && !!v['disabled']
+          @page_types << v['page_type']
+          self.parsers[v['page_type']] = v['file']
+        end
+        self.recollect_garbage
+        nil
+      end
+
+      # Print the message regardless of it being the same as the last message.
+      # @param [String] message Message to display.
+      def repeat_puts message
+        puts message
+        self.last_message = message
+        nil
+      end
+
+      # Print the message only when it is different from the last recorded
+      #   message.
+      # @param [String] message Message to display.
+      def no_repeat_puts message
+        return if message == self.last_message
+        puts message
+        self.last_message = message
+        nil
+      end
+
+      # Refresh dequeuer's still alive timestamp
+      def dequeuer_is_alive!
+        self.dequeue_mutex.synchronize do
+          @dequeuer_still_alive = self.class.timestamp
+        end
+        nil
+      end
+
+      # Load new pages by dequeuing from the API.
+      # @return [Integer] amount of pages loaded
+      def load_pages
+        self.dequeuer_is_alive!
+
+        # calculate dequeue size
+        max_dequeue_size = (self.worker_count * self.dequeue_scale).ceil
+        current_size = self.pages.length
+        dequeue_size = (self.dequeue_scale * (max_dequeue_size - current_size)).ceil
+        if dequeue_size < 1
+          return 0
+        end
+        dequeue_size = max_dequeue_size if dequeue_size > max_dequeue_size
+
+        # reserve and get to pages parse
+        response = nil
+        begin
+          response = client.dequeue self.job_id,
+            dequeue_size,
+            self.page_types,
+            config['parse_fetching_failed'],
+            timeout: self.dequeue_timeout
+        rescue Net::ReadTimeout, Net::OpenTimeout => e
+          self.repeat_puts "Dequeue API call timeout! Contact infra team, your job needs a profile change"
+          self.dequeuer_is_alive!
+          return 0
+        rescue => e
+          raise e
+        end
+        self.dequeuer_is_alive!
+
+        # ensure a valid response or try again
+        if response.nil? || response.response.code.to_i != 200
+          self.repeat_puts(response.nil? ? 'null' : response.body)
+          self.recollect_garbage
+          return 0
+        end
+
+        # add pages
+        count = 0
+        (JSON.parse(response.body) || []).each do |page|
+          count += 1
+          next if self.loaded_pages.has_key? page['gid']
+          self.pages << (self.loaded_pages[page['gid']] = page)
+        end
+        response = nil
+        self.dequeuer_is_alive!
+
+        # recolect garbage to free some memory before parsing
+        if count > 0
+          @not_found = false
+          self.recollect_garbage
+          self.repeat_puts "Found #{count} page(s) to parse"
+          self.second_dequeue_count += 1 unless self.second_dequeue_count > 1
+        else
+          @not_found = true
+          self.no_repeat_puts NOT_FOUND_MSG
+        end
+
+        # return how many pages were loaded
+        count
+      end
+
+      # Ensures that the dequeuer thread exists and is running.
+      # @return [Boolean] `true` if thread was alive, or `false` if had to
+      #   create a new thread
+      def ensure_dequeuer_thread
+        self.dequeue_mutex.synchronize do
+          # check if dequeuer thread is alive and healthy
+          if !self.dequeuer_thread.nil? && self.dequeuer_thread.alive?
+            still_alive_timeout = (self.dequeue_timeout + self.dequeue_interval) * 2 + self.dequeuer_still_alive
+            return true if self.class.timestamp < still_alive_timeout
+
+            # kill dequeuer thread
+            self.repeat_puts "Dequeuer isn't healthy, will restart it..."
+            self.dequeuer_thread.kill
+            @dequeuer_thread = nil
+            self.recollect_garbage
+            self.no_repeat_puts "Dequeuer thread was killed!"
+          end
+
+          # dequeuing on parallel (the ride never ends :D)
+          @dequeuer_thread = Thread.new do
+            while true
+              begin
+                self.load_pages
+                self.class.wait self.dequeue_interval
+              rescue => e
+                puts [e.message] + e.backtrace rescue 'error'
+              end
+            end
+            puts "Error: dequeuer died! D:"
+          end
+          self.repeat_puts "Dequeuer thread was started!"
+        end
+        false
+      end
+
+      # Dequeue one page from the previously loaded pages, and waits until there
+      #   are new pages whenever there are no loaded pages.
+      # @return [Hash] dequeued page
+      def dequeue_pages
+        # collect garbage
+        self.recollect_garbage
+
+        # return page if there are loeaded pages
+        is_waiting = false
+        while true do
+          page = self.pages.shift
+          unless page.nil?
+            puts "[Worker #{Parallel.worker_number}]: Finish waiting" if is_waiting
+            loaded_pages.delete(page['gid'])
+            return page
+          end
+
+          # be more verbose on worker waiting
+          unless is_waiting
+            is_waiting = true
+            puts "[Worker #{Parallel.worker_number}]: Is waiting for a page..."
+            if self.second_dequeue_count > 1 && !self.not_found
+              puts "\nWARNING: Your job is not optimized, increase your job's \"parser_dequeue_scale\"\n"
+            end
+          end
+          self.class.wait 1
+
+          # ensure the dequeuer thread is alive and healthy
+          self.ensure_dequeuer_thread
+        end
+      end
+
+      # Dequeue pages and execute the parsers associated to them on parallel.
+      def exec_parse save = false, keep_outputs = false
+        if self.worker_count < 1
+          self.no_repeat_puts NO_WORKERS_MSG
+          return
+        else
+          self.no_repeat_puts "Spawing #{self.worker_count} workers"
+        end
+
+        # start dequeuer
+        self.ensure_dequeuer_thread
+
+        # process the pages
+        dequeue = lambda{ self.dequeue_pages }
+        Parallel.each(dequeue, in_threads: (worker_count)) do |page|
+          parser_file = self.parsers[page['page_type']]
+          begin
+            puts Datahen::Scraper::Parser.exec_parser_by_page(
+              parser_file,
+              page,
+              job_id,
+              save,
+              nil,
+              keep_outputs
+            )
+          rescue Parallel::Kill => e
+            puts "[Worker #{Parallel.worker_number}]: Someone tried to kill Parallel!!!"
+          rescue Parallel::Break => e
+            puts "[Worker #{Parallel.worker_number}]: Someone tried to break Parallel!!!"
+          rescue => e
+            puts [e.message] + e.backtrace rescue 'error'
+          end
+        end
+
+        nil
+      end
+    end
+  end
+end

data/lib/datahen/scraper/executor.rb

@@ -6,7 +6,7 @@ module Datahen
       # Max allowed page size when query outputs (see #find_outputs).
       MAX_FIND_OUTPUTS_PER_PAGE = 500
 
-      attr_accessor :filename, :gid, :job_id
+      attr_accessor :filename, :page, :gid, :job_id
 
       include Datahen::Plugin::ContextExposer
 
@@ -15,6 +15,9 @@ module Datahen
       end
 
       def init_page()
+        # skip whenever a page is provided
+        return self.page unless self.page.nil?
+
         if job_id
           puts "getting Job Page"
           init_job_page

data/lib/datahen/scraper/parser.rb

@@ -18,6 +18,22 @@ module Datahen
         end
       end
 
+      def self.exec_parser_by_page(filename, page, job_id=nil, save=false, vars = {}, keep_outputs=false)
+        extname = File.extname(filename)
+        case extname
+        when '.rb'
+          executor = RubyParserExecutor.new(
+            filename: filename,
+            page: page,
+            job_id: job_id,
+            vars: vars,
+            keep_outputs: keep_outputs
+          )
+          executor.exec_parser(save)
+        else
+          puts "Unable to find a parser executor for file type \"#{extname}\""
+        end
+      end
 
     end
   end

data/lib/datahen/scraper/ruby_parser_executor.rb

@@ -12,7 +12,8 @@ module Datahen
 
       def initialize(options={})
         @filename = options.fetch(:filename) { raise "Filename is required"}
-        @gid = options.fetch(:gid) { raise "GID is required"}
+        @page = options.fetch(:page) { nil }
+        @gid = (self.page || {})['gid'] || options.fetch(:gid) { raise "GID or a page with a GID is required"}
         @job_id = options.fetch(:job_id)
         @page_vars = options.fetch(:vars) { {} }
         @keep_outputs = !!(options.fetch(:keep_outputs) { false })
@@ -46,6 +47,8 @@ module Datahen
       end
 
       def init_page_vars(page)
+        return self.page unless self.page.nil?
+
         if !@page_vars.nil? && !@page_vars.empty?
           page['vars'] = @page_vars
         end

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "0.14.26"
+  VERSION = "0.16.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 0.14.26
+  version: 0.16.0
 platform: ruby
 authors:
 - Parama Danoesubroto
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-04-20 00:00:00.000000000 Z
+date: 2021-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -52,6 +52,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.20'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.20'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -232,6 +260,7 @@ files:
 - lib/datahen/plugin.rb
 - lib/datahen/plugin/context_exposer.rb
 - lib/datahen/scraper.rb
+- lib/datahen/scraper/batch_parser.rb
 - lib/datahen/scraper/executor.rb
 - lib/datahen/scraper/finisher.rb
 - lib/datahen/scraper/parser.rb
@@ -247,7 +276,7 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://datahen.com
   source_code_uri: https://github.com/DataHenOfficial/datahen-ruby
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -255,7 +284,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.2.2
+      version: 2.4.4
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -263,7 +292,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key: 
+signing_key:
 specification_version: 4
 summary: DataHen toolbelt for developers
 test_files: []