datahen 0.14.24 → 0.15.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 54064cf5656d253f683540fa3704cdcae3991bc07a91ab5339bf9083eb0792f2
-  data.tar.gz: 3c82cedd06937454a9af5d91eea9b32f3ad744394763ca9fadbcdc6068eda683
+  metadata.gz: f77e31da8e2a7ff08086c4aa9d174608a9c3f186679d456b22310b48384d3572
+  data.tar.gz: 0bf53ae0886b16bf6fe08b0db07b1a631b69f31d8e3a6868a4d483549049e4ed
 SHA512:
-  metadata.gz: fb17b046f9dbd15cf7a278f68f550e8b3b84d8c16a030d0ef4df100df28c8e4dce29fe74ffc70e02f79af26a2d1e3f66ccb6890e06a342a70fac09d824169431
-  data.tar.gz: 732256fc714635896f444bed50e2d6f49c2a03c4868fd23eb84e52701935bc2bfbef25811055983691c5e0ff39af53328dbeb67faf91f127c1d06af450d7d666
+  metadata.gz: a491874347ed6ac97c0a0e4f0d2c5830140b9367c1e01b4c95e7a447b071df643d12793ed7d6e5a0224b8876905cf74bc13b987e6e3e03e937d1f821557b8ec3
+  data.tar.gz: c553a372790654726f2921b6d9d582ca90b314b3fe8d78625cf2442e01cf2ce96ae556e78812d5bd6a03f350beb32aa3158981554179410c12926d480c911887

data/datahen.gemspec

@@ -33,10 +33,12 @@ Gem::Specification.new do |spec|
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-  spec.required_ruby_version = '>= 2.2.2'
+  spec.required_ruby_version = '>= 2.4.4'
   spec.add_dependency "thor", "~> 0.20.3"
   spec.add_dependency 'httparty', '~> 0.16.2'
-  spec.add_dependency 'nokogiri', '~> 1.6', '< 1.10'
+  spec.add_dependency 'nokogiri', '~> 1.6'
+  spec.add_dependency 'concurrent-ruby', '~> 1.1'
+  spec.add_dependency 'parallel', '~> 1.20'
   spec.add_development_dependency 'bundler', '>= 1.16'
   spec.add_development_dependency 'rake', '>= 10.0'
   spec.add_development_dependency 'minitest', '>= 5.11'

data/lib/datahen/cli/parser.rb

@@ -43,17 +43,17 @@ module Datahen
       option :vars, :aliases => :v, type: :string, desc: 'Set user-defined page variables. Must be in json format. i.e: {"Foo":"bar"}'
       option :"keep-outputs", :aliases => :ko, type: :boolean, default: false, desc: "Don't delete existing outputs"
       def exec_parse(scraper_name, parser_file, *gids)
+        if options[:job]
+          job_id = options[:job]
+        else
+          job = Client::ScraperJob.new(options).find(scraper_name)
+          job_id = job['id']
+        end
+
         gids.each do |gid|
           begin
             puts "Parsing #{gid}"
 
-            if options[:job]
-              job_id = options[:job]
-            else
-              job = Client::ScraperJob.new(options).find(scraper_name)
-              job_id = job['id']
-            end
-
             vars = JSON.parse(options[:vars]) if options[:vars]
             puts Datahen::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, true, vars, options[:"keep-outputs"])
           rescue => e
@@ -61,6 +61,47 @@ module Datahen
           end
         end
       end
+
+      desc "batch <scraper_name> <config_file>", "Dequeue and execute Job Pages within a scraper's current job"
+      long_desc <<-LONGDESC
+            Dequeue pending job page(s) to execute their scripts and save the output to the scraper's current job\x5
+          LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :"keep-outputs", :aliases => :ko, type: :boolean, default: false, desc: "Don't delete existing outputs"
+      option :"workers", type: :numeric, default: 1, desc: "Worker count"
+      option :"max-garbage", type: :numeric, default: 5, desc: "Pages processed before calling the garbage collector"
+      option :"dequeue-interval", type: :numeric, default: 3, desc: "Seconds to wait between dequeueing"
+      option :"dequeue-scale", type: :numeric, default: 2, desc: "Scale vs worker count describing how many pages to dequeue"
+      option :"dequeue-timeout", type: :numeric, default: 30, desc: "Dequeue pages API request timeout"
+      def batch_exec_parse(scraper_name, config_file)
+        if options[:job]
+          job_id = options[:job]
+        else
+          job = Client::ScraperJob.new(options).find(scraper_name)
+          job_id = job['id']
+        end
+
+        # make the stdout and stderr sync to prevent buffering
+        old_stdout_sync = $stdout.sync
+        old_stderr_sync = $stderr.sync
+        $stdout.sync = true
+        $stderr.sync = true
+
+        begin
+          batch = Datahen::Scraper::BatchParser.new job_id, config_file,
+            worker_count: options[:"workers"],
+            max_garbage: options[:"max-garbage"],
+            dequeue_interval: options[:"dequeue-interval"],
+            dequeue_scale: options[:"dequeue-scale"]
+          batch.exec_parse true, options[:"keep-outputs"]
+        rescue => e
+          puts [e.message] + e.backtrace
+        end
+
+        # resume whatever state the stdout and stderr sync were
+        $stdout.sync = old_stdout_sync
+        $stderr.sync = old_stderr_sync
+      end
     end
   end
 

data/lib/datahen/client/backblaze_content.rb

@@ -7,7 +7,7 @@ module Datahen
       include HTTParty
 
       def get_content(url)
-        self.class.get(url, format: :plain)
+        self.class.get(url, format: :plain).response.body
       end
 
       def get_gunzipped_content(url)

data/lib/datahen/client/job_page.rb

@@ -42,6 +42,16 @@ module Datahen
         self.class.post("/jobs/#{job_id}/pages", params)
       end
 
+      def dequeue(job_id, limit, page_types, parse_fetching_failed, opts = {})
+        body = {
+          limit: limit,
+          page_types: page_types,
+          parse_fetching_failed: parse_fetching_failed
+        }
+        params = @options.merge(opts).merge({body: body.to_json})
+        self.class.put("/jobs/#{job_id}/pages/parse_dequeue", params)
+      end
+
       def parsing_update(job_id, gid, opts={})
         body = {}
         body[:outputs] = opts.fetch(:outputs) {[]}

data/lib/datahen/scraper.rb

@@ -1,6 +1,7 @@
 require "datahen/error"
 require "datahen/plugin"
 require "datahen/scraper/parser"
+require "datahen/scraper/batch_parser"
 require "datahen/scraper/seeder"
 require "datahen/scraper/finisher"
 require "datahen/scraper/executor"

data/lib/datahen/scraper/batch_parser.rb

@@ -0,0 +1,358 @@
+require 'concurrent'
+require 'parallel'
+
+module Datahen
+  module Scraper
+    class BatchParser
+      NOT_FOUND_MSG = "No more pages to parse found"
+      NO_DEQUEUE_COUNT_MSG = "\nWarning: Max page to parse dequeue count is 0, check pages to parse scale\n"
+      NO_WORKERS_MSG = "\nWarning: There are no parser workers\n"
+
+      # Configuration file path.
+      # @return [String] config file path
+      attr_accessor :config_file
+      # Garbage collector request counter.
+      # @return [Integer] garbage collector counter
+      attr_accessor :garbage_count
+      # Last printed message, useful to prevent duplicated log messages.
+      # @return [String] last printed message
+      attr_accessor :last_message
+      # Second dequeue counter used to prevent false negative warning messages.
+      # @return [Integer] second dequeue counter
+      attr_accessor :second_dequeue_count
+      # Dequeue API request timeout in seconds.
+      # @return [Integer] dequeue API request timeout in seconds
+      attr_accessor :dequeue_timeout
+      # Job id to be executed.
+      # @return [Integer] job id
+      attr_reader :job_id
+      # Parallel worker quantity.
+      # @return [Integer] parallel worker quantity
+      attr_reader :worker_count
+      # Loaded pages array.
+      # @return [Concurrent::Array<Hash>] loaded pages as an array
+      attr_reader :pages
+      # Loaded pages hash, useful to avoid duplicates on the loaded pages array.
+      # @return [Concurrent::Hash<String, Hash>] loaded pages as a concurrent hash
+      attr_reader :loaded_pages
+      # Max garbage collector requests before actually executing the garbage
+      #   collector.
+      # @return [Integer] max garbage request quantity before actually executing
+      #   it
+      attr_reader :max_garbage
+      # Dequeue interval in seconds.
+      # @return [Integer] dequeue interval in seconds
+      attr_reader :dequeue_interval
+      # Dequeue scale used to calculate the ideal dequeue size.
+      # @return [Numeric] dequeue scale
+      attr_reader :dequeue_scale
+      # Known page types extracted from the config file.
+      # @return [Array<String>] known page types
+      attr_reader :page_types
+      # Known parsers extracted from the config file.
+      # @return [Concurrent::Hash<String, String>] known parsers
+      attr_reader :parsers
+      # Current config file loaded.
+      # @return [Hash] current loaded configuration
+      attr_reader :config
+      # Datahen job pages client used for API pages dequeuing.
+      # @return [Datahen::Client::JobPage] datahen job pages API client
+      attr_reader :client
+      # Garbage collector mutex used to synchronize garbage collector requests.
+      # @return [Mutex] garbage collector mutex
+      attr_reader :garbage_mutex
+      # Current dequeuer thread.
+      # @return [Thread] dequeuer thread
+      attr_reader :dequeuer_thread
+      # Dequeuer mutext used to synchronize page dequeuing.
+      # @return [Mutex] dequeuer mutex
+      attr_reader :dequeue_mutex
+      # Dequeuer last run unix timestamp.
+      # @return [Integer] dequeuer last run unix timestamp
+      attr_reader :dequeuer_still_alive
+      # Indicates whenever the wait time is because there are no more pages.
+      # @return [Boolean] `true` when wait time is due to no more pages,
+      #   else `false`
+      attr_reader :not_found
+
+      # Wait a specific amount of seconds.
+      # @param [Integer] time_in_seconds Seconds to wait.
+      def self.wait time_in_seconds
+        Kernel.sleep time_in_seconds
+      end
+
+      # Get a unix timestamp.
+      # @return [Integer] unix timestamp
+      def self.timestamp
+        Time.new.utc.to_i
+      end
+
+      # Initialize a batch parser object.
+      # @param [Integer] job_id Job id.
+      # @param [String] config_file Config file path.
+      # @param [Hash] opts ({}) Configuration options
+        # @option opts [Integer] :worker_count (1) Parallel worker quantity.
+        # @option opts [Integer] :max_garbage (5) Max amount of times the garbage
+        #   collector can be requested before actually executing.
+        # @option opts [Integer] :dequeue_interval (3) Time in seconds to wait
+        #   between page dequeuing.
+        # @option opts [Numeric] :dequeue_scale (2) Scaling factor to used to
+        #   calculate page dequeue size.
+        # @option opts [Numeric] :dequeue_timeout (30) Page dequeue API request
+        #   timeout in seconds.
+        # @option opts [Hash] :client_options ({}) Datahen client gem additional
+        #   options (see Datahen::Client::Base#initialize method).
+      def initialize(job_id, config_file, opts = {})
+        opts = {
+          worker_count: 1,
+          max_garbage: 5,
+          dequeue_interval: 3,
+          dequeue_scale: 2,
+          dequeue_timeout: 30,
+          client_options: {}
+        }.merge opts
+
+        @job_id = job_id
+        @worker_count = opts[:worker_count]
+        @dequeue_interval = opts[:dequeue_interval]
+        @dequeue_scale = opts[:dequeue_scale]
+        @max_garbage = opts[:max_garbage]
+        @pages = Concurrent::Array.new
+        @loaded_pages = Concurrent::Hash.new
+        @garbage_mutex = Mutex.new
+        @dequeue_mutex = Mutex.new
+        @not_found = false
+        self.dequeue_timeout = opts[:dequeue_timeout]
+        self.second_dequeue_count = 0
+        self.garbage_count = 0
+        self.config_file = config_file
+        self.load_config
+
+        @client = Datahen::Client::JobPage.new(opts[:client_options])
+        nil
+      end
+
+      # Execute garbage collector after it is requested as many times as
+      #   described by #max_garbage.
+      def recollect_garbage
+        self.garbage_mutex.synchronize do
+          self.garbage_count += 1
+          if self.garbage_count > self.max_garbage
+            puts "Recollect garbage"
+            GC.start
+            self.garbage_count = 0
+          end
+        end
+        nil
+      end
+
+      # Loads the config file into a Hash.
+      def load_config
+        # build page type to script file map
+        @page_types = []
+        @parsers = Concurrent::Hash.new
+        @config = YAML.load_file(config_file)
+        self.config['parsers'].each do |v|
+          next if !v['disabled'].nil? && !!v['disabled']
+          @page_types << v['page_type']
+          self.parsers[v['page_type']] = v['file']
+        end
+        self.recollect_garbage
+        nil
+      end
+
+      # Print the message regardless of it being the same as the last message.
+      # @param [String] message Message to display.
+      def repeat_puts message
+        puts message
+        self.last_message = message
+        nil
+      end
+
+      # Print the message only when it is different from the last recorded
+      #   message.
+      # @param [String] message Message to display.
+      def no_repeat_puts message
+        return if message == self.last_message
+        puts message
+        self.last_message = message
+        nil
+      end
+
+      # Refresh dequeuer's still alive timestamp
+      def dequeuer_is_alive!
+        self.dequeue_mutex.synchronize do
+          @dequeuer_still_alive = self.class.timestamp
+        end
+        nil
+      end
+
+      # Load new pages by dequeuing from the API.
+      # @return [Integer] amount of pages loaded
+      def load_pages
+        self.dequeuer_is_alive!
+
+        # calculate dequeue size
+        max_dequeue_size = (self.worker_count * self.dequeue_scale).ceil
+        current_size = self.pages.length
+        dequeue_size = (self.dequeue_scale * (max_dequeue_size - current_size)).ceil
+        if dequeue_size < 1
+          return 0
+        end
+        dequeue_size = max_dequeue_size if dequeue_size > max_dequeue_size
+
+        # reserve and get to pages parse
+        response = nil
+        begin
+          response = client.dequeue self.job_id,
+            dequeue_size,
+            self.page_types,
+            config['parse_fetching_failed'],
+            timeout: self.dequeue_timeout
+        rescue Net::ReadTimeout, Net::OpenTimeout => e
+          self.repeat_puts "Dequeue API call timeout! Contact infra team, your job needs a profile change"
+          self.dequeuer_is_alive!
+          return 0
+        rescue => e
+          raise e
+        end
+        self.dequeuer_is_alive!
+
+        # ensure a valid response or try again
+        if response.nil? || response.response.code.to_i != 200
+          self.repeat_puts(response.nil? ? 'null' : response.body)
+          self.recollect_garbage
+          return 0
+        end
+
+        # add pages
+        count = 0
+        (JSON.parse(response.body) || []).each do |page|
+          count += 1
+          next if self.loaded_pages.has_key? page['gid']
+          self.pages << (self.loaded_pages[page['gid']] = page)
+        end
+        response = nil
+        self.dequeuer_is_alive!
+
+        # recolect garbage to free some memory before parsing
+        if count > 0
+          @not_found = false
+          self.recollect_garbage
+          self.repeat_puts "Found #{count} page(s) to parse"
+          self.second_dequeue_count += 1 unless self.second_dequeue_count > 1
+        else
+          @not_found = true
+          self.no_repeat_puts NOT_FOUND_MSG
+        end
+
+        # return how many pages were loaded
+        count
+      end
+
+      # Ensures that the dequeuer thread exists and is running.
+      # @return [Boolean] `true` if thread was alive, or `false` if had to
+      #   create a new thread
+      def ensure_dequeuer_thread
+        self.dequeue_mutex.synchronize do
+          # check if dequeuer thread is alive and healthy
+          if !self.dequeuer_thread.nil? && self.dequeuer_thread.alive?
+            still_alive_timeout = (self.dequeue_timeout + self.dequeue_interval) * 2 + self.dequeuer_still_alive
+            return true if self.class.timestamp < still_alive_timeout
+
+            # kill dequeuer thread
+            self.repeat_puts "Dequeuer isn't healthy, will restart it..."
+            self.dequeuer_thread.kill
+            @dequeuer_thread = nil
+            self.recollect_garbage
+            self.no_repeat_puts "Dequeuer thread was killed!"
+          end
+
+          # dequeuing on parallel (the ride never ends :D)
+          @dequeuer_thread = Thread.new do
+            while true
+              begin
+                self.load_pages
+                self.class.wait self.dequeue_interval
+              rescue => e
+                puts [e.message] + e.backtrace rescue 'error'
+              end
+            end
+            puts "Error: dequeuer died! D:"
+          end
+          self.repeat_puts "Dequeuer thread was started!"
+        end
+        false
+      end
+
+      # Dequeue one page from the previously loaded pages, and waits until there
+      #   are new pages whenever there are no loaded pages.
+      # @return [Hash] dequeued page
+      def dequeue_pages
+        # collect garbage
+        self.recollect_garbage
+
+        # return page if there are loeaded pages
+        is_waiting = false
+        while true do
+          page = self.pages.shift
+          unless page.nil?
+            puts "[Worker #{Parallel.worker_number}]: Finish waiting" if is_waiting
+            loaded_pages.delete(page['gid'])
+            return page
+          end
+
+          # be more verbose on worker waiting
+          unless is_waiting
+            is_waiting = true
+            puts "[Worker #{Parallel.worker_number}]: Is waiting for a page..."
+            if self.second_dequeue_count > 1 && !self.not_found
+              puts "\nWARNING: Your job is not optimized, increase your job's \"parser_dequeue_scale\"\n"
+            end
+          end
+          self.class.wait 1
+
+          # ensure the dequeuer thread is alive and healthy
+          self.ensure_dequeuer_thread
+        end
+      end
+
+      # Dequeue pages and execute the parsers associated to them on parallel.
+      def exec_parse save = false, keep_outputs = false
+        if self.worker_count < 1
+          self.no_repeat_puts NO_WORKERS_MSG
+          return
+        else
+          self.no_repeat_puts "Spawing #{self.worker_count} workers"
+        end
+
+        # start dequeuer
+        self.ensure_dequeuer_thread
+
+        # process the pages
+        dequeue = lambda{ self.dequeue_pages }
+        Parallel.each(dequeue, in_threads: (worker_count)) do |page|
+          parser_file = self.parsers[page['page_type']]
+          begin
+            puts Datahen::Scraper::Parser.exec_parser_by_page(
+              parser_file,
+              page,
+              job_id,
+              save,
+              nil,
+              keep_outputs
+            )
+          rescue Parallel::Kill => e
+            puts "[Worker #{Parallel.worker_number}]: Someone tried to kill Parallel!!!"
+          rescue Parallel::Break => e
+            puts "[Worker #{Parallel.worker_number}]: Someone tried to break Parallel!!!"
+          rescue => e
+            puts [e.message] + e.backtrace rescue 'error'
+          end
+        end
+
+        nil
+      end
+    end
+  end
+end

data/lib/datahen/scraper/executor.rb

@@ -6,7 +6,7 @@ module Datahen
       # Max allowed page size when query outputs (see #find_outputs).
       MAX_FIND_OUTPUTS_PER_PAGE = 500
 
-      attr_accessor :filename, :gid, :job_id
+      attr_accessor :filename, :page, :gid, :job_id
 
       include Datahen::Plugin::ContextExposer
 
@@ -15,6 +15,9 @@ module Datahen
       end
 
       def init_page()
+        # skip whenever a page is provided
+        return self.page unless self.page.nil?
+
         if job_id
           puts "getting Job Page"
           init_job_page

data/lib/datahen/scraper/parser.rb

@@ -18,6 +18,22 @@ module Datahen
         end
       end
 
+      def self.exec_parser_by_page(filename, page, job_id=nil, save=false, vars = {}, keep_outputs=false)
+        extname = File.extname(filename)
+        case extname
+        when '.rb'
+          executor = RubyParserExecutor.new(
+            filename: filename,
+            page: page,
+            job_id: job_id,
+            vars: vars,
+            keep_outputs: keep_outputs
+          )
+          executor.exec_parser(save)
+        else
+          puts "Unable to find a parser executor for file type \"#{extname}\""
+        end
+      end
 
     end
   end

data/lib/datahen/scraper/ruby_parser_executor.rb

@@ -12,7 +12,8 @@ module Datahen
 
       def initialize(options={})
         @filename = options.fetch(:filename) { raise "Filename is required"}
-        @gid = options.fetch(:gid) { raise "GID is required"}
+        @page = options.fetch(:page) { nil }
+        @gid = (self.page || {})['gid'] || options.fetch(:gid) { raise "GID or a page with a GID is required"}
         @job_id = options.fetch(:job_id)
         @page_vars = options.fetch(:vars) { {} }
         @keep_outputs = !!(options.fetch(:keep_outputs) { false })
@@ -46,6 +47,8 @@ module Datahen
       end
 
       def init_page_vars(page)
+        return self.page unless self.page.nil?
+
         if !@page_vars.nil? && !@page_vars.empty?
           page['vars'] = @page_vars
         end

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "0.14.24"
+  VERSION = "0.15.11"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 0.14.24
+  version: 0.15.11
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-03-16 00:00:00.000000000 Z
+date: 2021-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -45,9 +45,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '1.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -55,9 +52,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
-    - - "<"
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.20'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '1.20'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -238,6 +260,7 @@ files:
 - lib/datahen/plugin.rb
 - lib/datahen/plugin/context_exposer.rb
 - lib/datahen/scraper.rb
+- lib/datahen/scraper/batch_parser.rb
 - lib/datahen/scraper/executor.rb
 - lib/datahen/scraper/finisher.rb
 - lib/datahen/scraper/parser.rb
@@ -261,7 +284,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.2.2
+      version: 2.4.4
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="