datahen 0.14.22 → 0.15.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 25eb02598ac32462a53995a4b9e72b3bc466b54c2d74be02516f8d04f178a7b8
-  data.tar.gz: 5f6fcedfa7f4a477e18fc1a0ee80126b1a646a3ecefdd8258d1982bf7d7fe06f
+  metadata.gz: 84e5a734ca1b827228db1550d14f4e9b21e1369d88259a7067dc9209c9a5605b
+  data.tar.gz: 629b472f401b88dc4caabdb87264f9ab1d39d492ff27ac673a79d8ec2c411928
 SHA512:
-  metadata.gz: 69a71d740f9078a5a4c2a77211587c0099a4064cabda690cd6fb4803c153975e3e91f1c05f98278f2852a0bacf8cb444bba8f29f56c3cfbd0fba12cece39b9cd
-  data.tar.gz: df131c11592d2b6192fa74d26fc0e8d823b99f8073b907c82c8e9f04622c7d28aa5e1145419ac0377a99f6efcd3f46ff8fcef88fc436e802d51afc014fd4383a
+  metadata.gz: 320bd2aded5b02fa14a0e4f4cb1c1d4a7d64fe6f4fedcffc62bb04a4c71eab215174f874dc2408dc4416ebf357017f5e743e3f1120fd0e944daeca51deee5311
+  data.tar.gz: 8a86dbc795fae177e4bbd7bde44892ced937cd0fe74383ad962528f25d7d1ef9118897ae24219379299fdd7650fa433ffd436faca71af925c723679f8780db14

data/datahen.gemspec

@@ -33,10 +33,12 @@ Gem::Specification.new do |spec|
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-  spec.required_ruby_version = '>= 2.2.2'
+  spec.required_ruby_version = '>= 2.4.4'
   spec.add_dependency "thor", "~> 0.20.3"
   spec.add_dependency 'httparty', '~> 0.16.2'
-  spec.add_dependency 'nokogiri', '~> 1.6', '< 1.10'
+  spec.add_dependency 'nokogiri', '~> 1.6'
+  spec.add_dependency 'concurrent-ruby', '~> 1.1'
+  spec.add_dependency 'parallel', '~> 1.20'
   spec.add_development_dependency 'bundler', '>= 1.16'
   spec.add_development_dependency 'rake', '>= 10.0'
   spec.add_development_dependency 'minitest', '>= 5.11'

data/lib/datahen/cli/parser.rb

@@ -14,20 +14,19 @@ module Datahen
       def try_parse(scraper_name, parser_file, gid)
         begin
 
-            if options[:job]
-              job_id = options[:job]
-            elsif options[:global]
-              job_id = nil
-            else
-              job = Client::ScraperJob.new(options).find(scraper_name)
-              job_id = job['id']
-            end
-
+          if options[:job]
+            job_id = options[:job]
+          elsif options[:global]
+            job_id = nil
+          else
+            job = Client::ScraperJob.new(options).find(scraper_name)
+            job_id = job['id']
+          end
 
           vars = JSON.parse(options[:vars]) if options[:vars]
           puts Datahen::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, false, vars, options[:"keep-outputs"])
 
-          rescue JSON::ParserError
+        rescue JSON::ParserError
           if options[:vars]
             puts "Error: #{options[:vars]} on vars is not a valid JSON"
           end
@@ -44,17 +43,17 @@ module Datahen
       option :vars, :aliases => :v, type: :string, desc: 'Set user-defined page variables. Must be in json format. i.e: {"Foo":"bar"}'
       option :"keep-outputs", :aliases => :ko, type: :boolean, default: false, desc: "Don't delete existing outputs"
       def exec_parse(scraper_name, parser_file, *gids)
+        if options[:job]
+          job_id = options[:job]
+        else
+          job = Client::ScraperJob.new(options).find(scraper_name)
+          job_id = job['id']
+        end
+
         gids.each do |gid|
           begin
             puts "Parsing #{gid}"
 
-            if options[:job]
-              job_id = options[:job]
-            else
-              job = Client::ScraperJob.new(options).find(scraper_name)
-              job_id = job['id']
-            end
-
             vars = JSON.parse(options[:vars]) if options[:vars]
             puts Datahen::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, true, vars, options[:"keep-outputs"])
           rescue => e
@@ -62,6 +61,46 @@ module Datahen
           end
         end
       end
+
+      desc "batch <scraper_name> <config_file>", "Dequeue and execute Job Pages within a scraper's current job"
+      long_desc <<-LONGDESC
+            Dequeue pending job page(s) to execute their scripts and save the output to the scraper's current job\x5
+          LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :"keep-outputs", :aliases => :ko, type: :boolean, default: false, desc: "Don't delete existing outputs"
+      option :"workers", type: :numeric, default: 1, desc: "Worker count"
+      option :"max-garbage", type: :numeric, default: 5, desc: "Pages processed before calling the garbage collector"
+      option :"dequeue-interval", type: :numeric, default: 3, desc: "Seconds to wait between dequeueing"
+      option :"dequeue-scale", type: :numeric, default: 2, desc: "Scale vs worker count describing how many pages to dequeue"
+      def batch_exec_parse(scraper_name, config_file)
+        if options[:job]
+          job_id = options[:job]
+        else
+          job = Client::ScraperJob.new(options).find(scraper_name)
+          job_id = job['id']
+        end
+
+        # make the stdout and stderr sync to prevent buffering
+        old_stdout_sync = $stdout.sync
+        old_stderr_sync = $stderr.sync
+        $stdout.sync = true
+        $stderr.sync = true
+
+        begin
+          batch = Datahen::Scraper::BatchParser.new job_id, config_file,
+            worker_count: options[:"workers"],
+            max_garbage: options[:"max-garbage"],
+            dequeue_interval: options[:"dequeue-interval"],
+            dequeue_scale: options[:"dequeue-scale"]
+          batch.exec_parse true, options[:"keep-outputs"]
+        rescue => e
+          puts [e.message] + e.backtrace
+        end
+
+        # resume whatever state the stdout and stderr sync were
+        $stdout.sync = old_stdout_sync
+        $stderr.sync = old_stderr_sync
+      end
     end
   end
 

data/lib/datahen/client/backblaze_content.rb

@@ -4,10 +4,10 @@ require 'httparty'
 module Datahen
   module Client
     class BackblazeContent
-      include HTTParty    
-      
+      include HTTParty
+
       def get_content(url)
-        self.class.get(url, format: :plain)
+        self.class.get(url, format: :plain).response.body
       end
 
       def get_gunzipped_content(url)
@@ -19,19 +19,23 @@ module Datahen
         sio = StringIO.new(string)
         gz = Zlib::GzipReader.new(sio, encoding: Encoding::ASCII_8BIT)
         _content = ""
-        begin 
+        begin
           _content = gz.read
         rescue => e
           # if unexpected eof error, then readchar until error, and ignore it
           if e.to_s == 'unexpected end of file'
-            begin 
-              while !gz.eof?
-                _content += gz.readchar
-              end
+            # heavily improve content read recovery by using "String#<<",
+            #  reading all "good" lines and then concat the remaining chars
+            begin
+              gz.each_line{|line| _content << line}
             rescue => e
-              puts "Ignored Zlib error: #{e.to_s}"
+              begin
+                _content << gz.readchar while !gz.eof
+              rescue => e
+                puts "Ignored Zlib error: #{e.to_s}"
+              end
             end
-          else 
+          else
             raise e
           end
         end

data/lib/datahen/client/job_page.rb

@@ -42,6 +42,16 @@ module Datahen
         self.class.post("/jobs/#{job_id}/pages", params)
       end
 
+      def dequeue(job_id, limit, page_types, parse_fetching_failed, opts = {})
+        body = {
+          limit: limit,
+          page_types: page_types,
+          parse_fetching_failed: parse_fetching_failed
+        }
+        params = @options.merge({body: body.to_json, timeout: 30})
+        self.class.put("/jobs/#{job_id}/pages/parse_dequeue", params)
+      end
+
       def parsing_update(job_id, gid, opts={})
         body = {}
         body[:outputs] = opts.fetch(:outputs) {[]}

data/lib/datahen/scraper.rb

@@ -1,6 +1,7 @@
 require "datahen/error"
 require "datahen/plugin"
 require "datahen/scraper/parser"
+require "datahen/scraper/batch_parser"
 require "datahen/scraper/seeder"
 require "datahen/scraper/finisher"
 require "datahen/scraper/executor"

data/lib/datahen/scraper/batch_parser.rb

@@ -0,0 +1,203 @@
+require 'concurrent'
+require 'parallel'
+
+module Datahen
+  module Scraper
+    class BatchParser
+      NOT_FOUND_MSG = "No more pages to parse found"
+      NO_DEQUEUE_COUNT_MSG = "\nWarning: Max page to parse dequeue count is 0, check pages to parse scale\n"
+      NO_WORKERS_MSG = "\nWarning: There are no parser workers\n"
+
+      attr_accessor :config_file, :garbage_count, :last_message, :second_dequeue_count
+      attr_reader :job_id, :worker_count, :pages, :max_garbage
+      attr_reader :dequeue_interval, :dequeue_scale
+      attr_reader :page_types, :parsers
+      attr_reader :config, :client, :garbage_mutex
+
+      def self.wait time_in_seconds
+        Kernel.sleep time_in_seconds
+      end
+
+      def initialize(job_id, config_file, opts = {})
+        opts = {
+          worker_count: 1,
+          max_garbage: 5,
+          dequeue_interval: 3,
+          dequeue_scale: 2,
+          client_options: {}
+        }.merge opts
+
+        @job_id = job_id
+        @worker_count = opts[:worker_count]
+        @dequeue_interval = opts[:dequeue_interval]
+        @dequeue_scale = opts[:dequeue_scale]
+        @max_garbage = opts[:max_garbage]
+        @pages = Concurrent::Hash.new
+        @garbage_mutex = Mutex.new
+        self.second_dequeue_count = 0
+        self.garbage_count = 0
+        self.config_file = config_file
+        self.load_config
+
+        @client = Datahen::Client::JobPage.new(opts[:client_options])
+      end
+
+      def recollect_garbage
+        self.garbage_mutex.synchronize do
+          self.garbage_count += 1
+          if self.garbage_count > self.max_garbage
+            puts "Recollect garbage"
+            GC.start
+            self.garbage_count = 0
+          end
+        end
+      end
+
+      def load_config
+        # build page type to script file map
+        @page_types = []
+        @parsers = Concurrent::Hash.new
+        @config = YAML.load_file(config_file)
+        self.config['parsers'].each do |v|
+          next if !v['disabled'].nil? && !!v['disabled']
+          @page_types << v['page_type']
+          self.parsers[v['page_type']] = v['file']
+        end
+        self.recollect_garbage
+      end
+
+      def repeat_puts message
+        puts message
+        self.last_message = ''
+      end
+
+      def no_repeat_puts message
+        return if message == self.last_message
+        puts message
+        self.last_message = message
+      end
+
+      def load_pages
+        # calculate dequeue size
+        max_dequeue_size = (self.worker_count * self.dequeue_scale).ceil
+        current_size = self.pages.length
+        dequeue_size = (self.dequeue_scale * (max_dequeue_size - current_size)).ceil
+        if dequeue_size < 1
+          return 0
+        end
+        dequeue_size = max_dequeue_size if dequeue_size > max_dequeue_size
+
+        # reserve and get to pages parse
+        response = nil
+        begin
+          response = client.dequeue self.job_id,
+            dequeue_size,
+            self.page_types,
+            config['parse_fetching_failed']
+        rescue Net::ReadTimeout, Net::OpenTimeout => e
+          self.no_repeat_puts "Dequeue API call timeout! Contact infra team, your job needs a profile change"
+          return 0
+        rescue => e
+          raise e
+        end
+
+        # ensure a valid response or try again
+        if response.nil? || response.response.code.to_i != 200
+          self.repeat_puts(response.nil? ? 'null' : response.body)
+          self.recollect_garbage
+          return 0
+        end
+
+        # add pages
+        count = 0
+        (JSON.parse(response.body) || []).each do |page|
+          count += 1
+          next if self.pages.has_key? page['gid']
+          self.pages[page['gid']] = page
+        end
+        response = nil
+
+        # recolect garbage to free some memory before parsing
+        if count > 0
+          self.recollect_garbage
+          self.repeat_puts "Found #{count} page(s) to parse"
+          self.second_dequeue_count += 1 unless self.second_dequeue_count > 1
+        else
+          self.no_repeat_puts NOT_FOUND_MSG
+        end
+
+        # return how many pages were loaded
+        count
+      end
+
+      def dequeue_pages
+        # collect garbage
+        self.recollect_garbage
+
+        # return page if there are loeaded pages
+        is_waiting = false
+        while true do
+          key_value = self.pages.shift
+          unless key_value.nil?
+            puts "[Worker #{Parallel.worker_number}]: Finish waiting" if is_waiting
+            return key_value[1]
+          end
+
+          # be more verbose on worker waiting
+          unless is_waiting
+            is_waiting = true
+            puts "[Worker #{Parallel.worker_number}]: Is waiting for a page..."
+            if self.second_dequeue_count > 1
+              puts "\nWARNING: Your job is not optimized, increase your job's \"parser_dequeue_scale\"\n"
+            end
+          end
+          self.class.wait 1
+        end
+      end
+
+      def exec_parse save = false, keep_outputs = false
+        if self.worker_count < 1
+          self.no_repeat_puts NO_WORKERS_MSG
+          return
+        else
+          self.no_repeat_puts "Spawing #{self.worker_count} workers"
+        end
+
+        # dequeuing on parallel (the ride never ends :D)
+        Thread.new do
+          while true
+            begin
+              self.load_pages
+              self.class.wait self.dequeue_interval
+            rescue => e
+              puts [e.message] + e.backtrace rescue 'error'
+            end
+          end
+          puts "Error: dequeuer died! D:"
+        end
+
+        # process the pages
+        dequeue = lambda{ self.dequeue_pages }
+        Parallel.each(dequeue, in_threads: (worker_count)) do |page|
+          parser_file = self.parsers[page['page_type']]
+          begin
+            puts Datahen::Scraper::Parser.exec_parser_by_page(
+              parser_file,
+              page,
+              job_id,
+              save,
+              nil,
+              keep_outputs
+            )
+          rescue Parallel::Kill => e
+            puts "[Worker #{Parallel.worker_number}]: Someone tried to kill Parallel!!!"
+          rescue Parallel::Break => e
+            puts "[Worker #{Parallel.worker_number}]: Someone tried to break Parallel!!!"
+          rescue => e
+            puts [e.message] + e.backtrace rescue 'error'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datahen/scraper/executor.rb

@@ -6,7 +6,7 @@ module Datahen
       # Max allowed page size when query outputs (see #find_outputs).
       MAX_FIND_OUTPUTS_PER_PAGE = 500
 
-      attr_accessor :filename, :gid, :job_id
+      attr_accessor :filename, :page, :gid, :job_id
 
       include Datahen::Plugin::ContextExposer
 
@@ -15,6 +15,9 @@ module Datahen
       end
 
       def init_page()
+        # skip whenever a page is provided
+        return self.page unless self.page.nil?
+
         if job_id
           puts "getting Job Page"
           init_job_page
@@ -374,6 +377,11 @@ module Datahen
       def eval_with_context file_path, context
         eval(File.read(file_path), context, file_path)
       end
+
+      # Finish the executor execution
+      def finish
+        raise Error::SafeTerminateError
+      end
     end
   end
 end

data/lib/datahen/scraper/parser.rb

@@ -18,6 +18,22 @@ module Datahen
         end
       end
 
+      def self.exec_parser_by_page(filename, page, job_id=nil, save=false, vars = {}, keep_outputs=false)
+        extname = File.extname(filename)
+        case extname
+        when '.rb'
+          executor = RubyParserExecutor.new(
+            filename: filename,
+            page: page,
+            job_id: job_id,
+            vars: vars,
+            keep_outputs: keep_outputs
+          )
+          executor.exec_parser(save)
+        else
+          puts "Unable to find a parser executor for file type \"#{extname}\""
+        end
+      end
 
     end
   end

data/lib/datahen/scraper/ruby_finisher_executor.rb

@@ -40,6 +40,8 @@ module Datahen
               job_id: job_id
             })
             eval_with_context filename, context
+          rescue Error::SafeTerminateError => e
+            # do nothing, this is fine
           rescue SyntaxError => e
             handle_error(e) if save
             raise e
@@ -55,7 +57,7 @@ module Datahen
             handle_error(e) if save
             raise e
           end
-          
+
           update_finisher_done_status
         end
         proc.call

data/lib/datahen/scraper/ruby_parser_executor.rb

@@ -12,7 +12,8 @@ module Datahen
 
       def initialize(options={})
         @filename = options.fetch(:filename) { raise "Filename is required"}
-        @gid = options.fetch(:gid) { raise "GID is required"}
+        @page = options.fetch(:page) { nil }
+        @gid = (self.page || {})['gid'] || options.fetch(:gid) { raise "GID or a page with a GID is required"}
         @job_id = options.fetch(:job_id)
         @page_vars = options.fetch(:vars) { {} }
         @keep_outputs = !!(options.fetch(:keep_outputs) { false })
@@ -46,6 +47,8 @@ module Datahen
       end
 
       def init_page_vars(page)
+        return self.page unless self.page.nil?
+
         if !@page_vars.nil? && !@page_vars.empty?
           page['vars'] = @page_vars
         end

data/lib/datahen/scraper/ruby_seeder_executor.rb

@@ -44,6 +44,8 @@ module Datahen
               pages: pages
             })
             eval_with_context filename, context
+          rescue Error::SafeTerminateError => e
+            # do nothing, this is fine
           rescue SyntaxError => e
             handle_error(e) if save
             raise e

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "0.14.22"
+  VERSION = "0.15.10"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 0.14.22
+  version: 0.15.10
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-12-14 00:00:00.000000000 Z
+date: 2021-05-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -45,9 +45,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '1.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -55,9 +52,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
-    - - "<"
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.20'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '1.20'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -238,6 +260,7 @@ files:
 - lib/datahen/plugin.rb
 - lib/datahen/plugin/context_exposer.rb
 - lib/datahen/scraper.rb
+- lib/datahen/scraper/batch_parser.rb
 - lib/datahen/scraper/executor.rb
 - lib/datahen/scraper/finisher.rb
 - lib/datahen/scraper/parser.rb
@@ -261,7 +284,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.2.2
+      version: 2.4.4
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="