datahen 0.14.26 → 0.15.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dac57d98132102aa9ae8244b6528394473b2bdeb9992c7ea15d6979eaf87d4af
-  data.tar.gz: e68858d2f088b2d7b8538411dd59cf2ae2de7866416fc213c6a6fa009d93c556
+  metadata.gz: 7ad617ad097cf773fe496c3ccf801c49926524017784e5a51764ec88992b11ab
+  data.tar.gz: 8292cf447a3fedcad565bc1aa98041d68a51f0d93ff1598b8c31980170d1f66c
 SHA512:
-  metadata.gz: 857126b2f7ec4fa058aaa8d5b4a7095108224bdf3f6ece690dbfc930e0527a294853705227f0e63be5af3524982fff21f7d3c9d940c22b31caade5139a3d607b
-  data.tar.gz: 81ecf95378e6f4aa31a87e39a82bc815216fce1b84aa65d8f7f2aa8ee8b19b871f08eb8c86025d9dc8d84617f20864f5f39c21d7b8ac4900a739599c0aa6283c
+  metadata.gz: 3e883e9a53339e342446ed543bab41bf0ab60de6e1f5f7aaaad54125ec67275803bf1e7676e3764c59f0ba7b75e8fdf4adfe98733ab376c350b3a3247a678895
+  data.tar.gz: eaa4d12e93f31c7e516bb94f1222ecad2d6581609f77372de9078ac91a716438f6e998ec9b00ba4d3da76d02ccab5639fe06393c67346ab896e69d62d3fcc1e5

data/datahen.gemspec

@@ -33,10 +33,12 @@ Gem::Specification.new do |spec|
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-  spec.required_ruby_version = '>= 2.2.2'
+  spec.required_ruby_version = '>= 2.4.4'
   spec.add_dependency "thor", "~> 0.20.3"
   spec.add_dependency 'httparty', '~> 0.16.2'
   spec.add_dependency 'nokogiri', '~> 1.6'
+  spec.add_dependency 'concurrent-ruby', '~> 1.1'
+  spec.add_dependency 'parallel', '~> 1.20'
   spec.add_development_dependency 'bundler', '>= 1.16'
   spec.add_development_dependency 'rake', '>= 10.0'
   spec.add_development_dependency 'minitest', '>= 5.11'

data/lib/datahen/cli/parser.rb

@@ -43,17 +43,17 @@ module Datahen
       option :vars, :aliases => :v, type: :string, desc: 'Set user-defined page variables. Must be in json format. i.e: {"Foo":"bar"}'
       option :"keep-outputs", :aliases => :ko, type: :boolean, default: false, desc: "Don't delete existing outputs"
       def exec_parse(scraper_name, parser_file, *gids)
+        if options[:job]
+          job_id = options[:job]
+        else
+          job = Client::ScraperJob.new(options).find(scraper_name)
+          job_id = job['id']
+        end
+
         gids.each do |gid|
           begin
             puts "Parsing #{gid}"
 
-            if options[:job]
-              job_id = options[:job]
-            else
-              job = Client::ScraperJob.new(options).find(scraper_name)
-              job_id = job['id']
-            end
-
             vars = JSON.parse(options[:vars]) if options[:vars]
             puts Datahen::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, true, vars, options[:"keep-outputs"])
           rescue => e
@@ -61,6 +61,46 @@ module Datahen
           end
         end
       end
+
+      desc "batch <scraper_name> <config_file>", "Dequeue and execute Job Pages within a scraper's current job"
+      long_desc <<-LONGDESC
+            Dequeue pending job page(s) to execute their scripts and save the output to the scraper's current job\x5
+          LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :"keep-outputs", :aliases => :ko, type: :boolean, default: false, desc: "Don't delete existing outputs"
+      option :"workers", type: :numeric, default: 1, desc: "Worker count"
+      option :"max-garbage", type: :numeric, default: 5, desc: "Pages processed before calling the garbage collector"
+      option :"dequeue-interval", type: :numeric, default: 3, desc: "Seconds to wait between dequeueing"
+      option :"dequeue-scale", type: :numeric, default: 2, desc: "Scale vs worker count describing how many pages to dequeue"
+      def batch_exec_parse(scraper_name, config_file)
+        if options[:job]
+          job_id = options[:job]
+        else
+          job = Client::ScraperJob.new(options).find(scraper_name)
+          job_id = job['id']
+        end
+
+        # make the stdout and stderr sync to prevent buffering
+        old_stdout_sync = $stdout.sync
+        old_stderr_sync = $stderr.sync
+        $stdout.sync = true
+        $stderr.sync = true
+
+        begin
+          batch = Datahen::Scraper::BatchParser.new job_id, config_file,
+            worker_count: options[:"workers"],
+            max_garbage: options[:"max-garbage"],
+            dequeue_interval: options[:"dequeue-interval"],
+            dequeue_scale: options[:"dequeue-scale"]
+          batch.exec_parse true, options[:"keep-outputs"]
+        rescue => e
+          puts [e.message] + e.backtrace
+        end
+
+        # resume whatever state the stdout and stderr sync were
+        $stdout.sync = old_stdout_sync
+        $stderr.sync = old_stderr_sync
+      end
     end
   end
 

data/lib/datahen/client/job_page.rb

@@ -42,6 +42,16 @@ module Datahen
         self.class.post("/jobs/#{job_id}/pages", params)
       end
 
+      def dequeue(job_id, limit, page_types, parse_fetching_failed, opts = {})
+        body = {
+          limit: limit,
+          page_types: page_types,
+          parse_fetching_failed: parse_fetching_failed
+        }
+        params = @options.merge({body: body.to_json})
+        self.class.put("/jobs/#{job_id}/pages/parse_dequeue", params)
+      end
+
       def parsing_update(job_id, gid, opts={})
         body = {}
         body[:outputs] = opts.fetch(:outputs) {[]}

data/lib/datahen/scraper/batch_parser.rb

@@ -0,0 +1,177 @@
+require 'concurrent'
+require 'parallel'
+
+module Datahen
+  module Scraper
+    class BatchParser
+      NOT_FOUND_MSG = "No more pages to parse found"
+      NO_DEQUEUE_COUNT_MSG = "Warning: Max page to parse dequeue count is 0, check pages to parse scale"
+      NO_WORKERS_MSG = "Warning: There are no parser workers"
+
+      attr_accessor :config_file, :garbage_count, :last_message
+      attr_reader :job_id, :worker_count, :pages, :max_garbage
+      attr_reader :dequeue_interval, :dequeue_scale
+      attr_reader :page_types, :parsers
+      attr_reader :config, :client, :garbage_mutex
+
+      def self.wait time_in_seconds
+        Kernel.sleep time_in_seconds
+      end
+
+      def initialize(job_id, config_file, opts = {})
+        opts = {
+          worker_count: 1,
+          max_garbage: 5,
+          dequeue_interval: 3,
+          dequeue_scale: 2,
+          client_options: {}
+        }.merge opts
+
+        @job_id = job_id
+        @worker_count = opts[:worker_count]
+        @dequeue_interval = opts[:dequeue_interval]
+        @dequeue_scale = opts[:dequeue_scale]
+        @max_garbage = opts[:max_garbage]
+        @pages = Concurrent::Hash.new
+        @garbage_mutex = Mutex.new
+        self.garbage_count = 0
+        self.config_file = config_file
+        self.load_config
+
+        @client = Datahen::Client::JobPage.new(opts[:client_options])
+      end
+
+      def recollect_garbage
+        self.garbage_mutex.synchronize do
+          puts "Recollect garbage"
+          GC.start
+          self.garbage_count = 0
+        end
+      end
+
+      def load_config
+        # build page type to script file map
+        @page_types = []
+        @parsers = Concurrent::Hash.new
+        @config = YAML.load_file(config_file)
+        self.config['parsers'].each do |v|
+          next if !v['disabled'].nil? && !!v['disabled']
+          @page_types << v['page_type']
+          self.parsers[v['page_type']] = v['file']
+        end
+        self.recollect_garbage
+      end
+
+      def repeat_puts message
+        puts message
+        self.last_message = ''
+      end
+
+      def no_repeat_puts message
+        return if message == self.last_message
+        puts message
+        self.last_message = message
+      end
+
+      def load_pages
+        # calculate dequeue size
+        max_dequeue_size = (self.worker_count * self.dequeue_scale).ceil
+        current_size = self.pages.length
+        dequeue_size = (self.dequeue_scale * (max_dequeue_size - current_size)).ceil
+        if dequeue_size < 1
+          return 0
+        end
+        dequeue_size = max_dequeue_size if dequeue_size > max_dequeue_size
+
+        # reserve and get to pages parse
+        response = client.dequeue self.job_id,
+          dequeue_size,
+          self.page_types,
+          config['parse_fetching_failed']
+
+        # ensure a valid response or try again
+        if response.nil? || response.response.code.to_i != 200
+          self.repeat_puts(response.nil? ? 'null' : response.body)
+          self.recollect_garbage
+          return 0
+        end
+
+        # add pages
+        count = 0
+        (JSON.parse(response.body) || []).each do |page|
+          count += 1
+          next if self.pages.has_key? page['gid']
+          self.pages[page['gid']] = page
+        end
+        response = nil
+
+        # recolect garbage to free some memory before parsing
+        if count > 0
+          self.recollect_garbage
+          self.repeat_puts "Found #{count} page(s) to parse"
+        else
+          self.no_repeat_puts NOT_FOUND_MSG
+        end
+
+        # return how many pages were loaded
+        count
+      end
+
+      def dequeue_pages
+        # collect garbage
+        self.garbage_count += 1
+        if self.garbage_count > self.max_garbage
+          self.recollect_garbage
+        end
+
+        # return page if there are loeaded pages
+        while true do
+          key_value = self.pages.shift
+          return key_value[1] unless key_value.nil?
+          self.class.wait 1
+        end
+      end
+
+      def exec_parse save = false, keep_outputs = false
+        if self.worker_count < 1
+          self.no_repeat_puts NO_WORKERS_MSG
+          return
+        else
+          self.no_repeat_puts "Spawing #{self.worker_count} workers"
+        end
+
+        # dequeuing on parallel
+        keep_dequeue = Concurrent::Array.new
+        keep_dequeue[0] = true
+        Thread.new do
+          while keep_dequeue[0]
+            begin
+              self.load_pages
+              self.class.wait self.dequeue_interval
+            rescue => e
+              puts [e.message] + e.backtrace rescue 'error'
+            end
+          end
+        end
+
+        dequeue = lambda{ self.dequeue_pages }
+        Parallel.each(dequeue, in_threads: (worker_count)) do |page|
+          parser_file = self.parsers[page['page_type']]
+          begin
+            puts Datahen::Scraper::Parser.exec_parser_by_page(
+              parser_file,
+              page,
+              job_id,
+              save,
+              nil,
+              keep_outputs
+            )
+          rescue => e
+            puts [e.message] + e.backtrace rescue 'error'
+          end
+        end
+        keep_dequeue[0] = false
+      end
+    end
+  end
+end

data/lib/datahen/scraper/executor.rb

@@ -6,7 +6,7 @@ module Datahen
       # Max allowed page size when query outputs (see #find_outputs).
       MAX_FIND_OUTPUTS_PER_PAGE = 500
 
-      attr_accessor :filename, :gid, :job_id
+      attr_accessor :filename, :page, :gid, :job_id
 
       include Datahen::Plugin::ContextExposer
 
@@ -15,6 +15,9 @@ module Datahen
       end
 
       def init_page()
+        # skip whenever a page is provided
+        return self.page unless self.page.nil?
+
         if job_id
           puts "getting Job Page"
           init_job_page

data/lib/datahen/scraper/parser.rb

@@ -18,6 +18,22 @@ module Datahen
         end
       end
 
+      def self.exec_parser_by_page(filename, page, job_id=nil, save=false, vars = {}, keep_outputs=false)
+        extname = File.extname(filename)
+        case extname
+        when '.rb'
+          executor = RubyParserExecutor.new(
+            filename: filename,
+            page: page,
+            job_id: job_id,
+            vars: vars,
+            keep_outputs: keep_outputs
+          )
+          executor.exec_parser(save)
+        else
+          puts "Unable to find a parser executor for file type \"#{extname}\""
+        end
+      end
 
     end
   end

data/lib/datahen/scraper/ruby_parser_executor.rb

@@ -12,7 +12,8 @@ module Datahen
 
       def initialize(options={})
         @filename = options.fetch(:filename) { raise "Filename is required"}
-        @gid = options.fetch(:gid) { raise "GID is required"}
+        @page = options.fetch(:page) { nil }
+        @gid = (self.page || {})['gid'] || options.fetch(:gid) { raise "GID or a page with a GID is required"}
         @job_id = options.fetch(:job_id)
         @page_vars = options.fetch(:vars) { {} }
         @keep_outputs = !!(options.fetch(:keep_outputs) { false })
@@ -46,6 +47,8 @@ module Datahen
       end
 
       def init_page_vars(page)
+        return self.page unless self.page.nil?
+
         if !@page_vars.nil? && !@page_vars.empty?
           page['vars'] = @page_vars
         end

data/lib/datahen/scraper.rb

@@ -1,6 +1,7 @@
 require "datahen/error"
 require "datahen/plugin"
 require "datahen/scraper/parser"
+require "datahen/scraper/batch_parser"
 require "datahen/scraper/seeder"
 require "datahen/scraper/finisher"
 require "datahen/scraper/executor"

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "0.14.26"
+  VERSION = "0.15.9"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 0.14.26
+  version: 0.15.9
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-04-20 00:00:00.000000000 Z
+date: 2021-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -52,6 +52,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.20'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.20'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -232,6 +260,7 @@ files:
 - lib/datahen/plugin.rb
 - lib/datahen/plugin/context_exposer.rb
 - lib/datahen/scraper.rb
+- lib/datahen/scraper/batch_parser.rb
 - lib/datahen/scraper/executor.rb
 - lib/datahen/scraper/finisher.rb
 - lib/datahen/scraper/parser.rb
@@ -255,7 +284,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.2.2
+      version: 2.4.4
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="