datahen 0.14.19 → 0.14.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b483263384d6a00e51fd345499ad020d69c9d07c1a77e3521e6b55a4528195de
-  data.tar.gz: 167c0a417c402198f13151daf29bddb0f8429cac3876c8a84d6b7ba396b1ac90
+  metadata.gz: 54064cf5656d253f683540fa3704cdcae3991bc07a91ab5339bf9083eb0792f2
+  data.tar.gz: 3c82cedd06937454a9af5d91eea9b32f3ad744394763ca9fadbcdc6068eda683
 SHA512:
-  metadata.gz: 344168b7a8a4cb746347aba1640d7b4e75a901cd5a1e2ab7acd00f6e8eee3d8c297795812259f84a21757b444e653331e44dd9e1efbec372472b6aeb98c56619
-  data.tar.gz: 1ef00c32694830740ea477fb4291de9234957ee4b7f7ad02642aa9d5cb91d1b3b1cd6a56542db4cbf84d92d42039546f2e9bfb8a40cf22d103256e8d53b0ac28
+  metadata.gz: fb17b046f9dbd15cf7a278f68f550e8b3b84d8c16a030d0ef4df100df28c8e4dce29fe74ffc70e02f79af26a2d1e3f66ccb6890e06a342a70fac09d824169431
+  data.tar.gz: 732256fc714635896f444bed50e2d6f49c2a03c4868fd23eb84e52701935bc2bfbef25811055983691c5e0ff39af53328dbeb67faf91f127c1d06af450d7d666

data/lib/datahen/cli/parser.rb

@@ -14,20 +14,19 @@ module Datahen
       def try_parse(scraper_name, parser_file, gid)
         begin
 
-            if options[:job]
-              job_id = options[:job]
-            elsif options[:global]
-              job_id = nil
-            else
-              job = Client::ScraperJob.new(options).find(scraper_name)
-              job_id = job['id']
-            end
-
+          if options[:job]
+            job_id = options[:job]
+          elsif options[:global]
+            job_id = nil
+          else
+            job = Client::ScraperJob.new(options).find(scraper_name)
+            job_id = job['id']
+          end
 
           vars = JSON.parse(options[:vars]) if options[:vars]
           puts Datahen::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, false, vars, options[:"keep-outputs"])
 
-          rescue JSON::ParserError
+        rescue JSON::ParserError
           if options[:vars]
             puts "Error: #{options[:vars]} on vars is not a valid JSON"
           end

data/lib/datahen/cli/scraper.rb

@@ -93,6 +93,7 @@ module Datahen
       option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Default: 1'
       option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
       option :proxy_type, desc: 'Set the Proxy type. Default: standard'
+      option :vars, type: :string, banner: :JSON, desc: 'Set input vars. Must be in json format. i.e: [{"name":"foo", "value":"bar", "secret":false}] '
       def start(scraper_name)
         client = Client::ScraperJob.new(options)
         puts "Starting a scrape job..."

data/lib/datahen/client/backblaze_content.rb

@@ -4,8 +4,8 @@ require 'httparty'
 module Datahen
   module Client
     class BackblazeContent
-      include HTTParty    
-      
+      include HTTParty
+
       def get_content(url)
         self.class.get(url, format: :plain)
       end
@@ -19,19 +19,23 @@ module Datahen
         sio = StringIO.new(string)
         gz = Zlib::GzipReader.new(sio, encoding: Encoding::ASCII_8BIT)
         _content = ""
-        begin 
+        begin
           _content = gz.read
         rescue => e
           # if unexpected eof error, then readchar until error, and ignore it
           if e.to_s == 'unexpected end of file'
-            begin 
-              while !gz.eof?
-                _content += gz.readchar
-              end
+            # heavily improve content read recovery by using "String#<<",
+            #  reading all "good" lines and then concat the remaining chars
+            begin
+              gz.each_line{|line| _content << line}
             rescue => e
-              puts "Ignored Zlib error: #{e.to_s}"
+              begin
+                _content << gz.readchar while !gz.eof
+              rescue => e
+                puts "Ignored Zlib error: #{e.to_s}"
+              end
             end
-          else 
+          else
             raise e
           end
         end

data/lib/datahen/client/scraper_job.rb

@@ -11,6 +11,13 @@ module Datahen
         body[:standard_worker_count] = opts[:workers] if opts[:workers]
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
+        if opts[:vars]
+          if opts[:vars].is_a?(Array)
+            body[:vars] = opts[:vars]
+          elsif opts[:vars].is_a?(String)
+            body[:vars] = JSON.parse(opts[:vars])
+          end
+        end
         params = @options.merge({body: body.to_json})
         self.class.post("/scrapers/#{scraper_name}/jobs", params)
       end

data/lib/datahen/scraper/executor.rb

@@ -374,6 +374,11 @@ module Datahen
       def eval_with_context file_path, context
         eval(File.read(file_path), context, file_path)
       end
+
+      # Finish the executor execution
+      def finish
+        raise Error::SafeTerminateError
+      end
     end
   end
 end

data/lib/datahen/scraper/ruby_finisher_executor.rb

@@ -40,6 +40,8 @@ module Datahen
               job_id: job_id
             })
             eval_with_context filename, context
+          rescue Error::SafeTerminateError => e
+            # do nothing, this is fine
           rescue SyntaxError => e
             handle_error(e) if save
             raise e
@@ -55,7 +57,7 @@ module Datahen
             handle_error(e) if save
             raise e
           end
-          
+
           update_finisher_done_status
         end
         proc.call

data/lib/datahen/scraper/ruby_seeder_executor.rb

@@ -44,6 +44,8 @@ module Datahen
               pages: pages
             })
             eval_with_context filename, context
+          rescue Error::SafeTerminateError => e
+            # do nothing, this is fine
           rescue SyntaxError => e
             handle_error(e) if save
             raise e

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "0.14.19"
+  VERSION = "0.14.24"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 0.14.19
+  version: 0.14.24
 platform: ruby
 authors:
 - Parama Danoesubroto
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-10-27 00:00:00.000000000 Z
+date: 2021-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -253,7 +253,7 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://datahen.com
   source_code_uri: https://github.com/DataHenOfficial/datahen-ruby
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -269,7 +269,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key:
+signing_key: 
 specification_version: 4
 summary: DataHen toolbelt for developers
 test_files: []