datahen 0.14.19 → 0.14.25

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b483263384d6a00e51fd345499ad020d69c9d07c1a77e3521e6b55a4528195de
-  data.tar.gz: 167c0a417c402198f13151daf29bddb0f8429cac3876c8a84d6b7ba396b1ac90
+  metadata.gz: 566f3019f38a812f11eb5475e787a2ee0f4475dc0f8ee342accb78897974308f
+  data.tar.gz: d050afb771fd6f6d95054e587540c0dc5da0769a7369b55f407c3f727633fff2
 SHA512:
-  metadata.gz: 344168b7a8a4cb746347aba1640d7b4e75a901cd5a1e2ab7acd00f6e8eee3d8c297795812259f84a21757b444e653331e44dd9e1efbec372472b6aeb98c56619
-  data.tar.gz: 1ef00c32694830740ea477fb4291de9234957ee4b7f7ad02642aa9d5cb91d1b3b1cd6a56542db4cbf84d92d42039546f2e9bfb8a40cf22d103256e8d53b0ac28
+  metadata.gz: a5bd6b82b1982ea466a3b12f2d49daddd865d8aa45b30a6e5d5c8db6d921c7a9eef2af67c8fe9c79c7460d1df348c88572e8e8cfb7df9ba627e6cda3f5f37af5
+  data.tar.gz: e19ee95ad123ed2c345a1f90fe01421f7be9819f58a67bdf5b498eb93e512793b0b3e8d32c4d5ec9acd1751e7108f9e3f189d9eb7a0beefbb5ce6ba798296f66

data/datahen.gemspec

@@ -36,7 +36,7 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.2.2'
   spec.add_dependency "thor", "~> 0.20.3"
   spec.add_dependency 'httparty', '~> 0.16.2'
-  spec.add_dependency 'nokogiri', '~> 1.6', '< 1.10'
+  spec.add_dependency 'nokogiri', '~> 1.6'
   spec.add_development_dependency 'bundler', '>= 1.16'
   spec.add_development_dependency 'rake', '>= 10.0'
   spec.add_development_dependency 'minitest', '>= 5.11'

data/lib/datahen/cli/parser.rb

@@ -14,20 +14,19 @@ module Datahen
       def try_parse(scraper_name, parser_file, gid)
         begin
 
-            if options[:job]
-              job_id = options[:job]
-            elsif options[:global]
-              job_id = nil
-            else
-              job = Client::ScraperJob.new(options).find(scraper_name)
-              job_id = job['id']
-            end
-
+          if options[:job]
+            job_id = options[:job]
+          elsif options[:global]
+            job_id = nil
+          else
+            job = Client::ScraperJob.new(options).find(scraper_name)
+            job_id = job['id']
+          end
 
           vars = JSON.parse(options[:vars]) if options[:vars]
           puts Datahen::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, false, vars, options[:"keep-outputs"])
 
-          rescue JSON::ParserError
+        rescue JSON::ParserError
           if options[:vars]
             puts "Error: #{options[:vars]} on vars is not a valid JSON"
           end

data/lib/datahen/cli/scraper.rb

@@ -93,6 +93,7 @@ module Datahen
       option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Default: 1'
       option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
       option :proxy_type, desc: 'Set the Proxy type. Default: standard'
+      option :vars, type: :string, banner: :JSON, desc: 'Set input vars. Must be in json format. i.e: [{"name":"foo", "value":"bar", "secret":false}] '
       def start(scraper_name)
         client = Client::ScraperJob.new(options)
         puts "Starting a scrape job..."

data/lib/datahen/client/backblaze_content.rb

@@ -4,8 +4,8 @@ require 'httparty'
 module Datahen
   module Client
     class BackblazeContent
-      include HTTParty    
-      
+      include HTTParty
+
       def get_content(url)
         self.class.get(url, format: :plain)
       end
@@ -19,19 +19,23 @@ module Datahen
         sio = StringIO.new(string)
         gz = Zlib::GzipReader.new(sio, encoding: Encoding::ASCII_8BIT)
         _content = ""
-        begin 
+        begin
           _content = gz.read
         rescue => e
           # if unexpected eof error, then readchar until error, and ignore it
           if e.to_s == 'unexpected end of file'
-            begin 
-              while !gz.eof?
-                _content += gz.readchar
-              end
+            # heavily improve content read recovery by using "String#<<",
+            #  reading all "good" lines and then concat the remaining chars
+            begin
+              gz.each_line{|line| _content << line}
             rescue => e
-              puts "Ignored Zlib error: #{e.to_s}"
+              begin
+                _content << gz.readchar while !gz.eof
+              rescue => e
+                puts "Ignored Zlib error: #{e.to_s}"
+              end
             end
-          else 
+          else
             raise e
           end
         end

data/lib/datahen/client/scraper_job.rb

@@ -11,6 +11,13 @@ module Datahen
         body[:standard_worker_count] = opts[:workers] if opts[:workers]
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
+        if opts[:vars]
+          if opts[:vars].is_a?(Array)
+            body[:vars] = opts[:vars]
+          elsif opts[:vars].is_a?(String)
+            body[:vars] = JSON.parse(opts[:vars])
+          end
+        end
         params = @options.merge({body: body.to_json})
         self.class.post("/scrapers/#{scraper_name}/jobs", params)
       end

data/lib/datahen/scraper/executor.rb

@@ -374,6 +374,11 @@ module Datahen
       def eval_with_context file_path, context
         eval(File.read(file_path), context, file_path)
       end
+
+      # Finish the executor execution
+      def finish
+        raise Error::SafeTerminateError
+      end
     end
   end
 end

data/lib/datahen/scraper/ruby_finisher_executor.rb

@@ -40,6 +40,8 @@ module Datahen
               job_id: job_id
             })
             eval_with_context filename, context
+          rescue Error::SafeTerminateError => e
+            # do nothing, this is fine
           rescue SyntaxError => e
             handle_error(e) if save
             raise e
@@ -55,7 +57,7 @@ module Datahen
             handle_error(e) if save
             raise e
           end
-          
+
           update_finisher_done_status
         end
         proc.call

data/lib/datahen/scraper/ruby_seeder_executor.rb

@@ -44,6 +44,8 @@ module Datahen
               pages: pages
             })
             eval_with_context filename, context
+          rescue Error::SafeTerminateError => e
+            # do nothing, this is fine
           rescue SyntaxError => e
             handle_error(e) if save
             raise e

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "0.14.19"
+  VERSION = "0.14.25"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 0.14.19
+  version: 0.14.25
 platform: ruby
 authors:
 - Parama Danoesubroto
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-10-27 00:00:00.000000000 Z
+date: 2021-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -45,9 +45,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '1.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -55,9 +52,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '1.10'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -253,7 +247,7 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://datahen.com
   source_code_uri: https://github.com/DataHenOfficial/datahen-ruby
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -269,7 +263,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key:
+signing_key: 
 specification_version: 4
 summary: DataHen toolbelt for developers
 test_files: []