datahen 0.14.20 → 0.14.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4a2f75a84728b7e8c228c578e4a4af6253dcd19db445f509ba44f9866847936c
-  data.tar.gz: c37744322cc3e3035a31dd69bca340397ac906d27a3497604e3d98321187d972
+  metadata.gz: dac57d98132102aa9ae8244b6528394473b2bdeb9992c7ea15d6979eaf87d4af
+  data.tar.gz: e68858d2f088b2d7b8538411dd59cf2ae2de7866416fc213c6a6fa009d93c556
 SHA512:
-  metadata.gz: fcf774827b35bf23048b47241da29b1a17e2b1d691a360d2afcb5b34c206c26b7ae316c457e26b946ae3144b4d9075dda5db90a2fd0fb7e56b9f53b0cd612d13
-  data.tar.gz: 56fe24f850c0b695b87c61629ab6939f856ecf8f0ea6b6557d59646214b6ffd87557a6ade6bf59e03b993626663cc1af3071ca10858802272d75d47cf34de56a
+  metadata.gz: 857126b2f7ec4fa058aaa8d5b4a7095108224bdf3f6ece690dbfc930e0527a294853705227f0e63be5af3524982fff21f7d3c9d940c22b31caade5139a3d607b
+  data.tar.gz: 81ecf95378e6f4aa31a87e39a82bc815216fce1b84aa65d8f7f2aa8ee8b19b871f08eb8c86025d9dc8d84617f20864f5f39c21d7b8ac4900a739599c0aa6283c

data/datahen.gemspec

@@ -36,7 +36,7 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.2.2'
   spec.add_dependency "thor", "~> 0.20.3"
   spec.add_dependency 'httparty', '~> 0.16.2'
-  spec.add_dependency 'nokogiri', '~> 1.6', '< 1.10'
+  spec.add_dependency 'nokogiri', '~> 1.6'
   spec.add_development_dependency 'bundler', '>= 1.16'
   spec.add_development_dependency 'rake', '>= 10.0'
   spec.add_development_dependency 'minitest', '>= 5.11'

data/lib/datahen/cli/parser.rb

@@ -14,20 +14,19 @@ module Datahen
       def try_parse(scraper_name, parser_file, gid)
         begin
 
-            if options[:job]
-              job_id = options[:job]
-            elsif options[:global]
-              job_id = nil
-            else
-              job = Client::ScraperJob.new(options).find(scraper_name)
-              job_id = job['id']
-            end
-
+          if options[:job]
+            job_id = options[:job]
+          elsif options[:global]
+            job_id = nil
+          else
+            job = Client::ScraperJob.new(options).find(scraper_name)
+            job_id = job['id']
+          end
 
           vars = JSON.parse(options[:vars]) if options[:vars]
           puts Datahen::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, false, vars, options[:"keep-outputs"])
 
-          rescue JSON::ParserError
+        rescue JSON::ParserError
           if options[:vars]
             puts "Error: #{options[:vars]} on vars is not a valid JSON"
           end

data/lib/datahen/cli/scraper.rb

@@ -93,7 +93,7 @@ module Datahen
       option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Default: 1'
       option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
       option :proxy_type, desc: 'Set the Proxy type. Default: standard'
-      option :vars, type: :string, banner: :JSON, desc: 'Set input vars. Must be in json format. i.e: {"name":"foo", "value":"bar", "secret":false} '
+      option :vars, type: :string, banner: :JSON, desc: 'Set input vars. Must be in json format. i.e: [{"name":"foo", "value":"bar", "secret":false}] '
       def start(scraper_name)
         client = Client::ScraperJob.new(options)
         puts "Starting a scrape job..."

data/lib/datahen/client/backblaze_content.rb

@@ -4,10 +4,10 @@ require 'httparty'
 module Datahen
   module Client
     class BackblazeContent
-      include HTTParty    
-      
+      include HTTParty
+
       def get_content(url)
-        self.class.get(url, format: :plain)
+        self.class.get(url, format: :plain).response.body
       end
 
       def get_gunzipped_content(url)
@@ -19,19 +19,23 @@ module Datahen
         sio = StringIO.new(string)
         gz = Zlib::GzipReader.new(sio, encoding: Encoding::ASCII_8BIT)
         _content = ""
-        begin 
+        begin
           _content = gz.read
         rescue => e
           # if unexpected eof error, then readchar until error, and ignore it
           if e.to_s == 'unexpected end of file'
-            begin 
-              while !gz.eof?
-                _content += gz.readchar
-              end
+            # heavily improve content read recovery by using "String#<<",
+            #  reading all "good" lines and then concat the remaining chars
+            begin
+              gz.each_line{|line| _content << line}
             rescue => e
-              puts "Ignored Zlib error: #{e.to_s}"
+              begin
+                _content << gz.readchar while !gz.eof
+              rescue => e
+                puts "Ignored Zlib error: #{e.to_s}"
+              end
             end
-          else 
+          else
             raise e
           end
         end

data/lib/datahen/scraper/executor.rb

@@ -374,6 +374,11 @@ module Datahen
       def eval_with_context file_path, context
         eval(File.read(file_path), context, file_path)
       end
+
+      # Finish the executor execution
+      def finish
+        raise Error::SafeTerminateError
+      end
     end
   end
 end

data/lib/datahen/scraper/ruby_finisher_executor.rb

@@ -40,6 +40,8 @@ module Datahen
               job_id: job_id
             })
             eval_with_context filename, context
+          rescue Error::SafeTerminateError => e
+            # do nothing, this is fine
           rescue SyntaxError => e
             handle_error(e) if save
             raise e
@@ -55,7 +57,7 @@ module Datahen
             handle_error(e) if save
             raise e
           end
-          
+
           update_finisher_done_status
         end
         proc.call

data/lib/datahen/scraper/ruby_seeder_executor.rb

@@ -44,6 +44,8 @@ module Datahen
               pages: pages
             })
             eval_with_context filename, context
+          rescue Error::SafeTerminateError => e
+            # do nothing, this is fine
           rescue SyntaxError => e
             handle_error(e) if save
             raise e

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "0.14.20"
+  VERSION = "0.14.26"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 0.14.20
+  version: 0.14.26
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-12-14 00:00:00.000000000 Z
+date: 2021-04-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -45,9 +45,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '1.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -55,9 +52,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '1.10'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement