dh_easy-core 0.3.2 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2111211cde8e88762921d1aaeeb9c34129971915376761dfadf0a03c580ed214
-  data.tar.gz: d9d5279b0b66c750eb6311e007abe75dd609fd5d48600bfc484966b1338f4169
+  metadata.gz: 1f2ac7aed05a99e4908c4d69e5d7f523b371fa5bedafa28eb3ea366e79dccbb1
+  data.tar.gz: 0156a723554b7983b390cf06c204c306ac0d7e682c78a5064629983d1c5b47b5
 SHA512:
-  metadata.gz: ff5fb9f2f24eff5d3a35ac80a162be963f5f5f219e142cecbba9e9a49030d3f1742b664a86e72aaf676a8a801cb374d7b2b310cfd8ab187b4a09f63ff5536ee0
-  data.tar.gz: 7895293f6aa71bfd3ef6e83ff3eb4208dbdd184c4bf447023dc3e977c7e276f02ed062806dee3d94cd474800406372b99c41d6985555eb386dca1eac23c49120
+  metadata.gz: e66d2bb8d41916a6c54029050b9665c0d0be2f00b3821dd2d183604f120f33735e685580d689560419a6836d28cbbd3b15b2e3a81ca7c8ca62c76b1b2a0bc5a4
+  data.tar.gz: 13ab9cd4131501c917c4e2bd666dd4ff3f10ad5326dc5fc47b866160ecf0c949c52a5c3d3d0120b5aa01c424f1644177c89e91110ac89fabd4c4adfb534df779

data/lib/dh_easy/core/mock/fake_db.rb

@@ -102,7 +102,8 @@ module DhEasy
         # @return [URI::HTTPS]
         def self.clean_uri_obj raw_url
           url = URI.parse(raw_url)
-          url.hostname = url.hostname.downcase
+          return url if raw_url =~ /^\s*about:blank\s*$/i
+          url.hostname = url.hostname.downcase unless url.hostname.nil?
           url.fragment = nil
 
           # Sort query string keys
@@ -175,6 +176,7 @@ module DhEasy
           return false if driver['pre_code'].to_s.strip != ''
           return false if !driver['stealth'].nil? && !!driver['stealth']
           return false if !driver['enable_images'].nil? && !!driver['enable_images']
+          return false if !driver['disable_adblocker'].nil? && !!driver['disable_adblocker']
           return false if !driver['goto_options'].nil? && driver['goto_options'].is_a?(Hash) && driver['goto_options'].keys.length > 0
           true
         end
@@ -499,33 +501,61 @@ module DhEasy
           @page_defaults ||= {
             'job_id' => lambda{|page| job_id},
             'url' => nil,
+            'effective_url' => nil,
             'status' => 'to_fetch',
+            'hostname' => '',
             'page_type' => 'default',
             'method' => 'GET',
             'headers' => {},
             'fetch_type' => DEFAULT_FETCH_TYPE,
             'cookie' => nil,
+            'no_default_headers' => false,
             'no_redirect' => false,
             'body' => nil,
             'ua_type' => 'desktop',
+            'freshness' => nil,
+            'fresh' => nil,
+            'proxy_type' => '',
             'no_url_encode' => false,
             'http2' => false,
             'priority' => 0,
             'parsing_try_count' => 0,
             'parsing_fail_count' => 0,
+            'parsing_at' => nil,
+            'parsing_failed_at' => nil,
+            'parsed_at' => nil,
             'fetching_at' => '0001-01-01T00:00:00Z',
             'fetching_try_count' => 0,
             'refetch_count' => 0,
+            'to_fetch' => nil,
+            'fetched_at' => nil,
             'fetched_from' => '',
             'content_size' => 0,
+            'content_type' => nil,
             'force_fetch' => false,
+            'response_checksum' => nil,
+            'response_cookie' => nil,
+            'response_headers' => nil,
+            'response_proto' => nil,
+            'response_status' => nil,
+            'response_status_code' => nil,
+            'failed_at' => nil,
+            'failed_content_type' => nil,
+            'failed_effective_url' => nil,
+            'failed_response_checksum' => nil,
+            'failed_response_cookie' => nil,
+            'failed_response_headers' => nil,
+            'failed_response_proto' => nil,
+            'failed_response_status' => nil,
+            'failed_response_status_code' => nil,
             'driver' => {
               'name' => '',
               'pre_code' => '',
               'code' => '',
               'goto_options' => nil,
               'stealth' => false,
-              'enable_images' => false
+              'enable_images' => false,
+              'disable_adblocker' => false
             },
             'display' => {
               'width' => 0,
@@ -536,7 +566,11 @@ module DhEasy
               'options' => nil
             },
             'driver_log' => nil,
-            'vars' => {}
+            'max_size' => 0,
+            'enable_global_cache' => nil,
+            'retry_interval' => nil,
+            'vars' => {},
+            'created_at' => nil,
           }
         end
 
@@ -574,7 +608,7 @@ module DhEasy
             item['headers'] = nil if self.class.is_hash_empty? item['headers']
             item['vars'] = nil if self.class.is_hash_empty? item['vars']
             uri = self.class.clean_uri_obj(item['url'])
-            item['hostname'] = uri.hostname
+            item['hostname'] = (item['url'] =~ /^about:blank$/i) ? '127.0.0.1' : uri.hostname
             uri = nil
             if item['gid'].nil? || !allow_page_gid_override?
               item['gid'] = generate_page_gid item

data/lib/dh_easy/core/mock/fake_finisher.rb

@@ -15,7 +15,10 @@ module DhEasy
             :outputs,
             :save_outputs,
             :find_output,
-            :find_outputs
+            :find_outputs,
+            :get_content,
+            :get_failed_content,
+            :finish
           ]
           DhEasy::Core::Mock::FakeExecutor.check_compatibility real_methods, mock_methods
           mock_methods << :job_id

data/lib/dh_easy/core/mock/fake_parser.rb

@@ -22,7 +22,11 @@ module DhEasy
             :find_output,
             :find_outputs,
             :refetch,
-            :reparse
+            :reparse,
+            :limbo,
+            :get_content,
+            :get_failed_content,
+            :finish
           ].freeze
           DhEasy::Core::Mock::FakeExecutor.check_compatibility real_methods, mock_methods
           mock_methods

data/lib/dh_easy/core/mock/fake_seeder.rb

@@ -17,7 +17,8 @@ module DhEasy
             :save_pages,
             :save_outputs,
             :find_output,
-            :find_outputs
+            :find_outputs,
+            :finish
           ].freeze
           DhEasy::Core::Mock::FakeExecutor.check_compatibility real_methods, mock_methods
           mock_methods

data/lib/dh_easy/core/version.rb

@@ -1,6 +1,6 @@
 module DhEasy
   module Core
     # Gem version
-    VERSION = "0.3.2"
+    VERSION = "0.3.4"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dh_easy-core
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.4
 platform: ruby
 authors:
 - Eduardo Rosales
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-06-06 00:00:00.000000000 Z
+date: 2023-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: datahen