aranha-parsers 0.10.0 → 0.12.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f6ec27959167dfbae56e1fb00bfdbedad5bfe022a0708e493c623ca259df5c5d
-  data.tar.gz: c87db56022bfdf05f3f5cc99f58fbba4f0e73a5e8ddb7683d2569aabbf7bf28a
+  metadata.gz: 46d8f5c357e5bedf69c56ee590df3efec25f5eaf814af3456442d9b3d6323e9d
+  data.tar.gz: 1c0f0f1642003a1bb3d742b61f6d1d796fa27bc3518ac72c0bac83cbe43104f8
 SHA512:
-  metadata.gz: 9116a522005c0a01b650bc889ce1a4c70f7096b478febc2e1b24f6596a4d94c77e750d81e616bd70b6ca557a1c61e56d55fcfe2066ff4f62aea3a9a14237ed01
-  data.tar.gz: 5080f643d1da251f05afb094a1c7ac9c5cbbe140a15f0d05b77ec87591e700b148ba4766615ee1853284d24674e2720149f25e6858b485e0123ebd3add74c4db
+  metadata.gz: 61aae78c8114ba83cc4b4d9c81731dd478b1f0df851abcd9a0068309d79488ac4e639cdea78a358aebc66034547211ec46761aaa366b0d48c2cfb4162fe8f26e
+  data.tar.gz: a8fab9c04756953d1fd680f636067f14d4eff15396c99de585da710362a5d88e42bdf83be84169524e821a67fb23fae2617249ed9cf51e54bcaa125927071b37

data/lib/aranha/parsers/base.rb

@@ -12,7 +12,7 @@ module Aranha
         def from_content(content)
           ::EacRubyUtils::Fs::Temp.on_file do |path|
             ::File.open(path.to_s, 'w:UTF-8') do |f|
-              f.write content.force_encoding('UTF-8')
+              f.write content.dup.force_encoding('UTF-8')
             end
             r = new(path.to_path)
             r.content
@@ -37,21 +37,21 @@ module Aranha
       delegate :url, to: :source_address
 
       def content
-        @content ||= begin
-          s = source_address.content
-          log_content(s)
-          s
-        end
+        @content ||= log_content(source_address_content)
       end
 
+      # @return [String]
+      delegate :content, to: :source_address, prefix: true
+
       private
 
+      # @return [String]
       def log_content(content, suffix = '')
         path = log_file(suffix)
 
-        return unless path
+        File.open(path, 'wb') { |file| file.write(content) } if path
 
-        File.open(path, 'wb') { |file| file.write(content) }
+        content
       end
 
       def log_file(suffix)

data/lib/aranha/parsers/html/item_list.rb

@@ -10,11 +10,15 @@ module Aranha
           items_data
         end
 
+        def item_data(item)
+          item
+        end
+
         def items_data
           count = 0
           @data ||= nokogiri.xpath(items_xpath).map do |m|
             count += 1
-            node_parser.parse(m)
+            item_data(node_parser.parse(m))
           end
         rescue StandardError => e
           raise StandardError, "#{e.message} (Count: #{count})"

data/lib/aranha/parsers/source_address/hash_http_base.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/source_address/fetch_content_error'
+require 'aranha/parsers/source_address/hash_http_base'
+require 'eac_ruby_utils/core_ext'
+require 'faraday_middleware'
+require 'yaml'
+
+module Aranha
+  module Parsers
+    class SourceAddress
+      class HashHttpBase
+        class << self
+          def http_method
+            const_get 'HTTP_METHOD'
+          end
+
+          def valid_source?(source)
+            source.is_a?(::Hash) &&
+              source.with_indifferent_access[:method].to_s.downcase.strip == http_method.to_s
+          end
+        end
+
+        DEFAULT_BODY = ''
+        DEFAULT_FOLLOW_REDIRECT = true
+        DEFAULT_HEADERS = {}.freeze
+        DEFAULT_PARAMS = {}.freeze
+
+        common_constructor :source do
+          self.source = source.with_indifferent_access
+        end
+        compare_by :source
+
+        def body
+          param(:body, DEFAULT_BODY)
+        end
+
+        def follow_redirect?
+          param(:follow_redirect, DEFAULT_FOLLOW_REDIRECT)
+        end
+
+        def headers
+          param(:headers, DEFAULT_HEADERS)
+        end
+
+        def url
+          source.fetch(:url)
+        end
+
+        def serialize
+          source.to_yaml
+        end
+
+        # @return [Faraday]
+        def faraday_connection
+          ::Faraday.new do |f|
+            f.response :follow_redirects if follow_redirect?
+          end
+        end
+
+        def faraday_request
+          faraday_connection.send(self.class.http_method, url) do |req|
+            headers.if_present { |v| req.headers = v }
+            body.if_present { |v| req.body = v }
+          end
+        end
+
+        def content
+          req = faraday_request
+          return req.body if req.status == 200
+
+          raise ::Aranha::Parsers::SourceAddress::FetchContentError,
+                "Get #{url} returned #{req.status.to_i}"
+        end
+
+        def param(key, default_value)
+          source[key] || params[key] || default_value
+        end
+
+        def params
+          source[:params].if_present(DEFAULT_PARAMS)
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/source_address/hash_http_get.rb

@@ -1,24 +1,12 @@
 # frozen_string_literal: true
 
-require 'aranha/parsers/source_address/hash_http_post'
+require 'aranha/parsers/source_address/hash_http_base'
 
 module Aranha
   module Parsers
     class SourceAddress
-      class HashHttpGet < ::Aranha::Parsers::SourceAddress::HashHttpPost
-        class << self
-          def valid_source?(source)
-            source.is_a?(::Hash) &&
-              source.with_indifferent_access[:method].to_s.downcase.strip == 'get'
-          end
-        end
-
-        def content
-          HTTPClient.new.get_content(
-            source[:url],
-            source[:params]
-          )
-        end
+      class HashHttpGet < ::Aranha::Parsers::SourceAddress::HashHttpBase
+        HTTP_METHOD = :get
       end
     end
   end

data/lib/aranha/parsers/source_address/hash_http_post.rb

@@ -1,44 +1,12 @@
 # frozen_string_literal: true
 
-require 'active_support/core_ext/hash/indifferent_access'
-require 'httpclient'
-require 'yaml'
+require 'aranha/parsers/source_address/hash_http_base'
 
 module Aranha
   module Parsers
     class SourceAddress
-      class HashHttpPost
-        class << self
-          def valid_source?(source)
-            source.is_a?(::Hash) &&
-              source.with_indifferent_access[:method].to_s.downcase.strip == 'post'
-          end
-        end
-
-        attr_reader :source
-
-        def initialize(source)
-          @source = source.with_indifferent_access
-        end
-
-        def ==(other)
-          self.class == other.class && source == other.source
-        end
-
-        def url
-          source.fetch(:url)
-        end
-
-        def serialize
-          source.to_yaml
-        end
-
-        def content
-          HTTPClient.new.post_content(
-            source[:url],
-            source[:params].merge(follow_redirect: true)
-          )
-        end
+      class HashHttpPost < ::Aranha::Parsers::SourceAddress::HashHttpBase
+        HTTP_METHOD = :post
       end
     end
   end

data/lib/aranha/parsers/version.rb

@@ -2,6 +2,6 @@
 
 module Aranha
   module Parsers
-    VERSION = '0.10.0'
+    VERSION = '0.12.1'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aranha-parsers
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.12.1
 platform: ruby
 authors:
 - Esquilo Azul Company
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-27 00:00:00.000000000 Z
+date: 2022-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -58,14 +58,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.74'
+        version: '0.92'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.92.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.74'
+        version: '0.92'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.92.1
 - !ruby/object:Gem::Dependency
   name: faraday_middleware
   requirement: !ruby/object:Gem::Requirement
@@ -80,26 +86,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: httpclient
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.8'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.8.3
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.8'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.8.3
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -183,6 +169,7 @@ files:
 - lib/aranha/parsers/source_address.rb
 - lib/aranha/parsers/source_address/fetch_content_error.rb
 - lib/aranha/parsers/source_address/file.rb
+- lib/aranha/parsers/source_address/hash_http_base.rb
 - lib/aranha/parsers/source_address/hash_http_get.rb
 - lib/aranha/parsers/source_address/hash_http_post.rb
 - lib/aranha/parsers/source_address/http_get.rb