aranha-parsers 0.11.0 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ea4e3ac094b66f5e1b02e6af4f102c752a869e0a234354b0badb3b2d666368c7
-  data.tar.gz: 95de53aed0f9e7157894515f3b3c881c740c6a9d4e2e787d4c5bbb2bb4432294
+  metadata.gz: 2ec41f5c85e077409458835a5cbd8d244a2f74ee4edb5e3c8d02ae9a3919f66b
+  data.tar.gz: 1a64b3f50994e1ed230b36a90861895faea86875963693ff95256d49d563ec09
 SHA512:
-  metadata.gz: 176470a9a8163f44654485f96285c254ecf8d6661f3b435f7af0bc6ccf076c8081cb0554f7b31d77a751f2ce9caa4592454d7b3b5de4cd522a008c2798ecdc7d
-  data.tar.gz: ebc4b32f65dc83d6ba681a0af28babcd8375f4b4c9585985080fc5bcdb5f7bae653979bf21377f10075862c1ab7fa589cca4c1ce0b254b134f1d8d9e7c1f9d3e
+  metadata.gz: cfca3d50399e38f7eccdf8173e8f630ce7b05f255619bc603a65a1a0ca34c5534582b791e1d6e67a5dd91536e053dcb99df1acf32270e7b7ca12724e7fbbe302
+  data.tar.gz: 1deb6e809fe03032f681f79194e058709e822b31771293f6c796ab22ccc543836201771fccfc03ab6e1f3421370eefa1e28787a613972e248cb0ba6d3814bf40

data/lib/aranha/parsers/base.rb

@@ -12,7 +12,7 @@ module Aranha
         def from_content(content)
           ::EacRubyUtils::Fs::Temp.on_file do |path|
             ::File.open(path.to_s, 'w:UTF-8') do |f|
-              f.write content.force_encoding('UTF-8')
+              f.write content.dup.force_encoding('UTF-8')
             end
             r = new(path.to_path)
             r.content

data/lib/aranha/parsers/html/item_list.rb

@@ -10,11 +10,15 @@ module Aranha
           items_data
         end
 
+        def item_data(item)
+          item
+        end
+
         def items_data
           count = 0
           @data ||= nokogiri.xpath(items_xpath).map do |m|
             count += 1
-            node_parser.parse(m)
+            item_data(node_parser.parse(m))
           end
         rescue StandardError => e
           raise StandardError, "#{e.message} (Count: #{count})"

data/lib/aranha/parsers/source_address/hash_http_base.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/source_address/fetch_content_error'
+require 'aranha/parsers/source_address/hash_http_base'
+require 'eac_ruby_utils/core_ext'
+require 'faraday_middleware'
+require 'yaml'
+
+module Aranha
+  module Parsers
+    class SourceAddress
+      class HashHttpBase
+        class << self
+          def http_method
+            const_get 'HTTP_METHOD'
+          end
+
+          def valid_source?(source)
+            source.is_a?(::Hash) &&
+              source.with_indifferent_access[:method].to_s.downcase.strip == http_method.to_s
+          end
+        end
+
+        DEFAULT_BODY = ''
+        DEFAULT_FOLLOW_REDIRECT = true
+        DEFAULT_HEADERS = {}.freeze
+        DEFAULT_PARAMS = {}.freeze
+
+        enable_simple_cache
+
+        common_constructor :source do
+          self.source = source.with_indifferent_access
+        end
+        compare_by :source
+
+        def body
+          param(:body, DEFAULT_BODY)
+        end
+
+        def follow_redirect?
+          param(:follow_redirect, DEFAULT_FOLLOW_REDIRECT)
+        end
+
+        def headers
+          param(:headers, DEFAULT_HEADERS)
+        end
+
+        def url
+          source.fetch(:url)
+        end
+
+        def serialize
+          source.to_yaml
+        end
+
+        # @return [Faraday]
+        def faraday_connection
+          ::Faraday.new do |f|
+            f.response :follow_redirects if follow_redirect?
+          end
+        end
+
+        def content
+          req = faraday_request
+          return req.body if req.status == 200
+
+          raise ::Aranha::Parsers::SourceAddress::FetchContentError,
+                "Get #{url} returned #{req.status.to_i}"
+        end
+
+        def param(key, default_value)
+          source[key] || params[key] || default_value
+        end
+
+        def params
+          source[:params].if_present(DEFAULT_PARAMS)
+        end
+
+        private
+
+        def faraday_request_uncached
+          faraday_connection.send(self.class.http_method, url) do |req|
+            headers.if_present { |v| req.headers = v }
+            body.if_present { |v| req.body = v }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/source_address/hash_http_get.rb

@@ -1,24 +1,12 @@
 # frozen_string_literal: true
 
-require 'aranha/parsers/source_address/hash_http_post'
+require 'aranha/parsers/source_address/hash_http_base'
 
 module Aranha
   module Parsers
     class SourceAddress
-      class HashHttpGet < ::Aranha::Parsers::SourceAddress::HashHttpPost
-        class << self
-          def valid_source?(source)
-            source.is_a?(::Hash) &&
-              source.with_indifferent_access[:method].to_s.downcase.strip == 'get'
-          end
-        end
-
-        def content
-          HTTPClient.new.get_content(
-            source[:url],
-            source[:params]
-          )
-        end
+      class HashHttpGet < ::Aranha::Parsers::SourceAddress::HashHttpBase
+        HTTP_METHOD = :get
       end
     end
   end

data/lib/aranha/parsers/source_address/hash_http_post.rb

@@ -1,44 +1,12 @@
 # frozen_string_literal: true
 
-require 'active_support/core_ext/hash/indifferent_access'
-require 'httpclient'
-require 'yaml'
+require 'aranha/parsers/source_address/hash_http_base'
 
 module Aranha
   module Parsers
     class SourceAddress
-      class HashHttpPost
-        class << self
-          def valid_source?(source)
-            source.is_a?(::Hash) &&
-              source.with_indifferent_access[:method].to_s.downcase.strip == 'post'
-          end
-        end
-
-        attr_reader :source
-
-        def initialize(source)
-          @source = source.with_indifferent_access
-        end
-
-        def ==(other)
-          self.class == other.class && source == other.source
-        end
-
-        def url
-          source.fetch(:url)
-        end
-
-        def serialize
-          source.to_yaml
-        end
-
-        def content
-          HTTPClient.new.post_content(
-            source[:url],
-            source[:params].merge(follow_redirect: true)
-          )
-        end
+      class HashHttpPost < ::Aranha::Parsers::SourceAddress::HashHttpBase
+        HTTP_METHOD = :post
       end
     end
   end

data/lib/aranha/parsers/version.rb

@@ -2,6 +2,6 @@
 
 module Aranha
   module Parsers
-    VERSION = '0.11.0'
+    VERSION = '0.13.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aranha-parsers
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.13.0
 platform: ruby
 authors:
 - Esquilo Azul Company
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-04-06 00:00:00.000000000 Z
+date: 2022-05-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -58,14 +58,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.74'
+        version: '0.92'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.92.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.74'
+        version: '0.92'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.92.1
 - !ruby/object:Gem::Dependency
   name: faraday_middleware
   requirement: !ruby/object:Gem::Requirement
@@ -80,26 +86,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: httpclient
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.8'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.8.3
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.8'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.8.3
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -183,6 +169,7 @@ files:
 - lib/aranha/parsers/source_address.rb
 - lib/aranha/parsers/source_address/fetch_content_error.rb
 - lib/aranha/parsers/source_address/file.rb
+- lib/aranha/parsers/source_address/hash_http_base.rb
 - lib/aranha/parsers/source_address/hash_http_get.rb
 - lib/aranha/parsers/source_address/hash_http_post.rb
 - lib/aranha/parsers/source_address/http_get.rb