aranha-parsers 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/aranha/parsers/base.rb +1 -1
- data/lib/aranha/parsers/source_address/hash_http_base.rb +13 -7
- data/lib/aranha/parsers/version.rb +1 -1
- metadata +16 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 93e8bef4141b178291abc10e4d6c3fd6fdf4ad6a6e51191fef0839e041830c06
|
|
4
|
+
data.tar.gz: 92d9b0532f8bba3920c02a66910f59d98abb1aebc28f3d13bbbc94a835c74994
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: ec41a5338e09bb46fc69722cabc00b6aa3705443db6d7cbed003161042c3f111d923940789242ecd215c5ad853c70d8e5feb8513d391f555d2d916f38b7dd00b
|
|
7
|
+
data.tar.gz: 06d67ce47eff7cd0d7444ac2656d3a611cfd3615435410da2f3c6a28fa8e07e591fb70f361efd2295d0bce58808c2f07986004084984da2806d85b6f354c75fa
|
data/lib/aranha/parsers/base.rb
CHANGED
|
@@ -12,7 +12,7 @@ module Aranha
|
|
|
12
12
|
def from_content(content)
|
|
13
13
|
::EacRubyUtils::Fs::Temp.on_file do |path|
|
|
14
14
|
::File.open(path.to_s, 'w:UTF-8') do |f|
|
|
15
|
-
f.write content.force_encoding('UTF-8')
|
|
15
|
+
f.write content.dup.force_encoding('UTF-8')
|
|
16
16
|
end
|
|
17
17
|
r = new(path.to_path)
|
|
18
18
|
r.content
|
|
@@ -4,6 +4,7 @@ require 'aranha/parsers/source_address/fetch_content_error'
|
|
|
4
4
|
require 'aranha/parsers/source_address/hash_http_base'
|
|
5
5
|
require 'eac_ruby_utils/core_ext'
|
|
6
6
|
require 'faraday_middleware'
|
|
7
|
+
require 'faraday/gzip'
|
|
7
8
|
require 'yaml'
|
|
8
9
|
|
|
9
10
|
module Aranha
|
|
@@ -26,6 +27,8 @@ module Aranha
|
|
|
26
27
|
DEFAULT_HEADERS = {}.freeze
|
|
27
28
|
DEFAULT_PARAMS = {}.freeze
|
|
28
29
|
|
|
30
|
+
enable_simple_cache
|
|
31
|
+
|
|
29
32
|
common_constructor :source do
|
|
30
33
|
self.source = source.with_indifferent_access
|
|
31
34
|
end
|
|
@@ -54,17 +57,11 @@ module Aranha
|
|
|
54
57
|
# @return [Faraday]
|
|
55
58
|
def faraday_connection
|
|
56
59
|
::Faraday.new do |f|
|
|
60
|
+
f.request :gzip
|
|
57
61
|
f.response :follow_redirects if follow_redirect?
|
|
58
62
|
end
|
|
59
63
|
end
|
|
60
64
|
|
|
61
|
-
def faraday_request
|
|
62
|
-
faraday_connection.send(self.class.http_method, url) do |req|
|
|
63
|
-
headers.if_present { |v| req.headers = v }
|
|
64
|
-
body.if_present { |v| req.body = v }
|
|
65
|
-
end
|
|
66
|
-
end
|
|
67
|
-
|
|
68
65
|
def content
|
|
69
66
|
req = faraday_request
|
|
70
67
|
return req.body if req.status == 200
|
|
@@ -80,6 +77,15 @@ module Aranha
|
|
|
80
77
|
def params
|
|
81
78
|
source[:params].if_present(DEFAULT_PARAMS)
|
|
82
79
|
end
|
|
80
|
+
|
|
81
|
+
private
|
|
82
|
+
|
|
83
|
+
def faraday_request_uncached
|
|
84
|
+
faraday_connection.send(self.class.http_method, url) do |req|
|
|
85
|
+
headers.if_present { |v| req.headers = v }
|
|
86
|
+
body.if_present { |v| req.body = v }
|
|
87
|
+
end
|
|
88
|
+
end
|
|
83
89
|
end
|
|
84
90
|
end
|
|
85
91
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: aranha-parsers
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.14.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Esquilo Azul Company
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2022-05-
|
|
11
|
+
date: 2022-05-21 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -72,6 +72,20 @@ dependencies:
|
|
|
72
72
|
- - ">="
|
|
73
73
|
- !ruby/object:Gem::Version
|
|
74
74
|
version: 0.92.1
|
|
75
|
+
- !ruby/object:Gem::Dependency
|
|
76
|
+
name: faraday-gzip
|
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
|
78
|
+
requirements:
|
|
79
|
+
- - "~>"
|
|
80
|
+
- !ruby/object:Gem::Version
|
|
81
|
+
version: '0.1'
|
|
82
|
+
type: :runtime
|
|
83
|
+
prerelease: false
|
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
85
|
+
requirements:
|
|
86
|
+
- - "~>"
|
|
87
|
+
- !ruby/object:Gem::Version
|
|
88
|
+
version: '0.1'
|
|
75
89
|
- !ruby/object:Gem::Dependency
|
|
76
90
|
name: faraday_middleware
|
|
77
91
|
requirement: !ruby/object:Gem::Requirement
|